jemalloc: Import 5.3.0 54eaed1d8b56b1aa528be3bdd1877e59c56fa90cvendor/jemalloc/5.3.0vendor/jemalloc

Import jemalloc 5.3.0.

This import changes how manage the jemalloc vendor branch (which was
just started anyway). Starting with 5.3.0, we import a clean tree from
the upstream github, removing all the old files that are no longer
upstream, or that we've kept around for some reason. We do this because
we merge from this raw version of jemalloc into the FreeBSD
contrib/jemalloc, then we run autogen stuff, generate all the generated
.h files with gmake, then finally remove much of the generated files in
contrib/jemalloc using an update script.

Sponsored by:		Netflix

1 files changed, 143 insertions, 86 deletions

diff --git a/test/unit/stats.c b/test/unit/stats.c
index 646768e881da..bbdbd1809af4 100644
--- a/test/unit/stats.c
+++ b/test/unit/stats.c
@@ -1,25 +1,28 @@
 #include "test/jemalloc_test.h"
 
+#define STRINGIFY_HELPER(x) #x
+#define STRINGIFY(x) STRINGIFY_HELPER(x)
+
 TEST_BEGIN(test_stats_summary) {
 	size_t sz, allocated, active, resident, mapped;
 	int expected = config_stats ? 0 : ENOENT;
 
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("stats.allocated", (void *)&allocated, &sz, NULL,
+	expect_d_eq(mallctl("stats.allocated", (void *)&allocated, &sz, NULL,
 	    0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.active", (void *)&active, &sz, NULL, 0),
+	expect_d_eq(mallctl("stats.active", (void *)&active, &sz, NULL, 0),
 	    expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.resident", (void *)&resident, &sz, NULL, 0),
+	expect_d_eq(mallctl("stats.resident", (void *)&resident, &sz, NULL, 0),
 	    expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.mapped", (void *)&mapped, &sz, NULL, 0),
+	expect_d_eq(mallctl("stats.mapped", (void *)&mapped, &sz, NULL, 0),
 	    expected, "Unexpected mallctl() result");
 
 	if (config_stats) {
-		assert_zu_le(allocated, active,
+		expect_zu_le(allocated, active,
 		    "allocated should be no larger than active");
-		assert_zu_lt(active, resident,
+		expect_zu_lt(active, resident,
 		    "active should be less than resident");
-		assert_zu_lt(active, mapped,
+		expect_zu_lt(active, mapped,
 		    "active should be less than mapped");
 	}
 }
@@ -34,30 +37,30 @@ TEST_BEGIN(test_stats_large) {
 	int expected = config_stats ? 0 : ENOENT;
 
 	p = mallocx(SC_SMALL_MAXCLASS + 1, MALLOCX_ARENA(0));
-	assert_ptr_not_null(p, "Unexpected mallocx() failure");
+	expect_ptr_not_null(p, "Unexpected mallocx() failure");
 
-	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
+	expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
 	    0, "Unexpected mallctl() failure");
 
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("stats.arenas.0.large.allocated",
+	expect_d_eq(mallctl("stats.arenas.0.large.allocated",
 	    (void *)&allocated, &sz, NULL, 0), expected,
 	    "Unexpected mallctl() result");
 	sz = sizeof(uint64_t);
-	assert_d_eq(mallctl("stats.arenas.0.large.nmalloc", (void *)&nmalloc,
+	expect_d_eq(mallctl("stats.arenas.0.large.nmalloc", (void *)&nmalloc,
 	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.large.ndalloc", (void *)&ndalloc,
+	expect_d_eq(mallctl("stats.arenas.0.large.ndalloc", (void *)&ndalloc,
 	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.large.nrequests",
+	expect_d_eq(mallctl("stats.arenas.0.large.nrequests",
 	    (void *)&nrequests, &sz, NULL, 0), expected,
 	    "Unexpected mallctl() result");
 
 	if (config_stats) {
-		assert_zu_gt(allocated, 0,
+		expect_zu_gt(allocated, 0,
 		    "allocated should be greater than zero");
-		assert_u64_ge(nmalloc, ndalloc,
+		expect_u64_ge(nmalloc, ndalloc,
 		    "nmalloc should be at least as large as ndalloc");
-		assert_u64_le(nmalloc, nrequests,
+		expect_u64_le(nmalloc, nrequests,
 		    "nmalloc should no larger than nrequests");
 	}
 
@@ -75,54 +78,54 @@ TEST_BEGIN(test_stats_arenas_summary) {
 	uint64_t muzzy_npurge, muzzy_nmadvise, muzzy_purged;
 
 	little = mallocx(SC_SMALL_MAXCLASS, MALLOCX_ARENA(0));
-	assert_ptr_not_null(little, "Unexpected mallocx() failure");
+	expect_ptr_not_null(little, "Unexpected mallocx() failure");
 	large = mallocx((1U << SC_LG_LARGE_MINCLASS),
 	    MALLOCX_ARENA(0));
-	assert_ptr_not_null(large, "Unexpected mallocx() failure");
+	expect_ptr_not_null(large, "Unexpected mallocx() failure");
 
 	dallocx(little, 0);
 	dallocx(large, 0);
 
-	assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0),
+	expect_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0),
 	    opt_tcache ? 0 : EFAULT, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0,
+	expect_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0,
 	    "Unexpected mallctl() failure");
 
-	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
+	expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
 	    0, "Unexpected mallctl() failure");
 
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("stats.arenas.0.mapped", (void *)&mapped, &sz, NULL,
+	expect_d_eq(mallctl("stats.arenas.0.mapped", (void *)&mapped, &sz, NULL,
 	    0), expected, "Unexepected mallctl() result");
 
 	sz = sizeof(uint64_t);
-	assert_d_eq(mallctl("stats.arenas.0.dirty_npurge",
+	expect_d_eq(mallctl("stats.arenas.0.dirty_npurge",
 	    (void *)&dirty_npurge, &sz, NULL, 0), expected,
 	    "Unexepected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.dirty_nmadvise",
+	expect_d_eq(mallctl("stats.arenas.0.dirty_nmadvise",
 	    (void *)&dirty_nmadvise, &sz, NULL, 0), expected,
 	    "Unexepected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.dirty_purged",
+	expect_d_eq(mallctl("stats.arenas.0.dirty_purged",
 	    (void *)&dirty_purged, &sz, NULL, 0), expected,
 	    "Unexepected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.muzzy_npurge",
+	expect_d_eq(mallctl("stats.arenas.0.muzzy_npurge",
 	    (void *)&muzzy_npurge, &sz, NULL, 0), expected,
 	    "Unexepected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.muzzy_nmadvise",
+	expect_d_eq(mallctl("stats.arenas.0.muzzy_nmadvise",
 	    (void *)&muzzy_nmadvise, &sz, NULL, 0), expected,
 	    "Unexepected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.muzzy_purged",
+	expect_d_eq(mallctl("stats.arenas.0.muzzy_purged",
 	    (void *)&muzzy_purged, &sz, NULL, 0), expected,
 	    "Unexepected mallctl() result");
 
 	if (config_stats) {
-		if (!background_thread_enabled()) {
-			assert_u64_gt(dirty_npurge + muzzy_npurge, 0,
+		if (!is_background_thread_enabled() && !opt_hpa) {
+			expect_u64_gt(dirty_npurge + muzzy_npurge, 0,
 			    "At least one purge should have occurred");
 		}
-		assert_u64_le(dirty_nmadvise, dirty_purged,
+		expect_u64_le(dirty_nmadvise, dirty_purged,
 		    "dirty_nmadvise should be no greater than dirty_purged");
-		assert_u64_le(muzzy_nmadvise, muzzy_purged,
+		expect_u64_le(muzzy_nmadvise, muzzy_purged,
 		    "muzzy_nmadvise should be no greater than muzzy_purged");
 	}
 }
@@ -150,35 +153,35 @@ TEST_BEGIN(test_stats_arenas_small) {
 	no_lazy_lock(); /* Lazy locking would dodge tcache testing. */
 
 	p = mallocx(SC_SMALL_MAXCLASS, MALLOCX_ARENA(0));
-	assert_ptr_not_null(p, "Unexpected mallocx() failure");
+	expect_ptr_not_null(p, "Unexpected mallocx() failure");
 
-	assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0),
+	expect_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0),
 	    opt_tcache ? 0 : EFAULT, "Unexpected mallctl() result");
 
-	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
+	expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
 	    0, "Unexpected mallctl() failure");
 
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("stats.arenas.0.small.allocated",
+	expect_d_eq(mallctl("stats.arenas.0.small.allocated",
 	    (void *)&allocated, &sz, NULL, 0), expected,
 	    "Unexpected mallctl() result");
 	sz = sizeof(uint64_t);
-	assert_d_eq(mallctl("stats.arenas.0.small.nmalloc", (void *)&nmalloc,
+	expect_d_eq(mallctl("stats.arenas.0.small.nmalloc", (void *)&nmalloc,
 	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.small.ndalloc", (void *)&ndalloc,
+	expect_d_eq(mallctl("stats.arenas.0.small.ndalloc", (void *)&ndalloc,
 	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.small.nrequests",
+	expect_d_eq(mallctl("stats.arenas.0.small.nrequests",
 	    (void *)&nrequests, &sz, NULL, 0), expected,
 	    "Unexpected mallctl() result");
 
 	if (config_stats) {
-		assert_zu_gt(allocated, 0,
+		expect_zu_gt(allocated, 0,
 		    "allocated should be greater than zero");
-		assert_u64_gt(nmalloc, 0,
+		expect_u64_gt(nmalloc, 0,
 		    "nmalloc should be no greater than zero");
-		assert_u64_ge(nmalloc, ndalloc,
+		expect_u64_ge(nmalloc, ndalloc,
 		    "nmalloc should be at least as large as ndalloc");
-		assert_u64_gt(nrequests, 0,
+		expect_u64_gt(nrequests, 0,
 		    "nrequests should be greater than zero");
 	}
 
@@ -193,27 +196,27 @@ TEST_BEGIN(test_stats_arenas_large) {
 	int expected = config_stats ? 0 : ENOENT;
 
 	p = mallocx((1U << SC_LG_LARGE_MINCLASS), MALLOCX_ARENA(0));
-	assert_ptr_not_null(p, "Unexpected mallocx() failure");
+	expect_ptr_not_null(p, "Unexpected mallocx() failure");
 
-	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
+	expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
 	    0, "Unexpected mallctl() failure");
 
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("stats.arenas.0.large.allocated",
+	expect_d_eq(mallctl("stats.arenas.0.large.allocated",
 	    (void *)&allocated, &sz, NULL, 0), expected,
 	    "Unexpected mallctl() result");
 	sz = sizeof(uint64_t);
-	assert_d_eq(mallctl("stats.arenas.0.large.nmalloc", (void *)&nmalloc,
+	expect_d_eq(mallctl("stats.arenas.0.large.nmalloc", (void *)&nmalloc,
 	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.large.ndalloc", (void *)&ndalloc,
+	expect_d_eq(mallctl("stats.arenas.0.large.ndalloc", (void *)&ndalloc,
 	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
 
 	if (config_stats) {
-		assert_zu_gt(allocated, 0,
+		expect_zu_gt(allocated, 0,
 		    "allocated should be greater than zero");
-		assert_u64_gt(nmalloc, 0,
+		expect_u64_gt(nmalloc, 0,
 		    "nmalloc should be greater than zero");
-		assert_u64_ge(nmalloc, ndalloc,
+		expect_u64_ge(nmalloc, ndalloc,
 		    "nmalloc should be at least as large as ndalloc");
 	}
 
@@ -234,85 +237,85 @@ TEST_BEGIN(test_stats_arenas_bins) {
 	int expected = config_stats ? 0 : ENOENT;
 
 	/* Make sure allocation below isn't satisfied by tcache. */
-	assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0),
+	expect_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0),
 	    opt_tcache ? 0 : EFAULT, "Unexpected mallctl() result");
 
 	unsigned arena_ind, old_arena_ind;
 	sz = sizeof(unsigned);
-	assert_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0),
+	expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0),
 	    0, "Arena creation failure");
 	sz = sizeof(arena_ind);
-	assert_d_eq(mallctl("thread.arena", (void *)&old_arena_ind, &sz,
+	expect_d_eq(mallctl("thread.arena", (void *)&old_arena_ind, &sz,
 	    (void *)&arena_ind, sizeof(arena_ind)), 0,
 	    "Unexpected mallctl() failure");
 
 	p = malloc(bin_infos[0].reg_size);
-	assert_ptr_not_null(p, "Unexpected malloc() failure");
+	expect_ptr_not_null(p, "Unexpected malloc() failure");
 
-	assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0),
+	expect_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0),
 	    opt_tcache ? 0 : EFAULT, "Unexpected mallctl() result");
 
-	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
+	expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
 	    0, "Unexpected mallctl() failure");
 
 	char cmd[128];
 	sz = sizeof(uint64_t);
 	gen_mallctl_str(cmd, "nmalloc", arena_ind);
-	assert_d_eq(mallctl(cmd, (void *)&nmalloc, &sz, NULL, 0), expected,
+	expect_d_eq(mallctl(cmd, (void *)&nmalloc, &sz, NULL, 0), expected,
 	    "Unexpected mallctl() result");
 	gen_mallctl_str(cmd, "ndalloc", arena_ind);
-	assert_d_eq(mallctl(cmd, (void *)&ndalloc, &sz, NULL, 0), expected,
+	expect_d_eq(mallctl(cmd, (void *)&ndalloc, &sz, NULL, 0), expected,
 	    "Unexpected mallctl() result");
 	gen_mallctl_str(cmd, "nrequests", arena_ind);
-	assert_d_eq(mallctl(cmd, (void *)&nrequests, &sz, NULL, 0), expected,
+	expect_d_eq(mallctl(cmd, (void *)&nrequests, &sz, NULL, 0), expected,
 	    "Unexpected mallctl() result");
 	sz = sizeof(size_t);
 	gen_mallctl_str(cmd, "curregs", arena_ind);
-	assert_d_eq(mallctl(cmd, (void *)&curregs, &sz, NULL, 0), expected,
+	expect_d_eq(mallctl(cmd, (void *)&curregs, &sz, NULL, 0), expected,
 	    "Unexpected mallctl() result");
 
 	sz = sizeof(uint64_t);
 	gen_mallctl_str(cmd, "nfills", arena_ind);
-	assert_d_eq(mallctl(cmd, (void *)&nfills, &sz, NULL, 0), expected,
+	expect_d_eq(mallctl(cmd, (void *)&nfills, &sz, NULL, 0), expected,
 	    "Unexpected mallctl() result");
 	gen_mallctl_str(cmd, "nflushes", arena_ind);
-	assert_d_eq(mallctl(cmd, (void *)&nflushes, &sz, NULL, 0), expected,
+	expect_d_eq(mallctl(cmd, (void *)&nflushes, &sz, NULL, 0), expected,
 	    "Unexpected mallctl() result");
 
 	gen_mallctl_str(cmd, "nslabs", arena_ind);
-	assert_d_eq(mallctl(cmd, (void *)&nslabs, &sz, NULL, 0), expected,
+	expect_d_eq(mallctl(cmd, (void *)&nslabs, &sz, NULL, 0), expected,
 	    "Unexpected mallctl() result");
 	gen_mallctl_str(cmd, "nreslabs", arena_ind);
-	assert_d_eq(mallctl(cmd, (void *)&nreslabs, &sz, NULL, 0), expected,
+	expect_d_eq(mallctl(cmd, (void *)&nreslabs, &sz, NULL, 0), expected,
 	    "Unexpected mallctl() result");
 	sz = sizeof(size_t);
 	gen_mallctl_str(cmd, "curslabs", arena_ind);
-	assert_d_eq(mallctl(cmd, (void *)&curslabs, &sz, NULL, 0), expected,
+	expect_d_eq(mallctl(cmd, (void *)&curslabs, &sz, NULL, 0), expected,
 	    "Unexpected mallctl() result");
 	gen_mallctl_str(cmd, "nonfull_slabs", arena_ind);
-	assert_d_eq(mallctl(cmd, (void *)&nonfull_slabs, &sz, NULL, 0),
+	expect_d_eq(mallctl(cmd, (void *)&nonfull_slabs, &sz, NULL, 0),
 	    expected, "Unexpected mallctl() result");
 
 	if (config_stats) {
-		assert_u64_gt(nmalloc, 0,
+		expect_u64_gt(nmalloc, 0,
 		    "nmalloc should be greater than zero");
-		assert_u64_ge(nmalloc, ndalloc,
+		expect_u64_ge(nmalloc, ndalloc,
 		    "nmalloc should be at least as large as ndalloc");
-		assert_u64_gt(nrequests, 0,
+		expect_u64_gt(nrequests, 0,
 		    "nrequests should be greater than zero");
-		assert_zu_gt(curregs, 0,
+		expect_zu_gt(curregs, 0,
 		    "allocated should be greater than zero");
 		if (opt_tcache) {
-			assert_u64_gt(nfills, 0,
+			expect_u64_gt(nfills, 0,
 			    "At least one fill should have occurred");
-			assert_u64_gt(nflushes, 0,
+			expect_u64_gt(nflushes, 0,
 			    "At least one flush should have occurred");
 		}
-		assert_u64_gt(nslabs, 0,
+		expect_u64_gt(nslabs, 0,
 		    "At least one slab should have been allocated");
-		assert_zu_gt(curslabs, 0,
+		expect_zu_gt(curslabs, 0,
 		    "At least one slab should be currently allocated");
-		assert_zu_eq(nonfull_slabs, 0,
+		expect_zu_eq(nonfull_slabs, 0,
 		    "slabs_nonfull should be empty");
 	}
 
@@ -327,33 +330,33 @@ TEST_BEGIN(test_stats_arenas_lextents) {
 	int expected = config_stats ? 0 : ENOENT;
 
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("arenas.lextent.0.size", (void *)&hsize, &sz, NULL,
+	expect_d_eq(mallctl("arenas.lextent.0.size", (void *)&hsize, &sz, NULL,
 	    0), 0, "Unexpected mallctl() failure");
 
 	p = mallocx(hsize, MALLOCX_ARENA(0));
-	assert_ptr_not_null(p, "Unexpected mallocx() failure");
+	expect_ptr_not_null(p, "Unexpected mallocx() failure");
 
-	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
+	expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
 	    0, "Unexpected mallctl() failure");
 
 	sz = sizeof(uint64_t);
-	assert_d_eq(mallctl("stats.arenas.0.lextents.0.nmalloc",
+	expect_d_eq(mallctl("stats.arenas.0.lextents.0.nmalloc",
 	    (void *)&nmalloc, &sz, NULL, 0), expected,
 	    "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.lextents.0.ndalloc",
+	expect_d_eq(mallctl("stats.arenas.0.lextents.0.ndalloc",
 	    (void *)&ndalloc, &sz, NULL, 0), expected,
 	    "Unexpected mallctl() result");
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("stats.arenas.0.lextents.0.curlextents",
+	expect_d_eq(mallctl("stats.arenas.0.lextents.0.curlextents",
 	    (void *)&curlextents, &sz, NULL, 0), expected,
 	    "Unexpected mallctl() result");
 
 	if (config_stats) {
-		assert_u64_gt(nmalloc, 0,
+		expect_u64_gt(nmalloc, 0,
 		    "nmalloc should be greater than zero");
-		assert_u64_ge(nmalloc, ndalloc,
+		expect_u64_ge(nmalloc, ndalloc,
 		    "nmalloc should be at least as large as ndalloc");
-		assert_u64_gt(curlextents, 0,
+		expect_u64_gt(curlextents, 0,
 		    "At least one extent should be currently allocated");
 	}
 
@@ -361,6 +364,58 @@ TEST_BEGIN(test_stats_arenas_lextents) {
 }
 TEST_END
 
+static void
+test_tcache_bytes_for_usize(size_t usize) {
+	uint64_t epoch;
+	size_t tcache_bytes, tcache_stashed_bytes;
+	size_t sz = sizeof(tcache_bytes);
+
+	void *ptr = mallocx(usize, 0);
+
+	expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
+	    0, "Unexpected mallctl() failure");
+	assert_d_eq(mallctl(
+	    "stats.arenas." STRINGIFY(MALLCTL_ARENAS_ALL) ".tcache_bytes",
+	    &tcache_bytes, &sz, NULL, 0), 0, "Unexpected mallctl failure");
+	assert_d_eq(mallctl(
+	    "stats.arenas." STRINGIFY(MALLCTL_ARENAS_ALL)
+	    ".tcache_stashed_bytes", &tcache_stashed_bytes, &sz, NULL, 0), 0,
+	    "Unexpected mallctl failure");
+	size_t tcache_bytes_before = tcache_bytes + tcache_stashed_bytes;
+	dallocx(ptr, 0);
+
+	expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
+	    0, "Unexpected mallctl() failure");
+	assert_d_eq(mallctl(
+	    "stats.arenas." STRINGIFY(MALLCTL_ARENAS_ALL) ".tcache_bytes",
+	    &tcache_bytes, &sz, NULL, 0), 0, "Unexpected mallctl failure");
+	assert_d_eq(mallctl(
+	    "stats.arenas." STRINGIFY(MALLCTL_ARENAS_ALL)
+	    ".tcache_stashed_bytes", &tcache_stashed_bytes, &sz, NULL, 0), 0,
+	    "Unexpected mallctl failure");
+	size_t tcache_bytes_after = tcache_bytes + tcache_stashed_bytes;
+	assert_zu_eq(tcache_bytes_after - tcache_bytes_before,
+	    usize, "Incorrectly attributed a free");
+}
+
+TEST_BEGIN(test_stats_tcache_bytes_small) {
+	test_skip_if(!config_stats);
+	test_skip_if(!opt_tcache);
+	test_skip_if(opt_tcache_max < SC_SMALL_MAXCLASS);
+
+	test_tcache_bytes_for_usize(SC_SMALL_MAXCLASS);
+}
+TEST_END
+
+TEST_BEGIN(test_stats_tcache_bytes_large) {
+	test_skip_if(!config_stats);
+	test_skip_if(!opt_tcache);
+	test_skip_if(opt_tcache_max < SC_LARGE_MINCLASS);
+
+	test_tcache_bytes_for_usize(SC_LARGE_MINCLASS);
+}
+TEST_END
+
 int
 main(void) {
 	return test_no_reentrancy(
@@ -370,5 +425,7 @@ main(void) {
 	    test_stats_arenas_small,
 	    test_stats_arenas_large,
 	    test_stats_arenas_bins,
-	    test_stats_arenas_lextents);
+	    test_stats_arenas_lextents,
+	    test_stats_tcache_bytes_small,
+	    test_stats_tcache_bytes_large);
 }