Loading...
--- libmalloc/libmalloc-657.60.21/src/magazine_malloc.c
+++ libmalloc/libmalloc-140.1.1/src/magazine_malloc.c
@@ -26,12 +26,10 @@
/*
* Multithread enhancements for "tiny" allocations introduced February 2008.
* These are in the spirit of "Hoard". See:
- * Emery D. Berger, Kathryn S. McKinley, Robert D. Blumofe, and Paul R. Wilson. 2000.
- * Hoard: a scalable memory allocator for multithreaded applications.
- * In Proceedings of the ninth international conference on Architectural support for
- * programming languages and operating systems (ASPLOS IX).
- * ACM, New York, NY, USA, 117-128.
- * DOI: https://doi.org/10.1145/378993.379232
+ * Berger, E.D.; McKinley, K.S.; Blumofe, R.D.; Wilson, P.R. (2000).
+ * "Hoard: a scalable memory allocator for multithreaded applications".
+ * ACM SIGPLAN Notices 35 (11): 117-128. Berger2000.
+ * <http://portal.acm.org/citation.cfm?id=356989.357000>
* Retrieved on 2008-02-22.
*/
@@ -43,83 +41,101 @@
#define LOG(szone, ptr) 0
#endif
-// Maximum number of magazines, set from the number of logical CPUS and
-// possibly limited by the MallocMaxMagazines environment variable.
-int max_magazines;
-
-// Control whether medium is enabled at all when creating new magazine zones
-bool magazine_medium_enabled = DEFAULT_MEDIUM_ALLOCATOR_ENABLED;
-
-// Control the DRAM limit at which medium kicks in.
-uint64_t magazine_medium_active_threshold = MEDIUM_ACTIVATION_THRESHOLD;
-
-#if CONFIG_MEDIUM_ALLOCATOR
-
-// Control the dram divisor that's used to scale up medium's madvise window.
-// We'll double the window for each multiple of magazine_medium_madvise_dram_scale_divisor
-// bytes of dram on the system rounded down to the neareast power of 2.
-// This is done by setting magazine_medium_madvise_window_scale_factor.
-uint64_t magazine_medium_madvise_dram_scale_divisor = MEDIUM_MADVISE_DRAM_SCALE_DIVISOR;
-
-// Controls how much to scale up medium's madvise window.
-uint64_t magazine_medium_madvise_window_scale_factor = 1;
-#endif // CONFIG_MEDIUM_ALLOCATOR
-
-// Control the DRAM limit at which the expanded large cache kicks in.
-uint64_t magazine_large_expanded_cache_threshold = LARGE_CACHE_EXPANDED_THRESHOLD;
-
-#if CONFIG_AGGRESSIVE_MADVISE
-bool aggressive_madvise_enabled = DEFAULT_AGGRESSIVE_MADVISE_ENABLED;
-#endif // CONFIG_AGGRESSIVE_MADVISE
-
-#if CONFIG_LARGE_CACHE
-bool large_cache_enabled = DEFAULT_LARGE_CACHE_ENABLED;
-#endif // CONFIG_LARGE_CACHE
-
-// <rdar://problem/47353961> Maximum number of magzines that the medium
-// allocator will use. This addresses a 32-bit load-offset range issue found
-// in some apps when introducing medium.
-int max_medium_magazines;
-
-// Number of regions to retain in a recirc depot.
-#if CONFIG_RECIRC_DEPOT
-int recirc_retained_regions = DEFAULT_RECIRC_RETAINED_REGIONS;
-#endif // CONFIG_RECIRC_DEPOT
+/********************* VERY LOW LEVEL UTILITIES ************************/
+
+static void
+szone_sleep(void)
+{
+ if (getenv("MallocErrorStop")) {
+ _malloc_printf(ASL_LEVEL_NOTICE, "*** sending SIGSTOP to help debug\n");
+ kill(getpid(), SIGSTOP);
+ } else if (getenv("MallocErrorSleep")) {
+ _malloc_printf(ASL_LEVEL_NOTICE, "*** sleeping to help debug\n");
+ sleep(3600); // to help debug
+ }
+}
+
+// msg prints after fmt, ...
+void
+szone_error(uint32_t debug_flags, int is_corruption, const char *msg, const void *ptr, const char *fmt, ...)
+{
+ va_list ap;
+ _SIMPLE_STRING b = _simple_salloc();
+
+ if (b) {
+ if (fmt) {
+ va_start(ap, fmt);
+ _simple_vsprintf(b, fmt, ap);
+ va_end(ap);
+ }
+ if (ptr) {
+ _simple_sprintf(b, "*** error for object %p: %s\n", ptr, msg);
+ } else {
+ _simple_sprintf(b, "*** error: %s\n", msg);
+ }
+ malloc_printf("%s*** set a breakpoint in malloc_error_break to debug\n", _simple_string(b));
+ } else {
+ /*
+ * Should only get here if vm_allocate() can't get a single page of
+ * memory, implying _simple_asl_log() would also fail. So we just
+ * print to the file descriptor.
+ */
+ if (fmt) {
+ va_start(ap, fmt);
+ _malloc_vprintf(MALLOC_PRINTF_NOLOG, fmt, ap);
+ va_end(ap);
+ }
+ if (ptr) {
+ _malloc_printf(MALLOC_PRINTF_NOLOG, "*** error for object %p: %s\n", ptr, msg);
+ } else {
+ _malloc_printf(MALLOC_PRINTF_NOLOG, "*** error: %s\n", msg);
+ }
+ _malloc_printf(MALLOC_PRINTF_NOLOG, "*** set a breakpoint in malloc_error_break to debug\n");
+ }
+ malloc_error_break();
+ szone_sleep();
+ // Call abort() if this is a memory corruption error and the abort on
+ // corruption flag is set, or if any error should abort.
+ if ((is_corruption && (debug_flags & MALLOC_ABORT_ON_CORRUPTION)) ||
+ (debug_flags & MALLOC_ABORT_ON_ERROR)) {
+ _os_set_crash_log_message_dynamic(b ? _simple_string(b) : msg);
+ abort();
+ } else if (b) {
+ _simple_sfree(b);
+ }
+}
/********************* Zone call backs ************************/
/*
* Mark these MALLOC_NOINLINE to avoid bloating the purgeable zone call backs
*/
-static void
-_szone_free(szone_t *szone, void *ptr, bool try)
+void
+szone_free(szone_t *szone, void *ptr)
{
region_t tiny_region;
region_t small_region;
#if DEBUG_MALLOC
if (LOG(szone, ptr)) {
- malloc_report(ASL_LEVEL_INFO, "in _szone_free with %p\n", ptr);
+ malloc_printf("in szone_free with %p\n", ptr);
}
#endif
if (!ptr) {
return;
- }
- if ((uintptr_t)ptr & (TINY_QUANTUM - 1)) {
- if (!try) {
- malloc_zone_error(szone->debug_flags, true, "Non-aligned pointer %p being freed\n", ptr);
- return;
- }
- goto not_claimed;
}
/*
* Try to free to a tiny region.
*/
+ if ((uintptr_t)ptr & (TINY_QUANTUM - 1)) {
+ szone_error(szone->debug_flags, 1, "Non-aligned pointer being freed", ptr, NULL);
+ return;
+ }
if ((tiny_region = tiny_region_for_ptr_no_lock(&szone->tiny_rack, ptr)) != NULL) {
if (TINY_INDEX_FOR_PTR(ptr) >= NUM_TINY_BLOCKS) {
- malloc_zone_error(szone->debug_flags, true, "Pointer %p to metadata being freed\n", ptr);
+ szone_error(szone->debug_flags, 1, "Pointer to metadata being freed", ptr, NULL);
return;
}
- free_tiny(&szone->tiny_rack, ptr, tiny_region, 0, false);
+ free_tiny(&szone->tiny_rack, ptr, tiny_region, 0);
return;
}
@@ -127,62 +143,24 @@
* Try to free to a small region.
*/
if ((uintptr_t)ptr & (SMALL_QUANTUM - 1)) {
- if (!try) {
- malloc_zone_error(szone->debug_flags, true, "Non-aligned pointer %p being freed (2)\n", ptr);
- return;
- }
- goto not_claimed;
+ szone_error(szone->debug_flags, 1, "Non-aligned pointer being freed (2)", ptr, NULL);
+ return;
}
if ((small_region = small_region_for_ptr_no_lock(&szone->small_rack, ptr)) != NULL) {
if (SMALL_META_INDEX_FOR_PTR(ptr) >= NUM_SMALL_BLOCKS) {
- malloc_zone_error(szone->debug_flags, true, "Pointer %p to metadata being freed (2)\n", ptr);
+ szone_error(szone->debug_flags, 1, "Pointer to metadata being freed (2)", ptr, NULL);
return;
}
free_small(&szone->small_rack, ptr, small_region, 0);
return;
}
-#if CONFIG_MEDIUM_ALLOCATOR
- region_t medium_region;
-
- if (szone->is_medium_engaged &&
- (medium_region = medium_region_for_ptr_no_lock(&szone->medium_rack, ptr)) != NULL) {
- if (MEDIUM_META_INDEX_FOR_PTR(ptr) >= NUM_MEDIUM_BLOCKS) {
- malloc_zone_error(szone->debug_flags, true, "Pointer %p to metadata being freed (2)\n", ptr);
- return;
- }
- free_medium(&szone->medium_rack, ptr, medium_region, 0);
- return;
- }
-#endif // CONFIG_MEDIUM_ALLOCATOR
-
/* check that it's a legal large allocation */
if ((uintptr_t)ptr & (vm_page_quanta_size - 1)) {
- if (!try) {
- malloc_zone_error(szone->debug_flags, true, "non-page-aligned, non-allocated pointer %p being freed\n", ptr);
- return;
- }
- goto not_claimed;
- }
- bool claimed = free_large(szone, ptr, try);
- if (!try || claimed) {
+ szone_error(szone->debug_flags, 1, "non-page-aligned, non-allocated pointer being freed", ptr, NULL);
return;
}
-
-not_claimed:
- find_zone_and_free(ptr, true);
-}
-
-void
-szone_free(szone_t *szone, void *ptr)
-{
- _szone_free(szone, ptr, false);
-}
-
-static void
-szone_try_free_default(szone_t *szone, void *ptr)
-{
- _szone_free(szone, ptr, true);
+ free_large(szone, ptr);
}
void
@@ -190,11 +168,11 @@
{
#if DEBUG_MALLOC
if (LOG(szone, ptr)) {
- malloc_report(ASL_LEVEL_INFO, "in szone_free_definite_size with %p\n", ptr);
+ malloc_printf("in szone_free_definite_size with %p\n", ptr);
}
if (0 == size) {
- malloc_zone_error(szone->debug_flags, true, "pointer %p of size zero being freed\n", ptr);
+ szone_error(szone->debug_flags, 1, "pointer of size zero being freed", ptr, NULL);
return;
}
@@ -207,15 +185,15 @@
* Try to free to a tiny region.
*/
if ((uintptr_t)ptr & (TINY_QUANTUM - 1)) {
- malloc_zone_error(szone->debug_flags, true, "Non-aligned pointer %p being freed\n", ptr);
+ szone_error(szone->debug_flags, 1, "Non-aligned pointer being freed", ptr, NULL);
return;
}
- if (size <= TINY_LIMIT_THRESHOLD) {
+ if (size <= SMALL_THRESHOLD) {
if (TINY_INDEX_FOR_PTR(ptr) >= NUM_TINY_BLOCKS) {
- malloc_zone_error(szone->debug_flags, true, "Pointer %p to metadata being freed\n", ptr);
+ szone_error(szone->debug_flags, 1, "Pointer to metadata being freed", ptr, NULL);
return;
}
- free_tiny(&szone->tiny_rack, ptr, TINY_REGION_FOR_PTR(ptr), size, false);
+ free_tiny(&szone->tiny_rack, ptr, TINY_REGION_FOR_PTR(ptr), size);
return;
}
@@ -223,38 +201,24 @@
* Try to free to a small region.
*/
if ((uintptr_t)ptr & (SMALL_QUANTUM - 1)) {
- malloc_zone_error(szone->debug_flags, true, "Non-aligned pointer %p being freed (2)\n", ptr);
+ szone_error(szone->debug_flags, 1, "Non-aligned pointer being freed (2)", ptr, NULL);
return;
}
- if (size <= SMALL_LIMIT_THRESHOLD) {
+ if (size <= szone->large_threshold) {
if (SMALL_META_INDEX_FOR_PTR(ptr) >= NUM_SMALL_BLOCKS) {
- malloc_zone_error(szone->debug_flags, true, "Pointer %p to metadata being freed (2)\n", ptr);
+ szone_error(szone->debug_flags, 1, "Pointer to metadata being freed (2)", ptr, NULL);
return;
}
free_small(&szone->small_rack, ptr, SMALL_REGION_FOR_PTR(ptr), size);
return;
}
-#if CONFIG_MEDIUM_ALLOCATOR
- /*
- * Try to free to a medium region.
- */
- if (szone->is_medium_engaged && size <= MEDIUM_LIMIT_THRESHOLD) {
- if (MEDIUM_META_INDEX_FOR_PTR(ptr) >= NUM_MEDIUM_BLOCKS) {
- malloc_zone_error(szone->debug_flags, true, "Pointer %p to metadata being freed (2)\n", ptr);
- return;
- }
- free_medium(&szone->medium_rack, ptr, MEDIUM_REGION_FOR_PTR(ptr), size);
- return;
- }
-#endif // CONFIG_MEDIUM_ALLOCATOR
-
/* check that it's a legal large allocation */
if ((uintptr_t)ptr & (vm_page_quanta_size - 1)) {
- malloc_zone_error(szone->debug_flags, true, "non-page-aligned, non-allocated pointer %p being freed\n", ptr);
+ szone_error(szone->debug_flags, 1, "non-page-aligned, non-allocated pointer being freed", ptr, NULL);
return;
}
- free_large(szone, ptr, false);
+ free_large(szone, ptr);
}
MALLOC_NOINLINE void *
@@ -263,28 +227,25 @@
void *ptr;
msize_t msize;
- if (size <= TINY_LIMIT_THRESHOLD) {
+ if (size <= SMALL_THRESHOLD) {
+ // tiny size: <1024 bytes (64-bit), <512 bytes (32-bit)
+ // think tiny
msize = TINY_MSIZE_FOR_BYTES(size + TINY_QUANTUM - 1);
if (!msize) {
msize = 1;
}
ptr = tiny_malloc_should_clear(&szone->tiny_rack, msize, cleared_requested);
- } else if (size <= SMALL_LIMIT_THRESHOLD) {
+ } else if (size <= szone->large_threshold) {
+ // small size: <15k (<1GB machines), <127k (>1GB machines)
+ // think small
msize = SMALL_MSIZE_FOR_BYTES(size + SMALL_QUANTUM - 1);
if (!msize) {
msize = 1;
}
ptr = small_malloc_should_clear(&szone->small_rack, msize, cleared_requested);
-#if CONFIG_MEDIUM_ALLOCATOR
- } else if (szone->is_medium_engaged && size <= MEDIUM_LIMIT_THRESHOLD) {
- msize = MEDIUM_MSIZE_FOR_BYTES(size + MEDIUM_QUANTUM - 1);
- if (!msize) {
- msize = 1;
- }
- ptr = medium_malloc_should_clear(&szone->medium_rack, msize, cleared_requested);
-#endif
} else {
- size_t num_kernel_pages = round_large_page_quanta(size) >> large_vm_page_quanta_shift;
+ // large: all other allocations
+ size_t num_kernel_pages = round_page_quanta(size) >> vm_page_quanta_shift;
if (num_kernel_pages == 0) { /* Overflowed */
ptr = 0;
} else {
@@ -293,7 +254,7 @@
}
#if DEBUG_MALLOC
if (LOG(szone, ptr)) {
- malloc_report(ASL_LEVEL_INFO, "szone_malloc returned %p\n", ptr);
+ malloc_printf("szone_malloc returned %p\n", ptr);
}
#endif
/*
@@ -303,10 +264,6 @@
memset(ptr, SCRIBBLE_BYTE, szone_size(szone, ptr));
}
- if (os_unlikely(!ptr)) {
- malloc_set_errno_fast(MZ_POSIX, ENOMEM);
- }
-
return ptr;
}
@@ -316,27 +273,33 @@
return szone_malloc_should_clear(szone, size, 0);
}
-static void *
-szone_malloc_type_malloc(szone_t *szone, size_t size, malloc_type_id_t type_id)
-{
- return szone_malloc(szone, size);
-}
-
void *
szone_calloc(szone_t *szone, size_t num_items, size_t size)
{
- size_t total_bytes;
- if (calloc_get_size(num_items, size, 0, &total_bytes)) {
- return NULL;
- }
+ size_t total_bytes = num_items * size;
+
+ // Check for overflow of integer multiplication
+ if (num_items > 1) {
+#if __LP64__ /* size_t is uint64_t */
+ if ((num_items | size) & 0xffffffff00000000ul) {
+ // num_items or size equals or exceeds sqrt(2^64) == 2^32, appeal to wider arithmetic
+ __uint128_t product = ((__uint128_t)num_items) * ((__uint128_t)size);
+ if ((uint64_t)(product >> 64)) { // compiles to test on upper register of register pair
+ return NULL;
+ }
+ }
+#else /* size_t is uint32_t */
+ if ((num_items | size) & 0xffff0000ul) {
+ // num_items or size equals or exceeds sqrt(2^32) == 2^16, appeal to wider arithmetic
+ uint64_t product = ((uint64_t)num_items) * ((uint64_t)size);
+ if ((uint32_t)(product >> 32)) { // compiles to test on upper register of register pair
+ return NULL;
+ }
+ }
+#endif
+ }
+
return szone_malloc_should_clear(szone, total_bytes, 1);
-}
-
-static void *
-szone_malloc_type_calloc(szone_t *szone, size_t num_items, size_t size,
- malloc_type_id_t type_id)
-{
- return szone_calloc(szone, num_items, size);
}
void *
@@ -344,18 +307,18 @@
{
void *ptr;
- if (size <= MEDIUM_LIMIT_THRESHOLD) {
+ if (size <= szone->large_threshold) {
ptr = szone_memalign(szone, vm_page_quanta_size, size);
} else {
size_t num_kernel_pages;
- num_kernel_pages = round_large_page_quanta(size) >> large_vm_page_quanta_shift;
+ num_kernel_pages = round_page_quanta(size) >> vm_page_quanta_shift;
ptr = large_malloc(szone, num_kernel_pages, 0, 0);
}
#if DEBUG_MALLOC
if (LOG(szone, ptr)) {
- malloc_report(ASL_LEVEL_INFO, "szone_valloc returned %p\n", ptr);
+ malloc_printf("szone_valloc returned %p\n", ptr);
}
#endif
return ptr;
@@ -376,7 +339,7 @@
SZONE_UNLOCK(szone);
#if DEBUG_MALLOC
if (LOG(szone, ptr)) {
- malloc_report(ASL_LEVEL_INFO, "szone_size for %p returned %d\n", ptr, (unsigned)size);
+ malloc_printf("szone_size for %p returned %d\n", ptr, (unsigned)size);
}
#endif
return size;
@@ -392,7 +355,7 @@
}
#if DEBUG_MALLOC
if (LOG(szone, ptr)) {
- malloc_report(ASL_LEVEL_INFO, "in szone_size for %p (szone=%p)\n", ptr, szone);
+ malloc_printf("in szone_size for %p (szone=%p)\n", ptr, szone);
}
#endif
@@ -420,18 +383,6 @@
return sz;
}
-#if CONFIG_MEDIUM_ALLOCATOR
- /*
- * Look for it in a medium region.
- */
- if (szone->is_medium_engaged) {
- sz = medium_size(&szone->medium_rack, ptr);
- if (sz) {
- return sz;
- }
- }
-#endif // CONFIG_MEDIUM_ALLOCATOR
-
/*
* If not page-aligned, it cannot have come from a large allocation.
*/
@@ -453,7 +404,7 @@
#if DEBUG_MALLOC
if (LOG(szone, ptr)) {
- malloc_report(ASL_LEVEL_INFO, "in szone_realloc for %p, %d\n", ptr, (unsigned)new_size);
+ malloc_printf("in szone_realloc for %p, %d\n", ptr, (unsigned)new_size);
}
#endif
if (NULL == ptr) {
@@ -469,7 +420,7 @@
old_size = szone_size(szone, ptr);
if (!old_size) {
- malloc_zone_error(szone->debug_flags, true, "pointer %p being reallocated was not allocated\n", ptr);
+ szone_error(szone->debug_flags, 1, "pointer being reallocated was not allocated", ptr, NULL);
return NULL;
}
@@ -482,8 +433,8 @@
* If the new size suits the tiny allocator and the pointer being resized
* belongs to a tiny region, try to reallocate in-place.
*/
- if (new_good_size <= TINY_LIMIT_THRESHOLD) {
- if (old_size <= TINY_LIMIT_THRESHOLD) {
+ if (new_good_size <= SMALL_THRESHOLD) {
+ if (old_size <= SMALL_THRESHOLD) {
if (new_good_size <= (old_size >> 1)) {
/*
* Serious shrinkage (more than half). free() the excess.
@@ -511,8 +462,8 @@
* belongs to a small region, and we're not protecting the small allocations
* try to reallocate in-place.
*/
- } else if (new_good_size <= SMALL_LIMIT_THRESHOLD) {
- if (TINY_LIMIT_THRESHOLD < old_size && old_size <= SMALL_LIMIT_THRESHOLD) {
+ } else if (new_good_size <= szone->large_threshold) {
+ if (SMALL_THRESHOLD < old_size && old_size <= szone->large_threshold) {
if (new_good_size <= (old_size >> 1)) {
return small_try_shrink_in_place(&szone->small_rack, ptr, old_size, new_good_size);
} else if (new_good_size <= old_size) {
@@ -527,30 +478,11 @@
return ptr;
}
}
-
-#if CONFIG_MEDIUM_ALLOCATOR
- } else if (szone->is_medium_engaged && new_good_size <= MEDIUM_LIMIT_THRESHOLD) {
- if (SMALL_LIMIT_THRESHOLD < old_size && old_size <= MEDIUM_LIMIT_THRESHOLD) {
- if (new_good_size <= (old_size >> 1)) {
- return medium_try_shrink_in_place(&szone->medium_rack, ptr, old_size, new_good_size);
- } else if (new_good_size <= old_size) {
- if (szone->debug_flags & MALLOC_DO_SCRIBBLE) {
- memset(ptr + new_size, SCRIBBLE_BYTE, old_size - new_size);
- }
- } else if (medium_try_realloc_in_place(&szone->medium_rack, ptr, old_size, new_good_size)) {
- if (szone->debug_flags & MALLOC_DO_SCRIBBLE) {
- memset(ptr + old_size, SCRIBBLE_BYTE, new_good_size - old_size);
- }
- return ptr;
- }
- }
-#endif // CONFIG_MEDIUM_ALLOCATOR
-
/*
* Else if the allocation's a large allocation, try to reallocate in-place there.
*/
} else if (!(szone->debug_flags & MALLOC_PURGEABLE) && // purgeable needs fresh allocation
- (old_size > LARGE_THRESHOLD(szone)) && (new_good_size > LARGE_THRESHOLD(szone))) {
+ (old_size > szone->large_threshold) && (new_good_size > szone->large_threshold)) {
if (new_good_size <= (old_size >> 1)) {
return large_try_shrink_in_place(szone, ptr, old_size, new_good_size);
} else if (new_good_size <= old_size) {
@@ -588,28 +520,18 @@
* if it's too small, just copy by hand.
*/
valid_size = MIN(old_size, new_size);
-#if CONFIG_REALLOC_CAN_USE_VMCOPY
- if ((valid_size <= VM_COPY_THRESHOLD) ||
- vm_copy(mach_task_self(), (vm_address_t)ptr, valid_size, (vm_address_t)new_ptr))
-#endif
- {
+ if ((valid_size < szone->vm_copy_threshold) ||
+ vm_copy(mach_task_self(), (vm_address_t)ptr, valid_size, (vm_address_t)new_ptr)) {
memcpy(new_ptr, ptr, valid_size);
}
szone_free(szone, ptr);
#if DEBUG_MALLOC
if (LOG(szone, ptr)) {
- malloc_report(ASL_LEVEL_INFO, "szone_realloc returned %p for %d\n", new_ptr, (unsigned)new_size);
+ malloc_printf("szone_realloc returned %p for %d\n", new_ptr, (unsigned)new_size);
}
#endif
return new_ptr;
-}
-
-static void *
-szone_malloc_type_realloc(szone_t *szone, void *ptr, size_t size,
- malloc_type_id_t type_id)
-{
- return szone_realloc(szone, ptr, size);
}
void *
@@ -622,88 +544,98 @@
return NULL;
}
- // alignment is a power of 2 at least as large as sizeof(void *), hence
- // non-zero. Since size + alignment didn't wrap, 0 <= size + alignment - 1
- // < size + alignment
+ // alignment is gauranteed a power of 2 at least as large as sizeof(void *), hence non-zero.
+ // Since size + alignment didn't wrap, 0 <= size + alignment - 1 < size + alignment
size_t span = size + alignment - 1;
if (alignment <= TINY_QUANTUM) {
- // Trivially satisfied by tiny, small, medium, or large.
- return szone_malloc(szone, size);
- }
- if (span <= TINY_LIMIT_THRESHOLD) {
+ return szone_malloc(szone, size); // Trivially satisfied by tiny, small, or large
+
+ } else if (span <= SMALL_THRESHOLD) {
return tiny_memalign(szone, alignment, size, span);
- }
- if (TINY_LIMIT_THRESHOLD < size && alignment <= SMALL_QUANTUM) {
- // Trivially satisfied by small, medium or large.
- return szone_malloc(szone, size);
- }
- if (size <= TINY_LIMIT_THRESHOLD) {
- // The allocation asked for a size that TINY would normally fulfill
- // but it cannot guarantee the alignment. So bump it up to fit inside
- // SMALL and try again.
- size = TINY_LIMIT_THRESHOLD + TINY_QUANTUM;
- span = size + alignment - 1;
- }
- if (span <= SMALL_LIMIT_THRESHOLD) {
+
+ } else if (SMALL_THRESHOLD < size && alignment <= SMALL_QUANTUM) {
+ return szone_malloc(szone, size); // Trivially satisfied by small or large
+
+ } else if (span <= szone->large_threshold) {
return small_memalign(szone, alignment, size, span);
- }
-#if CONFIG_MEDIUM_ALLOCATOR
- if (szone->is_medium_engaged) {
- if (size <= SMALL_LIMIT_THRESHOLD) {
- size = SMALL_LIMIT_THRESHOLD + SMALL_QUANTUM;
- span = size + alignment - 1;
- }
- if (szone->is_medium_engaged && span <= MEDIUM_LIMIT_THRESHOLD) {
- return medium_memalign(szone, alignment, size, span);
- }
- }
-#endif // CONFIG_MEDIUM_ALLOCATOR
- if (LARGE_THRESHOLD(szone) < size && alignment <= vm_page_quanta_size) {
- // Trivially satisfied by large (which rounds to a whole page).
- return szone_malloc(szone, size);
- }
- // ensure block allocated by large does not have a small-possible size
- size_t num_kernel_pages = round_large_page_quanta(MAX(LARGE_THRESHOLD(szone) + 1,
- size)) >> large_vm_page_quanta_shift;
- if (num_kernel_pages == 0) { /* Overflowed */
- return NULL;
+
+ } else if (szone->large_threshold < size && alignment <= vm_page_quanta_size) {
+ return szone_malloc(szone, size); // Trivially satisfied by large
+
} else {
- MALLOC_STATIC_ASSERT(sizeof(size_t) == sizeof(long), "builtin_ctzl should be the right intrinsic for size_t");
-
- return large_malloc(szone, num_kernel_pages,
- MAX(vm_page_quanta_shift, __builtin_ctzl(alignment)), 0);
+ // ensure block allocated by large does not have a small-possible size
+ size_t num_kernel_pages = round_page_quanta(MAX(szone->large_threshold + 1, size)) >> vm_page_quanta_shift;
+ void *p;
+
+ if (num_kernel_pages == 0) { /* Overflowed */
+ p = NULL;
+ } else {
+ p = large_malloc(szone, num_kernel_pages, MAX(vm_page_quanta_shift, __builtin_ctz((unsigned)alignment)), 0);
+ }
+
+ return p;
}
/* NOTREACHED */
- __builtin_unreachable();
-}
-
-static void *
-szone_malloc_type_memalign(szone_t *szone, size_t align, size_t size,
- malloc_type_id_t type_id)
-{
- return szone_memalign(szone, align, size);
-}
-
-// Given a size, returns the number of pointers allocated capable of holding
+}
+
+// given a size, returns the number of pointers allocated capable of holding
// that size, up to the limit specified by the 'count' argument. These pointers
// are stored in the 'results' array, which must be allocated by the caller.
-// May return zero, since this function is only a best attempt at allocating
-// the pointers. Clients should be prepared to call malloc for any additional
+// may return zero, since this function is only a best attempt at allocating
+// the pointers. clients should be prepared to call malloc for any additional
// blocks they need.
unsigned
szone_batch_malloc(szone_t *szone, size_t size, void **results, unsigned count)
{
+ msize_t msize = TINY_MSIZE_FOR_BYTES(size + TINY_QUANTUM - 1);
+ unsigned found = 0;
+ mag_index_t mag_index = mag_get_thread_index() % szone->tiny_rack.num_magazines;
+ magazine_t *tiny_mag_ptr = &(szone->tiny_rack.magazines[mag_index]);
+
// only bother implementing this for tiny
- if (size <= TINY_LIMIT_THRESHOLD) {
- return tiny_batch_malloc(szone, size, results, count);
- }
- return 0;
-}
-
+ if (size > SMALL_THRESHOLD) {
+ return 0;
+ }
+ // make sure to return objects at least one quantum in size
+ if (!msize) {
+ msize = 1;
+ }
+
+ CHECK(szone, __PRETTY_FUNCTION__);
+
+ // We must lock the zone now, since tiny_malloc_from_free_list assumes that
+ // the caller has done so.
+ SZONE_MAGAZINE_PTR_LOCK(tiny_mag_ptr);
+
+ // with the zone locked, allocate objects from the free list until all
+ // sufficiently large objects have been exhausted, or we have met our quota
+ // of objects to allocate.
+ while (found < count) {
+ void *ptr = tiny_malloc_from_free_list(&szone->tiny_rack, tiny_mag_ptr, mag_index, msize);
+ if (!ptr) {
+ break;
+ }
+
+ *results++ = ptr;
+ found++;
+ }
+ SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
+ return found;
+}
+
+/* Try caching the tiny_region and checking if the next ptr hits there. */
void
szone_batch_free(szone_t *szone, void **to_be_freed, unsigned count)
{
+ unsigned cc = 0;
+ void *ptr;
+ region_t tiny_region = NULL;
+ boolean_t is_free;
+ msize_t msize;
+ magazine_t *tiny_mag_ptr = NULL;
+ mag_index_t mag_index = -1;
+
// frees all the pointers in to_be_freed
// note that to_be_freed may be overwritten during the process
if (!count) {
@@ -711,14 +643,55 @@
}
CHECK(szone, __PRETTY_FUNCTION__);
-
- // We only support batch malloc in tiny. Let it free all of the pointers
- // that belong to it, then let the standard free deal with the rest.
- tiny_batch_free(szone, to_be_freed, count);
+ while (cc < count) {
+ ptr = to_be_freed[cc];
+ if (ptr) {
+ if (NULL == tiny_region || tiny_region != TINY_REGION_FOR_PTR(ptr)) { // region same as last iteration?
+ if (tiny_mag_ptr) { // non-NULL iff magazine lock taken
+ SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
+ tiny_mag_ptr = NULL;
+ }
+
+ tiny_region = tiny_region_for_ptr_no_lock(&szone->tiny_rack, ptr);
+
+ if (tiny_region) {
+ tiny_mag_ptr = mag_lock_zine_for_region_trailer(szone->tiny_rack.magazines,
+ REGION_TRAILER_FOR_TINY_REGION(tiny_region),
+ MAGAZINE_INDEX_FOR_TINY_REGION(tiny_region));
+ mag_index = MAGAZINE_INDEX_FOR_TINY_REGION(tiny_region);
+ }
+ }
+ if (tiny_region) {
+ // this is a tiny pointer
+ if (TINY_INDEX_FOR_PTR(ptr) >= NUM_TINY_BLOCKS) {
+ break; // pointer to metadata; let the standard free deal with it
+ }
+ msize = get_tiny_meta_header(ptr, &is_free);
+ if (is_free) {
+ break; // a double free; let the standard free deal with it
+ }
+ if (!tiny_free_no_lock(&szone->tiny_rack, tiny_mag_ptr, mag_index, tiny_region, ptr, msize)) {
+ // Arrange to re-acquire magazine lock
+ tiny_mag_ptr = NULL;
+ tiny_region = NULL;
+ }
+ to_be_freed[cc] = NULL;
+ } else {
+ // No region in this zone claims ptr; let the standard free deal with it
+ break;
+ }
+ }
+ cc++;
+ }
+
+ if (tiny_mag_ptr) {
+ SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
+ tiny_mag_ptr = NULL;
+ }
CHECK(szone, __PRETTY_FUNCTION__);
while (count--) {
- void *ptr = to_be_freed[count];
+ ptr = to_be_freed[count];
if (ptr) {
szone_free(szone, ptr);
}
@@ -734,10 +707,36 @@
vm_range_t range_to_deallocate;
#if CONFIG_LARGE_CACHE
- if (large_cache_enabled) {
- large_destroy_cache(szone);
- }
-#endif // CONFIG_LARGE_CACHE
+ SZONE_LOCK(szone);
+
+ /* disable any memory pressure responder */
+ szone->flotsam_enabled = FALSE;
+
+ // stack allocated copy of the death-row cache
+ int idx = szone->large_entry_cache_oldest, idx_max = szone->large_entry_cache_newest;
+ large_entry_t local_entry_cache[LARGE_ENTRY_CACHE_SIZE];
+
+ memcpy((void *)local_entry_cache, (void *)szone->large_entry_cache, sizeof(local_entry_cache));
+
+ szone->large_entry_cache_oldest = szone->large_entry_cache_newest = 0;
+ szone->large_entry_cache[0].address = 0x0;
+ szone->large_entry_cache[0].size = 0;
+ szone->large_entry_cache_bytes = 0;
+ szone->large_entry_cache_reserve_bytes = 0;
+
+ SZONE_UNLOCK(szone);
+
+ // deallocate the death-row cache outside the zone lock
+ while (idx != idx_max) {
+ mvm_deallocate_pages((void *)local_entry_cache[idx].address, local_entry_cache[idx].size, 0);
+ if (++idx == LARGE_ENTRY_CACHE_SIZE) {
+ idx = 0;
+ }
+ }
+ if (0 != local_entry_cache[idx].address && 0 != local_entry_cache[idx].size) {
+ mvm_deallocate_pages((void *)local_entry_cache[idx].address, local_entry_cache[idx].size, 0);
+ }
+#endif
/* destroy large entries */
index = szone->num_large_entries;
@@ -750,7 +749,7 @@
}
large_entries_free_no_lock(szone, szone->large_entries, szone->num_large_entries, &range_to_deallocate);
if (range_to_deallocate.size) {
- mvm_deallocate_pages((void *)range_to_deallocate.address, (size_t)range_to_deallocate.size, szone->debug_flags);
+ mvm_deallocate_pages((void *)range_to_deallocate.address, (size_t)range_to_deallocate.size, 0);
}
/* destroy allocator regions */
@@ -761,13 +760,6 @@
rack_destroy(&szone->tiny_rack);
rack_destroy(&szone->small_rack);
-#if CONFIG_MEDIUM_ALLOCATOR
- if (szone->is_medium_engaged) {
- rack_destroy_regions(&szone->medium_rack, MEDIUM_REGION_SIZE);
- rack_destroy(&szone->medium_rack);
- }
-#endif // CONFIG_MEDIUM_ALLOCATOR
-
mvm_deallocate_pages((void *)szone, SZONE_PAGED_SIZE, 0);
}
@@ -777,7 +769,7 @@
msize_t msize;
// Find a good size for this tiny allocation.
- if (size <= TINY_LIMIT_THRESHOLD) {
+ if (size <= SMALL_THRESHOLD) {
msize = TINY_MSIZE_FOR_BYTES(size + TINY_QUANTUM - 1);
if (!msize) {
msize = 1;
@@ -786,7 +778,7 @@
}
// Find a good size for this small allocation.
- if (size <= SMALL_LIMIT_THRESHOLD) {
+ if (size <= szone->large_threshold) {
msize = SMALL_MSIZE_FOR_BYTES(size + SMALL_QUANTUM - 1);
if (!msize) {
msize = 1;
@@ -794,19 +786,9 @@
return SMALL_BYTES_FOR_MSIZE(msize);
}
-#if CONFIG_MEDIUM_ALLOCATOR
- if (szone->is_medium_engaged && size <= MEDIUM_LIMIT_THRESHOLD) {
- msize = MEDIUM_MSIZE_FOR_BYTES(size + MEDIUM_QUANTUM - 1);
- if (!msize) {
- msize = 1;
- }
- return MEDIUM_BYTES_FOR_MSIZE(msize);
- }
-#endif // CONFIG_MEDIUM_ALLOCATOR
-
// Check for integer overflow on the size, since unlike the two cases above,
// there is no upper bound on allocation size at this point.
- if (size > round_large_page_quanta(size)) {
+ if (size > round_page_quanta(size)) {
return (size_t)(-1LL);
}
@@ -815,22 +797,10 @@
// failed to catch a request for zero bytes in the tiny check, or the size
// overflowed to zero during some arithmetic.
if (size == 0) {
- malloc_report(ASL_LEVEL_INFO, "szone_good_size() invariant broken %y\n", size);
- }
-#endif
- return round_large_page_quanta(size);
-}
-
-boolean_t
-szone_claimed_address(szone_t *szone, void *ptr)
-{
- return tiny_claimed_address(&szone->tiny_rack, ptr)
- || small_claimed_address(&szone->small_rack, ptr)
-#if CONFIG_MEDIUM_ALLOCATOR
- || (szone->is_medium_engaged &&
- medium_claimed_address(&szone->medium_rack, ptr))
-#endif // CONFIG_MEDIUM_ALLOCATOR
- || large_claimed_address(szone, ptr);
+ malloc_printf("szone_good_size() invariant broken %y\n", size);
+ }
+#endif
+ return round_page_quanta(size);
}
unsigned szone_check_counter = 0;
@@ -842,10 +812,38 @@
{
size_t index;
- boolean_t tiny_result = tiny_check(&szone->tiny_rack, szone_check_counter);
- if (!tiny_result) {
- szone->debug_flags &= ~CHECK_REGIONS;
- return 0;
+ /* check tiny regions - chould check region count */
+ for (index = 0; index < szone->tiny_rack.region_generation->num_regions_allocated; ++index) {
+ region_t tiny = szone->tiny_rack.region_generation->hashed_regions[index];
+
+ if (HASHRING_REGION_DEALLOCATED == tiny) {
+ continue;
+ }
+
+ if (tiny) {
+ magazine_t *tiny_mag_ptr = mag_lock_zine_for_region_trailer(szone->tiny_rack.magazines,
+ REGION_TRAILER_FOR_TINY_REGION(tiny),
+ MAGAZINE_INDEX_FOR_TINY_REGION(tiny));
+
+ if (!tiny_check_region(&szone->tiny_rack, tiny)) {
+ SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
+ szone->debug_flags &= ~CHECK_REGIONS;
+ szone_error(szone->debug_flags, 1, "check: tiny region incorrect", NULL,
+ "*** tiny region %ld incorrect szone_check_all(%s) counter=%d\n", index, function, szone_check_counter);
+ return 0;
+ }
+ SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
+ }
+ }
+ /* check tiny free lists */
+ for (index = 0; index < NUM_TINY_SLOTS; ++index) {
+ if (!tiny_free_list_check(&szone->tiny_rack, (grain_t)index)) {
+ szone->debug_flags &= ~CHECK_REGIONS;
+ szone_error(szone->debug_flags, 1, "check: tiny free list incorrect", NULL,
+ "*** tiny free list incorrect (slot=%ld) szone_check_all(%s) counter=%d\n", index, function,
+ szone_check_counter);
+ return 0;
+ }
}
/* check small regions - could check region count */
@@ -861,9 +859,11 @@
REGION_TRAILER_FOR_SMALL_REGION(small),
MAGAZINE_INDEX_FOR_SMALL_REGION(small));
- if (!small_check_region(&szone->small_rack, small, index, szone_check_counter)) {
+ if (!small_check_region(&szone->small_rack, small)) {
SZONE_MAGAZINE_PTR_UNLOCK(small_mag_ptr);
szone->debug_flags &= ~CHECK_REGIONS;
+ szone_error(szone->debug_flags, 1, "check: small region incorrect", NULL,
+ "*** small region %ld incorrect szone_check_all(%s) counter=%d\n", index, function, szone_check_counter);
return 0;
}
SZONE_MAGAZINE_PTR_UNLOCK(small_mag_ptr);
@@ -871,44 +871,14 @@
}
/* check small free lists */
for (index = 0; index < SMALL_FREE_SLOT_COUNT(&szone->small_rack); ++index) {
- if (!small_free_list_check(&szone->small_rack, (grain_t)index, szone_check_counter)) {
+ if (!small_free_list_check(&szone->small_rack, (grain_t)index)) {
szone->debug_flags &= ~CHECK_REGIONS;
+ szone_error(szone->debug_flags, 1, "check: small free list incorrect", NULL,
+ "*** small free list incorrect (slot=%ld) szone_check_all(%s) counter=%d\n", index, function,
+ szone_check_counter);
return 0;
}
}
-
-#if CONFIG_MEDIUM_ALLOCATOR
- if (szone->is_medium_engaged) {
- /* check medium regions - could check region count */
- for (index = 0; index < szone->medium_rack.region_generation->num_regions_allocated; ++index) {
- region_t medium = szone->medium_rack.region_generation->hashed_regions[index];
-
- if (HASHRING_REGION_DEALLOCATED == medium) {
- continue;
- }
-
- if (medium) {
- magazine_t *medium_mag_ptr = mag_lock_zine_for_region_trailer(szone->medium_rack.magazines,
- REGION_TRAILER_FOR_MEDIUM_REGION(medium),
- MAGAZINE_INDEX_FOR_MEDIUM_REGION(medium));
-
- if (!medium_check_region(&szone->medium_rack, medium, index, szone_check_counter)) {
- SZONE_MAGAZINE_PTR_UNLOCK(medium_mag_ptr);
- szone->debug_flags &= ~CHECK_REGIONS;
- return 0;
- }
- SZONE_MAGAZINE_PTR_UNLOCK(medium_mag_ptr);
- }
- }
- /* check medium free lists */
- for (index = 0; index < MEDIUM_FREE_SLOT_COUNT(&szone->medium_rack); ++index) {
- if (!medium_free_list_check(&szone->medium_rack, (grain_t)index, szone_check_counter)) {
- szone->debug_flags &= ~CHECK_REGIONS;
- return 0;
- }
- }
- }
-#endif // CONFIG_MEDIUM_ALLOCATOR
return 1;
}
@@ -917,7 +887,7 @@
szone_check(szone_t *szone)
{
if ((++szone_check_counter % 10000) == 0) {
- malloc_report(ASL_LEVEL_NOTICE, "at szone_check counter=%d\n", szone_check_counter);
+ _malloc_printf(ASL_LEVEL_NOTICE, "at szone_check counter=%d\n", szone_check_counter);
}
if (szone_check_counter < szone_check_start) {
@@ -931,10 +901,6 @@
return szone_check_all(szone, "");
}
-// To support the sanitizer zone, we need to be able to perform zone enumeration across different
-// architecture slices on macOS, because ReportCrash is always running as a native (arm64e) process,
-// but we also need to be able to inspect x86_64 targets that are running under Rosetta. So the data
-// layout and zone logic needs to match between x86_64 and arm64(e).
static kern_return_t
szone_ptr_in_use_enumerator(task_t task,
void *context,
@@ -946,7 +912,9 @@
szone_t *szone;
kern_return_t err;
- reader = reader_or_in_memory_fallback(reader, task);
+ if (!reader) {
+ reader = _szone_default_reader;
+ }
err = reader(task, zone_address, sizeof(szone_t), (void **)&szone);
if (err) {
@@ -963,23 +931,14 @@
return err;
}
-#if CONFIG_MEDIUM_ALLOCATOR
- if (szone->is_medium_engaged) {
- err = medium_in_use_enumerator(task, context, type_mask, szone, reader, recorder);
- if (err) {
- return err;
- }
- }
-#endif // CONFIG_MEDIUM_ALLOCATOR
-
err = large_in_use_enumerator(
task, context, type_mask, (vm_address_t)szone->large_entries, szone->num_large_entries, reader, recorder);
return err;
}
-static boolean_t
-scalable_zone_info_task(task_t task, memory_reader_t reader,
- malloc_zone_t *zone, unsigned *info_to_fill, unsigned count)
+// Following method is deprecated: use scalable_zone_statistics instead
+void
+scalable_zone_info(malloc_zone_t *zone, unsigned *info_to_fill, unsigned count)
{
szone_t *szone = (void *)zone;
unsigned info[13];
@@ -991,30 +950,21 @@
size_t u = 0;
mag_index_t mag_index;
- magazine_t *mapped_magazines;
- if (reader(task, (vm_address_t)szone->tiny_rack.magazines,
- sizeof(magazine_t), (void **)&mapped_magazines)) {
- return false;
- }
for (mag_index = -1; mag_index < szone->tiny_rack.num_magazines; mag_index++) {
- s += mapped_magazines[mag_index].mag_bytes_free_at_start;
- s += mapped_magazines[mag_index].mag_bytes_free_at_end;
- t += mapped_magazines[mag_index].mag_num_objects;
- u += mapped_magazines[mag_index].mag_num_bytes_in_objects;
+ s += szone->tiny_rack.magazines[mag_index].mag_bytes_free_at_start;
+ s += szone->tiny_rack.magazines[mag_index].mag_bytes_free_at_end;
+ t += szone->tiny_rack.magazines[mag_index].mag_num_objects;
+ u += szone->tiny_rack.magazines[mag_index].mag_num_bytes_in_objects;
}
info[4] = (unsigned)t;
info[5] = (unsigned)u;
- if (reader(task, (vm_address_t)szone->small_rack.magazines,
- sizeof(magazine_t), (void **)&mapped_magazines)) {
- return false;
- }
for (t = 0, u = 0, mag_index = -1; mag_index < szone->small_rack.num_magazines; mag_index++) {
- s += mapped_magazines[mag_index].mag_bytes_free_at_start;
- s += mapped_magazines[mag_index].mag_bytes_free_at_end;
- t += mapped_magazines[mag_index].mag_num_objects;
- u += mapped_magazines[mag_index].mag_num_bytes_in_objects;
+ s += szone->small_rack.magazines[mag_index].mag_bytes_free_at_start;
+ s += szone->small_rack.magazines[mag_index].mag_bytes_free_at_end;
+ t += szone->small_rack.magazines[mag_index].mag_num_objects;
+ u += szone->small_rack.magazines[mag_index].mag_num_bytes_in_objects;
}
info[6] = (unsigned)t;
@@ -1036,275 +986,64 @@
info[2] = info[3] - (unsigned)s;
memcpy(info_to_fill, info, sizeof(unsigned) * count);
-
- return true;
-}
-
-// Following method is deprecated: use scalable_zone_statistics instead
-// Required for backward compatibility.
-void
-scalable_zone_info(malloc_zone_t *zone, unsigned *info_to_fill, unsigned count) {
- scalable_zone_info_task(mach_task_self(), _malloc_default_reader, zone,
- info_to_fill, count);
}
// FIXME: consistent picture requires locking!
static MALLOC_NOINLINE void
-szone_print(task_t task, unsigned level, vm_address_t zone_address,
- memory_reader_t reader, print_task_printer_t printer)
+szone_print(szone_t *szone, boolean_t verbose)
{
unsigned info[13];
size_t index;
region_t region;
- region_t mapped_region;
-
- szone_t *szone = (szone_t *)zone_address;
- szone_t *mapped_szone;
- if (reader(task, zone_address, sizeof(szone_t), (void **)&mapped_szone)) {
- printer("Failed to read szone structure\n");
- return;
- }
-
- if (!scalable_zone_info_task(task, reader, (void *)mapped_szone, info, 13)) {
- printer("Failed to get scalable zone info\n");
- return;
- }
- printer("Scalable zone %p: inUse=%u(%u) touched=%u allocated=%u flags=0x%x\n",
- zone_address, info[0], info[1], info[2], info[3], info[12]);
- printer("\ttiny=%u(%u) small=%u(%u) large=%u(%u)\n", info[4],
- info[5], info[6], info[7], info[8], info[9]);
-
- // FIXME: The rest of the code here assumes that regions have their normal
- // alignment, which isn't guaranteed when looking at regions mapped from
- // other processes
- if (!mach_task_is_self(task)) {
- printer("(unable to safely further examine remote process)\n");
- return;
- }
-
+
+ scalable_zone_info((void *)szone, info, 13);
+ _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX,
+ "Scalable zone %p: inUse=%u(%y) touched=%y allocated=%y flags=%d\n", szone, info[0], info[1], info[2], info[3],
+ info[12]);
+ _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX, "\ttiny=%u(%y) small=%u(%y) large=%u(%y) huge=%u(%y)\n", info[4],
+ info[5], info[6], info[7], info[8], info[9], info[10], info[11]);
// tiny
- printer("%lu tiny regions:\n", mapped_szone->tiny_rack.num_regions);
- if (mapped_szone->tiny_rack.num_regions_dealloc) {
- printer("[%lu tiny regions have been vm_deallocate'd]\n",
- mapped_szone->tiny_rack.num_regions_dealloc);
- }
-
- region_hash_generation_t *mapped_region_generation;
- region_t *mapped_hashed_regions;
- magazine_t *mapped_magazines;
- if (reader(task, (vm_address_t)mapped_szone->tiny_rack.region_generation,
- sizeof(region_hash_generation_t), (void **)&mapped_region_generation)) {
- printer("Failed to map tiny rack region_generation\n");
- return;
- }
- if (reader(task, (vm_address_t)mapped_region_generation->hashed_regions,
- sizeof(region_t), (void **)&mapped_hashed_regions)) {
- printer("Failed to map tiny rack hashed_regions\n");
- return;
- }
- if (reader(task, (vm_address_t)mapped_szone->tiny_rack.magazines,
- mapped_szone->tiny_rack.num_magazines * sizeof(magazine_t),
- (void **)&mapped_magazines)) {
- printer("Failed to map tiny rack magazines\n");
- return;
- }
-
- int recirc_regions = 0;
- for (index = 0; index < mapped_region_generation->num_regions_allocated; ++index) {
- region = mapped_hashed_regions[index];
+ _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX, "%lu tiny regions:\n", szone->tiny_rack.num_regions);
+ if (szone->tiny_rack.num_regions_dealloc) {
+ _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX, "[%lu tiny regions have been vm_deallocate'd]\n",
+ szone->tiny_rack.num_regions_dealloc);
+ }
+ for (index = 0; index < szone->tiny_rack.region_generation->num_regions_allocated; ++index) {
+ region = szone->tiny_rack.region_generation->hashed_regions[index];
if (HASHRING_OPEN_ENTRY != region && HASHRING_REGION_DEALLOCATED != region) {
- if (reader(task, (vm_address_t)region, sizeof(struct tiny_region),
- (void **)&mapped_region)) {
- printer("Failed to map region %p\n", region);
- return;
- }
- mag_index_t mag_index = MAGAZINE_INDEX_FOR_TINY_REGION(mapped_region);
- if (mag_index == DEPOT_MAGAZINE_INDEX) {
- recirc_regions++;
- }
- print_tiny_region(task, reader, printer, level, region,
- (region == mapped_magazines[mag_index].mag_last_region)
- ? mapped_magazines[mag_index].mag_bytes_free_at_start
- : 0,
- (region == mapped_magazines[mag_index].mag_last_region)
- ? mapped_magazines[mag_index].mag_bytes_free_at_end
- : 0);
- }
- }
-
-#if CONFIG_RECIRC_DEPOT
- magazine_t *mapped_recirc_depot = &mapped_magazines[DEPOT_MAGAZINE_INDEX];
- if (mapped_recirc_depot->mag_num_bytes_in_objects) {
- printer("Tiny recirc depot: total bytes: %llu, in-use bytes: %llu, "
- "allocations: %llu, regions: %d (min # retained regions: %d)\n",
- mapped_recirc_depot->num_bytes_in_magazine,
- mapped_recirc_depot->mag_num_bytes_in_objects,
- mapped_recirc_depot->mag_num_objects, recirc_regions,
- recirc_retained_regions);
- } else {
- printer("Tiny recirc depot is empty\n");
- }
-#else // CONFIG_RECIRC_DEPOT
- printer("Tiny recirc depot not configured\n");
-#endif // CONFIG_RECIRC_DEPOT
-
- if (level > 0) {
- print_tiny_free_list(task, reader, printer, &szone->tiny_rack);
- }
-
+ mag_index_t mag_index = MAGAZINE_INDEX_FOR_TINY_REGION(region);
+ print_tiny_region(verbose, region, (region == szone->tiny_rack.magazines[mag_index].mag_last_region)
+ ? szone->tiny_rack.magazines[mag_index].mag_bytes_free_at_start
+ : 0,
+ (region == szone->tiny_rack.magazines[mag_index].mag_last_region)
+ ? szone->tiny_rack.magazines[mag_index].mag_bytes_free_at_end
+ : 0);
+ }
+ }
+ if (verbose) {
+ print_tiny_free_list(&szone->tiny_rack);
+ }
// small
- printer("%lu small regions:\n", mapped_szone->small_rack.num_regions);
- if (mapped_szone->small_rack.num_regions_dealloc) {
- printer("[%lu small regions have been vm_deallocate'd]\n",
- mapped_szone->small_rack.num_regions_dealloc);
- }
- if (reader(task, (vm_address_t)mapped_szone->small_rack.region_generation,
- sizeof(region_hash_generation_t), (void **)&mapped_region_generation)) {
- printer("Failed to map small rack region_generation\n");
- return;
- }
- if (reader(task, (vm_address_t)mapped_region_generation->hashed_regions,
- sizeof(region_t), (void **)&mapped_hashed_regions)) {
- printer("Failed to map small rack hashed_regions\n");
- return;
- }
- if (reader(task, (vm_address_t)mapped_szone->small_rack.magazines,
- mapped_szone->small_rack.num_magazines * sizeof(magazine_t),
- (void **)&mapped_magazines)) {
- printer("Failed to map small rack magazines\n");
- return;
- }
-
- recirc_regions = 0;
- for (index = 0; index < mapped_region_generation->num_regions_allocated; ++index) {
- region = mapped_hashed_regions[index];
+ _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX, "%lu small regions:\n", szone->small_rack.num_regions);
+ if (szone->small_rack.num_regions_dealloc) {
+ _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX, "[%lu small regions have been vm_deallocate'd]\n",
+ szone->small_rack.num_regions_dealloc);
+ }
+ for (index = 0; index < szone->small_rack.region_generation->num_regions_allocated; ++index) {
+ region = szone->small_rack.region_generation->hashed_regions[index];
if (HASHRING_OPEN_ENTRY != region && HASHRING_REGION_DEALLOCATED != region) {
- if (reader(task, (vm_address_t)region, sizeof(struct small_region),
- (void **)&mapped_region)) {
- printer("Failed to map region %p\n", region);
- return;
- }
- mag_index_t mag_index = MAGAZINE_INDEX_FOR_SMALL_REGION(mapped_region);
- if (mag_index == DEPOT_MAGAZINE_INDEX) {
- recirc_regions++;
- }
- print_small_region(task, reader, printer, mapped_szone, level, region,
- (region == mapped_magazines[mag_index].mag_last_region)
- ? mapped_magazines[mag_index].mag_bytes_free_at_start
- : 0,
- (region == mapped_magazines[mag_index].mag_last_region)
- ? mapped_magazines[mag_index].mag_bytes_free_at_end
- : 0);
- }
- }
-
-#if CONFIG_RECIRC_DEPOT
- mapped_recirc_depot = &mapped_magazines[DEPOT_MAGAZINE_INDEX];
- if (mapped_recirc_depot->mag_num_bytes_in_objects) {
- printer("Small recirc depot: total bytes: %llu, in-use bytes: %llu, "
- "allocations: %llu, regions: %d (min # retained regions: %d)\n",
- mapped_recirc_depot->num_bytes_in_magazine,
- mapped_recirc_depot->mag_num_bytes_in_objects,
- mapped_recirc_depot->mag_num_objects, recirc_regions,
- recirc_retained_regions);
- } else {
- printer("Small recirc depot is empty\n");
- }
-#else // CONFIG_RECIRC_DEPOT
- printer("Small recirc depot not configured\n");
-#endif // CONFIG_RECIRC_DEPOT
-
- if (level > 0) {
- print_small_free_list(task, reader, printer, &szone->small_rack);
- }
-
-#if CONFIG_MEDIUM_ALLOCATOR
- if (szone->is_medium_engaged) {
- // medium
- printer("%lu medium regions:\n", mapped_szone->medium_rack.num_regions);
- if (mapped_szone->medium_rack.num_regions_dealloc) {
- printer("[%lu medium regions have been vm_deallocate'd]\n",
- mapped_szone->medium_rack.num_regions_dealloc);
- }
- if (reader(task, (vm_address_t)mapped_szone->medium_rack.region_generation,
- sizeof(region_hash_generation_t), (void **)&mapped_region_generation)) {
- printer("Failed to map medium rack region_generation\n");
- return;
- }
- if (reader(task, (vm_address_t)mapped_region_generation->hashed_regions,
- sizeof(region_t), (void **)&mapped_hashed_regions)) {
- printer("Failed to map medium rack hashed_regions\n");
- return;
- }
- if (reader(task, (vm_address_t)mapped_szone->medium_rack.magazines,
- mapped_szone->medium_rack.num_magazines * sizeof(magazine_t),
- (void **)&mapped_magazines)) {
- printer("Failed to map medium rack magazines\n");
- return;
- }
-
- recirc_regions = 0;
- for (index = 0; index < mapped_region_generation->num_regions_allocated; ++index) {
- region = mapped_hashed_regions[index];
- if (HASHRING_OPEN_ENTRY != region && HASHRING_REGION_DEALLOCATED != region) {
- if (reader(task, (vm_address_t)region, sizeof(struct medium_region),
- (void **)&mapped_region)) {
- printer("Failed to map region %p\n", region);
- return;
- }
- mag_index_t mag_index = MAGAZINE_INDEX_FOR_MEDIUM_REGION(mapped_region);
- if (mag_index == DEPOT_MAGAZINE_INDEX) {
- recirc_regions++;
- }
- print_medium_region(task, reader, printer, mapped_szone, level,
- region,
- (region == mapped_magazines[mag_index].mag_last_region)
- ? mapped_magazines[mag_index].mag_bytes_free_at_start
- : 0,
- (region == mapped_magazines[mag_index].mag_last_region)
- ? mapped_magazines[mag_index].mag_bytes_free_at_end
+ mag_index_t mag_index = MAGAZINE_INDEX_FOR_SMALL_REGION(region);
+ print_small_region(szone, verbose, region, (region == szone->small_rack.magazines[mag_index].mag_last_region)
+ ? szone->small_rack.magazines[mag_index].mag_bytes_free_at_start
+ : 0,
+ (region == szone->small_rack.magazines[mag_index].mag_last_region)
+ ? szone->small_rack.magazines[mag_index].mag_bytes_free_at_end
: 0);
- }
- }
-
-#if CONFIG_RECIRC_DEPOT
- mapped_recirc_depot = &mapped_magazines[DEPOT_MAGAZINE_INDEX];
- if (mapped_recirc_depot->mag_num_bytes_in_objects) {
- printer("Medium recirc depot: total bytes: %llu, in-use bytes: %llu, "
- "allocations: %llu, regions: %d (min # retained regions: %d)\n",
- mapped_recirc_depot->num_bytes_in_magazine,
- mapped_recirc_depot->mag_num_bytes_in_objects,
- mapped_recirc_depot->mag_num_objects, recirc_regions,
- recirc_retained_regions);
- } else {
- printer("Medium recirc depot is empty\n");
- }
-#else // CONFIG_RECIRC_DEPOT
- printer("Medium recirc depot not configured\n");
-#endif // CONFIG_RECIRC_DEPOT
-
- if (level > 0) {
- print_medium_free_list(task, reader, printer, &szone->medium_rack);
- }
- }
-#endif // CONFIG_MEDIUM_ALLOCATOR
-
- // Large
- large_debug_print(task, level, zone_address, reader, printer);
-}
-
-static void
-szone_print_self(szone_t *szone, boolean_t verbose)
-{
- szone_print(mach_task_self(), verbose ? MALLOC_VERBOSE_PRINT_LEVEL : 0,
- (vm_address_t)szone, _malloc_default_reader, malloc_report_simple);
-}
-
-static void
-szone_print_task(task_t task, unsigned level, vm_address_t zone_address,
- memory_reader_t reader, print_task_printer_t printer)
-{
- szone_print(task, level, zone_address, reader, printer);
+ }
+ }
+ if (verbose) {
+ print_small_free_list(&szone->small_rack);
+ }
}
static void
@@ -1320,9 +1059,15 @@
static MALLOC_INLINE void
szone_force_lock_magazine(szone_t *szone, magazine_t *mag)
{
- // Acquire the alloc lock first to avoid deadlocking with allocating threads
- _malloc_lock_lock(&mag->magazine_alloc_lock);
- SZONE_MAGAZINE_PTR_LOCK(mag);
+ while (1) {
+ SZONE_MAGAZINE_PTR_LOCK(mag);
+ if (!mag->alloc_underway) {
+ return;
+ }
+
+ SZONE_MAGAZINE_PTR_UNLOCK(mag);
+ yield();
+ }
}
static void
@@ -1340,15 +1085,6 @@
}
szone_force_lock_magazine(szone, &szone->small_rack.magazines[DEPOT_MAGAZINE_INDEX]);
-#if CONFIG_MEDIUM_ALLOCATOR
- if (szone->is_medium_engaged) {
- for (i = 0; i < szone->medium_rack.num_magazines; ++i) {
- szone_force_lock_magazine(szone, &szone->medium_rack.magazines[i]);
- }
- szone_force_lock_magazine(szone, &szone->medium_rack.magazines[DEPOT_MAGAZINE_INDEX]);
- }
-#endif
-
SZONE_LOCK(szone);
}
@@ -1359,23 +1095,12 @@
SZONE_UNLOCK(szone);
-#if CONFIG_MEDIUM_ALLOCATOR
- if (szone->is_medium_engaged) {
- for (i = -1; i < szone->medium_rack.num_magazines; ++i) {
- SZONE_MAGAZINE_PTR_UNLOCK((&(szone->medium_rack.magazines[i])));
- _malloc_lock_unlock(&szone->medium_rack.magazines[i].magazine_alloc_lock);
- }
- }
-#endif // CONFIG_MEDIUM_ALLOCATOR
-
for (i = -1; i < szone->small_rack.num_magazines; ++i) {
SZONE_MAGAZINE_PTR_UNLOCK((&(szone->small_rack.magazines[i])));
- _malloc_lock_unlock(&szone->small_rack.magazines[i].magazine_alloc_lock);
}
for (i = -1; i < szone->tiny_rack.num_magazines; ++i) {
SZONE_MAGAZINE_PTR_UNLOCK((&(szone->tiny_rack.magazines[i])));
- _malloc_lock_unlock(&szone->tiny_rack.magazines[i].magazine_alloc_lock);
}
}
@@ -1386,23 +1111,12 @@
SZONE_REINIT_LOCK(szone);
-#if CONFIG_MEDIUM_ALLOCATOR
- if (szone->is_medium_engaged) {
- for (i = -1; i < szone->medium_rack.num_magazines; ++i) {
- SZONE_MAGAZINE_PTR_REINIT_LOCK((&(szone->medium_rack.magazines[i])));
- _malloc_lock_init(&szone->medium_rack.magazines[i].magazine_alloc_lock);
- }
- }
-#endif // CONFIG_MEDIUM_ALLOCATOR
-
for (i = -1; i < szone->small_rack.num_magazines; ++i) {
SZONE_MAGAZINE_PTR_REINIT_LOCK((&(szone->small_rack.magazines[i])));
- _malloc_lock_init(&szone->small_rack.magazines[i].magazine_alloc_lock);
}
for (i = -1; i < szone->tiny_rack.num_magazines; ++i) {
SZONE_MAGAZINE_PTR_REINIT_LOCK((&(szone->tiny_rack.magazines[i])));
- _malloc_lock_init(&szone->tiny_rack.magazines[i].magazine_alloc_lock);
}
}
@@ -1417,23 +1131,6 @@
return 1;
}
SZONE_UNLOCK(szone);
-
-#if CONFIG_MEDIUM_ALLOCATOR
- if (szone->is_medium_engaged) {
- for (i = -1; i < szone->small_rack.num_magazines; ++i) {
- tookLock = SZONE_MAGAZINE_PTR_TRY_LOCK((&(szone->small_rack.magazines[i])));
- if (tookLock == 0) {
- return 1;
- }
- SZONE_MAGAZINE_PTR_UNLOCK((&(szone->small_rack.magazines[i])));
- tookLock = _malloc_lock_trylock(&szone->medium_rack.magazines[i].magazine_alloc_lock);
- if (tookLock == 0) {
- return 1;
- }
- _malloc_lock_unlock(&szone->medium_rack.magazines[i].magazine_alloc_lock);
- }
- }
-#endif // CONFIG_MEDIUM_ALLOCATOR
for (i = -1; i < szone->small_rack.num_magazines; ++i) {
tookLock = SZONE_MAGAZINE_PTR_TRY_LOCK((&(szone->small_rack.magazines[i])));
@@ -1441,11 +1138,6 @@
return 1;
}
SZONE_MAGAZINE_PTR_UNLOCK((&(szone->small_rack.magazines[i])));
- tookLock = _malloc_lock_trylock(&szone->small_rack.magazines[i].magazine_alloc_lock);
- if (tookLock == 0) {
- return 1;
- }
- _malloc_lock_unlock(&szone->small_rack.magazines[i].magazine_alloc_lock);
}
for (i = -1; i < szone->tiny_rack.num_magazines; ++i) {
@@ -1454,11 +1146,6 @@
return 1;
}
SZONE_MAGAZINE_PTR_UNLOCK((&(szone->tiny_rack.magazines[i])));
- tookLock = _malloc_lock_trylock(&szone->tiny_rack.magazines[i].magazine_alloc_lock);
- if (tookLock == 0) {
- return 1;
- }
- _malloc_lock_unlock(&szone->tiny_rack.magazines[i].magazine_alloc_lock);
}
return 0;
}
@@ -1468,27 +1155,168 @@
{
size_t total = 0;
- MAGMALLOC_PRESSURERELIEFBEGIN((void *)szone, szone->basic_zone.zone_name, (int)goal); // DTrace USDT Probe
- MALLOC_TRACE(TRACE_malloc_memory_pressure | DBG_FUNC_START, (uint64_t)szone, goal, 0, 0);
-
#if CONFIG_MADVISE_PRESSURE_RELIEF
- tiny_madvise_pressure_relief(&szone->tiny_rack);
- small_madvise_pressure_relief(&szone->small_rack);
-
-#if CONFIG_MEDIUM_ALLOCATOR
- if (szone->is_medium_engaged) {
- medium_madvise_pressure_relief(&szone->medium_rack);
- }
-#endif // CONFIG_MEDIUM_ALLOCATOR
-#endif // CONFIG_MADVISE_PRESSURE_RELIEF
-
-#if CONFIG_LARGE_CACHE && !CONFIG_DEFERRED_RECLAIM
- if (large_cache_enabled && szone->flotsam_enabled) {
+ mag_index_t mag_index;
+
+ magazine_t *tiny_depot_ptr = (&szone->tiny_rack.magazines[DEPOT_MAGAZINE_INDEX]);
+ magazine_t *small_depot_ptr = (&szone->small_rack.magazines[DEPOT_MAGAZINE_INDEX]);
+
+ for (mag_index = 0; mag_index < szone->tiny_rack.num_magazines; mag_index++) {
+ size_t index;
+ for (index = 0; index < szone->tiny_rack.region_generation->num_regions_allocated; ++index) {
+ SZONE_LOCK(szone);
+
+ region_t tiny = szone->tiny_rack.region_generation->hashed_regions[index];
+ if (!tiny || tiny == HASHRING_REGION_DEALLOCATED) {
+ SZONE_UNLOCK(szone);
+ continue;
+ }
+
+ magazine_t *mag_ptr = mag_lock_zine_for_region_trailer(szone->tiny_rack.magazines,
+ REGION_TRAILER_FOR_TINY_REGION(tiny),
+ MAGAZINE_INDEX_FOR_TINY_REGION(tiny));
+ SZONE_UNLOCK(szone);
+
+ /* Ordering is important here, the magazine of a region may potentially change
+ * during mag_lock_zine_for_region_trailer, so src_mag_index must be taken
+ * after we've obtained the lock.
+ */
+ mag_index_t src_mag_index = MAGAZINE_INDEX_FOR_TINY_REGION(tiny);
+
+ /* We can (and must) ignore magazines that are already in the recirc depot. */
+ if (src_mag_index == DEPOT_MAGAZINE_INDEX) {
+ SZONE_MAGAZINE_PTR_UNLOCK(mag_ptr);
+ continue;
+ }
+
+ if (tiny == mag_ptr->mag_last_region && (mag_ptr->mag_bytes_free_at_end || mag_ptr->mag_bytes_free_at_start)) {
+ tiny_finalize_region(&szone->tiny_rack, mag_ptr);
+ }
+
+ /* Because this region is currently in use, we can't safely madvise it while
+ * it's attached to the magazine. For this operation we have to remove it from
+ * the current mag, attach it to the depot and then madvise.
+ */
+
+ recirc_list_extract(&szone->tiny_rack, mag_ptr, REGION_TRAILER_FOR_TINY_REGION(tiny));
+ int objects_in_use = tiny_free_detach_region(&szone->tiny_rack, mag_ptr, tiny);
+
+ SZONE_MAGAZINE_PTR_LOCK(tiny_depot_ptr);
+ MAGAZINE_INDEX_FOR_TINY_REGION(tiny) = DEPOT_MAGAZINE_INDEX;
+ REGION_TRAILER_FOR_TINY_REGION(tiny)->pinned_to_depot = 0;
+
+ size_t bytes_inplay = tiny_free_reattach_region(&szone->tiny_rack, tiny_depot_ptr, tiny);
+
+ /* Fix up the metadata of the target magazine while the region is in the depot. */
+ mag_ptr->mag_num_bytes_in_objects -= bytes_inplay;
+ mag_ptr->num_bytes_in_magazine -= TINY_REGION_PAYLOAD_BYTES;
+ mag_ptr->mag_num_objects -= objects_in_use;
+
+ /* Now we can drop the magazine lock of the source mag. */
+ SZONE_MAGAZINE_PTR_UNLOCK(mag_ptr);
+
+ tiny_depot_ptr->mag_num_bytes_in_objects += bytes_inplay;
+ tiny_depot_ptr->num_bytes_in_magazine += TINY_REGION_PAYLOAD_BYTES;
+ tiny_depot_ptr->mag_num_objects -= objects_in_use;
+
+ recirc_list_splice_last(&szone->tiny_rack, tiny_depot_ptr, REGION_TRAILER_FOR_TINY_REGION(tiny));
+
+ /* Actually do the scan, done holding the depot lock, the call will drop the lock
+ * around the actual madvise syscalls.
+ */
+ tiny_free_scan_madvise_free(&szone->tiny_rack, tiny_depot_ptr, tiny);
+
+ /* Now the region is in the recirc depot, the next allocations to require more
+ * blocks will come along and take one of these regions back out of the depot.
+ * As OS X madvise's reuse on an per-region basis, we leave as many of these
+ * regions in the depot as possible after memory pressure.
+ */
+ SZONE_MAGAZINE_PTR_UNLOCK(tiny_depot_ptr);
+ }
+ }
+
+ for (mag_index = 0; mag_index < szone->small_rack.num_magazines; mag_index++) {
+ size_t index;
+ for (index = 0; index < szone->small_rack.region_generation->num_regions_allocated; ++index) {
+ SZONE_LOCK(szone);
+
+ region_t small = szone->small_rack.region_generation->hashed_regions[index];
+ if (!small || small == HASHRING_REGION_DEALLOCATED) {
+ SZONE_UNLOCK(szone);
+ continue;
+ }
+
+ magazine_t *mag_ptr = mag_lock_zine_for_region_trailer(szone->small_rack.magazines,
+ REGION_TRAILER_FOR_SMALL_REGION(small),
+ MAGAZINE_INDEX_FOR_SMALL_REGION(small));
+ SZONE_UNLOCK(szone);
+
+ /* Ordering is important here, the magazine of a region may potentially change
+ * during mag_lock_zine_for_region_trailer, so src_mag_index must be taken
+ * after we've obtained the lock.
+ */
+ mag_index_t src_mag_index = MAGAZINE_INDEX_FOR_SMALL_REGION(small);
+
+ /* We can (and must) ignore magazines that are already in the recirc depot. */
+ if (src_mag_index == DEPOT_MAGAZINE_INDEX) {
+ SZONE_MAGAZINE_PTR_UNLOCK(mag_ptr);
+ continue;
+ }
+
+ if (small == mag_ptr->mag_last_region && (mag_ptr->mag_bytes_free_at_end || mag_ptr->mag_bytes_free_at_start)) {
+ small_finalize_region(&szone->small_rack, mag_ptr);
+ }
+
+ /* Because this region is currently in use, we can't safely madvise it while
+ * it's attached to the magazine. For this operation we have to remove it from
+ * the current mag, attach it to the depot and then madvise.
+ */
+
+ recirc_list_extract(&szone->small_rack, mag_ptr, REGION_TRAILER_FOR_SMALL_REGION(small));
+ int objects_in_use = small_free_detach_region(&szone->small_rack, mag_ptr, small);
+
+ SZONE_MAGAZINE_PTR_LOCK(small_depot_ptr);
+ MAGAZINE_INDEX_FOR_SMALL_REGION(small) = DEPOT_MAGAZINE_INDEX;
+ REGION_TRAILER_FOR_SMALL_REGION(small)->pinned_to_depot = 0;
+
+ size_t bytes_inplay = small_free_reattach_region(&szone->small_rack, small_depot_ptr, small);
+
+ /* Fix up the metadata of the target magazine while the region is in the depot. */
+ mag_ptr->mag_num_bytes_in_objects -= bytes_inplay;
+ mag_ptr->num_bytes_in_magazine -= SMALL_REGION_PAYLOAD_BYTES;
+ mag_ptr->mag_num_objects -= objects_in_use;
+
+ /* Now we can drop the magazine lock of the source mag. */
+ SZONE_MAGAZINE_PTR_UNLOCK(mag_ptr);
+
+ small_depot_ptr->mag_num_bytes_in_objects += bytes_inplay;
+ small_depot_ptr->num_bytes_in_magazine += SMALL_REGION_PAYLOAD_BYTES;
+ small_depot_ptr->mag_num_objects -= objects_in_use;
+
+ recirc_list_splice_last(&szone->small_rack, small_depot_ptr, REGION_TRAILER_FOR_SMALL_REGION(small));
+
+ /* Actually do the scan, done holding the depot lock, the call will drop the lock
+ * around the actual madvise syscalls.
+ */
+ small_free_scan_madvise_free(&szone->small_rack, small_depot_ptr, small);
+
+ /* Now the region is in the recirc depot, the next allocations to require more
+ * blocks will come along and take one of these regions back out of the depot.
+ * As OS X madvise's reuse on an per-region basis, we leave as many of these
+ * regions in the depot as possible after memory pressure.
+ */
+ SZONE_MAGAZINE_PTR_UNLOCK(small_depot_ptr);
+ }
+ }
+#endif
+
+#if CONFIG_LARGE_CACHE
+ if (szone->flotsam_enabled) {
SZONE_LOCK(szone);
// stack allocated copy of the death-row cache
int idx = szone->large_entry_cache_oldest, idx_max = szone->large_entry_cache_newest;
- large_entry_t local_entry_cache[LARGE_ENTRY_CACHE_SIZE_HIGH];
+ large_entry_t local_entry_cache[LARGE_ENTRY_CACHE_SIZE];
memcpy((void *)local_entry_cache, (void *)szone->large_entry_cache, sizeof(local_entry_cache));
@@ -1505,22 +1333,20 @@
// deallocate the death-row cache outside the zone lock
size_t total = 0;
while (idx != idx_max) {
- mvm_deallocate_pages((void *)local_entry_cache[idx].address, local_entry_cache[idx].size, szone->debug_flags);
+ mvm_deallocate_pages((void *)local_entry_cache[idx].address, local_entry_cache[idx].size, 0);
total += local_entry_cache[idx].size;
- if (++idx == szone->large_cache_depth) {
+ if (++idx == LARGE_ENTRY_CACHE_SIZE) {
idx = 0;
}
}
if (0 != local_entry_cache[idx].address && 0 != local_entry_cache[idx].size) {
- mvm_deallocate_pages((void *)local_entry_cache[idx].address, local_entry_cache[idx].size, szone->debug_flags);
+ mvm_deallocate_pages((void *)local_entry_cache[idx].address, local_entry_cache[idx].size, 0);
total += local_entry_cache[idx].size;
}
}
-#endif // CONFIG_LARGE_CACHE && !CONFIG_DEFERRED_RECLAIM
-
- MAGMALLOC_PRESSURERELIEFEND((void *)szone, szone->basic_zone.zone_name, (int)goal, (int)total); // DTrace USDT Probe
- MALLOC_TRACE(TRACE_malloc_memory_pressure | DBG_FUNC_END, (uint64_t)szone, goal, total, 0);
-
+#endif
+
+ MAGMALLOC_PRESSURERELIEF((void *)szone, (int)goal, (int)total); // DTrace USDT Probe
return total;
}
@@ -1578,118 +1404,50 @@
stats->size_in_use = 0; // DEPRECATED szone->num_bytes_in_huge_objects;
stats->max_size_in_use = stats->size_allocated = 0;
return 1;
- case 4: {
- size_t s = 0;
- unsigned t = 0;
- size_t u = 0;
- size_t sa = 0;
-
-#if CONFIG_MEDIUM_ALLOCATOR
- mag_index_t mag_index;
- if (szone->is_medium_engaged) {
- for (mag_index = -1; mag_index < szone->medium_rack.num_magazines; mag_index++) {
- s += szone->medium_rack.magazines[mag_index].mag_bytes_free_at_start;
- s += szone->medium_rack.magazines[mag_index].mag_bytes_free_at_end;
- t += szone->medium_rack.magazines[mag_index].mag_num_objects;
- u += szone->medium_rack.magazines[mag_index].mag_num_bytes_in_objects;
- }
- }
-
- sa = (szone->medium_rack.num_regions - szone->medium_rack.num_regions_dealloc) * MEDIUM_REGION_SIZE;
-#endif // CONFIG_MEDIUM_ALLOCATOR
-
- stats->blocks_in_use = t;
- stats->size_in_use = u;
- stats->size_allocated = sa;
- stats->max_size_in_use = stats->size_allocated - s;
- return 1;
- }}
+ }
return 0;
}
-static kern_return_t
-szone_statistics_task(task_t task, vm_address_t zone_address,
- memory_reader_t reader, malloc_statistics_t *stats)
-{
- reader = reader_or_in_memory_fallback(reader, task);
-
- szone_t *szone;
- kern_return_t err;
-
- err = reader(task, zone_address, sizeof(szone_t), (void**)&szone);
- if (err) return err;
-
+static void
+szone_statistics(szone_t *szone, malloc_statistics_t *stats)
+{
size_t large;
+
size_t s = 0;
unsigned t = 0;
size_t u = 0;
mag_index_t mag_index;
- magazine_t *mags;
- err = reader(task, (vm_address_t)szone->tiny_rack.magazines, sizeof(magazine_t) * szone->tiny_rack.num_magazines, (void**)&mags);
- if (err) return err;
-
for (mag_index = -1; mag_index < szone->tiny_rack.num_magazines; mag_index++) {
- s += mags[mag_index].mag_bytes_free_at_start;
- s += mags[mag_index].mag_bytes_free_at_end;
- t += mags[mag_index].mag_num_objects;
- u += mags[mag_index].mag_num_bytes_in_objects;
- }
-
- err = reader(task, (vm_address_t)szone->small_rack.magazines, sizeof(magazine_t) * szone->small_rack.num_magazines, (void**)&mags);
- if (err) return err;
+ s += szone->tiny_rack.magazines[mag_index].mag_bytes_free_at_start;
+ s += szone->tiny_rack.magazines[mag_index].mag_bytes_free_at_end;
+ t += szone->tiny_rack.magazines[mag_index].mag_num_objects;
+ u += szone->tiny_rack.magazines[mag_index].mag_num_bytes_in_objects;
+ }
for (mag_index = -1; mag_index < szone->small_rack.num_magazines; mag_index++) {
- s += mags[mag_index].mag_bytes_free_at_start;
- s += mags[mag_index].mag_bytes_free_at_end;
- t += mags[mag_index].mag_num_objects;
- u += mags[mag_index].mag_num_bytes_in_objects;
- }
-
-#if CONFIG_MEDIUM_ALLOCATOR
- if (szone->is_medium_engaged) {
- for (mag_index = -1; mag_index < szone->medium_rack.num_magazines; mag_index++) {
- s += szone->medium_rack.magazines[mag_index].mag_bytes_free_at_start;
- s += szone->medium_rack.magazines[mag_index].mag_bytes_free_at_end;
- t += szone->medium_rack.magazines[mag_index].mag_num_objects;
- u += szone->medium_rack.magazines[mag_index].mag_num_bytes_in_objects;
- }
- }
-#endif // CONFIG_MEDIUM_ALLOCATOR
-
- large = szone->num_bytes_in_large_objects;
-
- stats->blocks_in_use = t + szone->num_large_objects_in_use;
+ s += szone->small_rack.magazines[mag_index].mag_bytes_free_at_start;
+ s += szone->small_rack.magazines[mag_index].mag_bytes_free_at_end;
+ t += szone->small_rack.magazines[mag_index].mag_num_objects;
+ u += szone->small_rack.magazines[mag_index].mag_num_bytes_in_objects;
+ }
+
+ large = szone->num_bytes_in_large_objects + 0; // DEPRECATED szone->num_bytes_in_huge_objects;
+
+ stats->blocks_in_use = t + szone->num_large_objects_in_use + 0; // DEPRECATED szone->num_huge_entries;
stats->size_in_use = u + large;
stats->max_size_in_use = stats->size_allocated =
(szone->tiny_rack.num_regions - szone->tiny_rack.num_regions_dealloc) * TINY_REGION_SIZE +
(szone->small_rack.num_regions - szone->small_rack.num_regions_dealloc) * SMALL_REGION_SIZE + large;
-
-#if CONFIG_MEDIUM_ALLOCATOR
- if (szone->is_medium_engaged) {
- stats->max_size_in_use += (szone->medium_rack.num_regions -
- szone->medium_rack.num_regions_dealloc) * MEDIUM_REGION_SIZE;
- }
-#endif
// Now we account for the untouched areas
stats->max_size_in_use -= s;
-
- return KERN_SUCCESS;
-}
-
-static void
-szone_statistics(szone_t *szone, malloc_statistics_t *stats)
-{
- szone_statistics_task(mach_task_self(), (vm_address_t)szone, NULL, stats);
}
const struct malloc_introspection_t szone_introspect = {
- (void *)szone_ptr_in_use_enumerator, (void *)szone_good_size, (void *)szone_check, (void *)szone_print_self, szone_log,
+ (void *)szone_ptr_in_use_enumerator, (void *)szone_good_size, (void *)szone_check, (void *)szone_print, szone_log,
(void *)szone_force_lock, (void *)szone_force_unlock, (void *)szone_statistics, (void *)szone_locked, NULL, NULL, NULL,
NULL, /* Zone enumeration version 7 and forward. */
- (void *)szone_reinit_lock, // reinit_lock version 9 and forward
- (void *)szone_print_task, // print task, version 11 and forward
- (void *)szone_statistics_task // stats for task, version 12 and forward
+ (void *)szone_reinit_lock, // reinit_lock version 9 and foward
}; // marked as const to spare the DATA section
szone_t *
@@ -1699,12 +1457,12 @@
#if defined(__i386__) || defined(__x86_64__)
if (_COMM_PAGE_VERSION_REQD > (*((uint16_t *)_COMM_PAGE_VERSION))) {
- MALLOC_REPORT_FATAL_ERROR((*((uint16_t *)_COMM_PAGE_VERSION)), "comm page version mismatch");
+ MALLOC_PRINTF_FATAL_ERROR((*((uint16_t *)_COMM_PAGE_VERSION)), "comm page version mismatch");
}
#endif
/* get memory for the zone. */
- szone = mvm_allocate_pages(SZONE_PAGED_SIZE, 0, DISABLE_ASLR, VM_MEMORY_MALLOC);
+ szone = mvm_allocate_pages(SZONE_PAGED_SIZE, 0, 0, VM_MEMORY_MALLOC);
if (!szone) {
return NULL;
}
@@ -1725,60 +1483,44 @@
debug_flags |= DISABLE_ASLR;
}
-#if CONFIG_MEDIUM_ALLOCATOR || CONFIG_LARGE_CACHE
- uint64_t memsize = platform_hw_memsize();
-#endif // CONFIG_MEDIUM_ALLOCATOR || CONFIG_LARGE_CACHE
-
-#if CONFIG_MEDIUM_ALLOCATOR
- szone->is_medium_engaged = (magazine_medium_enabled &&
- (memsize >= magazine_medium_active_threshold));
-#endif // CONFIG_MEDIUM_ALLOCATOR
+#if CONFIG_SMALL_CUTTOFF_127KB
+ debug_flags |= MALLOC_EXTENDED_SMALL_SLOTS;
+ szone->is_largemem = 1;
+ szone->large_threshold = LARGE_THRESHOLD_LARGEMEM;
+ szone->vm_copy_threshold = VM_COPY_THRESHOLD_LARGEMEM;
+#else // CONFIG_SMALL_CUTTOFF_127KB
+ debug_flags &= ~MALLOC_EXTENDED_SMALL_SLOTS;
+ szone->is_largemem = 0;
+ szone->large_threshold = LARGE_THRESHOLD;
+ szone->vm_copy_threshold = VM_COPY_THRESHOLD;
+#endif // CONFIG_SMALL_CUTTOFF_127KB
// Query the number of configured processors.
// Uniprocessor case gets just one tiny and one small magazine (whose index is zero). This gives
// the same behavior as the original scalable malloc. MP gets per-CPU magazines
// that scale (way) better.
- unsigned int max_mags = mag_max_magazines();
- uint32_t num_magazines = (max_mags > 1) ? MIN(max_mags, TINY_MAX_MAGAZINES) : 1;
+ uint32_t nproc = platform_cpu_count();
+ uint32_t num_magazines = (nproc > 1) ? MIN(nproc, TINY_MAX_MAGAZINES) : 1;
rack_init(&szone->tiny_rack, RACK_TYPE_TINY, num_magazines, debug_flags);
rack_init(&szone->small_rack, RACK_TYPE_SMALL, num_magazines, debug_flags);
-#if CONFIG_MEDIUM_ALLOCATOR
- if (szone->is_medium_engaged) {
- unsigned max_medium_mags = mag_max_medium_magazines();
- uint32_t num_medium_mags = (max_medium_mags > 1) ?
- MIN(max_medium_mags, TINY_MAX_MAGAZINES) : 1;
- rack_init(&szone->medium_rack, RACK_TYPE_MEDIUM, num_medium_mags,
- debug_flags);
- }
-#endif // CONFIG_MEDIUM_ALLOCATOR
-
#if CONFIG_LARGE_CACHE
- if (large_cache_enabled) {
- // madvise(..., MADV_REUSABLE) death-row arrivals above this threshold [~0.1%]
- szone->large_entry_cache_reserve_limit = (size_t)(memsize >> 10);
- if (memsize >= magazine_large_expanded_cache_threshold) {
- szone->large_cache_depth = LARGE_ENTRY_CACHE_SIZE_HIGH;
- szone->large_cache_entry_limit = LARGE_ENTRY_SIZE_ENTRY_LIMIT_HIGH;
- } else {
- szone->large_cache_depth = LARGE_ENTRY_CACHE_SIZE_LOW;
- szone->large_cache_entry_limit = LARGE_ENTRY_SIZE_ENTRY_LIMIT_LOW;
- }
-
- /* <rdar://problem/6610904> Reset protection when returning a previous large allocation? */
- int32_t libSystemVersion = NSVersionOfLinkTimeLibrary("System");
- if ((-1 != libSystemVersion) && ((libSystemVersion >> 16) < 112) /* CFSystemVersionSnowLeopard */) {
- szone->large_legacy_reset_mprotect = TRUE;
- } else {
- szone->large_legacy_reset_mprotect = FALSE;
- }
+ // madvise(..., MADV_REUSABLE) death-row arrivals above this threshold [~0.1%]
+ szone->large_entry_cache_reserve_limit = (size_t)(platform_hw_memsize() >> 10);
+
+ /* <rdar://problem/6610904> Reset protection when returning a previous large allocation? */
+ int32_t libSystemVersion = NSVersionOfLinkTimeLibrary("System");
+ if ((-1 != libSystemVersion) && ((libSystemVersion >> 16) < 112) /* CFSystemVersionSnowLeopard */) {
+ szone->large_legacy_reset_mprotect = TRUE;
+ } else {
+ szone->large_legacy_reset_mprotect = FALSE;
}
#endif
// Initialize the security token.
szone->cookie = (uintptr_t)malloc_entropy[0];
- szone->basic_zone.version = 16;
+ szone->basic_zone.version = 9;
szone->basic_zone.size = (void *)szone_size;
szone->basic_zone.malloc = (void *)szone_malloc;
szone->basic_zone.calloc = (void *)szone_calloc;
@@ -1792,13 +1534,6 @@
szone->basic_zone.memalign = (void *)szone_memalign;
szone->basic_zone.free_definite_size = (void *)szone_free_definite_size;
szone->basic_zone.pressure_relief = (void *)szone_pressure_relief;
- szone->basic_zone.claimed_address = (void *)szone_claimed_address;
- szone->basic_zone.try_free_default = (void *)szone_try_free_default;
-
- szone->basic_zone.malloc_type_malloc = (void *)szone_malloc_type_malloc;
- szone->basic_zone.malloc_type_calloc = (void *)szone_malloc_type_calloc;
- szone->basic_zone.malloc_type_realloc = (void *)szone_malloc_type_realloc;
- szone->basic_zone.malloc_type_memalign = (void *)szone_malloc_type_memalign;
/* Set to zero once and for all as required by CFAllocator. */
szone->basic_zone.reserved1 = 0;