Loading...
--- libmalloc/libmalloc-715.120.13/src/magazine_malloc.c
+++ libmalloc/libmalloc-283/src/magazine_malloc.c
@@ -48,33 +48,13 @@
int max_magazines;
// Control whether medium is enabled at all when creating new magazine zones
-bool magazine_medium_enabled = DEFAULT_MEDIUM_ALLOCATOR_ENABLED;
+bool magazine_medium_enabled = true;
// Control the DRAM limit at which medium kicks in.
uint64_t magazine_medium_active_threshold = MEDIUM_ACTIVATION_THRESHOLD;
-#if CONFIG_MEDIUM_ALLOCATOR
-
-// Control the dram divisor that's used to scale up medium's madvise window.
-// We'll double the window for each multiple of magazine_medium_madvise_dram_scale_divisor
-// bytes of dram on the system rounded down to the neareast power of 2.
-// This is done by setting magazine_medium_madvise_window_scale_factor.
-uint64_t magazine_medium_madvise_dram_scale_divisor = MEDIUM_MADVISE_DRAM_SCALE_DIVISOR;
-
-// Controls how much to scale up medium's madvise window.
-uint64_t magazine_medium_madvise_window_scale_factor = 1;
-#endif // CONFIG_MEDIUM_ALLOCATOR
-
// Control the DRAM limit at which the expanded large cache kicks in.
uint64_t magazine_large_expanded_cache_threshold = LARGE_CACHE_EXPANDED_THRESHOLD;
-
-#if CONFIG_AGGRESSIVE_MADVISE
-bool aggressive_madvise_enabled = DEFAULT_AGGRESSIVE_MADVISE_ENABLED;
-#endif // CONFIG_AGGRESSIVE_MADVISE
-
-#if CONFIG_LARGE_CACHE
-bool large_cache_enabled = DEFAULT_LARGE_CACHE_ENABLED;
-#endif // CONFIG_LARGE_CACHE
// <rdar://problem/47353961> Maximum number of magzines that the medium
// allocator will use. This addresses a 32-bit load-offset range issue found
@@ -90,30 +70,27 @@
/*
* Mark these MALLOC_NOINLINE to avoid bloating the purgeable zone call backs
*/
-static void
-_szone_free(szone_t *szone, void *ptr, bool try)
+void
+szone_free(szone_t *szone, void *ptr)
{
region_t tiny_region;
region_t small_region;
#if DEBUG_MALLOC
if (LOG(szone, ptr)) {
- malloc_report(ASL_LEVEL_INFO, "in _szone_free with %p\n", ptr);
+ malloc_report(ASL_LEVEL_INFO, "in szone_free with %p\n", ptr);
}
#endif
if (!ptr) {
return;
- }
- if ((uintptr_t)ptr & (TINY_QUANTUM - 1)) {
- if (!try) {
- malloc_zone_error(szone->debug_flags, true, "Non-aligned pointer %p being freed\n", ptr);
- return;
- }
- goto not_claimed;
}
/*
* Try to free to a tiny region.
*/
+ if ((uintptr_t)ptr & (TINY_QUANTUM - 1)) {
+ malloc_zone_error(szone->debug_flags, true, "Non-aligned pointer %p being freed\n", ptr);
+ return;
+ }
if ((tiny_region = tiny_region_for_ptr_no_lock(&szone->tiny_rack, ptr)) != NULL) {
if (TINY_INDEX_FOR_PTR(ptr) >= NUM_TINY_BLOCKS) {
malloc_zone_error(szone->debug_flags, true, "Pointer %p to metadata being freed\n", ptr);
@@ -127,11 +104,8 @@
* Try to free to a small region.
*/
if ((uintptr_t)ptr & (SMALL_QUANTUM - 1)) {
- if (!try) {
- malloc_zone_error(szone->debug_flags, true, "Non-aligned pointer %p being freed (2)\n", ptr);
- return;
- }
- goto not_claimed;
+ malloc_zone_error(szone->debug_flags, true, "Non-aligned pointer %p being freed (2)\n", ptr);
+ return;
}
if ((small_region = small_region_for_ptr_no_lock(&szone->small_rack, ptr)) != NULL) {
if (SMALL_META_INDEX_FOR_PTR(ptr) >= NUM_SMALL_BLOCKS) {
@@ -158,31 +132,10 @@
/* check that it's a legal large allocation */
if ((uintptr_t)ptr & (vm_page_quanta_size - 1)) {
- if (!try) {
- malloc_zone_error(szone->debug_flags, true, "non-page-aligned, non-allocated pointer %p being freed\n", ptr);
- return;
- }
- goto not_claimed;
- }
- bool claimed = free_large(szone, ptr, try);
- if (!try || claimed) {
- return;
- }
-
-not_claimed:
- find_zone_and_free(ptr, true);
-}
-
-void
-szone_free(szone_t *szone, void *ptr)
-{
- _szone_free(szone, ptr, false);
-}
-
-static void
-szone_try_free_default(szone_t *szone, void *ptr)
-{
- _szone_free(szone, ptr, true);
+ malloc_zone_error(szone->debug_flags, true, "non-page-aligned, non-allocated pointer %p being freed\n", ptr);
+ return;
+ }
+ free_large(szone, ptr);
}
void
@@ -254,7 +207,7 @@
malloc_zone_error(szone->debug_flags, true, "non-page-aligned, non-allocated pointer %p being freed\n", ptr);
return;
}
- free_large(szone, ptr, false);
+ free_large(szone, ptr);
}
MALLOC_NOINLINE void *
@@ -284,7 +237,7 @@
ptr = medium_malloc_should_clear(&szone->medium_rack, msize, cleared_requested);
#endif
} else {
- size_t num_kernel_pages = round_large_page_quanta(size) >> large_vm_page_quanta_shift;
+ size_t num_kernel_pages = round_page_quanta(size) >> vm_page_quanta_shift;
if (num_kernel_pages == 0) { /* Overflowed */
ptr = 0;
} else {
@@ -303,10 +256,6 @@
memset(ptr, SCRIBBLE_BYTE, szone_size(szone, ptr));
}
- if (os_unlikely(!ptr)) {
- malloc_set_errno_fast(MZ_POSIX, ENOMEM);
- }
-
return ptr;
}
@@ -314,12 +263,6 @@
szone_malloc(szone_t *szone, size_t size)
{
return szone_malloc_should_clear(szone, size, 0);
-}
-
-static void *
-szone_malloc_type_malloc(szone_t *szone, size_t size, malloc_type_id_t type_id)
-{
- return szone_malloc(szone, size);
}
void *
@@ -330,13 +273,6 @@
return NULL;
}
return szone_malloc_should_clear(szone, total_bytes, 1);
-}
-
-static void *
-szone_malloc_type_calloc(szone_t *szone, size_t num_items, size_t size,
- malloc_type_id_t type_id)
-{
- return szone_calloc(szone, num_items, size);
}
void *
@@ -349,7 +285,7 @@
} else {
size_t num_kernel_pages;
- num_kernel_pages = round_large_page_quanta(size) >> large_vm_page_quanta_shift;
+ num_kernel_pages = round_page_quanta(size) >> vm_page_quanta_shift;
ptr = large_malloc(szone, num_kernel_pages, 0, 0);
}
@@ -605,13 +541,6 @@
return new_ptr;
}
-static void *
-szone_malloc_type_realloc(szone_t *szone, void *ptr, size_t size,
- malloc_type_id_t type_id)
-{
- return szone_realloc(szone, ptr, size);
-}
-
void *
szone_memalign(szone_t *szone, size_t alignment, size_t size)
{
@@ -664,25 +593,16 @@
return szone_malloc(szone, size);
}
// ensure block allocated by large does not have a small-possible size
- size_t num_kernel_pages = round_large_page_quanta(MAX(LARGE_THRESHOLD(szone) + 1,
- size)) >> large_vm_page_quanta_shift;
+ size_t num_kernel_pages = round_page_quanta(MAX(LARGE_THRESHOLD(szone) + 1,
+ size)) >> vm_page_quanta_shift;
if (num_kernel_pages == 0) { /* Overflowed */
return NULL;
} else {
- MALLOC_STATIC_ASSERT(sizeof(size_t) == sizeof(long), "builtin_ctzl should be the right intrinsic for size_t");
-
return large_malloc(szone, num_kernel_pages,
- MAX(vm_page_quanta_shift, __builtin_ctzl(alignment)), 0);
+ MAX(vm_page_quanta_shift, __builtin_ctz((unsigned)alignment)), 0);
}
/* NOTREACHED */
__builtin_unreachable();
-}
-
-static void *
-szone_malloc_type_memalign(szone_t *szone, size_t align, size_t size,
- malloc_type_id_t type_id)
-{
- return szone_memalign(szone, align, size);
}
// Given a size, returns the number of pointers allocated capable of holding
@@ -734,10 +654,36 @@
vm_range_t range_to_deallocate;
#if CONFIG_LARGE_CACHE
- if (large_cache_enabled) {
- large_destroy_cache(szone);
- }
-#endif // CONFIG_LARGE_CACHE
+ SZONE_LOCK(szone);
+
+ /* disable any memory pressure responder */
+ szone->flotsam_enabled = FALSE;
+
+ // stack allocated copy of the death-row cache
+ int idx = szone->large_entry_cache_oldest, idx_max = szone->large_entry_cache_newest;
+ large_entry_t local_entry_cache[LARGE_ENTRY_CACHE_SIZE_HIGH];
+
+ memcpy((void *)local_entry_cache, (void *)szone->large_entry_cache, sizeof(local_entry_cache));
+
+ szone->large_entry_cache_oldest = szone->large_entry_cache_newest = 0;
+ szone->large_entry_cache[0].address = 0x0;
+ szone->large_entry_cache[0].size = 0;
+ szone->large_entry_cache_bytes = 0;
+ szone->large_entry_cache_reserve_bytes = 0;
+
+ SZONE_UNLOCK(szone);
+
+ // deallocate the death-row cache outside the zone lock
+ while (idx != idx_max) {
+ mvm_deallocate_pages((void *)local_entry_cache[idx].address, local_entry_cache[idx].size, 0);
+ if (++idx == szone->large_cache_depth) {
+ idx = 0;
+ }
+ }
+ if (0 != local_entry_cache[idx].address && 0 != local_entry_cache[idx].size) {
+ mvm_deallocate_pages((void *)local_entry_cache[idx].address, local_entry_cache[idx].size, 0);
+ }
+#endif
/* destroy large entries */
index = szone->num_large_entries;
@@ -750,7 +696,7 @@
}
large_entries_free_no_lock(szone, szone->large_entries, szone->num_large_entries, &range_to_deallocate);
if (range_to_deallocate.size) {
- mvm_deallocate_pages((void *)range_to_deallocate.address, (size_t)range_to_deallocate.size, szone->debug_flags);
+ mvm_deallocate_pages((void *)range_to_deallocate.address, (size_t)range_to_deallocate.size, 0);
}
/* destroy allocator regions */
@@ -806,7 +752,7 @@
// Check for integer overflow on the size, since unlike the two cases above,
// there is no upper bound on allocation size at this point.
- if (size > round_large_page_quanta(size)) {
+ if (size > round_page_quanta(size)) {
return (size_t)(-1LL);
}
@@ -818,7 +764,7 @@
malloc_report(ASL_LEVEL_INFO, "szone_good_size() invariant broken %y\n", size);
}
#endif
- return round_large_page_quanta(size);
+ return round_page_quanta(size);
}
boolean_t
@@ -842,10 +788,33 @@
{
size_t index;
- boolean_t tiny_result = tiny_check(&szone->tiny_rack, szone_check_counter);
- if (!tiny_result) {
- szone->debug_flags &= ~CHECK_REGIONS;
- return 0;
+ /* check tiny regions - chould check region count */
+ for (index = 0; index < szone->tiny_rack.region_generation->num_regions_allocated; ++index) {
+ region_t tiny = szone->tiny_rack.region_generation->hashed_regions[index];
+
+ if (HASHRING_REGION_DEALLOCATED == tiny) {
+ continue;
+ }
+
+ if (tiny) {
+ magazine_t *tiny_mag_ptr = mag_lock_zine_for_region_trailer(szone->tiny_rack.magazines,
+ REGION_TRAILER_FOR_TINY_REGION(tiny),
+ MAGAZINE_INDEX_FOR_TINY_REGION(tiny));
+
+ if (!tiny_check_region(&szone->tiny_rack, tiny, index, szone_check_counter)) {
+ SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
+ szone->debug_flags &= ~CHECK_REGIONS;
+ return 0;
+ }
+ SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
+ }
+ }
+ /* check tiny free lists */
+ for (index = 0; index < NUM_TINY_SLOTS; ++index) {
+ if (!tiny_free_list_check(&szone->tiny_rack, (grain_t)index, szone_check_counter)) {
+ szone->debug_flags &= ~CHECK_REGIONS;
+ return 0;
+ }
}
/* check small regions - could check region count */
@@ -931,10 +900,6 @@
return szone_check_all(szone, "");
}
-// To support the sanitizer zone, we need to be able to perform zone enumeration across different
-// architecture slices on macOS, because ReportCrash is always running as a native (arm64e) process,
-// but we also need to be able to inspect x86_64 targets that are running under Rosetta. So the data
-// layout and zone logic needs to match between x86_64 and arm64(e).
static kern_return_t
szone_ptr_in_use_enumerator(task_t task,
void *context,
@@ -946,7 +911,9 @@
szone_t *szone;
kern_return_t err;
- reader = reader_or_in_memory_fallback(reader, task);
+ if (!reader) {
+ reader = _malloc_default_reader;
+ }
err = reader(task, zone_address, sizeof(szone_t), (void **)&szone);
if (err) {
@@ -1073,15 +1040,6 @@
zone_address, info[0], info[1], info[2], info[3], info[12]);
printer("\ttiny=%u(%u) small=%u(%u) large=%u(%u)\n", info[4],
info[5], info[6], info[7], info[8], info[9]);
-
- // FIXME: The rest of the code here assumes that regions have their normal
- // alignment, which isn't guaranteed when looking at regions mapped from
- // other processes
- if (!mach_task_is_self(task)) {
- printer("(unable to safely further examine remote process)\n");
- return;
- }
-
// tiny
printer("%lu tiny regions:\n", mapped_szone->tiny_rack.num_regions);
if (mapped_szone->tiny_rack.num_regions_dealloc) {
@@ -1320,9 +1278,15 @@
static MALLOC_INLINE void
szone_force_lock_magazine(szone_t *szone, magazine_t *mag)
{
- // Acquire the alloc lock first to avoid deadlocking with allocating threads
- _malloc_lock_lock(&mag->magazine_alloc_lock);
- SZONE_MAGAZINE_PTR_LOCK(mag);
+ while (1) {
+ SZONE_MAGAZINE_PTR_LOCK(mag);
+ if (!mag->alloc_underway) {
+ return;
+ }
+
+ SZONE_MAGAZINE_PTR_UNLOCK(mag);
+ yield();
+ }
}
static void
@@ -1363,19 +1327,16 @@
if (szone->is_medium_engaged) {
for (i = -1; i < szone->medium_rack.num_magazines; ++i) {
SZONE_MAGAZINE_PTR_UNLOCK((&(szone->medium_rack.magazines[i])));
- _malloc_lock_unlock(&szone->medium_rack.magazines[i].magazine_alloc_lock);
}
}
#endif // CONFIG_MEDIUM_ALLOCATOR
for (i = -1; i < szone->small_rack.num_magazines; ++i) {
SZONE_MAGAZINE_PTR_UNLOCK((&(szone->small_rack.magazines[i])));
- _malloc_lock_unlock(&szone->small_rack.magazines[i].magazine_alloc_lock);
}
for (i = -1; i < szone->tiny_rack.num_magazines; ++i) {
SZONE_MAGAZINE_PTR_UNLOCK((&(szone->tiny_rack.magazines[i])));
- _malloc_lock_unlock(&szone->tiny_rack.magazines[i].magazine_alloc_lock);
}
}
@@ -1390,19 +1351,16 @@
if (szone->is_medium_engaged) {
for (i = -1; i < szone->medium_rack.num_magazines; ++i) {
SZONE_MAGAZINE_PTR_REINIT_LOCK((&(szone->medium_rack.magazines[i])));
- _malloc_lock_init(&szone->medium_rack.magazines[i].magazine_alloc_lock);
}
}
#endif // CONFIG_MEDIUM_ALLOCATOR
for (i = -1; i < szone->small_rack.num_magazines; ++i) {
SZONE_MAGAZINE_PTR_REINIT_LOCK((&(szone->small_rack.magazines[i])));
- _malloc_lock_init(&szone->small_rack.magazines[i].magazine_alloc_lock);
}
for (i = -1; i < szone->tiny_rack.num_magazines; ++i) {
SZONE_MAGAZINE_PTR_REINIT_LOCK((&(szone->tiny_rack.magazines[i])));
- _malloc_lock_init(&szone->tiny_rack.magazines[i].magazine_alloc_lock);
}
}
@@ -1426,11 +1384,6 @@
return 1;
}
SZONE_MAGAZINE_PTR_UNLOCK((&(szone->small_rack.magazines[i])));
- tookLock = _malloc_lock_trylock(&szone->medium_rack.magazines[i].magazine_alloc_lock);
- if (tookLock == 0) {
- return 1;
- }
- _malloc_lock_unlock(&szone->medium_rack.magazines[i].magazine_alloc_lock);
}
}
#endif // CONFIG_MEDIUM_ALLOCATOR
@@ -1441,11 +1394,6 @@
return 1;
}
SZONE_MAGAZINE_PTR_UNLOCK((&(szone->small_rack.magazines[i])));
- tookLock = _malloc_lock_trylock(&szone->small_rack.magazines[i].magazine_alloc_lock);
- if (tookLock == 0) {
- return 1;
- }
- _malloc_lock_unlock(&szone->small_rack.magazines[i].magazine_alloc_lock);
}
for (i = -1; i < szone->tiny_rack.num_magazines; ++i) {
@@ -1454,11 +1402,6 @@
return 1;
}
SZONE_MAGAZINE_PTR_UNLOCK((&(szone->tiny_rack.magazines[i])));
- tookLock = _malloc_lock_trylock(&szone->tiny_rack.magazines[i].magazine_alloc_lock);
- if (tookLock == 0) {
- return 1;
- }
- _malloc_lock_unlock(&szone->tiny_rack.magazines[i].magazine_alloc_lock);
}
return 0;
}
@@ -1482,8 +1425,8 @@
#endif // CONFIG_MEDIUM_ALLOCATOR
#endif // CONFIG_MADVISE_PRESSURE_RELIEF
-#if CONFIG_LARGE_CACHE && !CONFIG_MAGAZINE_DEFERRED_RECLAIM
- if (large_cache_enabled && szone->flotsam_enabled) {
+#if CONFIG_LARGE_CACHE
+ if (szone->flotsam_enabled) {
SZONE_LOCK(szone);
// stack allocated copy of the death-row cache
@@ -1505,18 +1448,18 @@
// deallocate the death-row cache outside the zone lock
size_t total = 0;
while (idx != idx_max) {
- mvm_deallocate_pages((void *)local_entry_cache[idx].address, local_entry_cache[idx].size, szone->debug_flags);
+ mvm_deallocate_pages((void *)local_entry_cache[idx].address, local_entry_cache[idx].size, 0);
total += local_entry_cache[idx].size;
if (++idx == szone->large_cache_depth) {
idx = 0;
}
}
if (0 != local_entry_cache[idx].address && 0 != local_entry_cache[idx].size) {
- mvm_deallocate_pages((void *)local_entry_cache[idx].address, local_entry_cache[idx].size, szone->debug_flags);
+ mvm_deallocate_pages((void *)local_entry_cache[idx].address, local_entry_cache[idx].size, 0);
total += local_entry_cache[idx].size;
}
}
-#endif // CONFIG_LARGE_CACHE && !CONFIG_MAGAZINE_DEFERRED_RECLAIM
+#endif
MAGMALLOC_PRESSURERELIEFEND((void *)szone, szone->basic_zone.zone_name, (int)goal, (int)total); // DTrace USDT Probe
MALLOC_TRACE(TRACE_malloc_memory_pressure | DBG_FUNC_END, (uint64_t)szone, goal, total, 0);
@@ -1611,7 +1554,7 @@
szone_statistics_task(task_t task, vm_address_t zone_address,
memory_reader_t reader, malloc_statistics_t *stats)
{
- reader = reader_or_in_memory_fallback(reader, task);
+ reader = !reader && task == mach_task_self() ? _malloc_default_reader : reader;
szone_t *szone;
kern_return_t err;
@@ -1704,7 +1647,7 @@
#endif
/* get memory for the zone. */
- szone = mvm_allocate_pages(SZONE_PAGED_SIZE, 0, DISABLE_ASLR, VM_MEMORY_MALLOC);
+ szone = mvm_allocate_pages(SZONE_PAGED_SIZE, 0, 0, VM_MEMORY_MALLOC);
if (!szone) {
return NULL;
}
@@ -1754,31 +1697,29 @@
#endif // CONFIG_MEDIUM_ALLOCATOR
#if CONFIG_LARGE_CACHE
- if (large_cache_enabled) {
- // madvise(..., MADV_REUSABLE) death-row arrivals above this threshold [~0.1%]
- szone->large_entry_cache_reserve_limit = (size_t)(memsize >> 10);
- if (memsize >= magazine_large_expanded_cache_threshold) {
- szone->large_cache_depth = LARGE_ENTRY_CACHE_SIZE_HIGH;
- szone->large_cache_entry_limit = LARGE_ENTRY_SIZE_ENTRY_LIMIT_HIGH;
- } else {
- szone->large_cache_depth = LARGE_ENTRY_CACHE_SIZE_LOW;
- szone->large_cache_entry_limit = LARGE_ENTRY_SIZE_ENTRY_LIMIT_LOW;
- }
-
- /* <rdar://problem/6610904> Reset protection when returning a previous large allocation? */
- int32_t libSystemVersion = NSVersionOfLinkTimeLibrary("System");
- if ((-1 != libSystemVersion) && ((libSystemVersion >> 16) < 112) /* CFSystemVersionSnowLeopard */) {
- szone->large_legacy_reset_mprotect = TRUE;
- } else {
- szone->large_legacy_reset_mprotect = FALSE;
- }
+ // madvise(..., MADV_REUSABLE) death-row arrivals above this threshold [~0.1%]
+ szone->large_entry_cache_reserve_limit = (size_t)(memsize >> 10);
+ if (memsize >= magazine_large_expanded_cache_threshold) {
+ szone->large_cache_depth = LARGE_ENTRY_CACHE_SIZE_HIGH;
+ szone->large_cache_entry_limit = LARGE_ENTRY_SIZE_ENTRY_LIMIT_HIGH;
+ } else {
+ szone->large_cache_depth = LARGE_ENTRY_CACHE_SIZE_LOW;
+ szone->large_cache_entry_limit = LARGE_ENTRY_SIZE_ENTRY_LIMIT_LOW;
+ }
+
+ /* <rdar://problem/6610904> Reset protection when returning a previous large allocation? */
+ int32_t libSystemVersion = NSVersionOfLinkTimeLibrary("System");
+ if ((-1 != libSystemVersion) && ((libSystemVersion >> 16) < 112) /* CFSystemVersionSnowLeopard */) {
+ szone->large_legacy_reset_mprotect = TRUE;
+ } else {
+ szone->large_legacy_reset_mprotect = FALSE;
}
#endif
// Initialize the security token.
szone->cookie = (uintptr_t)malloc_entropy[0];
- szone->basic_zone.version = 16;
+ szone->basic_zone.version = 12;
szone->basic_zone.size = (void *)szone_size;
szone->basic_zone.malloc = (void *)szone_malloc;
szone->basic_zone.calloc = (void *)szone_calloc;
@@ -1793,12 +1734,6 @@
szone->basic_zone.free_definite_size = (void *)szone_free_definite_size;
szone->basic_zone.pressure_relief = (void *)szone_pressure_relief;
szone->basic_zone.claimed_address = (void *)szone_claimed_address;
- szone->basic_zone.try_free_default = (void *)szone_try_free_default;
-
- szone->basic_zone.malloc_type_malloc = (void *)szone_malloc_type_malloc;
- szone->basic_zone.malloc_type_calloc = (void *)szone_malloc_type_calloc;
- szone->basic_zone.malloc_type_realloc = (void *)szone_malloc_type_realloc;
- szone->basic_zone.malloc_type_memalign = (void *)szone_malloc_type_memalign;
/* Set to zero once and for all as required by CFAllocator. */
szone->basic_zone.reserved1 = 0;