Loading...
--- libmalloc/libmalloc-116/src/purgeable_malloc.c
+++ libmalloc/libmalloc-715.140.5/src/purgeable_malloc.c
@@ -38,7 +38,7 @@
static void *
purgeable_malloc(szone_t *szone, size_t size)
{
- if (size <= szone->large_threshold) {
+ if (size <= LARGE_THRESHOLD(szone)) {
return szone_malloc(szone->helper_zone, size);
} else {
return szone_malloc(szone, size);
@@ -48,30 +48,13 @@
static void *
purgeable_calloc(szone_t *szone, size_t num_items, size_t size)
{
- size_t total_bytes = num_items * size;
-
- // Check for overflow of integer multiplication
- if (num_items > 1) {
-#if __LP64__ /* size_t is uint64_t */
- if ((num_items | size) & 0xffffffff00000000ul) {
- // num_items or size equals or exceeds sqrt(2^64) == 2^32, appeal to wider arithmetic
- __uint128_t product = ((__uint128_t)num_items) * ((__uint128_t)size);
- if ((uint64_t)(product >> 64)) { // compiles to test on upper register of register pair
- return NULL;
- }
- }
-#else /* size_t is uint32_t */
- if ((num_items | size) & 0xffff0000ul) {
- // num_items or size equals or exceeds sqrt(2^32) == 2^16, appeal to wider arithmetic
- uint64_t product = ((uint64_t)num_items) * ((uint64_t)size);
- if ((uint32_t)(product >> 32)) { // compiles to test on upper register of register pair
- return NULL;
- }
- }
-#endif
- }
-
- if (total_bytes <= szone->large_threshold) {
+ size_t total_bytes;
+
+ if (calloc_get_size(num_items, size, 0, &total_bytes)) {
+ return NULL;
+ }
+
+ if (total_bytes <= LARGE_THRESHOLD(szone)) {
return szone_calloc(szone->helper_zone, 1, total_bytes);
} else {
return szone_calloc(szone, 1, total_bytes);
@@ -81,7 +64,7 @@
static void *
purgeable_valloc(szone_t *szone, size_t size)
{
- if (size <= szone->large_threshold) {
+ if (size <= LARGE_THRESHOLD(szone)) {
return szone_valloc(szone->helper_zone, size);
} else {
return szone_valloc(szone, size);
@@ -97,7 +80,7 @@
entry = large_entry_for_pointer_no_lock(szone, ptr);
SZONE_UNLOCK(szone);
if (entry) {
- return free_large(szone, ptr);
+ return (void)free_large(szone, ptr, false);
} else {
return szone_free(szone->helper_zone, ptr);
}
@@ -106,7 +89,7 @@
static void
purgeable_free_definite_size(szone_t *szone, void *ptr, size_t size)
{
- if (size <= szone->large_threshold) {
+ if (size <= LARGE_THRESHOLD(szone)) {
return szone_free_definite_size(szone->helper_zone, ptr, size);
} else {
return szone_free_definite_size(szone, ptr, size);
@@ -135,14 +118,14 @@
}
if (!old_size) {
- szone_error(szone, 1, "pointer being reallocated was not allocated", ptr, NULL);
+ malloc_zone_error(szone->debug_flags, true, "pointer %p being reallocated was not allocated\n", ptr);
return NULL;
}
// Distinguish 4 cases: {oldsize, newsize} x { <= , > large_threshold }
// and deal with the allocation crossing from the purgeable zone to the helper zone and vice versa.
- if (old_size <= szone->large_threshold) {
- if (new_size <= szone->large_threshold) {
+ if (old_size <= LARGE_THRESHOLD(szone)) {
+ if (new_size <= LARGE_THRESHOLD(szone)) {
return szone_realloc(szone->helper_zone, ptr, new_size);
} else {
// allocation crosses from helper to purgeable zone
@@ -154,7 +137,7 @@
return new_ptr; // in state VM_PURGABLE_NONVOLATILE
}
} else {
- if (new_size <= szone->large_threshold) {
+ if (new_size <= LARGE_THRESHOLD(szone)) {
// allocation crosses from purgeable to helper zone
void *new_ptr = szone_malloc(szone->helper_zone, new_size);
if (new_ptr) {
@@ -186,16 +169,16 @@
large = szone->large_entries + index;
if (large->address) {
// we deallocate_pages, including guard pages
- deallocate_pages(szone, (void *)(large->address), large->size, szone->debug_flags);
+ mvm_deallocate_pages((void *)(large->address), large->size, szone->debug_flags);
}
}
large_entries_free_no_lock(szone, szone->large_entries, szone->num_large_entries, &range_to_deallocate);
if (range_to_deallocate.size) {
- deallocate_pages(szone, (void *)range_to_deallocate.address, (size_t)range_to_deallocate.size, 0);
+ mvm_deallocate_pages((void *)range_to_deallocate.address, (size_t)range_to_deallocate.size, 0);
}
/* Now destroy the separate szone region */
- deallocate_pages(szone, (void *)szone, SZONE_PAGED_SIZE, 0);
+ mvm_deallocate_pages((void *)szone, SZONE_PAGED_SIZE, 0);
}
static unsigned
@@ -213,7 +196,7 @@
static void *
purgeable_memalign(szone_t *szone, size_t alignment, size_t size)
{
- if (size <= szone->large_threshold) {
+ if (size <= LARGE_THRESHOLD(szone)) {
return szone_memalign(szone->helper_zone, alignment, size);
} else {
return szone_memalign(szone, alignment, size);
@@ -231,24 +214,23 @@
szone_t *szone;
kern_return_t err;
- if (!reader) {
- reader = _szone_default_reader;
- }
+ reader = reader_or_in_memory_fallback(reader, task);
err = reader(task, zone_address, sizeof(szone_t), (void **)&szone);
if (err) {
return err;
}
- err = large_in_use_enumerator(
- task, context, type_mask, (vm_address_t)szone->large_entries, szone->num_large_entries, reader, recorder);
+ err = large_in_use_enumerator(task, context, type_mask,
+ (vm_address_t)szone->large_entries, szone->num_large_entries,
+ reader, recorder);
return err;
}
static size_t
purgeable_good_size(szone_t *szone, size_t size)
{
- if (size <= szone->large_threshold) {
+ if (size <= LARGE_THRESHOLD(szone)) {
return szone_good_size(szone->helper_zone, size);
} else {
return szone_good_size(szone, size);
@@ -262,10 +244,31 @@
}
static void
-purgeable_print(szone_t *szone, boolean_t verbose)
-{
- _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX, "Scalable zone %p: inUse=%u(%y) flags=%d\n", szone,
- szone->num_large_objects_in_use, szone->num_bytes_in_large_objects, szone->debug_flags);
+purgeable_print(task_t task, unsigned level MALLOC_UNUSED,
+ vm_address_t zone_address, memory_reader_t reader,
+ print_task_printer_t printer)
+{
+ szone_t *szone;
+ if (reader(task, zone_address, sizeof(szone_t), (void **)&szone)) {
+ printer("Purgeable zone %p: inUse=%u(%y) flags=%d\n", zone_address,
+ szone->num_large_objects_in_use,
+ (int)szone->num_bytes_in_large_objects,
+ szone->debug_flags);
+ }
+}
+
+static void
+purgeable_print_self(szone_t *szone, boolean_t verbose)
+{
+ purgeable_print(mach_task_self(), verbose ? MALLOC_VERBOSE_PRINT_LEVEL : 0,
+ (vm_address_t)szone, _malloc_default_reader, malloc_report_simple);
+}
+
+static void
+purgeable_print_task(task_t task, unsigned level, vm_address_t zone_address,
+ memory_reader_t reader, print_task_printer_t printer)
+{
+ purgeable_print(task, level, zone_address, reader, printer);
}
static void
@@ -321,11 +324,26 @@
}
static const struct malloc_introspection_t purgeable_introspect = {
- (void *)purgeable_ptr_in_use_enumerator, (void *)purgeable_good_size, (void *)purgeable_check, (void *)purgeable_print,
- purgeable_log, (void *)purgeable_force_lock, (void *)purgeable_force_unlock, (void *)purgeable_statistics,
- (void *)purgeable_locked, NULL, NULL, NULL, NULL, /* Zone enumeration version 7 and forward. */
- (void *)purgeable_reinit_lock, // reinit_lock version 9 and foward
+ (void *)purgeable_ptr_in_use_enumerator,
+ (void *)purgeable_good_size,
+ (void *)purgeable_check,
+ (void *)purgeable_print_self,
+ (void *)purgeable_log,
+ (void *)purgeable_force_lock,
+ (void *)purgeable_force_unlock,
+ (void *)purgeable_statistics,
+ (void *)purgeable_locked,
+ NULL, NULL, NULL, NULL, /* Zone enumeration version 7 and forward. */
+ (void *)purgeable_reinit_lock, // reinit_lock version 9 and forward
+ (void *)purgeable_print_task, // print_task version 11 and forward
}; // marked as const to spare the DATA section
+
+
+static boolean_t
+purgeable_claimed_address(szone_t *szone, void *ptr)
+{
+ return szone_claimed_address(szone->helper_zone, ptr);
+}
malloc_zone_t *
create_purgeable_zone(size_t initial_size, malloc_zone_t *malloc_default_zone, unsigned debug_flags)
@@ -334,7 +352,7 @@
uint64_t hw_memsize = 0;
/* get memory for the zone. */
- szone = allocate_pages(NULL, SZONE_PAGED_SIZE, 0, 0, VM_MEMORY_MALLOC);
+ szone = mvm_allocate_pages(SZONE_PAGED_SIZE, 0, DISABLE_ASLR, VM_MEMORY_MALLOC);
if (!szone) {
return NULL;
}
@@ -353,41 +371,25 @@
sysctlbyname("hw.memsize", &hw_memsize, &uint64_t_size, 0, 0);
#endif
- szone->trg[0].nextgen = &(szone->trg[1]);
- szone->trg[1].nextgen = &(szone->trg[0]);
- szone->tiny_region_generation = &(szone->trg[0]);
-
- szone->tiny_region_generation->hashed_regions = szone->initial_tiny_regions;
- szone->tiny_region_generation->num_regions_allocated = INITIAL_NUM_REGIONS;
- szone->tiny_region_generation->num_regions_allocated_shift = INITIAL_NUM_REGIONS_SHIFT;
-
- szone->srg[0].nextgen = &(szone->srg[1]);
- szone->srg[1].nextgen = &(szone->srg[0]);
- szone->small_region_generation = &(szone->srg[0]);
-
- szone->small_region_generation->hashed_regions = szone->initial_small_regions;
- szone->small_region_generation->num_regions_allocated = INITIAL_NUM_REGIONS;
- szone->small_region_generation->num_regions_allocated_shift = INITIAL_NUM_REGIONS_SHIFT;
-
- /* Purgeable zone does not participate in the adaptive "largemem" sizing. */
- szone->is_largemem = 0;
- szone->large_threshold = LARGE_THRESHOLD;
- szone->vm_copy_threshold = VM_COPY_THRESHOLD;
+ rack_init(&szone->tiny_rack, RACK_TYPE_TINY, 0, debug_flags | MALLOC_PURGEABLE);
+ rack_init(&szone->small_rack, RACK_TYPE_SMALL, 0, debug_flags | MALLOC_PURGEABLE);
#if CONFIG_LARGE_CACHE
- szone->large_entry_cache_reserve_limit =
- hw_memsize >> 10; // madvise(..., MADV_REUSABLE) death-row arrivals above this threshold [~0.1%]
-
- /* <rdar://problem/6610904> Reset protection when returning a previous large allocation? */
- int32_t libSystemVersion = NSVersionOfLinkTimeLibrary("System");
- if ((-1 != libSystemVersion) && ((libSystemVersion >> 16) < 112) /* CFSystemVersionSnowLeopard */) {
- szone->large_legacy_reset_mprotect = TRUE;
- } else {
- szone->large_legacy_reset_mprotect = FALSE;
+ if (large_cache_enabled) {
+ // madvise(..., MADV_REUSABLE) death-row arrivals above this threshold [~0.1%]
+ szone->large_entry_cache_reserve_limit = (size_t)(hw_memsize >> 10);
+
+ /* <rdar://problem/6610904> Reset protection when returning a previous large allocation? */
+ int32_t libSystemVersion = NSVersionOfLinkTimeLibrary("System");
+ if ((-1 != libSystemVersion) && ((libSystemVersion >> 16) < 112) /* CFSystemVersionSnowLeopard */) {
+ szone->large_legacy_reset_mprotect = TRUE;
+ } else {
+ szone->large_legacy_reset_mprotect = FALSE;
+ }
}
#endif
- szone->basic_zone.version = 9;
+ szone->basic_zone.version = 11;
szone->basic_zone.size = (void *)purgeable_size;
szone->basic_zone.malloc = (void *)purgeable_malloc;
szone->basic_zone.calloc = (void *)purgeable_calloc;
@@ -401,6 +403,7 @@
szone->basic_zone.memalign = (void *)purgeable_memalign;
szone->basic_zone.free_definite_size = (void *)purgeable_free_definite_size;
szone->basic_zone.pressure_relief = (void *)purgeable_pressure_relief;
+ szone->basic_zone.claimed_address = (void *)purgeable_claimed_address;
szone->basic_zone.reserved1 = 0; /* Set to zero once and for all as required by CFAllocator. */
szone->basic_zone.reserved2 = 0; /* Set to zero once and for all as required by CFAllocator. */
@@ -409,11 +412,13 @@
szone->debug_flags = debug_flags | MALLOC_PURGEABLE;
/* Purgeable zone does not support MALLOC_ADD_GUARD_PAGES. */
- if (szone->debug_flags & MALLOC_ADD_GUARD_PAGES) {
- _malloc_printf(ASL_LEVEL_INFO, "purgeable zone does not support guard pages\n");
- szone->debug_flags &= ~MALLOC_ADD_GUARD_PAGES;
- }
-
+ if (szone->debug_flags & MALLOC_ALL_GUARD_PAGE_FLAGS) {
+ if (!(szone->debug_flags & MALLOC_GUARD_ALL)) {
+ // Don't report when MallocGuardEdges == "all".
+ malloc_report(ASL_LEVEL_INFO, "purgeable zone does not support guard pages\n");
+ }
+ szone->debug_flags &= ~MALLOC_ALL_GUARD_PAGE_FLAGS;
+ }
_malloc_lock_init(&szone->large_szone_lock);
szone->helper_zone = (struct szone_s *)malloc_default_zone;