Loading...
--- libmalloc/libmalloc-116/src/magazine_large.c
+++ libmalloc/libmalloc-166.200.60/src/magazine_large.c
@@ -38,11 +38,48 @@
}
}
- _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX, "%s\n", _simple_string(b));
+ malloc_report(MALLOC_REPORT_NOLOG | MALLOC_REPORT_NOPREFIX, "%s\n", _simple_string(b));
_simple_sfree(b);
}
}
#endif
+
+/*
+ * Scan the hash ring looking for an entry containing a given pointer.
+ */
+static large_entry_t *
+large_entry_containing_pointer_no_lock(szone_t *szone, const void *ptr)
+{
+ // result only valid with lock held
+ unsigned num_large_entries = szone->num_large_entries;
+ unsigned hash_index;
+ unsigned index;
+ large_entry_t *range;
+
+ if (!num_large_entries) {
+ return NULL;
+ }
+
+ hash_index = ((uintptr_t)ptr >> vm_page_quanta_shift) % num_large_entries;
+ index = hash_index;
+
+ do {
+ range = szone->large_entries + index;
+ if (range->address == (vm_address_t)ptr) {
+ return range;
+ } else if ((vm_address_t)ptr >= range->address
+ && (vm_address_t)ptr < range->address + range->size) {
+ return range;
+ }
+
+ // Since we may be looking for an inner pointer, we might not get an
+ // exact match on the address, so we need to scan further and to skip
+ // over empty entries. It will usually be faster to scan backwards.
+ index = index == 0 ? num_large_entries - 1 : index - 1;
+ } while (index != hash_index);
+
+ return NULL;
+}
/*
* Scan the hash ring looking for an entry for the given pointer.
@@ -138,14 +175,14 @@
// FIXME: num should probably be a size_t, since you can theoretically allocate
// more than 2^32-1 large_threshold objects in 64 bit.
static MALLOC_INLINE large_entry_t *
-large_entries_alloc_no_lock(szone_t *szone, unsigned num)
+large_entries_alloc_no_lock(unsigned num)
{
size_t size = num * sizeof(large_entry_t);
// Note that we allocate memory (via a system call) under a spin lock
// That is certainly evil, however it's very rare in the lifetime of a process
// The alternative would slow down the normal case
- return allocate_pages(szone, round_page_quanta(size), 0, 0, VM_MEMORY_MALLOC_LARGE);
+ return mvm_allocate_pages(round_page_quanta(size), 0, 0, VM_MEMORY_MALLOC_LARGE);
}
void
@@ -166,7 +203,7 @@
// always an odd number for good hashing
unsigned new_num_entries =
(old_num_entries) ? old_num_entries * 2 + 1 : (unsigned)((vm_page_quanta_size / sizeof(large_entry_t)) - 1);
- large_entry_t *new_entries = large_entries_alloc_no_lock(szone, new_num_entries);
+ large_entry_t *new_entries = large_entries_alloc_no_lock(new_num_entries);
unsigned index = old_num_entries;
large_entry_t oldRange;
@@ -209,7 +246,7 @@
range.size = entry->size;
if (szone->debug_flags & MALLOC_ADD_GUARD_PAGES) {
- protect((void *)range.address, range.size, PROT_READ | PROT_WRITE, szone->debug_flags);
+ mvm_protect((void *)range.address, range.size, PROT_READ | PROT_WRITE, szone->debug_flags);
range.address -= vm_page_quanta_size;
range.size += 2 * vm_page_quanta_size;
}
@@ -221,7 +258,7 @@
#if DEBUG_MALLOC
if (large_entry_for_pointer_no_lock(szone, (void *)range.address)) {
- malloc_printf("*** freed entry %p still in use; num_large_entries=%d\n", range.address, szone->num_large_entries);
+ malloc_report(ASL_LEVEL_ERR, "*** freed entry %p still in use; num_large_entries=%d\n", range.address, szone->num_large_entries);
large_debug_print(szone);
szone_sleep();
}
@@ -409,48 +446,14 @@
if (range_to_deallocate.size) {
// we deallocate outside the lock
- deallocate_pages(szone, (void *)range_to_deallocate.address, range_to_deallocate.size, 0);
- }
-
- // Perform the madvise() outside the lock.
- // Typically the madvise() is successful and we'll quickly return from this routine.
- // In the unusual case of failure, reacquire the lock to unwind.
-#if TARGET_OS_EMBEDDED
- // Ok to do this madvise on embedded because we won't call MADV_FREE_REUSABLE on a large
- // cache block twice without MADV_FREE_REUSE in between.
-#endif
- if (was_madvised_reusable && -1 == madvise(addr, size, MADV_FREE_REUSE)) {
- /* -1 return: VM map entry change makes this unfit for reuse. */
-#if DEBUG_MADVISE
- szone_error(szone, 0, "large_malloc madvise(..., MADV_FREE_REUSE) failed", addr, "length=%d\n", size);
-#endif
-
- SZONE_LOCK(szone);
- szone->num_large_objects_in_use--;
- szone->num_bytes_in_large_objects -= large_entry.size;
-
- // Re-acquire "entry" after interval just above where we let go the lock.
- large_entry_t *entry = large_entry_for_pointer_no_lock(szone, addr);
- if (NULL == entry) {
- szone_error(szone, 1, "entry for pointer being discarded from death-row vanished", addr, NULL);
- SZONE_UNLOCK(szone);
- } else {
- range_to_deallocate = large_entry_free_no_lock(szone, entry);
- SZONE_UNLOCK(szone);
-
- if (range_to_deallocate.size) {
- // we deallocate outside the lock
- deallocate_pages(szone, (void *)range_to_deallocate.address, range_to_deallocate.size, 0);
- }
- }
- /* Fall through to allocate_pages() afresh. */
- } else {
- if (cleared_requested) {
- memset(addr, 0, size);
- }
-
- return addr;
- }
+ mvm_deallocate_pages((void *)range_to_deallocate.address, range_to_deallocate.size, 0);
+ }
+
+ if (cleared_requested) {
+ memset(addr, 0, size);
+ }
+
+ return addr;
} else {
SZONE_UNLOCK(szone);
}
@@ -460,7 +463,7 @@
range_to_deallocate.address = 0;
#endif /* CONFIG_LARGE_CACHE */
- addr = allocate_pages(szone, size, alignment, szone->debug_flags, VM_MEMORY_MALLOC_LARGE);
+ addr = mvm_allocate_pages(size, alignment, szone->debug_flags, VM_MEMORY_MALLOC_LARGE);
if (addr == NULL) {
return NULL;
}
@@ -487,7 +490,7 @@
if (range_to_deallocate.size) {
// we deallocate outside the lock
- deallocate_pages(szone, (void *)range_to_deallocate.address, range_to_deallocate.size, 0);
+ mvm_deallocate_pages((void *)range_to_deallocate.address, range_to_deallocate.size, 0);
}
return addr;
}
@@ -518,7 +521,7 @@
// is accommodated, matching the behavior of the previous implementation.]
while (1) { // Scan large_entry_cache starting with most recent entry
if (szone->large_entry_cache[idx].address == entry->address) {
- szone_error(szone, 1, "pointer being freed already on death-row", ptr, NULL);
+ malloc_zone_error(szone->debug_flags, true, "pointer %p being freed already on death-row\n", ptr);
SZONE_UNLOCK(szone);
return;
}
@@ -540,8 +543,8 @@
int state = VM_PURGABLE_NONVOLATILE; // restore to default condition
if (KERN_SUCCESS != vm_purgable_control(mach_task_self(), this_entry.address, VM_PURGABLE_SET_STATE, &state)) {
- malloc_printf("*** can't vm_purgable_control(..., VM_PURGABLE_SET_STATE) for large freed block at %p\n",
- this_entry.address);
+ malloc_report(ASL_LEVEL_ERR, "*** can't vm_purgable_control(..., VM_PURGABLE_SET_STATE) for large freed block at %p\n",
+ (void *)this_entry.address);
reusable = FALSE;
}
}
@@ -550,7 +553,7 @@
// Accomodate Leopard apps that (illegally) mprotect() their own guard pages on large malloc'd allocations
int err = mprotect((void *)(this_entry.address), this_entry.size, PROT_READ | PROT_WRITE);
if (err) {
- malloc_printf("*** can't reset protection for large freed block at %p\n", this_entry.address);
+ malloc_report(ASL_LEVEL_ERR, "*** can't reset protection for large freed block at %p\n", (void *)this_entry.address);
reusable = FALSE;
}
}
@@ -558,18 +561,17 @@
// madvise(..., MADV_REUSABLE) death-row arrivals if hoarding would exceed large_entry_cache_reserve_limit
if (should_madvise) {
// Issue madvise to avoid paging out the dirtied free()'d pages in "entry"
- MAGMALLOC_MADVFREEREGION(
- (void *)szone, (void *)0, (void *)(this_entry.address), this_entry.size); // DTrace USDT Probe
-
-#if TARGET_OS_EMBEDDED
+ MAGMALLOC_MADVFREEREGION((void *)szone, (void *)0, (void *)(this_entry.address), (int)this_entry.size); // DTrace USDT Probe
+
// Ok to do this madvise on embedded because we won't call MADV_FREE_REUSABLE on a large
// cache block twice without MADV_FREE_REUSE in between.
-#endif
+
if (-1 == madvise((void *)(this_entry.address), this_entry.size, MADV_FREE_REUSABLE)) {
/* -1 return: VM map entry change makes this unfit for reuse. */
#if DEBUG_MADVISE
- szone_error(szone, 0, "free_large madvise(..., MADV_FREE_REUSABLE) failed", (void *)this_entry.address,
- "length=%d\n", this_entry.size);
+ malloc_zone_error(szone->debug_flags, false,
+ "free_large madvise(..., MADV_FREE_REUSABLE) failed for %p, length=%d\n",
+ (void *)this_entry.address, this_entry.size);
#endif
reusable = FALSE;
}
@@ -580,7 +582,7 @@
// Re-acquire "entry" after interval just above where we let go the lock.
entry = large_entry_for_pointer_no_lock(szone, ptr);
if (NULL == entry) {
- szone_error(szone, 1, "entry for pointer being freed from death-row vanished", ptr, NULL);
+ malloc_zone_error(szone->debug_flags, true, "entry for pointer %p being freed from death-row vanished\n", ptr);
SZONE_UNLOCK(szone);
return;
}
@@ -658,7 +660,7 @@
// we deallocate_pages, including guard pages, outside the lock
SZONE_UNLOCK(szone);
- deallocate_pages(szone, (void *)addr, (size_t)adjsize, 0);
+ mvm_deallocate_pages((void *)addr, (size_t)adjsize, 0);
return;
} else {
/* fall through to discard an allocation that is not reusable */
@@ -674,7 +676,7 @@
#if DEBUG_MALLOC
large_debug_print(szone);
#endif
- szone_error(szone, 1, "pointer being freed was not allocated", ptr, NULL);
+ malloc_zone_error(szone->debug_flags, true, "pointer %p being freed was not allocated\n", ptr);
SZONE_UNLOCK(szone);
return;
}
@@ -686,13 +688,13 @@
#if DEBUG_MALLOC
// FIXME: large_entry_for_pointer_no_lock() needs the lock held ...
if (large_entry_for_pointer_no_lock(szone, (void *)vm_range_to_deallocate.address)) {
- malloc_printf("*** invariant broken: %p still in use num_large_entries=%d\n", vm_range_to_deallocate.address,
- szone->num_large_entries);
+ malloc_report(ASL_LEVEL_ERR, "*** invariant broken: %p still in use num_large_entries=%d\n",
+ vm_range_to_deallocate.address, szone->num_large_entries);
large_debug_print(szone);
szone_sleep();
}
#endif
- deallocate_pages(szone, (void *)vm_range_to_deallocate.address, (size_t)vm_range_to_deallocate.size, 0);
+ mvm_deallocate_pages((void *)vm_range_to_deallocate.address, (size_t)vm_range_to_deallocate.size, 0);
}
}
@@ -706,7 +708,7 @@
/* contract existing large entry */
large_entry_t *large_entry = large_entry_for_pointer_no_lock(szone, ptr);
if (!large_entry) {
- szone_error(szone, 1, "large entry reallocated is not properly in table", ptr, NULL);
+ malloc_zone_error(szone->debug_flags, true, "large entry %p reallocated is not properly in table\n", ptr);
SZONE_UNLOCK(szone);
return ptr;
}
@@ -714,9 +716,23 @@
large_entry->address = (vm_address_t)ptr;
large_entry->size = new_good_size;
szone->num_bytes_in_large_objects -= shrinkage;
+ boolean_t guarded = szone->debug_flags & MALLOC_ADD_GUARD_PAGES;
SZONE_UNLOCK(szone); // we release the lock asap
- deallocate_pages(szone, (void *)((uintptr_t)ptr + new_good_size), shrinkage, 0);
+ if (guarded) {
+ // Keep the page above the new end of the allocation as the
+ // postlude guard page.
+ kern_return_t err;
+ err = mprotect((void *)((uintptr_t)ptr + new_good_size), vm_page_quanta_size, 0);
+ if (err) {
+ malloc_report(ASL_LEVEL_ERR, "*** can't mvm_protect(0x0) region for new postlude guard page at %p\n",
+ ptr + new_good_size);
+ }
+ new_good_size += vm_page_quanta_size;
+ shrinkage -= vm_page_quanta_size;
+ }
+
+ mvm_deallocate_pages((void *)((uintptr_t)ptr + new_good_size), shrinkage, 0);
}
return ptr;
}
@@ -750,7 +766,7 @@
/* extend existing large entry */
large_entry = large_entry_for_pointer_no_lock(szone, ptr);
if (!large_entry) {
- szone_error(szone, 1, "large entry reallocated is not properly in table", ptr, NULL);
+ malloc_zone_error(szone->debug_flags, true, "large entry %p reallocated is not properly in table\n", ptr);
SZONE_UNLOCK(szone);
return 0; // Bail, leaking "addr"
}
@@ -762,3 +778,13 @@
return 1;
}
+
+boolean_t
+large_claimed_address(szone_t *szone, void *ptr)
+{
+ SZONE_LOCK(szone);
+ boolean_t result = large_entry_containing_pointer_no_lock(szone,
+ (void *)trunc_page((uintptr_t)ptr)) != NULL;
+ SZONE_UNLOCK(szone);
+ return result;
+}