Loading...
--- libmalloc/libmalloc-140.50.6/src/magazine_malloc.c
+++ libmalloc/libmalloc-116.50.8/src/magazine_malloc.c
@@ -35,6 +35,17 @@
#include "internal.h"
+static volatile uintptr_t entropic_address = 0;
+static volatile uintptr_t entropic_limit = 0;
+#define ENTROPIC_KABILLION 0x10000000 /* 256Mb */
+
+// <rdar://problem/22277891> align 64bit ARM shift to 32MB PTE entries
+#if MALLOC_TARGET_IOS && MALLOC_TARGET_64BIT
+#define ENTROPIC_SHIFT 25
+#else // MALLOC_TARGET_IOS && MALLOC_TARGET_64BIT
+#define ENTROPIC_SHIFT SMALL_BLOCKS_ALIGN
+#endif
+
#if DEBUG_MALLOC
#define LOG(szone, ptr) (szone->log_address && (((uintptr_t)szone->log_address == -1) || (szone->log_address == (void *)(ptr))))
#else
@@ -57,7 +68,7 @@
// msg prints after fmt, ...
void
-szone_error(uint32_t debug_flags, int is_corruption, const char *msg, const void *ptr, const char *fmt, ...)
+szone_error(szone_t *szone, int is_corruption, const char *msg, const void *ptr, const char *fmt, ...)
{
va_list ap;
_SIMPLE_STRING b = _simple_salloc();
@@ -93,16 +104,245 @@
_malloc_printf(MALLOC_PRINTF_NOLOG, "*** set a breakpoint in malloc_error_break to debug\n");
}
malloc_error_break();
+#if DEBUG_MALLOC
+ szone_print(szone, 1);
+#endif
szone_sleep();
// Call abort() if this is a memory corruption error and the abort on
// corruption flag is set, or if any error should abort.
- if ((is_corruption && (debug_flags & MALLOC_ABORT_ON_CORRUPTION)) ||
- (debug_flags & MALLOC_ABORT_ON_ERROR)) {
+ if ((is_corruption && (szone->debug_flags & MALLOC_ABORT_ON_CORRUPTION)) ||
+ (szone->debug_flags & MALLOC_ABORT_ON_ERROR)) {
_os_set_crash_log_message_dynamic(b ? _simple_string(b) : msg);
abort();
} else if (b) {
_simple_sfree(b);
}
+}
+
+void
+protect(void *address, size_t size, unsigned protection, unsigned debug_flags)
+{
+ kern_return_t err;
+
+ if (!(debug_flags & MALLOC_DONT_PROTECT_PRELUDE)) {
+ err = mprotect((void *)((uintptr_t)address - vm_page_quanta_size), vm_page_quanta_size, protection);
+ if (err) {
+ malloc_printf("*** can't protect(%p) region for prelude guard page at %p\n", protection,
+ (uintptr_t)address - vm_page_quanta_size);
+ }
+ }
+ if (!(debug_flags & MALLOC_DONT_PROTECT_POSTLUDE)) {
+ err = mprotect((void *)(round_page_quanta(((uintptr_t)address + size))), vm_page_quanta_size, protection);
+ if (err) {
+ malloc_printf("*** can't protect(%p) region for postlude guard page at %p\n", protection, (uintptr_t)address + size);
+ }
+ }
+}
+
+void *
+allocate_pages(szone_t *szone, size_t size, unsigned char align, unsigned debug_flags, int vm_page_label)
+{
+ boolean_t add_guard_pages = debug_flags & MALLOC_ADD_GUARD_PAGES;
+ boolean_t purgeable = debug_flags & MALLOC_PURGEABLE;
+ mach_vm_address_t vm_addr;
+ uintptr_t addr;
+ mach_vm_size_t allocation_size = round_page_quanta(size);
+ mach_vm_offset_t allocation_mask = ((mach_vm_offset_t)1 << align) - 1;
+ int alloc_flags = VM_FLAGS_ANYWHERE | VM_MAKE_TAG(vm_page_label);
+ kern_return_t kr;
+
+ if (!allocation_size) {
+ allocation_size = vm_page_quanta_size;
+ }
+ if (add_guard_pages) {
+ if (align > vm_page_quanta_shift) {
+ /* <rdar://problem/16601499> alignment greater than pagesize needs more work */
+ allocation_size += (1 << align) + vm_page_quanta_size;
+ } else {
+ allocation_size += 2 * vm_page_quanta_size;
+ }
+ }
+
+ if (purgeable) {
+ alloc_flags |= VM_FLAGS_PURGABLE;
+ }
+ if (allocation_size < size) { // size_t arithmetic wrapped!
+ return NULL;
+ }
+
+ vm_addr = vm_page_quanta_size;
+ kr = mach_vm_map(mach_task_self(), &vm_addr, allocation_size, allocation_mask, alloc_flags, MEMORY_OBJECT_NULL, 0, FALSE,
+ VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
+ if (kr) {
+ szone_error(szone, 0, "can't allocate region", NULL, "*** mach_vm_map(size=%lu) failed (error code=%d)\n", size, kr);
+ return NULL;
+ }
+ addr = (uintptr_t)vm_addr;
+
+ if (add_guard_pages) {
+ if (align > vm_page_quanta_shift) {
+ /* <rdar://problem/16601499> calculate the first address inside the alignment padding
+ * where we can place the guard page and still be aligned.
+ *
+ * |-----------------------------------------------------------|
+ * |leading|gp| alloc |gp| t |
+ * |-----------------------------------------------------------|
+ */
+ uintptr_t alignaddr = ((addr + vm_page_quanta_size) + (1 << align) - 1) & ~((1 << align) - 1);
+ size_t leading = alignaddr - addr - vm_page_quanta_size;
+ size_t trailing = (1 << align) - vm_page_quanta_size - leading;
+
+ /* Unmap the excess area. */
+ kr = mach_vm_deallocate(mach_task_self(), addr, leading);
+ if (kr) {
+ szone_error(szone, 0, "can't unmap excess guard region", NULL,
+ "*** mach_vm_deallocate(addr=%p, size=%lu) failed (code=%d)", (void *)addr, leading, kr);
+ return NULL;
+ }
+
+ kr = mach_vm_deallocate(mach_task_self(), addr + allocation_size - trailing, trailing);
+ if (kr) {
+ szone_error(szone, 0, "can't unmap excess trailing guard region", NULL,
+ "*** mach_vm_deallocate(addr=%p, size=%lu) failed (code=%d)", (void *)(addr + allocation_size - trailing),
+ trailing, kr);
+ return NULL;
+ }
+
+ addr = alignaddr;
+ } else {
+ addr += vm_page_quanta_size;
+ }
+ protect((void *)addr, size, PROT_NONE, debug_flags);
+ }
+ return (void *)addr;
+}
+
+void *
+allocate_pages_securely(szone_t *szone, size_t size, unsigned char align, int vm_page_label)
+{
+ mach_vm_address_t vm_addr;
+ uintptr_t addr;
+ mach_vm_size_t allocation_size = round_page_quanta(size);
+ mach_vm_offset_t allocation_mask = ((mach_vm_offset_t)1 << align) - 1;
+ int alloc_flags = VM_FLAGS_ANYWHERE | VM_MAKE_TAG(vm_page_label);
+ kern_return_t kr;
+
+ if (szone->debug_flags & DISABLE_ASLR) {
+ return allocate_pages(szone, size, align, 0, vm_page_label);
+ }
+
+ if (!allocation_size) {
+ allocation_size = vm_page_quanta_size;
+ }
+ if (allocation_size < size) { // size_t arithmetic wrapped!
+ return NULL;
+ }
+
+retry:
+ vm_addr = entropic_address;
+ kr = mach_vm_map(mach_task_self(), &vm_addr, allocation_size, allocation_mask, alloc_flags, MEMORY_OBJECT_NULL, 0, FALSE,
+ VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
+ if (kr == KERN_NO_SPACE) {
+ vm_addr = vm_page_quanta_size;
+ kr = mach_vm_map(mach_task_self(), &vm_addr, allocation_size, allocation_mask, alloc_flags, MEMORY_OBJECT_NULL, 0, FALSE,
+ VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
+ }
+ if (kr) {
+ szone_error(
+ szone, 0, "can't allocate region securely", NULL, "*** mach_vm_map(size=%lu) failed (error code=%d)\n", size, kr);
+ return NULL;
+ }
+ addr = (uintptr_t)vm_addr;
+
+ // Don't allow allocation to rise above entropic_limit (for tidiness).
+ if (addr + allocation_size > entropic_limit) { // Exhausted current range?
+ uintptr_t t = entropic_address;
+ uintptr_t u = t - ENTROPIC_KABILLION;
+
+ if (u < t) { // provided we don't wrap, deallocate and retry, in the expanded entropic range
+ mach_vm_deallocate(mach_task_self(), vm_addr, allocation_size);
+ (void)__sync_bool_compare_and_swap(&entropic_address, t, u); // Just one reduction please
+ goto retry;
+ }
+ // fall through to use what we got
+ }
+
+ if (addr < entropic_address) { // we wrapped to find this allocation, expand the entropic range
+ uintptr_t t = entropic_address;
+ uintptr_t u = t - ENTROPIC_KABILLION;
+ if (u < t) {
+ (void)__sync_bool_compare_and_swap(&entropic_address, t, u); // Just one reduction please
+ }
+ // fall through to use what we got
+ }
+ return (void *)addr;
+}
+
+void
+deallocate_pages(szone_t *szone, void *addr, size_t size, unsigned debug_flags)
+{
+ boolean_t add_guard_pages = debug_flags & MALLOC_ADD_GUARD_PAGES;
+ mach_vm_address_t vm_addr = (mach_vm_address_t)addr;
+ mach_vm_size_t allocation_size = size;
+ kern_return_t kr;
+
+ if (add_guard_pages) {
+ vm_addr -= vm_page_quanta_size;
+ allocation_size += 2 * vm_page_quanta_size;
+ }
+ kr = mach_vm_deallocate(mach_task_self(), vm_addr, allocation_size);
+ if (kr && szone) {
+ szone_error(szone, 0, "Can't deallocate_pages region", addr, NULL);
+ }
+}
+
+int
+madvise_free_range(szone_t *szone, region_t r, uintptr_t pgLo, uintptr_t pgHi, uintptr_t *last)
+{
+ if (pgHi > pgLo) {
+ size_t len = pgHi - pgLo;
+
+ if (szone->debug_flags & MALLOC_DO_SCRIBBLE) {
+ memset((void *)pgLo, SCRUBBLE_BYTE, len); // Scribble on MADV_FREEd memory
+ }
+
+#if TARGET_OS_EMBEDDED
+ if (last) {
+ if (*last == pgLo) {
+ return 0;
+ }
+
+ *last = pgLo;
+ }
+#endif
+
+ MAGMALLOC_MADVFREEREGION((void *)szone, (void *)r, (void *)pgLo, (int)len); // DTrace USDT Probe
+ if (-1 == madvise((void *)pgLo, len, CONFIG_MADVISE_STYLE)) {
+/* -1 return: VM map entry change makes this unfit for reuse. Something evil lurks. */
+#if DEBUG_MADVISE
+ szone_error(szone, 0, "madvise_free_range madvise(..., MADV_FREE_REUSABLE) failed", (void *)pgLo, "length=%d\n", len);
+#endif
+ }
+ }
+ return 0;
+}
+
+int
+madvise_reuse_range(szone_t *szone, region_t r, uintptr_t pgLo, uintptr_t phHi)
+{
+ if (phHi > pgLo) {
+ size_t len = phHi - pgLo;
+
+ if (madvise((void *)pgLo, len, MADV_FREE_REUSE) == -1) {
+/* -1 return: VM map entry change makes this unfit for reuse. Something evil lurks. */
+#if DEBUG_MADVISE
+ szone_error(szone, 0, "madvise_reuse_range madvise(..., MADV_FREE_REUSE) failed", sparse_region, "length=%d\n",
+ TINY_REGION_PAYLOAD_BYTES);
+#endif
+ return 1;
+ }
+ }
+ return 0;
}
/********************* Zone call backs ************************/
@@ -127,15 +367,15 @@
* Try to free to a tiny region.
*/
if ((uintptr_t)ptr & (TINY_QUANTUM - 1)) {
- szone_error(szone->debug_flags, 1, "Non-aligned pointer being freed", ptr, NULL);
+ szone_error(szone, 1, "Non-aligned pointer being freed", ptr, NULL);
return;
}
- if ((tiny_region = tiny_region_for_ptr_no_lock(&szone->tiny_rack, ptr)) != NULL) {
+ if ((tiny_region = tiny_region_for_ptr_no_lock(szone, ptr)) != NULL) {
if (TINY_INDEX_FOR_PTR(ptr) >= NUM_TINY_BLOCKS) {
- szone_error(szone->debug_flags, 1, "Pointer to metadata being freed", ptr, NULL);
+ szone_error(szone, 1, "Pointer to metadata being freed", ptr, NULL);
return;
}
- free_tiny(&szone->tiny_rack, ptr, tiny_region, 0);
+ free_tiny(szone, ptr, tiny_region, 0);
return;
}
@@ -143,21 +383,21 @@
* Try to free to a small region.
*/
if ((uintptr_t)ptr & (SMALL_QUANTUM - 1)) {
- szone_error(szone->debug_flags, 1, "Non-aligned pointer being freed (2)", ptr, NULL);
+ szone_error(szone, 1, "Non-aligned pointer being freed (2)", ptr, NULL);
return;
}
- if ((small_region = small_region_for_ptr_no_lock(&szone->small_rack, ptr)) != NULL) {
+ if ((small_region = small_region_for_ptr_no_lock(szone, ptr)) != NULL) {
if (SMALL_META_INDEX_FOR_PTR(ptr) >= NUM_SMALL_BLOCKS) {
- szone_error(szone->debug_flags, 1, "Pointer to metadata being freed (2)", ptr, NULL);
+ szone_error(szone, 1, "Pointer to metadata being freed (2)", ptr, NULL);
return;
}
- free_small(&szone->small_rack, ptr, small_region, 0);
+ free_small(szone, ptr, small_region, 0);
return;
}
/* check that it's a legal large allocation */
if ((uintptr_t)ptr & (vm_page_quanta_size - 1)) {
- szone_error(szone->debug_flags, 1, "non-page-aligned, non-allocated pointer being freed", ptr, NULL);
+ szone_error(szone, 1, "non-page-aligned, non-allocated pointer being freed", ptr, NULL);
return;
}
free_large(szone, ptr);
@@ -172,7 +412,7 @@
}
if (0 == size) {
- szone_error(szone->debug_flags, 1, "pointer of size zero being freed", ptr, NULL);
+ szone_error(szone, 1, "pointer of size zero being freed", ptr, NULL);
return;
}
@@ -185,15 +425,15 @@
* Try to free to a tiny region.
*/
if ((uintptr_t)ptr & (TINY_QUANTUM - 1)) {
- szone_error(szone->debug_flags, 1, "Non-aligned pointer being freed", ptr, NULL);
+ szone_error(szone, 1, "Non-aligned pointer being freed", ptr, NULL);
return;
}
if (size <= SMALL_THRESHOLD) {
if (TINY_INDEX_FOR_PTR(ptr) >= NUM_TINY_BLOCKS) {
- szone_error(szone->debug_flags, 1, "Pointer to metadata being freed", ptr, NULL);
+ szone_error(szone, 1, "Pointer to metadata being freed", ptr, NULL);
return;
}
- free_tiny(&szone->tiny_rack, ptr, TINY_REGION_FOR_PTR(ptr), size);
+ free_tiny(szone, ptr, TINY_REGION_FOR_PTR(ptr), size);
return;
}
@@ -201,21 +441,21 @@
* Try to free to a small region.
*/
if ((uintptr_t)ptr & (SMALL_QUANTUM - 1)) {
- szone_error(szone->debug_flags, 1, "Non-aligned pointer being freed (2)", ptr, NULL);
+ szone_error(szone, 1, "Non-aligned pointer being freed (2)", ptr, NULL);
return;
}
if (size <= szone->large_threshold) {
if (SMALL_META_INDEX_FOR_PTR(ptr) >= NUM_SMALL_BLOCKS) {
- szone_error(szone->debug_flags, 1, "Pointer to metadata being freed (2)", ptr, NULL);
+ szone_error(szone, 1, "Pointer to metadata being freed (2)", ptr, NULL);
return;
}
- free_small(&szone->small_rack, ptr, SMALL_REGION_FOR_PTR(ptr), size);
+ free_small(szone, ptr, SMALL_REGION_FOR_PTR(ptr), size);
return;
}
/* check that it's a legal large allocation */
if ((uintptr_t)ptr & (vm_page_quanta_size - 1)) {
- szone_error(szone->debug_flags, 1, "non-page-aligned, non-allocated pointer being freed", ptr, NULL);
+ szone_error(szone, 1, "non-page-aligned, non-allocated pointer being freed", ptr, NULL);
return;
}
free_large(szone, ptr);
@@ -234,7 +474,7 @@
if (!msize) {
msize = 1;
}
- ptr = tiny_malloc_should_clear(&szone->tiny_rack, msize, cleared_requested);
+ ptr = tiny_malloc_should_clear(szone, msize, cleared_requested);
} else if (size <= szone->large_threshold) {
// small size: <15k (<1GB machines), <127k (>1GB machines)
// think small
@@ -242,7 +482,7 @@
if (!msize) {
msize = 1;
}
- ptr = small_malloc_should_clear(&szone->small_rack, msize, cleared_requested);
+ ptr = small_malloc_should_clear(szone, msize, cleared_requested);
} else {
// large: all other allocations
size_t num_kernel_pages = round_page_quanta(size) >> vm_page_quanta_shift;
@@ -348,7 +588,8 @@
size_t
szone_size(szone_t *szone, const void *ptr)
{
- size_t sz = 0;
+ boolean_t is_free;
+ msize_t msize, msize_and_free;
if (!ptr) {
return 0;
@@ -365,10 +606,35 @@
if ((uintptr_t)ptr & (TINY_QUANTUM - 1)) {
return 0;
}
-
- sz = tiny_size(&szone->tiny_rack, ptr);
- if (sz) {
- return sz;
+ if (tiny_region_for_ptr_no_lock(szone, ptr)) {
+ if (TINY_INDEX_FOR_PTR(ptr) >= NUM_TINY_BLOCKS) {
+ return 0;
+ }
+ msize = get_tiny_meta_header(ptr, &is_free);
+ if (is_free) {
+ return 0;
+ }
+#if CONFIG_TINY_CACHE
+ {
+ mag_index_t mag_index = MAGAZINE_INDEX_FOR_TINY_REGION(TINY_REGION_FOR_PTR(ptr));
+ if (DEPOT_MAGAZINE_INDEX != mag_index) {
+ magazine_t *tiny_mag_ptr = &(szone->tiny_magazines[mag_index]);
+
+ if (msize < TINY_QUANTUM && ptr == (void *)((uintptr_t)(tiny_mag_ptr->mag_last_free) & ~(TINY_QUANTUM - 1))) {
+ return 0;
+ }
+ } else {
+ for (mag_index = 0; mag_index < szone->num_tiny_magazines; mag_index++) {
+ magazine_t *tiny_mag_ptr = &(szone->tiny_magazines[mag_index]);
+
+ if (msize < TINY_QUANTUM && ptr == (void *)((uintptr_t)(tiny_mag_ptr->mag_last_free) & ~(TINY_QUANTUM - 1))) {
+ return 0;
+ }
+ }
+ }
+ }
+#endif
+ return TINY_BYTES_FOR_MSIZE(msize);
}
/*
@@ -377,10 +643,35 @@
if ((uintptr_t)ptr & (SMALL_QUANTUM - 1)) {
return 0;
}
-
- sz = small_size(&szone->small_rack, ptr);
- if (sz) {
- return sz;
+ if (small_region_for_ptr_no_lock(szone, ptr)) {
+ if (SMALL_META_INDEX_FOR_PTR(ptr) >= NUM_SMALL_BLOCKS) {
+ return 0;
+ }
+ msize_and_free = *SMALL_METADATA_FOR_PTR(ptr);
+ if (msize_and_free & SMALL_IS_FREE) {
+ return 0;
+ }
+#if CONFIG_SMALL_CACHE
+ {
+ mag_index_t mag_index = MAGAZINE_INDEX_FOR_SMALL_REGION(SMALL_REGION_FOR_PTR(ptr));
+ if (DEPOT_MAGAZINE_INDEX != mag_index) {
+ magazine_t *small_mag_ptr = &(szone->small_magazines[mag_index]);
+
+ if (ptr == (void *)((uintptr_t)(small_mag_ptr->mag_last_free) & ~(SMALL_QUANTUM - 1))) {
+ return 0;
+ }
+ } else {
+ for (mag_index = 0; mag_index < szone->num_small_magazines; mag_index++) {
+ magazine_t *small_mag_ptr = &(szone->small_magazines[mag_index]);
+
+ if (ptr == (void *)((uintptr_t)(small_mag_ptr->mag_last_free) & ~(SMALL_QUANTUM - 1))) {
+ return 0;
+ }
+ }
+ }
+ }
+#endif
+ return SMALL_BYTES_FOR_MSIZE(msize_and_free);
}
/*
@@ -420,7 +711,7 @@
old_size = szone_size(szone, ptr);
if (!old_size) {
- szone_error(szone->debug_flags, 1, "pointer being reallocated was not allocated", ptr, NULL);
+ szone_error(szone, 1, "pointer being reallocated was not allocated", ptr, NULL);
return NULL;
}
@@ -439,7 +730,7 @@
/*
* Serious shrinkage (more than half). free() the excess.
*/
- return tiny_try_shrink_in_place(&szone->tiny_rack, ptr, old_size, new_good_size);
+ return tiny_try_shrink_in_place(szone, ptr, old_size, new_good_size);
} else if (new_good_size <= old_size) {
/*
* new_good_size smaller than old_size but not by much (less than half).
@@ -449,7 +740,7 @@
memset(ptr + new_size, SCRIBBLE_BYTE, old_size - new_size);
}
return ptr;
- } else if (tiny_try_realloc_in_place(&szone->tiny_rack, ptr, old_size, new_good_size)) { // try to grow the allocation
+ } else if (tiny_try_realloc_in_place(szone, ptr, old_size, new_good_size)) { // try to grow the allocation
if (szone->debug_flags & MALLOC_DO_SCRIBBLE) {
memset(ptr + old_size, SCRIBBLE_BYTE, new_good_size - old_size);
}
@@ -465,13 +756,13 @@
} else if (new_good_size <= szone->large_threshold) {
if (SMALL_THRESHOLD < old_size && old_size <= szone->large_threshold) {
if (new_good_size <= (old_size >> 1)) {
- return small_try_shrink_in_place(&szone->small_rack, ptr, old_size, new_good_size);
+ return small_try_shrink_in_place(szone, ptr, old_size, new_good_size);
} else if (new_good_size <= old_size) {
if (szone->debug_flags & MALLOC_DO_SCRIBBLE) {
memset(ptr + new_size, SCRIBBLE_BYTE, old_size - new_size);
}
return ptr;
- } else if (small_try_realloc_in_place(&szone->small_rack, ptr, old_size, new_good_size)) {
+ } else if (small_try_realloc_in_place(szone, ptr, old_size, new_good_size)) {
if (szone->debug_flags & MALLOC_DO_SCRIBBLE) {
memset(ptr + old_size, SCRIBBLE_BYTE, new_good_size - old_size);
}
@@ -590,8 +881,8 @@
{
msize_t msize = TINY_MSIZE_FOR_BYTES(size + TINY_QUANTUM - 1);
unsigned found = 0;
- mag_index_t mag_index = mag_get_thread_index() % szone->tiny_rack.num_magazines;
- magazine_t *tiny_mag_ptr = &(szone->tiny_rack.magazines[mag_index]);
+ mag_index_t mag_index = mag_get_thread_index(szone);
+ magazine_t *tiny_mag_ptr = &(szone->tiny_magazines[mag_index]);
// only bother implementing this for tiny
if (size > SMALL_THRESHOLD) {
@@ -606,13 +897,13 @@
// We must lock the zone now, since tiny_malloc_from_free_list assumes that
// the caller has done so.
- SZONE_MAGAZINE_PTR_LOCK(tiny_mag_ptr);
+ SZONE_MAGAZINE_PTR_LOCK(szone, tiny_mag_ptr);
// with the zone locked, allocate objects from the free list until all
// sufficiently large objects have been exhausted, or we have met our quota
// of objects to allocate.
while (found < count) {
- void *ptr = tiny_malloc_from_free_list(&szone->tiny_rack, tiny_mag_ptr, mag_index, msize);
+ void *ptr = tiny_malloc_from_free_list(szone, tiny_mag_ptr, mag_index, msize);
if (!ptr) {
break;
}
@@ -620,7 +911,7 @@
*results++ = ptr;
found++;
}
- SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
+ SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
return found;
}
@@ -648,16 +939,15 @@
if (ptr) {
if (NULL == tiny_region || tiny_region != TINY_REGION_FOR_PTR(ptr)) { // region same as last iteration?
if (tiny_mag_ptr) { // non-NULL iff magazine lock taken
- SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
+ SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
tiny_mag_ptr = NULL;
}
- tiny_region = tiny_region_for_ptr_no_lock(&szone->tiny_rack, ptr);
+ tiny_region = tiny_region_for_ptr_no_lock(szone, ptr);
if (tiny_region) {
- tiny_mag_ptr = mag_lock_zine_for_region_trailer(szone->tiny_rack.magazines,
- REGION_TRAILER_FOR_TINY_REGION(tiny_region),
- MAGAZINE_INDEX_FOR_TINY_REGION(tiny_region));
+ tiny_mag_ptr = mag_lock_zine_for_region_trailer(szone, szone->tiny_magazines,
+ REGION_TRAILER_FOR_TINY_REGION(tiny_region), MAGAZINE_INDEX_FOR_TINY_REGION(tiny_region));
mag_index = MAGAZINE_INDEX_FOR_TINY_REGION(tiny_region);
}
}
@@ -670,7 +960,7 @@
if (is_free) {
break; // a double free; let the standard free deal with it
}
- if (!tiny_free_no_lock(&szone->tiny_rack, tiny_mag_ptr, mag_index, tiny_region, ptr, msize)) {
+ if (!tiny_free_no_lock(szone, tiny_mag_ptr, mag_index, tiny_region, ptr, msize)) {
// Arrange to re-acquire magazine lock
tiny_mag_ptr = NULL;
tiny_region = NULL;
@@ -685,7 +975,7 @@
}
if (tiny_mag_ptr) {
- SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
+ SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
tiny_mag_ptr = NULL;
}
@@ -728,13 +1018,13 @@
// deallocate the death-row cache outside the zone lock
while (idx != idx_max) {
- mvm_deallocate_pages((void *)local_entry_cache[idx].address, local_entry_cache[idx].size, 0);
+ deallocate_pages(szone, (void *)local_entry_cache[idx].address, local_entry_cache[idx].size, 0);
if (++idx == LARGE_ENTRY_CACHE_SIZE) {
idx = 0;
}
}
if (0 != local_entry_cache[idx].address && 0 != local_entry_cache[idx].size) {
- mvm_deallocate_pages((void *)local_entry_cache[idx].address, local_entry_cache[idx].size, 0);
+ deallocate_pages(szone, (void *)local_entry_cache[idx].address, local_entry_cache[idx].size, 0);
}
#endif
@@ -744,23 +1034,44 @@
large = szone->large_entries + index;
if (large->address) {
// we deallocate_pages, including guard pages
- mvm_deallocate_pages((void *)(large->address), large->size, szone->debug_flags);
+ deallocate_pages(szone, (void *)(large->address), large->size, szone->debug_flags);
}
}
large_entries_free_no_lock(szone, szone->large_entries, szone->num_large_entries, &range_to_deallocate);
if (range_to_deallocate.size) {
- mvm_deallocate_pages((void *)range_to_deallocate.address, (size_t)range_to_deallocate.size, 0);
- }
-
- /* destroy allocator regions */
- rack_destroy_regions(&szone->tiny_rack, TINY_REGION_SIZE);
- rack_destroy_regions(&szone->small_rack, SMALL_REGION_SIZE);
-
- /* destroy rack region hash rings and racks themselves */
- rack_destroy(&szone->tiny_rack);
- rack_destroy(&szone->small_rack);
-
- mvm_deallocate_pages((void *)szone, SZONE_PAGED_SIZE, 0);
+ deallocate_pages(szone, (void *)range_to_deallocate.address, (size_t)range_to_deallocate.size, 0);
+ }
+
+ /* destroy tiny regions */
+ for (index = 0; index < szone->tiny_region_generation->num_regions_allocated; ++index) {
+ if ((HASHRING_OPEN_ENTRY != szone->tiny_region_generation->hashed_regions[index]) &&
+ (HASHRING_REGION_DEALLOCATED != szone->tiny_region_generation->hashed_regions[index])) {
+ deallocate_pages(szone, szone->tiny_region_generation->hashed_regions[index], TINY_REGION_SIZE, 0);
+ }
+ }
+
+ /* destroy small regions */
+ for (index = 0; index < szone->small_region_generation->num_regions_allocated; ++index) {
+ if ((HASHRING_OPEN_ENTRY != szone->small_region_generation->hashed_regions[index]) &&
+ (HASHRING_REGION_DEALLOCATED != szone->small_region_generation->hashed_regions[index])) {
+ deallocate_pages(szone, szone->small_region_generation->hashed_regions[index], SMALL_REGION_SIZE, 0);
+ }
+ }
+
+ /* destroy region hash rings, if any */
+ if (szone->tiny_region_generation->hashed_regions != szone->initial_tiny_regions) {
+ size_t size = round_page_quanta(szone->tiny_region_generation->num_regions_allocated * sizeof(region_t));
+ deallocate_pages(szone, szone->tiny_region_generation->hashed_regions, size, 0);
+ }
+ if (szone->small_region_generation->hashed_regions != szone->initial_small_regions) {
+ size_t size = round_page_quanta(szone->small_region_generation->num_regions_allocated * sizeof(region_t));
+ deallocate_pages(szone, szone->small_region_generation->hashed_regions, size, 0);
+ }
+
+ /* Now destroy the separate szone region */
+ deallocate_pages(szone, (void *)&(szone->tiny_magazines[-1]), TINY_MAGAZINE_PAGED_SIZE, MALLOC_ADD_GUARD_PAGES);
+ deallocate_pages(szone, (void *)&(szone->small_magazines[-1]), SMALL_MAGAZINE_PAGED_SIZE, MALLOC_ADD_GUARD_PAGES);
+ deallocate_pages(szone, (void *)szone, SZONE_PAGED_SIZE, 0);
}
size_t
@@ -813,33 +1124,32 @@
size_t index;
/* check tiny regions - chould check region count */
- for (index = 0; index < szone->tiny_rack.region_generation->num_regions_allocated; ++index) {
- region_t tiny = szone->tiny_rack.region_generation->hashed_regions[index];
+ for (index = 0; index < szone->tiny_region_generation->num_regions_allocated; ++index) {
+ region_t tiny = szone->tiny_region_generation->hashed_regions[index];
if (HASHRING_REGION_DEALLOCATED == tiny) {
continue;
}
if (tiny) {
- magazine_t *tiny_mag_ptr = mag_lock_zine_for_region_trailer(szone->tiny_rack.magazines,
- REGION_TRAILER_FOR_TINY_REGION(tiny),
- MAGAZINE_INDEX_FOR_TINY_REGION(tiny));
-
- if (!tiny_check_region(&szone->tiny_rack, tiny)) {
- SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
+ magazine_t *tiny_mag_ptr = mag_lock_zine_for_region_trailer(
+ szone, szone->tiny_magazines, REGION_TRAILER_FOR_TINY_REGION(tiny), MAGAZINE_INDEX_FOR_TINY_REGION(tiny));
+
+ if (!tiny_check_region(szone, tiny)) {
+ SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
szone->debug_flags &= ~CHECK_REGIONS;
- szone_error(szone->debug_flags, 1, "check: tiny region incorrect", NULL,
+ szone_error(szone, 1, "check: tiny region incorrect", NULL,
"*** tiny region %ld incorrect szone_check_all(%s) counter=%d\n", index, function, szone_check_counter);
return 0;
}
- SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
+ SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
}
}
/* check tiny free lists */
for (index = 0; index < NUM_TINY_SLOTS; ++index) {
- if (!tiny_free_list_check(&szone->tiny_rack, (grain_t)index)) {
+ if (!tiny_free_list_check(szone, (grain_t)index)) {
szone->debug_flags &= ~CHECK_REGIONS;
- szone_error(szone->debug_flags, 1, "check: tiny free list incorrect", NULL,
+ szone_error(szone, 1, "check: tiny free list incorrect", NULL,
"*** tiny free list incorrect (slot=%ld) szone_check_all(%s) counter=%d\n", index, function,
szone_check_counter);
return 0;
@@ -847,33 +1157,32 @@
}
/* check small regions - could check region count */
- for (index = 0; index < szone->small_rack.region_generation->num_regions_allocated; ++index) {
- region_t small = szone->small_rack.region_generation->hashed_regions[index];
+ for (index = 0; index < szone->small_region_generation->num_regions_allocated; ++index) {
+ region_t small = szone->small_region_generation->hashed_regions[index];
if (HASHRING_REGION_DEALLOCATED == small) {
continue;
}
if (small) {
- magazine_t *small_mag_ptr = mag_lock_zine_for_region_trailer(szone->small_rack.magazines,
- REGION_TRAILER_FOR_SMALL_REGION(small),
- MAGAZINE_INDEX_FOR_SMALL_REGION(small));
-
- if (!small_check_region(&szone->small_rack, small)) {
- SZONE_MAGAZINE_PTR_UNLOCK(small_mag_ptr);
+ magazine_t *small_mag_ptr = mag_lock_zine_for_region_trailer(
+ szone, szone->small_magazines, REGION_TRAILER_FOR_SMALL_REGION(small), MAGAZINE_INDEX_FOR_SMALL_REGION(small));
+
+ if (!small_check_region(szone, small)) {
+ SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
szone->debug_flags &= ~CHECK_REGIONS;
- szone_error(szone->debug_flags, 1, "check: small region incorrect", NULL,
+ szone_error(szone, 1, "check: small region incorrect", NULL,
"*** small region %ld incorrect szone_check_all(%s) counter=%d\n", index, function, szone_check_counter);
return 0;
}
- SZONE_MAGAZINE_PTR_UNLOCK(small_mag_ptr);
+ SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
}
}
/* check small free lists */
- for (index = 0; index < SMALL_FREE_SLOT_COUNT(&szone->small_rack); ++index) {
- if (!small_free_list_check(&szone->small_rack, (grain_t)index)) {
+ for (index = 0; index < szone->num_small_slots; ++index) {
+ if (!small_free_list_check(szone, (grain_t)index)) {
szone->debug_flags &= ~CHECK_REGIONS;
- szone_error(szone->debug_flags, 1, "check: small free list incorrect", NULL,
+ szone_error(szone, 1, "check: small free list incorrect", NULL,
"*** small free list incorrect (slot=%ld) szone_check_all(%s) counter=%d\n", index, function,
szone_check_counter);
return 0;
@@ -950,21 +1259,21 @@
size_t u = 0;
mag_index_t mag_index;
- for (mag_index = -1; mag_index < szone->tiny_rack.num_magazines; mag_index++) {
- s += szone->tiny_rack.magazines[mag_index].mag_bytes_free_at_start;
- s += szone->tiny_rack.magazines[mag_index].mag_bytes_free_at_end;
- t += szone->tiny_rack.magazines[mag_index].mag_num_objects;
- u += szone->tiny_rack.magazines[mag_index].mag_num_bytes_in_objects;
+ for (mag_index = -1; mag_index < szone->num_tiny_magazines; mag_index++) {
+ s += szone->tiny_magazines[mag_index].mag_bytes_free_at_start;
+ s += szone->tiny_magazines[mag_index].mag_bytes_free_at_end;
+ t += szone->tiny_magazines[mag_index].mag_num_objects;
+ u += szone->tiny_magazines[mag_index].mag_num_bytes_in_objects;
}
info[4] = (unsigned)t;
info[5] = (unsigned)u;
- for (t = 0, u = 0, mag_index = -1; mag_index < szone->small_rack.num_magazines; mag_index++) {
- s += szone->small_rack.magazines[mag_index].mag_bytes_free_at_start;
- s += szone->small_rack.magazines[mag_index].mag_bytes_free_at_end;
- t += szone->small_rack.magazines[mag_index].mag_num_objects;
- u += szone->small_rack.magazines[mag_index].mag_num_bytes_in_objects;
+ for (t = 0, u = 0, mag_index = -1; mag_index < szone->num_small_magazines; mag_index++) {
+ s += szone->small_magazines[mag_index].mag_bytes_free_at_start;
+ s += szone->small_magazines[mag_index].mag_bytes_free_at_end;
+ t += szone->small_magazines[mag_index].mag_num_objects;
+ u += szone->small_magazines[mag_index].mag_num_bytes_in_objects;
}
info[6] = (unsigned)t;
@@ -981,8 +1290,8 @@
info[0] = info[4] + info[6] + info[8] + info[10];
info[1] = info[5] + info[7] + info[9] + info[11];
- info[3] = (unsigned)(szone->tiny_rack.num_regions - szone->tiny_rack.num_regions_dealloc) * TINY_REGION_SIZE +
- (unsigned)(szone->small_rack.num_regions - szone->small_rack.num_regions_dealloc) * SMALL_REGION_SIZE + info[9] + info[11];
+ info[3] = (unsigned)(szone->num_tiny_regions - szone->num_tiny_regions_dealloc) * TINY_REGION_SIZE +
+ (unsigned)(szone->num_small_regions - szone->num_small_regions_dealloc) * SMALL_REGION_SIZE + info[9] + info[11];
info[2] = info[3] - (unsigned)s;
memcpy(info_to_fill, info, sizeof(unsigned) * count);
@@ -1003,46 +1312,46 @@
_malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX, "\ttiny=%u(%y) small=%u(%y) large=%u(%y) huge=%u(%y)\n", info[4],
info[5], info[6], info[7], info[8], info[9], info[10], info[11]);
// tiny
- _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX, "%lu tiny regions:\n", szone->tiny_rack.num_regions);
- if (szone->tiny_rack.num_regions_dealloc) {
+ _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX, "%lu tiny regions:\n", szone->num_tiny_regions);
+ if (szone->num_tiny_regions_dealloc) {
_malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX, "[%lu tiny regions have been vm_deallocate'd]\n",
- szone->tiny_rack.num_regions_dealloc);
- }
- for (index = 0; index < szone->tiny_rack.region_generation->num_regions_allocated; ++index) {
- region = szone->tiny_rack.region_generation->hashed_regions[index];
+ szone->num_tiny_regions_dealloc);
+ }
+ for (index = 0; index < szone->tiny_region_generation->num_regions_allocated; ++index) {
+ region = szone->tiny_region_generation->hashed_regions[index];
if (HASHRING_OPEN_ENTRY != region && HASHRING_REGION_DEALLOCATED != region) {
mag_index_t mag_index = MAGAZINE_INDEX_FOR_TINY_REGION(region);
- print_tiny_region(verbose, region, (region == szone->tiny_rack.magazines[mag_index].mag_last_region)
- ? szone->tiny_rack.magazines[mag_index].mag_bytes_free_at_start
+ print_tiny_region(verbose, region, (region == szone->tiny_magazines[mag_index].mag_last_region)
+ ? szone->tiny_magazines[mag_index].mag_bytes_free_at_start
: 0,
- (region == szone->tiny_rack.magazines[mag_index].mag_last_region)
- ? szone->tiny_rack.magazines[mag_index].mag_bytes_free_at_end
+ (region == szone->tiny_magazines[mag_index].mag_last_region)
+ ? szone->tiny_magazines[mag_index].mag_bytes_free_at_end
: 0);
}
}
if (verbose) {
- print_tiny_free_list(&szone->tiny_rack);
+ print_tiny_free_list(szone);
}
// small
- _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX, "%lu small regions:\n", szone->small_rack.num_regions);
- if (szone->small_rack.num_regions_dealloc) {
+ _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX, "%lu small regions:\n", szone->num_small_regions);
+ if (szone->num_small_regions_dealloc) {
_malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX, "[%lu small regions have been vm_deallocate'd]\n",
- szone->small_rack.num_regions_dealloc);
- }
- for (index = 0; index < szone->small_rack.region_generation->num_regions_allocated; ++index) {
- region = szone->small_rack.region_generation->hashed_regions[index];
+ szone->num_small_regions_dealloc);
+ }
+ for (index = 0; index < szone->small_region_generation->num_regions_allocated; ++index) {
+ region = szone->small_region_generation->hashed_regions[index];
if (HASHRING_OPEN_ENTRY != region && HASHRING_REGION_DEALLOCATED != region) {
mag_index_t mag_index = MAGAZINE_INDEX_FOR_SMALL_REGION(region);
- print_small_region(szone, verbose, region, (region == szone->small_rack.magazines[mag_index].mag_last_region)
- ? szone->small_rack.magazines[mag_index].mag_bytes_free_at_start
+ print_small_region(szone, verbose, region, (region == szone->small_magazines[mag_index].mag_last_region)
+ ? szone->small_magazines[mag_index].mag_bytes_free_at_start
: 0,
- (region == szone->small_rack.magazines[mag_index].mag_last_region)
- ? szone->small_rack.magazines[mag_index].mag_bytes_free_at_end
+ (region == szone->small_magazines[mag_index].mag_last_region)
+ ? szone->small_magazines[mag_index].mag_bytes_free_at_end
: 0);
}
}
if (verbose) {
- print_small_free_list(&szone->small_rack);
+ print_small_free_list(szone);
}
}
@@ -1060,12 +1369,12 @@
szone_force_lock_magazine(szone_t *szone, magazine_t *mag)
{
while (1) {
- SZONE_MAGAZINE_PTR_LOCK(mag);
+ SZONE_MAGAZINE_PTR_LOCK(szone, mag);
if (!mag->alloc_underway) {
return;
}
- SZONE_MAGAZINE_PTR_UNLOCK(mag);
+ SZONE_MAGAZINE_PTR_UNLOCK(szone, mag);
yield();
}
}
@@ -1075,15 +1384,15 @@
{
mag_index_t i;
- for (i = 0; i < szone->tiny_rack.num_magazines; ++i) {
- szone_force_lock_magazine(szone, &szone->tiny_rack.magazines[i]);
- }
- szone_force_lock_magazine(szone, &szone->tiny_rack.magazines[DEPOT_MAGAZINE_INDEX]);
-
- for (i = 0; i < szone->small_rack.num_magazines; ++i) {
- szone_force_lock_magazine(szone, &szone->small_rack.magazines[i]);
- }
- szone_force_lock_magazine(szone, &szone->small_rack.magazines[DEPOT_MAGAZINE_INDEX]);
+ for (i = 0; i < szone->num_tiny_magazines; ++i) {
+ szone_force_lock_magazine(szone, &szone->tiny_magazines[i]);
+ }
+ szone_force_lock_magazine(szone, &szone->tiny_magazines[DEPOT_MAGAZINE_INDEX]);
+
+ for (i = 0; i < szone->num_small_magazines; ++i) {
+ szone_force_lock_magazine(szone, &szone->small_magazines[i]);
+ }
+ szone_force_lock_magazine(szone, &szone->small_magazines[DEPOT_MAGAZINE_INDEX]);
SZONE_LOCK(szone);
}
@@ -1095,12 +1404,12 @@
SZONE_UNLOCK(szone);
- for (i = -1; i < szone->small_rack.num_magazines; ++i) {
- SZONE_MAGAZINE_PTR_UNLOCK((&(szone->small_rack.magazines[i])));
- }
-
- for (i = -1; i < szone->tiny_rack.num_magazines; ++i) {
- SZONE_MAGAZINE_PTR_UNLOCK((&(szone->tiny_rack.magazines[i])));
+ for (i = -1; i < szone->num_small_magazines; ++i) {
+ SZONE_MAGAZINE_PTR_UNLOCK(szone, (&(szone->small_magazines[i])));
+ }
+
+ for (i = -1; i < szone->num_tiny_magazines; ++i) {
+ SZONE_MAGAZINE_PTR_UNLOCK(szone, (&(szone->tiny_magazines[i])));
}
}
@@ -1111,12 +1420,12 @@
SZONE_REINIT_LOCK(szone);
- for (i = -1; i < szone->small_rack.num_magazines; ++i) {
- SZONE_MAGAZINE_PTR_REINIT_LOCK((&(szone->small_rack.magazines[i])));
- }
-
- for (i = -1; i < szone->tiny_rack.num_magazines; ++i) {
- SZONE_MAGAZINE_PTR_REINIT_LOCK((&(szone->tiny_rack.magazines[i])));
+ for (i = -1; i < szone->num_small_magazines; ++i) {
+ SZONE_MAGAZINE_PTR_REINIT_LOCK(szone, (&(szone->small_magazines[i])));
+ }
+
+ for (i = -1; i < szone->num_tiny_magazines; ++i) {
+ SZONE_MAGAZINE_PTR_REINIT_LOCK(szone, (&(szone->tiny_magazines[i])));
}
}
@@ -1132,20 +1441,20 @@
}
SZONE_UNLOCK(szone);
- for (i = -1; i < szone->small_rack.num_magazines; ++i) {
- tookLock = SZONE_MAGAZINE_PTR_TRY_LOCK((&(szone->small_rack.magazines[i])));
+ for (i = -1; i < szone->num_small_magazines; ++i) {
+ tookLock = SZONE_MAGAZINE_PTR_TRY_LOCK(szone, (&(szone->small_magazines[i])));
if (tookLock == 0) {
return 1;
}
- SZONE_MAGAZINE_PTR_UNLOCK((&(szone->small_rack.magazines[i])));
- }
-
- for (i = -1; i < szone->tiny_rack.num_magazines; ++i) {
- tookLock = SZONE_MAGAZINE_PTR_TRY_LOCK((&(szone->tiny_rack.magazines[i])));
+ SZONE_MAGAZINE_PTR_UNLOCK(szone, (&(szone->small_magazines[i])));
+ }
+
+ for (i = -1; i < szone->num_tiny_magazines; ++i) {
+ tookLock = SZONE_MAGAZINE_PTR_TRY_LOCK(szone, (&(szone->tiny_magazines[i])));
if (tookLock == 0) {
return 1;
}
- SZONE_MAGAZINE_PTR_UNLOCK((&(szone->tiny_rack.magazines[i])));
+ SZONE_MAGAZINE_PTR_UNLOCK(szone, (&(szone->tiny_magazines[i])));
}
return 0;
}
@@ -1158,23 +1467,22 @@
#if CONFIG_MADVISE_PRESSURE_RELIEF
mag_index_t mag_index;
- magazine_t *tiny_depot_ptr = (&szone->tiny_rack.magazines[DEPOT_MAGAZINE_INDEX]);
- magazine_t *small_depot_ptr = (&szone->small_rack.magazines[DEPOT_MAGAZINE_INDEX]);
-
- for (mag_index = 0; mag_index < szone->tiny_rack.num_magazines; mag_index++) {
+ magazine_t *tiny_depot_ptr = (&szone->tiny_magazines[DEPOT_MAGAZINE_INDEX]);
+ magazine_t *small_depot_ptr = (&szone->small_magazines[DEPOT_MAGAZINE_INDEX]);
+
+ for (mag_index = 0; mag_index < szone->num_tiny_magazines; mag_index++) {
size_t index;
- for (index = 0; index < szone->tiny_rack.region_generation->num_regions_allocated; ++index) {
+ for (index = 0; index < szone->tiny_region_generation->num_regions_allocated; ++index) {
SZONE_LOCK(szone);
- region_t tiny = szone->tiny_rack.region_generation->hashed_regions[index];
+ region_t tiny = szone->tiny_region_generation->hashed_regions[index];
if (!tiny || tiny == HASHRING_REGION_DEALLOCATED) {
SZONE_UNLOCK(szone);
continue;
}
- magazine_t *mag_ptr = mag_lock_zine_for_region_trailer(szone->tiny_rack.magazines,
- REGION_TRAILER_FOR_TINY_REGION(tiny),
- MAGAZINE_INDEX_FOR_TINY_REGION(tiny));
+ magazine_t *mag_ptr = mag_lock_zine_for_region_trailer(
+ szone, szone->tiny_magazines, REGION_TRAILER_FOR_TINY_REGION(tiny), MAGAZINE_INDEX_FOR_TINY_REGION(tiny));
SZONE_UNLOCK(szone);
/* Ordering is important here, the magazine of a region may potentially change
@@ -1185,12 +1493,12 @@
/* We can (and must) ignore magazines that are already in the recirc depot. */
if (src_mag_index == DEPOT_MAGAZINE_INDEX) {
- SZONE_MAGAZINE_PTR_UNLOCK(mag_ptr);
+ SZONE_MAGAZINE_PTR_UNLOCK(szone, mag_ptr);
continue;
}
if (tiny == mag_ptr->mag_last_region && (mag_ptr->mag_bytes_free_at_end || mag_ptr->mag_bytes_free_at_start)) {
- tiny_finalize_region(&szone->tiny_rack, mag_ptr);
+ tiny_finalize_region(szone, mag_ptr);
}
/* Because this region is currently in use, we can't safely madvise it while
@@ -1198,14 +1506,14 @@
* the current mag, attach it to the depot and then madvise.
*/
- recirc_list_extract(&szone->tiny_rack, mag_ptr, REGION_TRAILER_FOR_TINY_REGION(tiny));
- int objects_in_use = tiny_free_detach_region(&szone->tiny_rack, mag_ptr, tiny);
-
- SZONE_MAGAZINE_PTR_LOCK(tiny_depot_ptr);
+ recirc_list_extract(szone, mag_ptr, REGION_TRAILER_FOR_TINY_REGION(tiny));
+ int objects_in_use = tiny_free_detach_region(szone, mag_ptr, tiny);
+
+ SZONE_MAGAZINE_PTR_LOCK(szone, tiny_depot_ptr);
MAGAZINE_INDEX_FOR_TINY_REGION(tiny) = DEPOT_MAGAZINE_INDEX;
REGION_TRAILER_FOR_TINY_REGION(tiny)->pinned_to_depot = 0;
- size_t bytes_inplay = tiny_free_reattach_region(&szone->tiny_rack, tiny_depot_ptr, tiny);
+ size_t bytes_inplay = tiny_free_reattach_region(szone, tiny_depot_ptr, tiny);
/* Fix up the metadata of the target magazine while the region is in the depot. */
mag_ptr->mag_num_bytes_in_objects -= bytes_inplay;
@@ -1213,42 +1521,41 @@
mag_ptr->mag_num_objects -= objects_in_use;
/* Now we can drop the magazine lock of the source mag. */
- SZONE_MAGAZINE_PTR_UNLOCK(mag_ptr);
+ SZONE_MAGAZINE_PTR_UNLOCK(szone, mag_ptr);
tiny_depot_ptr->mag_num_bytes_in_objects += bytes_inplay;
tiny_depot_ptr->num_bytes_in_magazine += TINY_REGION_PAYLOAD_BYTES;
tiny_depot_ptr->mag_num_objects -= objects_in_use;
- recirc_list_splice_last(&szone->tiny_rack, tiny_depot_ptr, REGION_TRAILER_FOR_TINY_REGION(tiny));
+ recirc_list_splice_last(szone, tiny_depot_ptr, REGION_TRAILER_FOR_TINY_REGION(tiny));
/* Actually do the scan, done holding the depot lock, the call will drop the lock
* around the actual madvise syscalls.
*/
- tiny_free_scan_madvise_free(&szone->tiny_rack, tiny_depot_ptr, tiny);
+ tiny_free_scan_madvise_free(szone, tiny_depot_ptr, tiny);
/* Now the region is in the recirc depot, the next allocations to require more
* blocks will come along and take one of these regions back out of the depot.
* As OS X madvise's reuse on an per-region basis, we leave as many of these
* regions in the depot as possible after memory pressure.
*/
- SZONE_MAGAZINE_PTR_UNLOCK(tiny_depot_ptr);
- }
- }
-
- for (mag_index = 0; mag_index < szone->small_rack.num_magazines; mag_index++) {
+ SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_depot_ptr);
+ }
+ }
+
+ for (mag_index = 0; mag_index < szone->num_small_magazines; mag_index++) {
size_t index;
- for (index = 0; index < szone->small_rack.region_generation->num_regions_allocated; ++index) {
+ for (index = 0; index < szone->small_region_generation->num_regions_allocated; ++index) {
SZONE_LOCK(szone);
- region_t small = szone->small_rack.region_generation->hashed_regions[index];
+ region_t small = szone->small_region_generation->hashed_regions[index];
if (!small || small == HASHRING_REGION_DEALLOCATED) {
SZONE_UNLOCK(szone);
continue;
}
- magazine_t *mag_ptr = mag_lock_zine_for_region_trailer(szone->small_rack.magazines,
- REGION_TRAILER_FOR_SMALL_REGION(small),
- MAGAZINE_INDEX_FOR_SMALL_REGION(small));
+ magazine_t *mag_ptr = mag_lock_zine_for_region_trailer(
+ szone, szone->small_magazines, REGION_TRAILER_FOR_SMALL_REGION(small), MAGAZINE_INDEX_FOR_SMALL_REGION(small));
SZONE_UNLOCK(szone);
/* Ordering is important here, the magazine of a region may potentially change
@@ -1259,12 +1566,12 @@
/* We can (and must) ignore magazines that are already in the recirc depot. */
if (src_mag_index == DEPOT_MAGAZINE_INDEX) {
- SZONE_MAGAZINE_PTR_UNLOCK(mag_ptr);
+ SZONE_MAGAZINE_PTR_UNLOCK(szone, mag_ptr);
continue;
}
if (small == mag_ptr->mag_last_region && (mag_ptr->mag_bytes_free_at_end || mag_ptr->mag_bytes_free_at_start)) {
- small_finalize_region(&szone->small_rack, mag_ptr);
+ small_finalize_region(szone, mag_ptr);
}
/* Because this region is currently in use, we can't safely madvise it while
@@ -1272,14 +1579,14 @@
* the current mag, attach it to the depot and then madvise.
*/
- recirc_list_extract(&szone->small_rack, mag_ptr, REGION_TRAILER_FOR_SMALL_REGION(small));
- int objects_in_use = small_free_detach_region(&szone->small_rack, mag_ptr, small);
-
- SZONE_MAGAZINE_PTR_LOCK(small_depot_ptr);
+ recirc_list_extract(szone, mag_ptr, REGION_TRAILER_FOR_SMALL_REGION(small));
+ int objects_in_use = small_free_detach_region(szone, mag_ptr, small);
+
+ SZONE_MAGAZINE_PTR_LOCK(szone, small_depot_ptr);
MAGAZINE_INDEX_FOR_SMALL_REGION(small) = DEPOT_MAGAZINE_INDEX;
REGION_TRAILER_FOR_SMALL_REGION(small)->pinned_to_depot = 0;
- size_t bytes_inplay = small_free_reattach_region(&szone->small_rack, small_depot_ptr, small);
+ size_t bytes_inplay = small_free_reattach_region(szone, small_depot_ptr, small);
/* Fix up the metadata of the target magazine while the region is in the depot. */
mag_ptr->mag_num_bytes_in_objects -= bytes_inplay;
@@ -1287,25 +1594,25 @@
mag_ptr->mag_num_objects -= objects_in_use;
/* Now we can drop the magazine lock of the source mag. */
- SZONE_MAGAZINE_PTR_UNLOCK(mag_ptr);
+ SZONE_MAGAZINE_PTR_UNLOCK(szone, mag_ptr);
small_depot_ptr->mag_num_bytes_in_objects += bytes_inplay;
small_depot_ptr->num_bytes_in_magazine += SMALL_REGION_PAYLOAD_BYTES;
small_depot_ptr->mag_num_objects -= objects_in_use;
- recirc_list_splice_last(&szone->small_rack, small_depot_ptr, REGION_TRAILER_FOR_SMALL_REGION(small));
+ recirc_list_splice_last(szone, small_depot_ptr, REGION_TRAILER_FOR_SMALL_REGION(small));
/* Actually do the scan, done holding the depot lock, the call will drop the lock
* around the actual madvise syscalls.
*/
- small_free_scan_madvise_free(&szone->small_rack, small_depot_ptr, small);
+ small_free_scan_madvise_free(szone, small_depot_ptr, small);
/* Now the region is in the recirc depot, the next allocations to require more
* blocks will come along and take one of these regions back out of the depot.
* As OS X madvise's reuse on an per-region basis, we leave as many of these
* regions in the depot as possible after memory pressure.
*/
- SZONE_MAGAZINE_PTR_UNLOCK(small_depot_ptr);
+ SZONE_MAGAZINE_PTR_UNLOCK(szone, small_depot_ptr);
}
}
#endif
@@ -1333,14 +1640,14 @@
// deallocate the death-row cache outside the zone lock
size_t total = 0;
while (idx != idx_max) {
- mvm_deallocate_pages((void *)local_entry_cache[idx].address, local_entry_cache[idx].size, 0);
+ deallocate_pages(szone, (void *)local_entry_cache[idx].address, local_entry_cache[idx].size, 0);
total += local_entry_cache[idx].size;
if (++idx == LARGE_ENTRY_CACHE_SIZE) {
idx = 0;
}
}
if (0 != local_entry_cache[idx].address && 0 != local_entry_cache[idx].size) {
- mvm_deallocate_pages((void *)local_entry_cache[idx].address, local_entry_cache[idx].size, 0);
+ deallocate_pages(szone, (void *)local_entry_cache[idx].address, local_entry_cache[idx].size, 0);
total += local_entry_cache[idx].size;
}
}
@@ -1362,16 +1669,16 @@
size_t u = 0;
mag_index_t mag_index;
- for (mag_index = -1; mag_index < szone->tiny_rack.num_magazines; mag_index++) {
- s += szone->tiny_rack.magazines[mag_index].mag_bytes_free_at_start;
- s += szone->tiny_rack.magazines[mag_index].mag_bytes_free_at_end;
- t += szone->tiny_rack.magazines[mag_index].mag_num_objects;
- u += szone->tiny_rack.magazines[mag_index].mag_num_bytes_in_objects;
+ for (mag_index = -1; mag_index < szone->num_tiny_magazines; mag_index++) {
+ s += szone->tiny_magazines[mag_index].mag_bytes_free_at_start;
+ s += szone->tiny_magazines[mag_index].mag_bytes_free_at_end;
+ t += szone->tiny_magazines[mag_index].mag_num_objects;
+ u += szone->tiny_magazines[mag_index].mag_num_bytes_in_objects;
}
stats->blocks_in_use = t;
stats->size_in_use = u;
- stats->size_allocated = (szone->tiny_rack.num_regions - szone->tiny_rack.num_regions_dealloc) * TINY_REGION_SIZE;
+ stats->size_allocated = (szone->num_tiny_regions - szone->num_tiny_regions_dealloc) * TINY_REGION_SIZE;
stats->max_size_in_use = stats->size_allocated - s;
return 1;
}
@@ -1381,16 +1688,16 @@
size_t u = 0;
mag_index_t mag_index;
- for (mag_index = -1; mag_index < szone->small_rack.num_magazines; mag_index++) {
- s += szone->small_rack.magazines[mag_index].mag_bytes_free_at_start;
- s += szone->small_rack.magazines[mag_index].mag_bytes_free_at_end;
- t += szone->small_rack.magazines[mag_index].mag_num_objects;
- u += szone->small_rack.magazines[mag_index].mag_num_bytes_in_objects;
+ for (mag_index = -1; mag_index < szone->num_small_magazines; mag_index++) {
+ s += szone->small_magazines[mag_index].mag_bytes_free_at_start;
+ s += szone->small_magazines[mag_index].mag_bytes_free_at_end;
+ t += szone->small_magazines[mag_index].mag_num_objects;
+ u += szone->small_magazines[mag_index].mag_num_bytes_in_objects;
}
stats->blocks_in_use = t;
stats->size_in_use = u;
- stats->size_allocated = (szone->small_rack.num_regions - szone->small_rack.num_regions_dealloc) * SMALL_REGION_SIZE;
+ stats->size_allocated = (szone->num_small_regions - szone->num_small_regions_dealloc) * SMALL_REGION_SIZE;
stats->max_size_in_use = stats->size_allocated - s;
return 1;
}
@@ -1418,18 +1725,18 @@
size_t u = 0;
mag_index_t mag_index;
- for (mag_index = -1; mag_index < szone->tiny_rack.num_magazines; mag_index++) {
- s += szone->tiny_rack.magazines[mag_index].mag_bytes_free_at_start;
- s += szone->tiny_rack.magazines[mag_index].mag_bytes_free_at_end;
- t += szone->tiny_rack.magazines[mag_index].mag_num_objects;
- u += szone->tiny_rack.magazines[mag_index].mag_num_bytes_in_objects;
- }
-
- for (mag_index = -1; mag_index < szone->small_rack.num_magazines; mag_index++) {
- s += szone->small_rack.magazines[mag_index].mag_bytes_free_at_start;
- s += szone->small_rack.magazines[mag_index].mag_bytes_free_at_end;
- t += szone->small_rack.magazines[mag_index].mag_num_objects;
- u += szone->small_rack.magazines[mag_index].mag_num_bytes_in_objects;
+ for (mag_index = -1; mag_index < szone->num_tiny_magazines; mag_index++) {
+ s += szone->tiny_magazines[mag_index].mag_bytes_free_at_start;
+ s += szone->tiny_magazines[mag_index].mag_bytes_free_at_end;
+ t += szone->tiny_magazines[mag_index].mag_num_objects;
+ u += szone->tiny_magazines[mag_index].mag_num_bytes_in_objects;
+ }
+
+ for (mag_index = -1; mag_index < szone->num_small_magazines; mag_index++) {
+ s += szone->small_magazines[mag_index].mag_bytes_free_at_start;
+ s += szone->small_magazines[mag_index].mag_bytes_free_at_end;
+ t += szone->small_magazines[mag_index].mag_num_objects;
+ u += szone->small_magazines[mag_index].mag_num_bytes_in_objects;
}
large = szone->num_bytes_in_large_objects + 0; // DEPRECATED szone->num_bytes_in_huge_objects;
@@ -1437,8 +1744,8 @@
stats->blocks_in_use = t + szone->num_large_objects_in_use + 0; // DEPRECATED szone->num_huge_entries;
stats->size_in_use = u + large;
stats->max_size_in_use = stats->size_allocated =
- (szone->tiny_rack.num_regions - szone->tiny_rack.num_regions_dealloc) * TINY_REGION_SIZE +
- (szone->small_rack.num_regions - szone->small_rack.num_regions_dealloc) * SMALL_REGION_SIZE + large;
+ (szone->num_tiny_regions - szone->num_tiny_regions_dealloc) * TINY_REGION_SIZE +
+ (szone->num_small_regions - szone->num_small_regions_dealloc) * SMALL_REGION_SIZE + large;
// Now we account for the untouched areas
stats->max_size_in_use -= s;
}
@@ -1462,7 +1769,7 @@
#endif
/* get memory for the zone. */
- szone = mvm_allocate_pages(SZONE_PAGED_SIZE, 0, 0, VM_MEMORY_MALLOC);
+ szone = allocate_pages(NULL, SZONE_PAGED_SIZE, 0, 0, VM_MEMORY_MALLOC);
if (!szone) {
return NULL;
}
@@ -1476,37 +1783,47 @@
#warning LOG enabled
szone->log_address = ~0;
#endif
-
- if (mvm_aslr_enabled()) {
- debug_flags &= ~DISABLE_ASLR;
- } else {
- debug_flags |= DISABLE_ASLR;
- }
-
-#if CONFIG_SMALL_CUTTOFF_127KB
- debug_flags |= MALLOC_EXTENDED_SMALL_SLOTS;
+ szone->trg[0].nextgen = &(szone->trg[1]);
+ szone->trg[1].nextgen = &(szone->trg[0]);
+ szone->tiny_region_generation = &(szone->trg[0]);
+
+ szone->tiny_region_generation->hashed_regions = szone->initial_tiny_regions;
+ szone->tiny_region_generation->num_regions_allocated = INITIAL_NUM_REGIONS;
+ szone->tiny_region_generation->num_regions_allocated_shift = INITIAL_NUM_REGIONS_SHIFT;
+
+ szone->srg[0].nextgen = &(szone->srg[1]);
+ szone->srg[1].nextgen = &(szone->srg[0]);
+ szone->small_region_generation = &(szone->srg[0]);
+
+ szone->small_region_generation->hashed_regions = szone->initial_small_regions;
+ szone->small_region_generation->num_regions_allocated = INITIAL_NUM_REGIONS;
+ szone->small_region_generation->num_regions_allocated_shift = INITIAL_NUM_REGIONS_SHIFT;
+
+#if CONFIG_SMALL_CUTTOFF_127KB
szone->is_largemem = 1;
+ szone->num_small_slots = NUM_SMALL_SLOTS_LARGEMEM;
szone->large_threshold = LARGE_THRESHOLD_LARGEMEM;
szone->vm_copy_threshold = VM_COPY_THRESHOLD_LARGEMEM;
#else // CONFIG_SMALL_CUTTOFF_127KB
- debug_flags &= ~MALLOC_EXTENDED_SMALL_SLOTS;
szone->is_largemem = 0;
+ szone->num_small_slots = NUM_SMALL_SLOTS;
szone->large_threshold = LARGE_THRESHOLD;
szone->vm_copy_threshold = VM_COPY_THRESHOLD;
#endif // CONFIG_SMALL_CUTTOFF_127KB
- // Query the number of configured processors.
- // Uniprocessor case gets just one tiny and one small magazine (whose index is zero). This gives
- // the same behavior as the original scalable malloc. MP gets per-CPU magazines
- // that scale (way) better.
- uint32_t nproc = platform_cpu_count();
- uint32_t num_magazines = (nproc > 1) ? MIN(nproc, TINY_MAX_MAGAZINES) : 1;
- rack_init(&szone->tiny_rack, RACK_TYPE_TINY, num_magazines, debug_flags);
- rack_init(&szone->small_rack, RACK_TYPE_SMALL, num_magazines, debug_flags);
-
#if CONFIG_LARGE_CACHE
+ uint64_t hw_memsize = 0;
+
+#if CONFIG_HAS_COMMPAGE_MEMSIZE
+ hw_memsize = *(uint64_t *)(uintptr_t)_COMM_PAGE_MEMORY_SIZE;
+#else
+ size_t uint64_t_size = sizeof(hw_memsize);
+ // hw_memsize was always 0 if sysctlbyname failed, so preserve that behaviour
+ (void)sysctlbyname("hw.memsize", &hw_memsize, &uint64_t_size, 0, 0);
+#endif
+
// madvise(..., MADV_REUSABLE) death-row arrivals above this threshold [~0.1%]
- szone->large_entry_cache_reserve_limit = (size_t)(platform_hw_memsize() >> 10);
+ szone->large_entry_cache_reserve_limit = hw_memsize >> 10;
/* <rdar://problem/6610904> Reset protection when returning a previous large allocation? */
int32_t libSystemVersion = NSVersionOfLinkTimeLibrary("System");
@@ -1515,6 +1832,44 @@
} else {
szone->large_legacy_reset_mprotect = FALSE;
}
+#endif
+
+// Prepare ASLR
+#if __i386__ || __x86_64__ || __arm64__ || TARGET_OS_EMBEDDED
+#if __i386__
+ uintptr_t stackbase = 0x8fe00000;
+ int entropic_bits = 3;
+#elif __x86_64__
+ uintptr_t stackbase = USRSTACK64;
+ int entropic_bits = 16;
+#elif __arm64__
+ uintptr_t stackbase = USRSTACK64;
+ int entropic_bits = 7;
+#else
+ uintptr_t stackbase = USRSTACK;
+ int entropic_bits = 3;
+#endif
+
+ // assert(((1 << entropic_bits) - 1) << SMALL_BLOCKS_ALIGN < (stackbase - MAXSSIZ - ENTROPIC_KABILLION));
+
+ if (0 != _dyld_get_image_slide((const struct mach_header *)_NSGetMachExecuteHeader())) {
+ if (0 == entropic_address) {
+ uintptr_t t = stackbase - MAXSSIZ - ((uintptr_t)(malloc_entropy[1] & ((1 << entropic_bits) - 1)) << ENTROPIC_SHIFT);
+ (void)__sync_bool_compare_and_swap(&entropic_limit, 0, t); // Just one initialization please
+ (void)__sync_bool_compare_and_swap(&entropic_address, 0, t - ENTROPIC_KABILLION); // Just one initialization please
+ }
+ debug_flags &= ~DISABLE_ASLR;
+ } else {
+ // zero slide when ASLR has been disabled by boot-arg. Eliminate cloaking.
+ malloc_entropy[0] = 0;
+ malloc_entropy[1] = 0;
+ debug_flags |= DISABLE_ASLR;
+ }
+
+#else
+ malloc_entropy[0] = 0;
+ malloc_entropy[1] = 0;
+ debug_flags |= DISABLE_ASLR;
#endif
// Initialize the security token.
@@ -1535,18 +1890,94 @@
szone->basic_zone.free_definite_size = (void *)szone_free_definite_size;
szone->basic_zone.pressure_relief = (void *)szone_pressure_relief;
- /* Set to zero once and for all as required by CFAllocator. */
- szone->basic_zone.reserved1 = 0;
- /* Set to zero once and for all as required by CFAllocator. */
- szone->basic_zone.reserved2 = 0;
-
- /* Prevent overwriting the function pointers in basic_zone. */
- mprotect(szone, sizeof(szone->basic_zone), PROT_READ);
+ szone->basic_zone.reserved1 = 0; /* Set to zero once and for all as required by CFAllocator. */
+ szone->basic_zone.reserved2 = 0; /* Set to zero once and for all as required by CFAllocator. */
+ mprotect(szone, sizeof(szone->basic_zone), PROT_READ); /* Prevent overwriting the function pointers in basic_zone. */
szone->debug_flags = debug_flags;
_malloc_lock_init(&szone->large_szone_lock);
szone->cpu_id_key = -1UL; // Unused.
+
+// Query the number of configured processors.
+// Uniprocessor case gets just one tiny and one small magazine (whose index is zero). This gives
+// the same behavior as the original scalable malloc. MP gets per-CPU magazines
+// that scale (way) better.
+#if defined(__i386__) || defined(__x86_64__) || defined(__arm__) || defined(__arm64__)
+ int nproc = *(uint8_t *)(uintptr_t)_COMM_PAGE_NCPUS;
+#else
+ int nproc = sysconf(_SC_NPROCESSORS_CONF);
+#endif
+ szone->num_tiny_magazines = (nproc > 1) ? MIN(nproc, TINY_MAX_MAGAZINES) : 1;
+
+ // FIXME vm_allocate() based on number of configured CPUs
+ magazine_t *tiny_magazines =
+ allocate_pages(NULL, TINY_MAGAZINE_PAGED_SIZE, 0, MALLOC_ADD_GUARD_PAGES, VM_MEMORY_MALLOC);
+ if (NULL == tiny_magazines) {
+ return NULL;
+ }
+
+ szone->tiny_magazines = &(tiny_magazines[1]); // szone->tiny_magazines[-1] is the Depot
+
+ // The magazines are indexed in [0 .. (num_tiny_magazines - 1)]
+ // Find the smallest power of 2 that exceeds (num_tiny_magazines - 1)
+ szone->num_tiny_magazines_mask_shift = 0;
+ int i = 1;
+ while (i <= (szone->num_tiny_magazines - 1)) {
+ szone->num_tiny_magazines_mask_shift++;
+ i <<= 1;
+ }
+
+ // Now if i <= TINY_MAX_MAGAZINES we'll never access tiny_magazines[] out of bounds.
+ if (i > TINY_MAX_MAGAZINES) {
+ MALLOC_PRINTF_FATAL_ERROR(i, "magazine mask exceeds allocated magazines");
+ }
+
+ // Reduce i by 1 to obtain a mask covering [0 .. (num_tiny_magazines - 1)]
+ szone->num_tiny_magazines_mask = i - 1; // A mask used for hashing to a magazine index (and a safety aid)
+ szone->last_tiny_advise = 0;
+
+ // Init the tiny_magazine locks
+ _malloc_lock_init(&szone->tiny_regions_lock);
+ _malloc_lock_init(&szone->tiny_magazines[DEPOT_MAGAZINE_INDEX].magazine_lock);
+ for (i = 0; i < szone->num_tiny_magazines; ++i) {
+ _malloc_lock_init(&szone->tiny_magazines[i].magazine_lock);
+ }
+
+ szone->num_small_magazines = (nproc > 1) ? MIN(nproc, SMALL_MAX_MAGAZINES) : 1;
+
+ // FIXME vm_allocate() based on number of configured CPUs
+ magazine_t *small_magazines =
+ allocate_pages(NULL, SMALL_MAGAZINE_PAGED_SIZE, 0, MALLOC_ADD_GUARD_PAGES, VM_MEMORY_MALLOC);
+ if (NULL == small_magazines) {
+ return NULL;
+ }
+
+ szone->small_magazines = &(small_magazines[1]); // szone->small_magazines[-1] is the Depot
+
+ // The magazines are indexed in [0 .. (num_small_magazines - 1)]
+ // Find the smallest power of 2 that exceeds (num_small_magazines - 1)
+ szone->num_small_magazines_mask_shift = 0;
+ while (i <= (szone->num_small_magazines - 1)) {
+ szone->num_small_magazines_mask_shift++;
+ i <<= 1;
+ }
+
+ // Now if i <= SMALL_MAX_MAGAZINES we'll never access small_magazines[] out of bounds.
+ if (i > SMALL_MAX_MAGAZINES) {
+ MALLOC_PRINTF_FATAL_ERROR(i, "magazine mask exceeds allocated magazines");
+ }
+
+ // Reduce i by 1 to obtain a mask covering [0 .. (num_small_magazines - 1)]
+ szone->num_small_magazines_mask = i - 1; // A mask used for hashing to a magazine index (and a safety aid)
+ szone->last_small_advise = 0;
+
+ // Init the small_magazine locks
+ _malloc_lock_init(&szone->small_regions_lock);
+ _malloc_lock_init(&szone->small_magazines[DEPOT_MAGAZINE_INDEX].magazine_lock);
+ for (i = 0; i < szone->num_small_magazines; ++i) {
+ _malloc_lock_init(&szone->small_magazines[i].magazine_lock);
+ }
CHECK(szone, __PRETTY_FUNCTION__);
return szone;