Loading...
--- libmalloc/libmalloc-317.140.5/src/nanov2_malloc.c
+++ libmalloc/libmalloc-657.60.21/src/nanov2_malloc.c
@@ -41,19 +41,39 @@
#pragma mark -
#pragma mark Externals for resolved functions
-MALLOC_NOEXPORT extern void *nanov2_allocate(nanozonev2_t *nanozone, size_t rounded_size,
- boolean_t clear);
-MALLOC_NOEXPORT extern void nanov2_free_to_block(nanozonev2_t *nanozone, void *ptr,
- nanov2_size_class_t size_class);
-MALLOC_NOEXPORT extern boolean_t nanov2_madvise_block(nanozonev2_t *nanozone,
- nanov2_block_meta_t *block_metap, nanov2_block_t *blockp,
- nanov2_size_class_t size_class);
MALLOC_NOEXPORT extern size_t nanov2_pointer_size(nanozonev2_t *nanozone, void *ptr,
boolean_t allow_inner);
MALLOC_NOEXPORT extern size_t nanov2_pressure_relief(nanozonev2_t *nanozone, size_t goal);
#if OS_VARIANT_RESOLVED
-MALLOC_NOEXPORT extern boolean_t nanov2_allocate_new_region(nanozonev2_t *nanozone);
+MALLOC_ALWAYS_INLINE MALLOC_INLINE size_t
+nanov2_pointer_size_inline(nanozonev2_t *nanozone, void *ptr, boolean_t allow_inner,
+ nanov2_size_class_t *size_class_out, nanov2_block_meta_t **block_metap_out);
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE void *
+nanov2_allocate_from_block_inline(nanozonev2_t *nanozone,
+ nanov2_block_meta_t *block_metap, nanov2_size_class_t size_class,
+ nanov2_block_meta_t **madvise_block_metapp_out, bool *corruption);
+
+static void *
+nanov2_allocate_outlined(nanozonev2_t *nanozone,
+ nanov2_block_meta_t **block_metapp, size_t rounded_size,
+ nanov2_size_class_t size_class, int allocation_index,
+ nanov2_block_meta_t *madvise_block_metap, void *corrupt_slot,
+ bool clear, bool typed, malloc_type_id_t type_id);
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_meta_t *
+nanov2_free_to_block_inline(nanozonev2_t *nanozone, void *ptr,
+ nanov2_size_class_t size_class, nanov2_block_meta_t *block_metap);
+
+static boolean_t nanov2_madvise_block_locked(
+ nanozonev2_t *nanozone, nanov2_block_meta_t *block_metap,
+ nanov2_block_t *blockp, nanov2_size_class_t size_class, uint32_t expected_state);
+static void nanov2_madvise_block(nanozonev2_t *nanozone,
+ nanov2_block_meta_t *block_metap, nanov2_size_class_t size_class,
+ uint32_t expected_state);
+
+MALLOC_NOEXPORT extern nanov2_arena_t *nanov2_allocate_new_region(nanozonev2_t *nanozone);
#endif // OS_VARIANT_RESOLVED
#pragma mark -
@@ -313,7 +333,6 @@
return (void *)(((uintptr_t)ptr) & NANOV2_ARENA_ADDRESS_MASK);
}
-#if OS_VARIANT_RESOLVED
// Given a pointer that is assumed to be in the Nano zone, returns the address
// of its containing region. Works for both real and logical pointers.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_region_t *
@@ -321,7 +340,6 @@
{
return (nanov2_region_t *)(((uintptr_t)ptr) & NANOV2_REGION_ADDRESS_MASK);
}
-#endif // OS_VARIANT_RESOLVED
// Given a pointer that is assumed to be in the Nano zone, returns the real
// address of its metadata block. Works for both real and logical pointers.
@@ -395,15 +413,40 @@
return (nanov2_arena_t *)region;
}
+#if OS_VARIANT_RESOLVED
+// Given an atomically-observed current_region_next_arena pointer, returns
+// whether or not it's a usable arena or a limit arena (indicating exhaustion of
+// the current region).
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE bool
+nanov2_current_region_next_arena_is_limit(
+ nanov2_arena_t *current_region_next_arena)
+{
+ // The first arena of a region is never stored in current_region_next_arena,
+ // so a value at the beginning of a region must be a limit arena.
+ return current_region_next_arena == (nanov2_arena_t *)(
+ nanov2_region_address_for_ptr(current_region_next_arena));
+}
+#endif // OS_VARIANT_RESOLVED
+
+// Given an atomically-observed current_region_next_arena pointer, returns the
+// base of the current region at the time of the observation.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_region_t *
+nanov2_current_region_base(nanov2_arena_t *current_region_next_arena)
+{
+ return nanov2_region_address_for_ptr(
+ (void *)(((uintptr_t)current_region_next_arena) - 1));
+}
+
// Given a region pointer, returns a pointer to the arena after the last
// active arena in the region.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_arena_t *
-nanov2_limit_arena_for_region(nanozonev2_t *nanozone, nanov2_region_t *region)
+nanov2_limit_arena_for_region(nanozonev2_t __unused *nanozone,
+ nanov2_region_t *region, nanov2_arena_t *current_region_next_arena)
{
// The first arena is colocated with the region itself.
nanov2_arena_t *limit_arena;
- if (region == nanozone->current_region_base) {
- limit_arena = nanozone->current_region_next_arena;
+ if (region == nanov2_current_region_base(current_region_next_arena)) {
+ limit_arena = current_region_next_arena;
} else {
limit_arena = nanov2_first_arena_for_region(region + 1);
}
@@ -424,14 +467,23 @@
#if OS_VARIANT_RESOLVED
// Given a pointer to a region, returns a pointer to the region that follows it,
-// or NULL if there isn't one.
+// or NULL if there isn't one. We may observe linkage to a new region that
+// hasn't yet actually been installed into current_region_next_arena; ignore the
+// linkage in this case.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_region_t *
-nanov2_next_region_for_region(nanozonev2_t *nanozone, nanov2_region_t *region)
+nanov2_next_region_for_region(nanozonev2_t *nanozone, nanov2_region_t *region,
+ nanov2_arena_t *current_region_next_arena)
{
nanov2_region_linkage_t *linkage =
nanov2_region_linkage_for_region(nanozone, region);
- int offset = linkage->next_region_offset;
- return offset ? region + offset : NULL;
+ int offset = os_atomic_load(&linkage->next_region_offset, relaxed);
+ if (!offset) {
+ return NULL;
+ }
+
+ nanov2_region_t *next_region = region + offset;
+ return (nanov2_arena_t *)next_region < current_region_next_arena ?
+ next_region : NULL;
}
#endif // OS_VARIANT_RESOLVED
@@ -441,14 +493,21 @@
// for another process.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_region_t *
nanov2_next_region_for_region_offset(nanozonev2_t *nanozone,
- nanov2_region_t *region, off_t region_offset)
-{
- nanov2_region_linkage_t *linkage =
- nanov2_region_linkage_for_region(nanozone, region);
- nanov2_region_linkage_t *mapped_linkage = (nanov2_region_linkage_t *)
- ((uintptr_t)linkage + region_offset);
- int offset = mapped_linkage->next_region_offset;
- return offset ? region + offset : NULL;
+ nanov2_region_t *region, off_t region_offset,
+ nanov2_arena_t *current_region_next_arena)
+{
+ nanov2_region_linkage_t *linkage =
+ nanov2_region_linkage_for_region(nanozone, region);
+ nanov2_region_linkage_t *mapped_linkage = (nanov2_region_linkage_t *)(
+ ((uintptr_t)linkage + region_offset));
+ int offset = os_atomic_load(&mapped_linkage->next_region_offset, relaxed);
+ if (!offset) {
+ return NULL;
+ }
+
+ nanov2_region_t *next_region = region + offset;
+ return (nanov2_arena_t *)next_region < current_region_next_arena ?
+ next_region : NULL;
}
#endif // OS_VARIANT_NOTRESOLVED
@@ -598,13 +657,13 @@
// Default case is max magazines == physical number of CPUs, which
// must be > _malloc_cpu_number() >> hyper_shift, so the modulo
// operation is not required.
- return _malloc_cpu_number() >> hyper_shift;
+ return (_malloc_cpu_number() >> hyper_shift) & MAX_CURRENT_BLOCKS_MASK;
}
#else // CONFIG_NANO_USES_HYPER_SHIFT
if (os_likely(nano_common_max_magazines_is_ncpu)) {
// Default case is max magazines == logical number of CPUs, which
// must be > _malloc_cpu_number() so the modulo operation is not required.
- return _malloc_cpu_number();
+ return _malloc_cpu_number() & MAX_CURRENT_BLOCKS_MASK;
}
#endif // CONFIG_NANO_USES_HYPER_SHIFT
@@ -614,9 +673,11 @@
#endif // CONFIG_NANO_USES_HYPER_SHIFT
if (os_likely(_os_cpu_number_override == -1)) {
- return (_malloc_cpu_number() >> shift) % nano_common_max_magazines;
- }
- return (_os_cpu_number_override >> shift) % nano_common_max_magazines;
+ return ((_malloc_cpu_number() >> shift) % nano_common_max_magazines) &
+ MAX_CURRENT_BLOCKS_MASK;
+ }
+ return ((_os_cpu_number_override >> shift) % nano_common_max_magazines) &
+ MAX_CURRENT_BLOCKS_MASK;
}
#endif // OS_VARIANT_RESOLVED
@@ -1001,8 +1062,64 @@
MALLOC_NOEXPORT size_t
nanov2_size(nanozonev2_t *nanozone, const void *ptr)
{
- size_t size = nanov2_pointer_size(nanozone, (void *)ptr, FALSE);
+ size_t size = nanov2_pointer_size_inline(nanozone, (void *)ptr, FALSE,
+ NULL, NULL);
return size ? size : nanozone->helper_zone->size(nanozone->helper_zone, ptr);
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+void
+nanov2_bzero(void *ptr, size_t size)
+{
+ // TODO: inline bzero from libplatform
+ bzero(ptr, size);
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static void *
+_nanov2_allocate(nanozonev2_t *nanozone, size_t rounded_size, bool clear,
+ bool typed, malloc_type_id_t type_id)
+{
+ nanov2_block_meta_t *madvise_block_metap = NULL;
+ nanov2_size_class_t size_class = nanov2_size_class_from_size(rounded_size);
+
+ // Get the index of the pointer to the block from which we are should be
+ // allocating. This currently depends on the physical CPU number.
+ int allocation_index = nanov2_get_allocation_block_index();
+
+ // Get the current allocation block meta data pointer. If this is NULL, we
+ // need to find a new allocation block.
+ nanov2_block_meta_t **block_metapp =
+ &nanozone->current_block[size_class][allocation_index];
+ nanov2_block_meta_t *block_metap = os_atomic_load(block_metapp, relaxed);
+ bool corruption = false;
+ void *ptr = NULL;
+ if (block_metap) {
+ // Fast path: we have a block -- try to allocate from it.
+ ptr = nanov2_allocate_from_block_inline(nanozone, block_metap,
+ size_class, &madvise_block_metap, &corruption);
+ if (ptr && !corruption) {
+ if (clear && malloc_zero_policy != MALLOC_ZERO_ON_FREE) {
+ nanov2_bzero(ptr, rounded_size);
+ } else {
+ // Always clear the double-free guard so that we can recognize that
+ // this block is not on the free list.
+ nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
+ os_atomic_store(&slotp->double_free_guard, 0, relaxed);
+
+ // We know the body of the allocation is already clear, so we just
+ // need to clean up the next_slot word to get to all-zero. Do so in
+ // all cases, even if a cleared allocation is not requested, to
+ // prevent any leakage through the next_slot bits.
+ os_atomic_store(&slotp->next_slot, 0, relaxed);
+ }
+ return ptr;
+ }
+ }
+
+ return nanov2_allocate_outlined(nanozone, block_metapp, rounded_size,
+ size_class, allocation_index, madvise_block_metap, ptr, clear,
+ typed, type_id);
}
MALLOC_NOEXPORT void *
@@ -1010,54 +1127,94 @@
{
size_t rounded_size = _nano_common_good_size(size);
if (rounded_size <= NANO_MAX_SIZE) {
- void *ptr = nanov2_allocate(nanozone, rounded_size, FALSE);
- if (ptr) {
- if (os_unlikely(size && (nanozone->debug_flags & MALLOC_DO_SCRIBBLE))) {
- memset(ptr, SCRIBBLE_BYTE, size);
- }
- return ptr;
- }
- }
-
- // If we reach this point, we couldn't allocate, so delegate to the
- // helper zone.
+ return _nanov2_allocate(nanozone, rounded_size, false, false,
+ MALLOC_TYPE_ID_NONE);
+ }
+
+ // Too big for nano, so delegate to the helper zone.
return nanozone->helper_zone->malloc(nanozone->helper_zone, size);
+}
+
+MALLOC_NOEXPORT void *
+nanov2_malloc_type(nanozonev2_t *nanozone, size_t size,
+ malloc_type_id_t type_id)
+{
+ size_t rounded_size = _nano_common_good_size(size);
+ if (rounded_size <= NANO_MAX_SIZE) {
+ return _nanov2_allocate(nanozone, rounded_size, false, true, type_id);
+ }
+
+ // Too big for nano, so delegate to the helper zone.
+ malloc_zone_t *helper_zone = nanozone->helper_zone;
+ return helper_zone->malloc_type_malloc(helper_zone, size, type_id);
}
MALLOC_NOEXPORT void
nanov2_free_definite_size(nanozonev2_t *nanozone, void *ptr, size_t size)
{
- // Check whether it's a Nano pointer and get the size. We should only get
- // here if it is and furthermore we already know that "size" is the actual
- // rounded size, so don't waste time rechecking that. This is just a
- // sanity check.
if (ptr && nanov2_has_valid_signature(ptr)) {
- if (os_unlikely(nanozone->debug_flags & MALLOC_DO_SCRIBBLE)) {
- memset(ptr, SCRABBLE_BYTE, size);
- }
- nanov2_free_to_block(nanozone, ptr, nanov2_size_class_from_size(size));
+ nanov2_size_class_t size_class = nanov2_size_class_from_size(size);
+
+ if (malloc_zero_policy == MALLOC_ZERO_ON_FREE) {
+ if (size_class != 0) {
+ nanov2_bzero((char *)ptr + sizeof(nanov2_free_slot_t),
+ size - sizeof(nanov2_free_slot_t));
+ }
+ }
+
+ nanov2_block_meta_t *madvise_block_metap = nanov2_free_to_block_inline(
+ nanozone, ptr, size_class, NULL);
+ if (madvise_block_metap) {
+ nanov2_madvise_block(nanozone, madvise_block_metap, size_class,
+ SLOT_CAN_MADVISE);
+ }
return;
}
return nanozone->helper_zone->free_definite_size(nanozone->helper_zone, ptr,
size);
}
+static void
+_nanov2_free(nanozonev2_t *nanozone, void *ptr, bool try)
+{
+ if (ptr) {
+ // Check whether it's a Nano pointer and get the size. If it's not
+ // Nano, pass it to the helper zone.
+ nanov2_size_class_t size_class;
+ nanov2_block_meta_t *block_metap;
+ size_t size = nanov2_pointer_size_inline(nanozone, ptr, FALSE,
+ &size_class, &block_metap);
+ if (size) {
+ if (malloc_zero_policy == MALLOC_ZERO_ON_FREE) {
+ if (size > sizeof(nanov2_free_slot_t)) {
+ nanov2_bzero((char *)ptr + sizeof(nanov2_free_slot_t),
+ size - sizeof(nanov2_free_slot_t));
+ }
+ }
+
+ nanov2_block_meta_t *madvise_block_metap = nanov2_free_to_block_inline(
+ nanozone, ptr, size_class, block_metap);
+ if (madvise_block_metap) {
+ nanov2_madvise_block(nanozone, madvise_block_metap, size_class,
+ SLOT_CAN_MADVISE);
+ }
+ return;
+ }
+ }
+ return try ? nanozone->helper_zone->try_free_default(nanozone->helper_zone, ptr) :
+ nanozone->helper_zone->free(nanozone->helper_zone, ptr);
+}
+
MALLOC_NOEXPORT void
nanov2_free(nanozonev2_t *nanozone, void *ptr)
{
- if (ptr && nanov2_has_valid_signature(ptr)) {
- // Check whether it's a Nano pointer and get the size. If it's not
- // Nano, pass it to the helper zone.
- size_t size = nanov2_pointer_size(nanozone, ptr, FALSE);
- if (size) {
- if (os_unlikely(nanozone->debug_flags & MALLOC_DO_SCRIBBLE)) {
- memset(ptr, SCRABBLE_BYTE, size);
- }
- nanov2_free_to_block(nanozone, ptr, nanov2_size_class_from_size(size));
- return;
- }
- }
- return nanozone->helper_zone->free(nanozone->helper_zone, ptr);
+ _nanov2_free(nanozone, ptr, false);
+}
+
+MALLOC_NOEXPORT void
+nanov2_try_free_default(nanozonev2_t *nanozone, void *ptr)
+{
+ _nanov2_free(nanozone, ptr, true);
}
MALLOC_NOEXPORT void *
@@ -1069,15 +1226,58 @@
}
size_t rounded_size = _nano_common_good_size(total_bytes);
if (total_bytes <= NANO_MAX_SIZE) {
- void *ptr = nanov2_allocate(nanozone, rounded_size, TRUE);
- if (ptr) {
- return ptr;
- }
- }
-
- // If we reach this point, we couldn't allocate, so delegate to the
- // helper zone.
+ return _nanov2_allocate(nanozone, rounded_size, true, false,
+ MALLOC_TYPE_ID_NONE);
+ }
+
+ // Too big for nano, so delegate to the helper zone.
return nanozone->helper_zone->calloc(nanozone->helper_zone, 1, total_bytes);
+}
+
+MALLOC_NOEXPORT void *
+nanov2_calloc_type(nanozonev2_t *nanozone, size_t num_items, size_t size,
+ malloc_type_id_t type_id)
+{
+ size_t total_bytes;
+ if (calloc_get_size(num_items, size, 0, &total_bytes)) {
+ return NULL;
+ }
+ size_t rounded_size = _nano_common_good_size(total_bytes);
+ if (total_bytes <= NANO_MAX_SIZE) {
+ return _nanov2_allocate(nanozone, rounded_size, true, true, type_id);
+ }
+
+ // Too big for nano, so delegate to the helper zone.
+ malloc_zone_t *helper_zone = nanozone->helper_zone;
+ return helper_zone->malloc_type_calloc(helper_zone, 1, total_bytes,
+ type_id);
+}
+
+MALLOC_NOEXPORT void *
+nanov2_malloc_zero_on_alloc(nanozonev2_t *nanozone, size_t size)
+{
+ size_t rounded_size = _nano_common_good_size(size);
+ if (rounded_size <= NANO_MAX_SIZE) {
+ return _nanov2_allocate(nanozone, rounded_size, true, false,
+ MALLOC_TYPE_ID_NONE);
+ }
+
+ // Too big for nano, so delegate to the helper zone.
+ return nanozone->helper_zone->malloc(nanozone->helper_zone, size);
+}
+
+MALLOC_NOEXPORT void *
+nanov2_malloc_type_zero_on_alloc(nanozonev2_t *nanozone, size_t size,
+ malloc_type_id_t type_id)
+{
+ size_t rounded_size = _nano_common_good_size(size);
+ if (rounded_size <= NANO_MAX_SIZE) {
+ return _nanov2_allocate(nanozone, rounded_size, true, true, type_id);
+ }
+
+ // Too big for nano, so delegate to the helper zone.
+ malloc_zone_t *helper_zone = nanozone->helper_zone;
+ return helper_zone->malloc_type_malloc(helper_zone, size, type_id);
}
#endif // OS_VARIANT_RESOLVED
@@ -1091,25 +1291,44 @@
#endif // OS_VARIANT_NOTRESOLVED
#if OS_VARIANT_RESOLVED
-MALLOC_NOEXPORT void *
-nanov2_realloc(nanozonev2_t *nanozone, void *ptr, size_t new_size)
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static void *
+_nanov2_realloc(nanozonev2_t *nanozone, void *ptr, size_t new_size, bool typed,
+ malloc_type_id_t type_id)
{
// If we are given a NULL pointer, just allocate memory of the requested
// size.
if (ptr == NULL) {
- return nanov2_malloc(nanozone, new_size);
- }
+ if (typed) {
+ return nanov2_malloc_type(nanozone, new_size, type_id);
+ } else {
+ return nanov2_malloc(nanozone, new_size);
+ }
+ }
+
+ malloc_zone_t *helper_zone = nanozone->helper_zone;
size_t old_size = nanov2_pointer_size(nanozone, ptr, FALSE);
if (!old_size) {
// Not a Nano pointer - let the helper deal with it
- return nanozone->helper_zone->realloc(nanozone->helper_zone, ptr, new_size);
+ if (typed) {
+ return helper_zone->malloc_type_realloc(helper_zone, ptr, new_size,
+ type_id);
+ } else {
+ return helper_zone->realloc(helper_zone, ptr, new_size);
+ }
}
void *new_ptr;
if (new_size > NANO_MAX_SIZE) {
// Too large for Nano. Try to allocate from the helper zone.
- new_ptr = nanozone->helper_zone->malloc(nanozone->helper_zone, new_size);
+ if (typed) {
+ new_ptr = helper_zone->malloc_type_malloc(helper_zone, new_size,
+ type_id);
+ } else {
+ new_ptr = helper_zone->malloc(helper_zone, new_size);
+ }
+
if (!new_ptr) {
// Failed to allocate - leave the existing allocation alone.
return NULL;
@@ -1121,17 +1340,24 @@
return nanov2_malloc(nanozone, 0);
} else {
size_t new_good_size = _nano_common_good_size(new_size);
- if (new_good_size > old_size || new_good_size <= old_size/2) {
+ if (new_good_size > old_size || new_good_size <= old_size / 2) {
// Growing or shrinking to less than half size - we need to
// reallocate.
- new_ptr = nanov2_malloc(nanozone, new_good_size);
+ if (typed) {
+ new_ptr = nanov2_malloc_type(nanozone, new_good_size, type_id);
+ } else {
+ new_ptr = nanov2_malloc(nanozone, new_good_size);
+ }
+
if (!new_ptr) {
// Failed to allocate - leave the existing allocation alone.
return NULL;
}
} else {
// Same size or shrinking by less than half size. Keep the same
- // allocation and clear the area that's being released.
+ // allocation and scribble the area that's being released. Nothing
+ // to do for zero-on-free yet; that will be taken care of when the
+ // shrunk allocation is freed.
if (new_size != old_size) {
MALLOC_ASSERT(new_size < old_size);
if (os_unlikely(nanozone->debug_flags & MALLOC_DO_SCRIBBLE)) {
@@ -1150,6 +1376,19 @@
return new_ptr;
}
+
+MALLOC_NOEXPORT void *
+nanov2_realloc(nanozonev2_t *nanozone, void *ptr, size_t new_size)
+{
+ return _nanov2_realloc(nanozone, ptr, new_size, false, MALLOC_TYPE_ID_NONE);
+}
+
+MALLOC_NOEXPORT void *
+nanov2_realloc_type(nanozonev2_t *nanozone, void *ptr, size_t new_size,
+ malloc_type_id_t type_id)
+{
+ return _nanov2_realloc(nanozone, ptr, new_size, true, type_id);
+}
#endif // OS_VARIANT_RESOLVED
#if OS_VARIANT_NOTRESOLVED
@@ -1178,7 +1417,8 @@
size_t rounded_size = _nano_common_good_size(size);
if (rounded_size <= NANO_MAX_SIZE) {
while (allocated < count) {
- void *ptr = nanov2_allocate(nanozone, rounded_size, FALSE);
+ // TODO: nanov2_malloc will redo _nano_common_good_size
+ void *ptr = nanov2_malloc(nanozone, rounded_size);
if (!ptr) {
break;
}
@@ -1209,19 +1449,36 @@
}
}
}
-#endif // OS_VARIANT_RESOLVED
-
-#if OS_VARIANT_NOTRESOLVED
-static void *
+
+MALLOC_NOEXPORT void *
nanov2_memalign(nanozonev2_t *nanozone, size_t alignment, size_t size)
{
- // Always delegate this to the helper zone.
+ // Serve directly if the requested alignment is trivially satisfied by our
+ // baseline alignment (16 bytes)
+ if (alignment <= NANO_REGIME_QUANTA_SIZE) {
+ return nanov2_malloc(nanozone, size);
+ }
+
+ // Otherwise delegate to the helper zone
return nanozone->helper_zone->memalign(nanozone->helper_zone, alignment,
size);
}
-#endif // OS_VARIANT_NOTRESOLVED
-
-#if OS_VARIANT_RESOLVED
+
+MALLOC_NOEXPORT void *
+nanov2_memalign_type(nanozonev2_t *nanozone, size_t alignment, size_t size,
+ malloc_type_id_t type_id)
+{
+ // Serve directly if the requested alignment is trivially satisfied by our
+ // baseline alignment (16 bytes)
+ if (alignment <= NANO_REGIME_QUANTA_SIZE) {
+ return nanov2_malloc_type(nanozone, size, type_id);
+ }
+
+ // Otherwise delegate to the helper zone
+ malloc_zone_t *helper_zone = nanozone->helper_zone;
+ return helper_zone->malloc_type_memalign(helper_zone, alignment, size,
+ type_id);
+}
size_t
nanov2_pressure_relief(nanozonev2_t *nanozone, size_t goal)
@@ -1243,9 +1500,12 @@
// until we reach our goal.
nanov2_region_t *region = nanozone->first_region_base;
nanov2_meta_index_t metablock_meta_index = nanov2_metablock_meta_index(nanozone);
+ nanov2_arena_t *current_region_next_arena = os_atomic_load(
+ &nanozone->current_region_next_arena, acquire);
while (region) {
nanov2_arena_t *arena = nanov2_first_arena_for_region(region);
- nanov2_arena_t *arena_after_region = nanov2_limit_arena_for_region(nanozone, region);
+ nanov2_arena_t *arena_after_region = nanov2_limit_arena_for_region(
+ nanozone, region, current_region_next_arena);
while (arena < arena_after_region) {
// Scan all of the blocks in the arena, skipping the metadata block.
nanov2_arena_metablock_t *meta_blockp =
@@ -1264,8 +1524,9 @@
if (meta.next_slot == SLOT_CAN_MADVISE) {
nanov2_block_t *blockp = nanov2_block_address_from_meta_index(
nanozone, arena, i);
- if (nanov2_madvise_block(nanozone, block_metap,
- blockp, nanov2_size_class_for_ptr(nanozone, blockp))) {
+ if (nanov2_madvise_block_locked(nanozone, block_metap,
+ blockp, nanov2_size_class_for_ptr(nanozone, blockp),
+ SLOT_CAN_MADVISE)) {
total += NANOV2_BLOCK_SIZE;
}
}
@@ -1277,7 +1538,8 @@
}
arena++;
}
- region = nanov2_next_region_for_region(nanozone, region);
+ region = nanov2_next_region_for_region(nanozone, region,
+ current_region_next_arena);
}
done:
@@ -1326,15 +1588,13 @@
kern_return_t kr;
bitarray_t slots;
- if (!reader) {
- reader = nano_common_default_reader;
- }
+ reader = reader_or_in_memory_fallback(reader, task);
kr = reader(task, zone_address, sizeof(nanozonev2_t), (void **)&nanozone);
if (kr) {
return kr;
}
- boolean_t self_zone = (task == mach_task_self() && (nanozonev2_t *)zone_address == nanozone);
+ boolean_t self_zone = mach_task_is_self(task) && (nanozonev2_t *)zone_address == nanozone;
memcpy(&zone_copy, nanozone, sizeof(zone_copy));
nanozone = &zone_copy;
nanov2_meta_index_t metablock_meta_index = nanov2_metablock_meta_index(nanozone);
@@ -1342,6 +1602,8 @@
// Process the zone one region at a time. Report each in-use block as a
// pointer range and each in-use slot as a pointer.
nanov2_region_t *region = nanozone->first_region_base;
+ nanov2_arena_t *current_region_next_arena = os_atomic_load(
+ &nanozone->current_region_next_arena, acquire);
while (region) {
mach_vm_address_t vm_addr = (mach_vm_address_t)NULL;
kern_return_t kr = reader(task, (vm_address_t)region, NANOV2_REGION_SIZE, (void **)&vm_addr);
@@ -1353,7 +1615,8 @@
// and its mapped address in this process.
mach_vm_offset_t ptr_offset = (mach_vm_address_t)region - vm_addr;
nanov2_arena_t *arena = nanov2_first_arena_for_region(region);
- nanov2_arena_t *limit_arena = nanov2_limit_arena_for_region(nanozone, region);
+ nanov2_arena_t *limit_arena = nanov2_limit_arena_for_region(nanozone, region,
+ current_region_next_arena);
vm_range_t ptr_range;
while (arena < limit_arena) {
// Find the metadata block and process every entry, apart from the
@@ -1430,7 +1693,7 @@
bitarray_zap(slots, log_size, next_slot);
void *ptr = nanov2_slot_in_block_ptr(blockp, size_class, next_slot);
nanov2_free_slot_t *slotp = NANOV2_ZONE_PTR_TO_MAPPED_PTR(nanov2_free_slot_t *, ptr, ptr_offset);
- next_slot = slotp->next_slot;
+ next_slot = (uint16_t)slotp->next_slot;
free_list_count++;
}
// Add a range for each slot that is not on the freelist,
@@ -1449,7 +1712,7 @@
ranges[range_count].size = slot_size;
range_count++;
}
- free(slots);
+ _free(slots);
}
if (range_count) {
// Notify the in-use pointers that we found.
@@ -1467,7 +1730,8 @@
nanov2_region_linkage_t *mapped_region_linkagep =
NANOV2_ZONE_PTR_TO_MAPPED_PTR(nanov2_region_linkage_t *,
region_linkagep, ptr_offset);
- int offset = mapped_region_linkagep->next_region_offset;
+ int offset = os_atomic_load(&mapped_region_linkagep->next_region_offset,
+ relaxed);
region = offset ? region + offset : NULL;
}
return 0;
@@ -1486,7 +1750,7 @@
static boolean_t
nanov2_check(nanozonev2_t *nanozone)
{
- // Does nothing, just like Nano V1.
+ // Does nothing
return 1;
}
@@ -1546,6 +1810,10 @@
nanov2_meta_index_t metablock_meta_index =
nanov2_metablock_meta_index(mapped_nanozone);
nanov2_region_t *region = mapped_nanozone->first_region_base;
+ // Use a single, consistent snapshot of current_region_next_arena throughout
+ // iteration, ignoring any arenas or regions allocated after it.
+ nanov2_arena_t *current_region_next_arena = os_atomic_load(
+ &mapped_nanozone->current_region_next_arena, acquire);
int region_index = 0;
while (region) {
printer("\nRegion %d: base address %p\n", region_index, region);
@@ -1559,7 +1827,7 @@
nanov2_arena_t *arena = nanov2_first_arena_for_region(region);
nanov2_arena_t *limit_arena = nanov2_limit_arena_for_region(
- mapped_nanozone, region);
+ mapped_nanozone, region, current_region_next_arena);
int arena_index = 0;
while (arena < limit_arena) {
// Find the metadata block and process every entry, apart from the
@@ -1700,7 +1968,7 @@
}
region = nanov2_next_region_for_region_offset(mapped_nanozone, region,
- region_offset);
+ region_offset, current_region_next_arena);
region_index++;
}
}
@@ -1722,7 +1990,7 @@
static void
nanov2_log(malloc_zone_t *zone, void *log_address)
{
- // Does nothing, just like Nano V1.
+ // Does nothing
}
static void
@@ -1761,7 +2029,7 @@
malloc_statistics_t *stats)
{
printer = printer ? printer : nanov2_null_printer;
- reader = !reader && task == mach_task_self() ? _malloc_default_reader : reader;
+ reader = reader_or_in_memory_fallback(reader, task);
kern_return_t err;
@@ -1788,6 +2056,8 @@
// Iterate over each arena in each region. Within each region, add
// statistics for each slot in each block, excluding the meta data block.
+ nanov2_arena_t *current_region_next_arena = os_atomic_load(
+ &mapped_nanozone->current_region_next_arena, acquire);
for (region = mapped_nanozone->first_region_base; region;) {
nanov2_region_t *mapped_region;
err = reader(task, (vm_address_t)region, sizeof(nanov2_region_t), (void **)&mapped_region);
@@ -1797,7 +2067,8 @@
}
off_t region_offset = (uintptr_t)mapped_region - (uintptr_t)region;
for (arena = nanov2_first_arena_for_region(region);
- arena < nanov2_limit_arena_for_region(mapped_nanozone, region);
+ arena < nanov2_limit_arena_for_region(mapped_nanozone, region,
+ current_region_next_arena);
arena++) {
nanov2_arena_metablock_t *meta_block =
nanov2_metablock_address_for_ptr(mapped_nanozone, arena);
@@ -1849,7 +2120,7 @@
}
}
region = nanov2_next_region_for_region_offset(mapped_nanozone,
- region, region_offset);
+ region, region_offset, current_region_next_arena);
}
return KERN_SUCCESS;
}
@@ -1909,8 +2180,10 @@
// the allocation, or 0 if the pointer does not correspond to an active
// allocation. If allow_inner is true, the pointer need not point to the start
// of the allocation.
-size_t
-nanov2_pointer_size(nanozonev2_t *nanozone, void *ptr, boolean_t allow_inner)
+MALLOC_ALWAYS_INLINE MALLOC_INLINE size_t
+nanov2_pointer_size_inline(nanozonev2_t *nanozone, void *ptr,
+ boolean_t allow_inner, nanov2_size_class_t *size_class_out,
+ nanov2_block_meta_t **block_metap_out)
{
// First check the address signature.
if (!nanov2_has_valid_signature((void *)ptr)) {
@@ -1922,25 +2195,40 @@
return 0;
}
+ // Atomically load the value of current_region_next_arena. No thread is
+ // allowed to allocate from an arena until it observes a greater value of
+ // current_region_next_arena, which must have happened before now if we're
+ // being called in the context of a deallocation, so we can safely use it as
+ // the upper bound for an overall address range check.
+ nanov2_arena_t *current_region_next_arena = os_atomic_load(
+ &nanozone->current_region_next_arena, relaxed);
+
// Bounds check against the active address space.
if (ptr < (void *)nanozone->first_region_base ||
- ptr > (void *)nanozone->current_region_next_arena) {
+ ptr > (void *)current_region_next_arena) {
return 0;
}
#if NANOV2_MULTIPLE_REGIONS
// Need to check that the region part is valid because there could be holes.
// Do this only if we know there is a hole.
- // NOTE: in M2 convergence, use a hashed structure to make this more
- // efficient.
- if (nanozone->statistics.region_address_clashes) {
+ //
+ // If we're looking at a legitimately-allocated nano pointer, a load-acquire
+ // of current_region_next_arena must have already happened when its
+ // containing arena was first allocated from, so any region_address_clashes
+ // increment that preceded the store-release of current_region_next_arena
+ // should be visible.
+ //
+ // TODO: use a hashed structure to make this more efficient.
+ if (os_atomic_load(&nanozone->statistics.region_address_clashes, relaxed)) {
nanov2_region_t *ptr_region = nanov2_region_address_for_ptr(ptr);
nanov2_region_t *region = nanozone->first_region_base;
while (region) {
if (ptr_region == region) {
break;
}
- region = nanov2_next_region_for_region(nanozone, region);
+ region = nanov2_next_region_for_region(nanozone, region,
+ current_region_next_arena);
}
if (!region) {
// Reached the end of the region list without matching - not a
@@ -1977,27 +2265,42 @@
return 0;
}
+ if (size_class_out) {
+ *size_class_out = size_class;
+ }
+ if (block_metap_out) {
+ *block_metap_out = block_metap;
+ }
return size;
+}
+
+size_t
+nanov2_pointer_size(nanozonev2_t *nanozone, void *ptr, boolean_t allow_inner)
+{
+ return nanov2_pointer_size_inline(nanozone, ptr, allow_inner, NULL, NULL);
}
#pragma mark -
#pragma mark Madvise Management
-// Given a pointer to a block and its metadata, calls madvise() on that block
-// if it is in state SLOT_CAN_MADVISE. Returns true on success, false if the
-// block is not in the correct state or if the state changed during the
-// operation.
+// Given a pointer to a block and its metadata, calls madvise() on that block if
+// it is still in the state we expect, either SLOT_CAN_MADVISE or SLOT_MADVISED
+// (the latter expected when we need to pessimistically re-madvise a block we
+// may have touched while racing to allocate against a transition to
+// SLOT_CAN_MADVISE). Returns true on success, false if the block is not in the
+// correct state or if the state changed during the operation.
//
// This function must be called with the zone's madvise_lock held
-boolean_t
-nanov2_madvise_block(nanozonev2_t *nanozone, nanov2_block_meta_t *block_metap,
- nanov2_block_t *blockp, nanov2_size_class_t size_class)
+static boolean_t
+nanov2_madvise_block_locked(nanozonev2_t *nanozone,
+ nanov2_block_meta_t *block_metap, nanov2_block_t *blockp,
+ nanov2_size_class_t size_class, uint32_t expected_state)
{
_malloc_lock_assert_owner(&nanozone->madvise_lock);
boolean_t madvised = FALSE;
nanov2_block_meta_t old_meta = os_atomic_load(block_metap, relaxed);
- if (old_meta.next_slot == SLOT_CAN_MADVISE) {
+ if (old_meta.next_slot == expected_state) {
// Nobody raced with us. We can safely madvise this block. First change
// the state to SLOT_MADVISING so that other threads don't try to
// grab the block for new allocations.
@@ -2039,6 +2342,18 @@
return madvised;
}
+static void
+nanov2_madvise_block(nanozonev2_t *nanozone, nanov2_block_meta_t *block_metap,
+ nanov2_size_class_t size_class, uint32_t expected_state)
+{
+ nanov2_block_t *blockp = nanov2_block_address_from_meta_ptr(nanozone,
+ block_metap);
+ _malloc_lock_lock(&nanozone->madvise_lock);
+ nanov2_madvise_block_locked(nanozone, block_metap, blockp, size_class,
+ expected_state);
+ _malloc_lock_unlock(&nanozone->madvise_lock);
+}
+
#endif // OS_VARIANT_RESOLVED
#pragma mark -
@@ -2046,68 +2361,124 @@
#if OS_VARIANT_NOTRESOLVED
-#if NANOV2_MULTIPLE_REGIONS
-static nanov2_addr_t nanov2_max_region_base = {
- .fields.nano_signature = NANOZONE_SIGNATURE,
- .fields.nano_region = NANOV2_MAX_REGION_NUMBER
-};
-#endif // NANOV2_MULTIPLE_REGIONS
-
+#if CONFIG_NANO_RESERVE_REGIONS
+// Update protection for region to DEFAULT
+static bool
+nanov2_unprotect_region(nanov2_region_t *region)
+{
+ MALLOC_TRACE(TRACE_nanov2_region_protection | DBG_FUNC_START,
+ (uint64_t)region, 0, 0, 0);
+ bool result = nano_common_unprotect_vm_space((mach_vm_address_t)region,
+ NANOV2_REGION_SIZE);
+ MALLOC_TRACE(TRACE_nanov2_region_protection | DBG_FUNC_END,
+ (uint64_t)region, result, 0, 0);
+ return result;
+}
+
+// Reserve VA at [base, base+num_regions*REGION_SIZE].
+// Note: permissions must still be granted on reserved region with `nanov2_unprotect_region`
+static bool
+nanov2_reserve_regions(nanov2_region_t *base, unsigned int num_regions)
+{
+ MALLOC_TRACE(TRACE_nanov2_region_reservation | DBG_FUNC_START,
+ (uint64_t)base, num_regions, 0, 0);
+ bool result = nano_common_reserve_vm_space((mach_vm_address_t)base,
+ (NANOV2_REGION_SIZE * (mach_vm_size_t)num_regions));
+ MALLOC_TRACE(TRACE_nanov2_region_reservation | DBG_FUNC_END,
+ (uint64_t)base, num_regions, result, 0);
+
+ return result;
+}
+#else
// Attempts to allocate VM space for a region at a given address and returns
// whether the allocation succeeded.
-static boolean_t
+static bool
nanov2_allocate_region(nanov2_region_t *region)
{
MALLOC_TRACE(TRACE_nanov2_region_allocation | DBG_FUNC_START,
(uint64_t)region, 0, 0, 0);
- boolean_t result = nano_common_allocate_vm_space((mach_vm_address_t)region,
+ bool result = nano_common_allocate_vm_space((mach_vm_address_t)region,
NANOV2_REGION_SIZE);
MALLOC_TRACE(TRACE_nanov2_region_allocation | DBG_FUNC_END,
(uint64_t)region, result, 0, 0);
return result;
}
+#endif // CONFIG_NANO_RESERVE_REGIONS
// Allocates a new region adjacent to the current one. If the allocation fails,
// keep sliding up by the size of a region until we either succeed or run out of
-// address space. The caller must own the Nanozone regions lock.
-MALLOC_NOEXPORT boolean_t
+// address space. The caller must own the Nanozone regions lock. Returns the
+// first arena of the newly-allocated region if successful, or NULL otherwise.
+MALLOC_NOEXPORT nanov2_arena_t *
nanov2_allocate_new_region(nanozonev2_t *nanozone)
{
#if NANOV2_MULTIPLE_REGIONS
- boolean_t result = FALSE;
+ bool allocated = false;
+
+ nanov2_addr_t nanov2_max_region_base = {
+ .fields.nano_signature = NANOZONE_SIGNATURE,
+ .fields.nano_region = nano_max_region,
+ };
_malloc_lock_assert_owner(&nanozone->regions_lock);
- nanov2_region_t *current_region = nanozone->current_region_base;
- nanov2_region_t *next_region = (nanov2_region_t *)nanozone->current_region_limit;
+ nanov2_region_t *current_region = nanov2_current_region_base(
+ os_atomic_load(&nanozone->current_region_next_arena, relaxed));
+ nanov2_region_t *next_region = current_region + 1;
+
while ((void *)next_region <= nanov2_max_region_base.addr) {
+#if CONFIG_NANO_RESERVE_REGIONS
+ if (!nanov2_unprotect_region(next_region)) {
+ MALLOC_REPORT_FATAL_ERROR(next_region,
+ "Nano: Unable to raise protection on pre-allocated region");
+ }
+ nanozone->statistics.allocated_regions++;
+ allocated = true;
+ break;
+#else // CONFIG_NANO_RESERVE_REGIONS
if (nanov2_allocate_region(next_region)) {
- nanozone->current_region_base = next_region;
- nanozone->current_region_next_arena = (nanov2_arena_t *)next_region;
- nanozone->current_region_limit = next_region + 1;
nanozone->statistics.allocated_regions++;
- result = TRUE;
+ allocated = true;
break;
}
next_region++;
- nanozone->statistics.region_address_clashes++;
- }
-
- if (result) {
- // Link this region to the previous one.
- nanov2_region_linkage_t *current_region_linkage =
- nanov2_region_linkage_for_region(nanozone, current_region);
- nanov2_region_linkage_t *next_region_linkage =
- nanov2_region_linkage_for_region(nanozone, next_region);
- uint16_t offset = next_region - current_region;
- current_region_linkage->next_region_offset = offset;
- next_region_linkage->next_region_offset = 0;
- }
-
- return result;
+
+ // Loaded atomically in nanov2_pointer_size() to determine whether or
+ // not it's necessary to walk the region list, so we need to increment
+ // atomically here. Published by the store-release of
+ // current_region_next_arena.
+ os_atomic_inc(&nanozone->statistics.region_address_clashes, relaxed);
+#endif // CONFIG_NANO_RESERVE_REGIONS
+ }
+
+ if (!allocated) {
+ return NULL;
+ }
+
+ // Link this region to the previous one.
+ nanov2_region_linkage_t *current_region_linkage =
+ nanov2_region_linkage_for_region(nanozone, current_region);
+
+ // The linkage of the next region is in pristine memory, so already zero -
+ // don't touch it.
+
+ // Store-release the linkage update so any dependent loads through it
+ // observe the (implicit zero-)initialization of the next region.
+ uint16_t offset = next_region - current_region;
+ os_atomic_store(¤t_region_linkage->next_region_offset, offset,
+ release);
+
+ // Store-release the update to current_region_next_arena to publish the
+ // linkage update. Pairs with load-acquires of current_region_next_arena
+ // followed by walks of the region list.
+ nanov2_arena_t *first_arena = nanov2_first_arena_for_region(next_region);
+ os_atomic_store(&nanozone->current_region_next_arena, first_arena + 1,
+ release);
+
+ return first_arena;
#else // NANOV2_MULTIPLE_REGIONS
// On iOS, only one region is supported, so we fail since the first
// region is allocated separately.
- return FALSE;
+ return NULL;
#endif // CONFIG_NANOV2_MULTIPLE_REGIONS
}
#endif // OS_VARIANT_NOTRESOLVED
@@ -2116,19 +2487,40 @@
#pragma mark Allocation
#if OS_VARIANT_RESOLVED
+
+MALLOC_NOINLINE MALLOC_NORETURN
+static void
+nanov2_guard_corruption_detected(void *corrupt_slot)
+{
+ uint64_t guard = *(uint64_t *)corrupt_slot;
+ malloc_zone_error(MALLOC_ABORT_ON_CORRUPTION, true,
+ "Heap corruption detected, free list is damaged at %p\n"
+ "*** Incorrect guard value: %llu\n", corrupt_slot,
+ (unsigned long long)guard);
+ __builtin_unreachable();
+}
// Allocates memory from the block that corresponds to a given block meta data
// pointer. The memory is taken from the free list if possible, or from the
// unused region of the block if not. If the block is no longer in use or is
// full, NULL is returned and the caller is expected to find another block to
// allocate from.
-MALLOC_NOEXPORT
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
void *
-nanov2_allocate_from_block(nanozonev2_t *nanozone,
- nanov2_block_meta_t *block_metap, nanov2_size_class_t size_class)
+nanov2_allocate_from_block_inline(nanozonev2_t *nanozone,
+ nanov2_block_meta_t *block_metap, nanov2_size_class_t size_class,
+ nanov2_block_meta_t **madvise_block_metap_out, bool *corruption)
{
nanov2_block_meta_view_t old_meta_view;
- old_meta_view.meta = os_atomic_load(block_metap, relaxed);
+
+ // Our loads of the block metadata use dependency ordering, which guarantees
+ // that any loads we do from a slot pointer derived from the metadata value
+ // as we do below will observe all of the stores preceding the store-release
+ // of that value we observed. This allows us to safely rely on the contents
+ // of the slot updated when it was last freed, including the double-free
+ // guard and zeroing done by zero-on-free (which is required for correctness
+ // in the case of calloc).
+ old_meta_view.meta = os_atomic_load(block_metap, dependency);
// Calculating blockp and ptr is relatively expensive. Do both lazily to
// minimize the time in the block starting with "again:" and ending with the
@@ -2169,7 +2561,7 @@
slot = old_meta_view.meta.next_slot - 1; // meta.next_slot is 1-based.
ptr = nanov2_slot_in_block_ptr(blockp, size_class, slot);
nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
- new_meta.next_slot = slot_full ? SLOT_FULL : slotp->next_slot;
+ new_meta.next_slot = slot_full ? SLOT_FULL : (uint16_t)slotp->next_slot;
}
// Write the updated meta data; try again if we raced with another thread.
@@ -2178,21 +2570,8 @@
if (old_meta_view.meta.next_slot == SLOT_CAN_MADVISE ||
old_meta_view.meta.next_slot == SLOT_MADVISING ||
old_meta_view.meta.next_slot == SLOT_MADVISED) {
- _malloc_lock_lock(&nanozone->madvise_lock);
- if (old_meta_view.meta.next_slot == SLOT_MADVISED) {
- // We raced against another thread madvising this block. We need
- // to redo the madvise because we may have touched it when
- // reading the next pointer in the freelist.
- if (!blockp) {
- blockp = nanov2_block_address_from_meta_ptr(nanozone, block_metap);
- }
- if (mvm_madvise_free(nanozone, nanov2_region_address_for_ptr(blockp),
- (uintptr_t)blockp, (uintptr_t)(blockp + 1), NULL, FALSE)) {
- malloc_zone_error(0, false,
- "Failed to remadvise block at blockp: %p, error: %d\n", blockp, errno);
- }
- }
- _malloc_lock_unlock(&nanozone->madvise_lock);
+ *madvise_block_metap_out = block_metap;
+ return NULL;
}
goto again;
}
@@ -2204,26 +2583,40 @@
ptr = nanov2_slot_in_block_ptr(blockp, size_class, slot);
}
- nanov2_free_slot_t *slotp = os_atomic_inject_dependency(ptr,
- (unsigned long)old_meta_view.bits);
if (from_free_list) {
// We grabbed the item from the free list. Check the free list canary
// and crash if it's not valid. We can't do this check before the
// cmpxchgv because another thread may race with us, claim the slot and
// write to it.
+ nanov2_free_slot_t *slotp = ptr;
uintptr_t guard = os_atomic_load(&slotp->double_free_guard, relaxed);
- if ((guard ^ nanozone->slot_freelist_cookie) != (uintptr_t)ptr) {
- malloc_zone_error(MALLOC_ABORT_ON_CORRUPTION, true,
- "Heap corruption detected, free list is damaged at %p\n"
- "*** Incorrect guard value: %lu\n", ptr, guard);
- __builtin_unreachable();
- }
- }
-
+ if (os_unlikely((guard ^ nanozone->slot_freelist_cookie) != (uintptr_t)ptr)) {
+ *corruption = true;
+ }
+ }
+
#if DEBUG_MALLOC
nanozone->statistics.size_class_statistics[size_class].total_allocations++;
#endif // DEBUG_MALLOC
+ return ptr;
+}
+
+static void *
+nanov2_allocate_from_block(nanozonev2_t *nanozone,
+ nanov2_block_meta_t *block_metap, nanov2_size_class_t size_class)
+{
+ nanov2_block_meta_t *madvise_block_metap = NULL;
+ bool corruption = false;
+ void *ptr = nanov2_allocate_from_block_inline(nanozone, block_metap,
+ size_class, &madvise_block_metap, &corruption);
+ if (os_unlikely(corruption)) {
+ nanov2_guard_corruption_detected(ptr);
+ }
+ if (madvise_block_metap) {
+ nanov2_madvise_block(nanozone, madvise_block_metap, size_class,
+ SLOT_MADVISED);
+ }
return ptr;
}
@@ -2458,8 +2851,13 @@
start_region = nanov2_region_address_for_ptr(arena);
nanov2_arena_t *start_arena = arena;
nanov2_region_t *region = start_region;
- nanov2_arena_t *limit_arena = nanov2_limit_arena_for_region(nanozone, start_region);
- nanov2_arena_t *initial_region_next_arena = nanozone->current_region_next_arena;
+ // The load-acquire pairs with store-release in nanov2_allocate_new_region()
+ // to make the most recent region linkage update visible when we load it in
+ // nanov2_next_region_for_region() below.
+ nanov2_arena_t *initial_region_next_arena = os_atomic_load(
+ &nanozone->current_region_next_arena, acquire);
+ nanov2_arena_t *limit_arena = nanov2_limit_arena_for_region(nanozone,
+ start_region, initial_region_next_arena);
do {
nanov2_block_meta_t *block_metap = nanov2_find_block_in_arena(nanozone,
arena, size_class, start_block);
@@ -2495,13 +2893,15 @@
start_block = NULL;
arena++;
if (arena >= limit_arena) {
- region = nanov2_next_region_for_region(nanozone, region);
+ region = nanov2_next_region_for_region(nanozone, region,
+ initial_region_next_arena);
if (!region) {
// Reached the last region -- loop back to the first.
region = nanozone->first_region_base;
}
arena = nanov2_first_arena_for_region(region);
- limit_arena = nanov2_limit_arena_for_region(nanozone, region);
+ limit_arena = nanov2_limit_arena_for_region(nanozone, region,
+ initial_region_next_arena);
}
} while (arena != start_arena);
@@ -2512,29 +2912,42 @@
}
// Allocate a new arena and maybe a new region. To do either of those
- // things, we need to take the regions_lock. After doing so, check that
- // the state is unchanged. If it has, just assume that we might have some
- // new space to allocate into and try again.
+ // things, we need to take the regions_lock. After doing so, check that the
+ // state is unchanged. If it has, just assume that we might have some new
+ // space to allocate into and try again.
+
boolean_t failed = FALSE;
- arena = initial_region_next_arena;
+
_malloc_lock_lock(&nanozone->regions_lock);
- if (nanozone->current_region_next_arena == arena) {
- if ((void *)arena >= nanozone->current_region_limit) {
+ nanov2_arena_t *current_region_next_arena = os_atomic_load(
+ &nanozone->current_region_next_arena, relaxed);
+ if (current_region_next_arena == initial_region_next_arena) {
+ if (nanov2_current_region_next_arena_is_limit(
+ current_region_next_arena)) {
// Reached the end of the region. Allocate a new one, if we can.
- if (nanov2_allocate_new_region(nanozone)) {
- arena = nanozone->current_region_next_arena++;
- } else {
+ arena = nanov2_allocate_new_region(nanozone);
+ if (!arena) {
failed = TRUE;
}
} else {
- // Assign the new arena, in the same region.
- nanozone->current_region_next_arena = arena + 1;
+ // Assign the new arena, in the current region.
+ arena = current_region_next_arena;
+
+ // Bump current_region_next_arena by 1. No need for an atomic add
+ // because we're under the regions_lock.
+ os_atomic_store(&nanozone->current_region_next_arena,
+ current_region_next_arena + 1, relaxed);
}
// Set up the guard blocks for the new arena, if requested
if (!failed) {
nanov2_init_guard_blocks(nanozone, arena);
}
+ } else {
+ // The arena just before current_region_next_arena is always the most
+ // recently allocated arena. Let's retry from that arena, which was
+ // allocated in the time since we started our last try.
+ arena = current_region_next_arena - 1;
}
_malloc_lock_unlock(&nanozone->regions_lock);
@@ -2551,46 +2964,49 @@
return NULL;
}
-// Allocates memory of a given size (which must be a multiple of the Nano
-// quantum size) and optionally clears it (for calloc).
+// This function is called when a fast-path allocation from a given (size_class,
+// allocation_index) has been tried and failed, and we need to act on
+// observations from that attempt and/or retry the allocation. Its rather
+// tortured calling contract is designed to allow the caller to avoid pushing a
+// frame and pass along as much of what it has already computed as possible.
//
-// Allocation is attempted first from the block last used for the caller's
-// context (which is initially the physical CPU by default). If there is no
-// last block, or the block is full or now out of use, find another one, if
-// possible. See the comments for nanov2_get_allocation_block() for the details.
+// If @corrupt_slot is non-NULL it means we detected corruption of the slot's
+// guard on the fast path, and we need to report that corruption.
+//
+// If @madvise_block_metap is non-NULL it means we raced with another thread
+// madvising the block we tried to allocate from and need to re-madvise it.
//
// If the allocation fails, NULL is returned.
-void *
-nanov2_allocate(nanozonev2_t *nanozone, size_t rounded_size, boolean_t clear)
+static void *
+nanov2_allocate_outlined(nanozonev2_t *nanozone, nanov2_block_meta_t **block_metapp,
+ size_t rounded_size, nanov2_size_class_t size_class,
+ int allocation_index, nanov2_block_meta_t *madvise_block_metap,
+ void *corrupt_slot, bool clear, bool typed, malloc_type_id_t type_id)
{
void *ptr = NULL;
- nanov2_size_class_t size_class = nanov2_size_class_from_size(rounded_size);
- MALLOC_ASSERT(size_class < NANO_SIZE_CLASSES);
- MALLOC_ASSERT(rounded_size != 0);
- nanov2_block_meta_t *block_metap;
- nanov2_block_meta_t **block_metapp;
-
- // Get the index of the pointer to the block from which we are should be
- // allocating. This currently depends on the physical CPU number.
- int allocation_index = nanov2_get_allocation_block_index() & MAX_CURRENT_BLOCKS_MASK;
-
- // Get the current allocation block meta data pointer. If this is NULL,
- // we need to find a new allocation block.
- block_metapp = &nanozone->current_block[size_class][allocation_index];
- block_metap = os_atomic_load(block_metapp, relaxed);
- if (block_metap) {
- // Fast path: we have a block -- try to allocate from it.
- ptr = nanov2_allocate_from_block(nanozone, block_metap, size_class);
- if (ptr) {
- goto done;
- }
+
+ if (os_unlikely(corrupt_slot)) {
+ nanov2_guard_corruption_detected(corrupt_slot);
+ }
+
+ // If we need to re-madvise the old block that we might have accidentally
+ // touched out of turn, do so now.
+ if (madvise_block_metap) {
+ nanov2_madvise_block(nanozone, madvise_block_metap, size_class,
+ SLOT_MADVISED);
}
// No current allocation block, or we were unable to allocate. We need to
// get a new block. Before doing so, delegate to the helper allocator if
// the size class was full and has not released enough memory yet.
if (nanozone->delegate_allocations & (1 << size_class)) {
- ptr = nanozone->helper_zone->malloc(nanozone->helper_zone, rounded_size);
+ malloc_zone_t *helper_zone = nanozone->helper_zone;
+ if (typed) {
+ ptr = helper_zone->malloc_type_malloc(helper_zone, rounded_size,
+ type_id);
+ } else {
+ ptr = helper_zone->malloc(helper_zone, rounded_size);
+ }
goto done;
}
@@ -2600,7 +3016,7 @@
_malloc_lock_s *lock = &nanozone->current_block_lock[size_class][allocation_index];
_malloc_lock_lock(lock);
- block_metap = os_atomic_load(block_metapp, relaxed);
+ nanov2_block_meta_t *block_metap = os_atomic_load(block_metapp, relaxed);
if (block_metap) {
ptr = nanov2_allocate_from_block(nanozone, block_metap, size_class);
if (ptr) {
@@ -2622,21 +3038,49 @@
// We could not find a block to allocate from -- make future
// allocations for this size class go to the helper zone until
// we have enough free space.
- _malloc_lock_lock(&nanozone->delegate_allocations_lock);
- nanozone->delegate_allocations |= 1 << size_class;
- _malloc_lock_unlock(&nanozone->delegate_allocations_lock);
+ os_atomic_or(&nanozone->delegate_allocations,
+ (uint16_t)(1 << size_class), relaxed);
+
+ malloc_zone_t *helper_zone = nanozone->helper_zone;
+ if (typed) {
+ ptr = helper_zone->malloc_type_malloc(helper_zone, rounded_size,
+ type_id);
+ } else {
+ ptr = helper_zone->malloc(helper_zone, rounded_size);
+ }
}
done:
- if (ptr) {
- if (clear) {
- memset(ptr, '\0', rounded_size);
- } else {
+ if (os_likely(ptr)) {
+ switch (malloc_zero_policy) {
+ case MALLOC_ZERO_ON_FREE: {
// Always clear the double-free guard so that we can recognize that
// this block is not on the free list.
nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
os_atomic_store(&slotp->double_free_guard, 0, relaxed);
- }
+
+ // We know the body of the allocation is already clear, so we just
+ // need to clean up the next_slot word to get to all-zero. Do so in
+ // all cases, even if a cleared allocation is not requested, to
+ // prevent any leakage through the next_slot bits.
+ os_atomic_store(&slotp->next_slot, 0, relaxed);
+ break;
+ }
+ case MALLOC_ZERO_NONE:
+ if (!clear) {
+ // Always clear the double-free guard so that we can recognize that
+ // this block is not on the free list.
+ nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
+ os_atomic_store(&slotp->double_free_guard, 0, relaxed);
+ break;
+ }
+ MALLOC_FALLTHROUGH;
+ case MALLOC_ZERO_ON_ALLOC:
+ memset(ptr, '\0', rounded_size);
+ break;
+ }
+ } else {
+ malloc_set_errno_fast(MZ_POSIX, ENOMEM);
}
return ptr;
}
@@ -2645,20 +3089,32 @@
#pragma mark Freeing
// Frees an allocation to its owning block and updates the block's state.
-// If the block becomes empty, it is marked as SLOT_CAN_MADVISE and is
-// madvised immediately if the policy is NANO_MADVISE_IMMEDIATE.
-void
-nanov2_free_to_block(nanozonev2_t *nanozone, void *ptr,
- nanov2_size_class_t size_class)
+//
+// If the block becomes empty, it is marked as SLOT_CAN_MADVISE and we return
+// the block to the caller to madvise if dictated by policy.
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+nanov2_block_meta_t *
+nanov2_free_to_block_inline(nanozonev2_t *nanozone, void *ptr,
+ nanov2_size_class_t size_class, nanov2_block_meta_t *block_metap)
{
nanov2_block_t *blockp = nanov2_block_address_for_ptr(ptr);
- nanov2_block_meta_t *block_metap = nanov2_meta_ptr_for_ptr(nanozone, ptr);
+ if (!block_metap) {
+ block_metap = nanov2_meta_ptr_for_ptr(nanozone, ptr);
+ }
// Release the slot memory onto the block's freelist.
nanov2_block_meta_t old_meta = os_atomic_load(block_metap, relaxed);
int slot_count = slots_by_size_class[size_class];
nanov2_block_meta_t new_meta;
boolean_t was_full;
+
+ nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
+ // All of the free slot content (double_free_guard, next_slot word and the
+ // zeroed remainder of the slot) must be visible when the os_atomic_cmpxchgv
+ // completes, so the metadata updates on either path below need a release
+ // barrier.
+ os_atomic_store(&slotp->double_free_guard,
+ nanozone->slot_freelist_cookie ^ (uintptr_t)ptr, relaxed);
again:
was_full = old_meta.next_slot == SLOT_FULL;
@@ -2668,12 +3124,14 @@
boolean_t freeing_last_active_slot = !was_full &&
new_meta.free_count == slots_by_size_class[size_class] - 1;
if (freeing_last_active_slot) {
+ os_atomic_store(&slotp->next_slot, SLOT_NULL, relaxed);
+
// Releasing the last active slot onto the free list. Mark the block as
// ready to be madvised if it's not in use, otherwise reset next_slot
// to SLOT_BUMP.
new_meta.next_slot = new_meta.in_use ? SLOT_BUMP : SLOT_CAN_MADVISE;
// Write the updated meta data; try again if we raced with another thread.
- if (!os_atomic_cmpxchgv(block_metap, old_meta, new_meta, &old_meta, relaxed)) {
+ if (!os_atomic_cmpxchgv(block_metap, old_meta, new_meta, &old_meta, release)) {
goto again;
}
@@ -2681,20 +3139,14 @@
// is to do so immediately.
if (new_meta.next_slot == SLOT_CAN_MADVISE &&
nanov2_madvise_policy == NANO_MADVISE_IMMEDIATE) {
- _malloc_lock_lock(&nanozone->madvise_lock);
- nanov2_madvise_block(nanozone, block_metap, blockp, size_class);
- _malloc_lock_unlock(&nanozone->madvise_lock);
+ return block_metap;
}
} else {
int slot_index = nanov2_slot_index_in_block(blockp, size_class, ptr);
new_meta.next_slot = slot_index + 1; // meta.next_slot is 1-based
- nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
- slotp->next_slot = was_full ? SLOT_BUMP : old_meta.next_slot;
- os_atomic_store(&slotp->double_free_guard,
- nanozone->slot_freelist_cookie ^ (uintptr_t)ptr, relaxed);
-
- // The double_free_guard change must be visible when the os_atomic_cmpxchgv
- // completes.
+ os_atomic_store(&slotp->next_slot,
+ was_full ? SLOT_BUMP : old_meta.next_slot, relaxed);
+
// Write the updated meta data; try again if we raced with another thread.
if (!os_atomic_cmpxchgv(block_metap, old_meta, new_meta, &old_meta, release)) {
goto again;
@@ -2708,14 +3160,14 @@
uint16_t class_mask = 1 << size_class;
if (!new_meta.in_use && (nanozone->delegate_allocations & class_mask) &&
(new_meta.free_count >= 0.75 * slot_count)) {
- _malloc_lock_lock(&nanozone->delegate_allocations_lock);
- nanozone->delegate_allocations &= ~class_mask;
- _malloc_lock_unlock(&nanozone->delegate_allocations_lock);
+ os_atomic_and(&nanozone->delegate_allocations, ~class_mask, relaxed);
}
#if DEBUG_MALLOC
nanozone->statistics.size_class_statistics[size_class].total_frees++;
#endif // DEBUG_MALLOC
+
+ return NULL;
}
#endif // OS_VARIANT_RESOLVED
@@ -2728,11 +3180,9 @@
malloc_zone_t *
nanov2_create_zone(malloc_zone_t *helper_zone, unsigned debug_flags)
{
- // Note: It is important that nanov2_create_zone resets _malloc_engaged_nano
- // if it is unable to enable the nanozone (and chooses not to abort). As
- // several functions rely on _malloc_engaged_nano to determine if they
- // should manipulate the nanozone, and these should not run if we failed
- // to create the zone.
+ // Note: It is not necessary that nanov2_create_zone resets _malloc_engaged_nano
+ // if it is unable to enable the nanozone - functions that need to determine
+ // whether the nanozone is preset should test initial_nano_zone.
MALLOC_ASSERT(_malloc_engaged_nano == NANO_V2);
// Get memory for the zone and disable Nano if we fail.
@@ -2744,9 +3194,18 @@
}
// Set up the basic_zone portion of the nanozonev2 structure
- nanozone->basic_zone.version = 12;
+ nanozone->basic_zone.version = 16;
nanozone->basic_zone.size = OS_RESOLVED_VARIANT_ADDR(nanov2_size);
- nanozone->basic_zone.malloc = OS_RESOLVED_VARIANT_ADDR(nanov2_malloc);
+ if (malloc_zero_policy == MALLOC_ZERO_ON_ALLOC) {
+ nanozone->basic_zone.malloc =
+ OS_RESOLVED_VARIANT_ADDR(nanov2_malloc_zero_on_alloc);
+ nanozone->basic_zone.malloc_type_malloc =
+ OS_RESOLVED_VARIANT_ADDR(nanov2_malloc_type_zero_on_alloc);
+ } else {
+ nanozone->basic_zone.malloc = OS_RESOLVED_VARIANT_ADDR(nanov2_malloc);
+ nanozone->basic_zone.malloc_type_malloc =
+ OS_RESOLVED_VARIANT_ADDR(nanov2_malloc_type);
+ }
nanozone->basic_zone.calloc = OS_RESOLVED_VARIANT_ADDR(nanov2_calloc);
nanozone->basic_zone.valloc = (void *)nanov2_valloc;
nanozone->basic_zone.free = OS_RESOLVED_VARIANT_ADDR(nanov2_free);
@@ -2756,10 +3215,18 @@
nanozone->basic_zone.batch_free = OS_RESOLVED_VARIANT_ADDR(nanov2_batch_free);
nanozone->basic_zone.introspect =
(struct malloc_introspection_t *)&nanov2_introspect;
- nanozone->basic_zone.memalign = (void *)nanov2_memalign;
+ nanozone->basic_zone.memalign = OS_RESOLVED_VARIANT_ADDR(nanov2_memalign);
nanozone->basic_zone.free_definite_size = OS_RESOLVED_VARIANT_ADDR(nanov2_free_definite_size);
nanozone->basic_zone.pressure_relief = OS_RESOLVED_VARIANT_ADDR(nanov2_pressure_relief);
nanozone->basic_zone.claimed_address = OS_RESOLVED_VARIANT_ADDR(nanov2_claimed_address);
+ nanozone->basic_zone.try_free_default = OS_RESOLVED_VARIANT_ADDR(nanov2_try_free_default);
+
+ // zone_type == MALLOC_ZONE_TYPE_UNKNOWN
+ // No malloc_with_options
+
+ nanozone->basic_zone.malloc_type_calloc = OS_RESOLVED_VARIANT_ADDR(nanov2_calloc_type);
+ nanozone->basic_zone.malloc_type_realloc = OS_RESOLVED_VARIANT_ADDR(nanov2_realloc_type);
+ nanozone->basic_zone.malloc_type_memalign = OS_RESOLVED_VARIANT_ADDR(nanov2_memalign_type);
// Set these both to zero as required by CFAllocator.
nanozone->basic_zone.reserved1 = 0;
@@ -2798,27 +3265,41 @@
_malloc_lock_init(&nanozone->madvise_lock);
// Allocate the initial region. If this does not succeed, we disable Nano.
- nanov2_addr_t p = {.fields.nano_signature = NANOZONE_SIGNATURE};
- nanov2_region_t *region = (nanov2_region_t *)p.addr;
- boolean_t result = nanov2_allocate_region(region);
+ nanov2_region_t *region = (nanov2_region_t *)NANOZONE_BASE_REGION_ADDRESS;
+
+ bool result;
+#if CONFIG_NANO_RESERVE_REGIONS
+ unsigned int num_regions = (nano_max_region + 1);
+ result = nanov2_reserve_regions(region, num_regions);
+ if (result) {
+ result = nanov2_unprotect_region(region);
+ if (!result) {
+ malloc_report(ASL_LEVEL_ERR,
+ "unable to protect initial region\n");
+ nano_common_deallocate_pages((void *)region,
+ num_regions * (size_t)NANOV2_REGION_SIZE, 0);
+ }
+ }
+#else // CONFIG_NANO_RESERVE_REGIONS
+ result = nanov2_allocate_region(region);
+#endif // CONFIG_NANO_RESERVE_REGIONS
if (!result) {
- nano_common_deallocate_pages(nanozone, NANOZONEV2_ZONE_PAGED_SIZE, 0);
+ nano_common_deallocate_pages((void *)nanozone,
+ NANOZONEV2_ZONE_PAGED_SIZE, 0);
_malloc_engaged_nano = NANO_NONE;
malloc_report(ASL_LEVEL_NOTICE, "nano zone abandoned due to inability "
- "to preallocate reserved vm space.\n");
+ "to reserve vm space.\n");
return NULL;
}
nanov2_region_linkage_t *region_linkage =
nanov2_region_linkage_for_region(nanozone, region);
- region_linkage->next_region_offset = 0;
+ os_atomic_store(®ion_linkage->next_region_offset, 0, relaxed);
// Install the first region and pre-allocate the first arena.
nanozone->first_region_base = region;
- nanozone->current_region_base = region;
- nanozone->current_region_next_arena = ((nanov2_arena_t *)region) + 1;
- nanozone->current_region_limit = region + 1;
+ os_atomic_store(&nanozone->current_region_next_arena,
+ ((nanov2_arena_t *)region) + 1, release);
nanozone->statistics.allocated_regions = 1;
-
// Set up the guard blocks for the initial arena, if requested
nanov2_init_guard_blocks(nanozone, (nanov2_arena_t *)region);
@@ -2843,6 +3324,15 @@
// Just hand to the helper zone.
return nanozone->helper_zone->malloc(nanozone->helper_zone, size);
}
+
+MALLOC_NOEXPORT void *
+nanov2_forked_malloc_type(nanozonev2_t *nanozone, size_t size,
+ malloc_type_id_t type_id)
+{
+ // Just hand to the helper zone.
+ malloc_zone_t *helper_zone = nanozone->helper_zone;
+ return helper_zone->malloc_type_malloc(helper_zone, size, type_id);
+}
#endif // OS_VARIANT_RESOLVED
#if OS_VARIANT_NOTRESOLVED
@@ -2853,6 +3343,33 @@
// Just hand to the helper zone.
return nanozone->helper_zone->calloc(nanozone->helper_zone, num_items,
size);
+}
+
+static void *
+nanov2_forked_calloc_type(nanozonev2_t *nanozone, size_t num_items, size_t size,
+ malloc_type_id_t type_id)
+{
+ // Just hand to the helper zone.
+ malloc_zone_t *helper_zone = nanozone->helper_zone;
+ return helper_zone->malloc_type_calloc(helper_zone, num_items, size, type_id);
+}
+
+static void *
+nanov2_forked_memalign(nanozonev2_t *nanozone, size_t alignment, size_t size)
+{
+ // Just hand to the helper zone.
+ return nanozone->helper_zone->memalign(nanozone->helper_zone, alignment,
+ size);
+}
+
+static void *
+nanov2_forked_memalign_type(nanozonev2_t *nanozone, size_t alignment,
+ size_t size, malloc_type_id_t type_id)
+{
+ // Just hand to the helper zone.
+ malloc_zone_t *helper_zone = nanozone->helper_zone;
+ return helper_zone->malloc_type_memalign(helper_zone, alignment, size,
+ type_id);
}
#endif // OS_VARIANT_NOTRESOLVED
@@ -2887,21 +3404,31 @@
nanov2_forked_free(nanozone, ptr);
}
-MALLOC_NOEXPORT void *
-nanov2_forked_realloc(nanozonev2_t *nanozone, void *ptr, size_t new_size)
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static void *
+_nanov2_forked_realloc(nanozonev2_t *nanozone, void *ptr, size_t new_size,
+ bool typed, malloc_type_id_t type_id)
{
// could occur through malloc_zone_realloc() path
if (!ptr) {
// If ptr is a null pointer, realloc() shall be equivalent to malloc()
// for the specified size.
- return nanov2_forked_malloc(nanozone, new_size);
+ if (typed) {
+ return nanov2_forked_malloc_type(nanozone, new_size, type_id);
+ } else {
+ return nanov2_forked_malloc(nanozone, new_size);
+ }
}
size_t old_size = nanov2_pointer_size(nanozone, ptr, FALSE);
if (!old_size) {
// not-nano pointer, hand down to helper zone
malloc_zone_t *zone = (malloc_zone_t *)(nanozone->helper_zone);
- return zone->realloc(zone, ptr, new_size);
+ if (typed) {
+ return zone->malloc_type_realloc(zone, ptr, new_size, type_id);
+ } else {
+ return zone->realloc(zone, ptr, new_size);
+ }
} else {
if (!new_size) {
// If size is 0 and ptr is not a null pointer, the object pointed to
@@ -2913,8 +3440,15 @@
return nanov2_forked_malloc(nanozone, 1);
}
- void *new_ptr = nanozone->helper_zone->malloc(nanozone->helper_zone,
+ void *new_ptr;
+ if (typed) {
+ malloc_zone_t *helper_zone = nanozone->helper_zone;
+ new_ptr = helper_zone->malloc_type_malloc(helper_zone, new_size,
+ type_id);
+ } else {
+ new_ptr = nanozone->helper_zone->malloc(nanozone->helper_zone,
new_size);
+ }
if (new_ptr) {
size_t valid_size = MIN(old_size, new_size);
memcpy(new_ptr, ptr, valid_size);
@@ -2928,6 +3462,20 @@
/* NOTREACHED */
}
/* NOTREACHED */
+}
+
+MALLOC_NOEXPORT void *
+nanov2_forked_realloc(nanozonev2_t *nanozone, void *ptr, size_t new_size)
+{
+ return _nanov2_forked_realloc(nanozone, ptr, new_size, false,
+ MALLOC_TYPE_ID_NONE);
+}
+
+MALLOC_NOEXPORT void *
+nanov2_forked_realloc_type(nanozonev2_t *nanozone, void *ptr, size_t new_size,
+ malloc_type_id_t type_id)
+{
+ return _nanov2_forked_realloc(nanozone, ptr, new_size, true, type_id);
}
#endif // OS_VARIANT_RESOLVED
@@ -2993,10 +3541,18 @@
nanozone->basic_zone.batch_free = OS_RESOLVED_VARIANT_ADDR(nanov2_forked_batch_free);
nanozone->basic_zone.introspect =
(struct malloc_introspection_t *)&nanov2_introspect;// Unchanged
- nanozone->basic_zone.memalign = (void *)nanov2_memalign; // Unchanged
+ nanozone->basic_zone.memalign = (void *)nanov2_forked_memalign;
nanozone->basic_zone.free_definite_size =
OS_RESOLVED_VARIANT_ADDR(nanov2_forked_free_definite_size);
nanozone->basic_zone.claimed_address = nanov2_forked_claimed_address;
+ nanozone->basic_zone.try_free_default = NULL; // Fall back to old protocol
+ nanozone->basic_zone.malloc_type_malloc =
+ OS_RESOLVED_VARIANT_ADDR(nanov2_forked_malloc_type);
+ nanozone->basic_zone.malloc_type_calloc = (void *)nanov2_forked_calloc_type;
+ nanozone->basic_zone.malloc_type_realloc =
+ OS_RESOLVED_VARIANT_ADDR(nanov2_forked_realloc_type);
+ nanozone->basic_zone.malloc_type_memalign =
+ (void *)nanov2_forked_memalign_type;
mprotect(nanozone, sizeof(nanozone->basic_zone), PROT_READ);
}