Loading...
--- libmalloc/libmalloc-409.40.6/src/nanov2_malloc.c
+++ libmalloc/libmalloc-166.200.60/src/nanov2_malloc.c
@@ -29,51 +29,25 @@
#pragma mark Forward Declarations
#if OS_VARIANT_NOTRESOLVED
-static kern_return_t
-nanov2_statistics_task_printer(task_t task, vm_address_t zone_address,
- memory_reader_t reader, print_task_printer_t printer,
- malloc_statistics_t *stats);
-static kern_return_t
-nanov2_statistics_task(task_t task, vm_address_t zone_address,
- memory_reader_t reader, malloc_statistics_t *stats);
+static void nanov2_statistics(nanozonev2_t *nanozone, malloc_statistics_t *stats);
#endif // OS_VARIANT_NOTRESOLVED
#pragma mark -
#pragma mark Externals for resolved functions
-MALLOC_NOEXPORT extern size_t nanov2_pointer_size(nanozonev2_t *nanozone, void *ptr,
+extern void *nanov2_allocate(nanozonev2_t *nanozone, size_t rounded_size,
+ boolean_t clear);
+extern void nanov2_free_to_block(nanozonev2_t *nanozone, void *ptr,
+ nanov2_size_class_t size_class);
+extern boolean_t nanov2_madvise_block(nanozonev2_t *nanozone,
+ nanov2_block_meta_t *block_metap, nanov2_block_t *blockp,
+ nanov2_size_class_t size_class);
+extern size_t nanov2_pointer_size(nanozonev2_t *nanozone, void *ptr,
boolean_t allow_inner);
-MALLOC_NOEXPORT extern size_t nanov2_pressure_relief(nanozonev2_t *nanozone, size_t goal);
+extern size_t nanov2_pressure_relief(nanozonev2_t *nanozone, size_t goal);
#if OS_VARIANT_RESOLVED
-MALLOC_ALWAYS_INLINE MALLOC_INLINE size_t
-nanov2_pointer_size_inline(nanozonev2_t *nanozone, void *ptr, boolean_t allow_inner,
- nanov2_size_class_t *size_class_out, nanov2_block_meta_t **block_metap_out);
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE void *
-nanov2_allocate_from_block_inline(nanozonev2_t *nanozone,
- nanov2_block_meta_t *block_metap, nanov2_size_class_t size_class,
- nanov2_block_meta_t **madvise_block_metapp_out, bool *corruption);
-
-static void *
-nanov2_allocate_outlined(nanozonev2_t *nanozone,
- nanov2_block_meta_t **block_metapp, size_t rounded_size,
- nanov2_size_class_t size_class, int allocation_index,
- nanov2_block_meta_t *madvise_block_metap, void *corrupt_slot,
- bool clear);
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_meta_t *
-nanov2_free_to_block_inline(nanozonev2_t *nanozone, void *ptr,
- nanov2_size_class_t size_class, nanov2_block_meta_t *block_metap);
-
-static boolean_t nanov2_madvise_block_locked(
- nanozonev2_t *nanozone, nanov2_block_meta_t *block_metap,
- nanov2_block_t *blockp, nanov2_size_class_t size_class, uint32_t expected_state);
-static void nanov2_madvise_block(nanozonev2_t *nanozone,
- nanov2_block_meta_t *block_metap, nanov2_size_class_t size_class,
- uint32_t expected_state);
-
-MALLOC_NOEXPORT extern nanov2_arena_t *nanov2_allocate_new_region(nanozonev2_t *nanozone);
+extern boolean_t nanov2_allocate_new_region(nanozonev2_t *nanozone);
#endif // OS_VARIANT_RESOLVED
#pragma mark -
@@ -130,9 +104,9 @@
#if OS_VARIANT_NOTRESOLVED
// Madvise policy. Set from the MallocNanoMadvisePolicy environment variable.
-MALLOC_NOEXPORT nanov2_madvise_policy_t nanov2_madvise_policy;
-
-MALLOC_NOEXPORT nanov2_policy_config_t nanov2_policy_config = {
+nanov2_madvise_policy_t nanov2_madvise_policy;
+
+nanov2_policy_config_t nanov2_policy_config = {
.block_scan_policy = NANO_SCAN_CAPACITY_BASED,
.block_scan_min_capacity = DEFAULT_SCAN_MIN_CAPACITY,
.block_scan_max_capacity = DEFAULT_SCAN_MAX_CAPACITY,
@@ -143,8 +117,8 @@
#else // OS_VARIANT_NOTRESOLVED
-MALLOC_NOEXPORT extern nanov2_policy_config_t nanov2_policy_config;
-MALLOC_NOEXPORT extern nanov2_madvise_policy_t nanov2_madvise_policy;
+extern nanov2_policy_config_t nanov2_policy_config;
+extern nanov2_madvise_policy_t nanov2_madvise_policy;
#endif // OS_VARIANT_NOTRESOLVED
@@ -159,7 +133,7 @@
// up to 64. One unit corresponds to BLOCKS_PER_UNIT blocks in the corresponding
// size class, so 64 units maps to a total of 64 * 64 = 4096 blocks and each
// block is 16K, making a total of 64MB, which is the size of an arena.
-static int block_units_by_size_class[] = {
+int block_units_by_size_class[] = {
2, // 16-byte allocations (less 1 for the metadata block)
10, // 32-byte allocations
11, // 48-byte allocations
@@ -189,18 +163,18 @@
// Offsets to the first and last blocks for each size class within an arena, in
// the logical address space. These tables are constructed from the values in
// the block_units_by_size_class table.
-MALLOC_NOEXPORT int first_block_offset_by_size_class[NANO_SIZE_CLASSES];
-MALLOC_NOEXPORT int last_block_offset_by_size_class[NANO_SIZE_CLASSES];
+int first_block_offset_by_size_class[NANO_SIZE_CLASSES];
+int last_block_offset_by_size_class[NANO_SIZE_CLASSES];
// Table mapping the part of a logical address that depends on size class to
// the size class. Also built from the block_units_by_size_class table.
-MALLOC_NOEXPORT int ptr_offset_to_size_class[TOTAL_BLOCK_UNITS];
+int ptr_offset_to_size_class[TOTAL_BLOCK_UNITS];
// Number of slots in a block, indexed by size class. Note that there is a small
// amount of wastage in some size classes because the block size is not always
// exactly divisible by the allocation size. The number of wasted bytes is shown
// in parentheses in the comments below.
-MALLOC_NOEXPORT const int slots_by_size_class[] = {
+const int slots_by_size_class[] = {
NANOV2_BLOCK_SIZE/(1 * NANO_REGIME_QUANTA_SIZE), // 16 bytes: 1024 (0)
NANOV2_BLOCK_SIZE/(2 * NANO_REGIME_QUANTA_SIZE), // 32 bytes: 512 (0)
NANOV2_BLOCK_SIZE/(3 * NANO_REGIME_QUANTA_SIZE), // 48 bytes: 341 (16)
@@ -259,14 +233,13 @@
// Given a block metadata pointer, returns whether the block is active (that is,
// it is being used for allocations, it has allocations that have not been freed,
-// or is waiting to be madvised and is not a guard block).
+// or is waiting to be madvised).
static MALLOC_ALWAYS_INLINE MALLOC_INLINE boolean_t
nanov2_is_block_active(nanov2_block_meta_t block_meta)
{
return block_meta.next_slot != SLOT_NULL
&& block_meta.next_slot != SLOT_MADVISING
- && block_meta.next_slot != SLOT_MADVISED
- && block_meta.next_slot != SLOT_GUARD;
+ && block_meta.next_slot != SLOT_MADVISED;
}
#if OS_VARIANT_RESOLVED
@@ -333,6 +306,7 @@
return (void *)(((uintptr_t)ptr) & NANOV2_ARENA_ADDRESS_MASK);
}
+#if OS_VARIANT_RESOLVED
// Given a pointer that is assumed to be in the Nano zone, returns the address
// of its containing region. Works for both real and logical pointers.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_region_t *
@@ -340,6 +314,7 @@
{
return (nanov2_region_t *)(((uintptr_t)ptr) & NANOV2_REGION_ADDRESS_MASK);
}
+#endif // OS_VARIANT_RESOLVED
// Given a pointer that is assumed to be in the Nano zone, returns the real
// address of its metadata block. Works for both real and logical pointers.
@@ -413,40 +388,15 @@
return (nanov2_arena_t *)region;
}
-#if OS_VARIANT_RESOLVED
-// Given an atomically-observed current_region_next_arena pointer, returns
-// whether or not it's a usable arena or a limit arena (indicating exhaustion of
-// the current region).
-static MALLOC_ALWAYS_INLINE MALLOC_INLINE bool
-nanov2_current_region_next_arena_is_limit(
- nanov2_arena_t *current_region_next_arena)
-{
- // The first arena of a region is never stored in current_region_next_arena,
- // so a value at the beginning of a region must be a limit arena.
- return current_region_next_arena == (nanov2_arena_t *)(
- nanov2_region_address_for_ptr(current_region_next_arena));
-}
-#endif // OS_VARIANT_RESOLVED
-
-// Given an atomically-observed current_region_next_arena pointer, returns the
-// base of the current region at the time of the observation.
-static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_region_t *
-nanov2_current_region_base(nanov2_arena_t *current_region_next_arena)
-{
- return nanov2_region_address_for_ptr(
- (void *)(((uintptr_t)current_region_next_arena) - 1));
-}
-
// Given a region pointer, returns a pointer to the arena after the last
// active arena in the region.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_arena_t *
-nanov2_limit_arena_for_region(nanozonev2_t __unused *nanozone,
- nanov2_region_t *region, nanov2_arena_t *current_region_next_arena)
+nanov2_limit_arena_for_region(nanozonev2_t *nanozone, nanov2_region_t *region)
{
// The first arena is colocated with the region itself.
nanov2_arena_t *limit_arena;
- if (region == nanov2_current_region_base(current_region_next_arena)) {
- limit_arena = current_region_next_arena;
+ if (region == nanozone->current_region_base) {
+ limit_arena = nanozone->current_region_next_arena;
} else {
limit_arena = nanov2_first_arena_for_region(region + 1);
}
@@ -465,51 +415,16 @@
nanov2_metablock_meta_index(nanozone)];
}
-#if OS_VARIANT_RESOLVED
// Given a pointer to a region, returns a pointer to the region that follows it,
-// or NULL if there isn't one. We may observe linkage to a new region that
-// hasn't yet actually been installed into current_region_next_arena; ignore the
-// linkage in this case.
+// or NULL if there isn't one.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_region_t *
-nanov2_next_region_for_region(nanozonev2_t *nanozone, nanov2_region_t *region,
- nanov2_arena_t *current_region_next_arena)
+nanov2_next_region_for_region(nanozonev2_t *nanozone, nanov2_region_t *region)
{
nanov2_region_linkage_t *linkage =
nanov2_region_linkage_for_region(nanozone, region);
- int offset = os_atomic_load(&linkage->next_region_offset, relaxed);
- if (!offset) {
- return NULL;
- }
-
- nanov2_region_t *next_region = region + offset;
- return (nanov2_arena_t *)next_region < current_region_next_arena ?
- next_region : NULL;
-}
-#endif // OS_VARIANT_RESOLVED
-
-#if OS_VARIANT_NOTRESOLVED
-// Given a pointer to a region, returns a pointer to the region that follows it,
-// or NULL if there isn't one. This variant is used when mapping the nanozone
-// for another process.
-static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_region_t *
-nanov2_next_region_for_region_offset(nanozonev2_t *nanozone,
- nanov2_region_t *region, off_t region_offset,
- nanov2_arena_t *current_region_next_arena)
-{
- nanov2_region_linkage_t *linkage =
- nanov2_region_linkage_for_region(nanozone, region);
- nanov2_region_linkage_t *mapped_linkage = (nanov2_region_linkage_t *)(
- ((uintptr_t)linkage + region_offset));
- int offset = os_atomic_load(&mapped_linkage->next_region_offset, relaxed);
- if (!offset) {
- return NULL;
- }
-
- nanov2_region_t *next_region = region + offset;
- return (nanov2_arena_t *)next_region < current_region_next_arena ?
- next_region : NULL;
-}
-#endif // OS_VARIANT_NOTRESOLVED
+ int offset = linkage->next_region_offset;
+ return offset ? region + offset : NULL;
+}
// Given the index of a slot in a block of a given size and the base address of
// the block, returns a pointer to the start of the slot. This works for both
@@ -652,85 +567,18 @@
static MALLOC_ALWAYS_INLINE MALLOC_INLINE int
nanov2_get_allocation_block_index(void)
{
-#if CONFIG_NANO_USES_HYPER_SHIFT
if (os_likely(nano_common_max_magazines_is_ncpu)) {
// Default case is max magazines == physical number of CPUs, which
- // must be > _malloc_cpu_number() >> hyper_shift, so the modulo
+ // must be > _os_cpu_number() >> hyper_shift, so the modulo
// operation is not required.
- return (_malloc_cpu_number() >> hyper_shift) & MAX_CURRENT_BLOCKS_MASK;
- }
-#else // CONFIG_NANO_USES_HYPER_SHIFT
- if (os_likely(nano_common_max_magazines_is_ncpu)) {
- // Default case is max magazines == logical number of CPUs, which
- // must be > _malloc_cpu_number() so the modulo operation is not required.
- return _malloc_cpu_number() & MAX_CURRENT_BLOCKS_MASK;
- }
-#endif // CONFIG_NANO_USES_HYPER_SHIFT
-
- unsigned int shift = 0;
-#if CONFIG_NANO_USES_HYPER_SHIFT
- shift = hyper_shift;
-#endif // CONFIG_NANO_USES_HYPER_SHIFT
-
+ return _os_cpu_number() >> hyper_shift;
+ }
if (os_likely(_os_cpu_number_override == -1)) {
- return ((_malloc_cpu_number() >> shift) % nano_common_max_magazines) &
- MAX_CURRENT_BLOCKS_MASK;
- }
- return ((_os_cpu_number_override >> shift) % nano_common_max_magazines) &
- MAX_CURRENT_BLOCKS_MASK;
+ return (_os_cpu_number() >> hyper_shift) % nano_common_max_magazines;
+ }
+ return (_os_cpu_number_override >> hyper_shift) % nano_common_max_magazines;
}
#endif // OS_VARIANT_RESOLVED
-
-#pragma mark -
-#pragma mark Guard Blocks
-
-// Converts a given block (specified by absolute block number) in an arena into
-// a guard block. The block will be marked as in-use so that it is not available
-// for allocations and its permissions are set to PROT_READ. Note that
-// PROT_READ is used instead of PROT_NONE because the latter breaks the
-// enumerator, which tries to map the whole region and fails if there are
-// PROT_NONE pages in the range. We can't fix that in the allocator because the
-// code that does the mapping is part of the sampling tools and is simply
-// invoked as a callback from the enumerator.
-static MALLOC_ALWAYS_INLINE MALLOC_INLINE void
-nanov2_create_guard_block(nanozonev2_t *nanozone, nanov2_arena_t *arena,
- nanov2_block_index_t block_index) {
- // Mark the block as in-use in the meta data
- static nanov2_block_meta_t in_use_block = {
- .in_use = 1,
- .next_slot = SLOT_GUARD
- };
- nanov2_meta_index_t block_meta_index =
- nanov2_block_index_to_meta_index(block_index);
- nanov2_arena_metablock_t *block_metap = nanov2_metablock_address_for_ptr(
- nanozone, arena);
- block_metap->arena_block_meta[block_meta_index] = in_use_block;
- void *block_ptr = &arena->blocks[block_index];
-
- // Apply PROT_NONE to the block itself.
- kern_return_t err = mprotect(block_ptr, NANOV2_BLOCK_SIZE, PROT_READ);
- if (err != KERN_SUCCESS) {
- malloc_report(ASL_LEVEL_ERR, "Failed to create guard block at %p (%d)\n",
- block_ptr, err);
- }
-}
-
-// Creates the guard blocks for an arena, if required. The guard blocks are
-// the first and last physical blocks in the arena that are not the metadata
-// block.
-static MALLOC_ALWAYS_INLINE MALLOC_INLINE void
-nanov2_init_guard_blocks(nanozonev2_t *nanozone, nanov2_arena_t *arena)
-{
- if (nanozone->debug_flags & MALLOC_ALL_GUARD_PAGE_FLAGS) {
- // Use the first and last blocks in the arena as guard regions,
- // avoiding the metadata block.
- nanov2_meta_index_t meta_index = nanov2_metablock_meta_index(nanozone);
- nanov2_create_guard_block(nanozone, arena, meta_index == 0 ? 1 : 0);
- nanov2_create_guard_block(nanozone, arena,
- meta_index == NANOV2_BLOCKS_PER_ARENA - 1 ?
- NANOV2_BLOCKS_PER_ARENA - 2 : NANOV2_BLOCKS_PER_ARENA - 1);
- }
-}
#pragma mark -
#pragma mark Allocator Initialization
@@ -840,7 +688,7 @@
boolean_t max_found = FALSE;
boolean_t lim_found = FALSE;
const char *value = ptr;
-
+
if (ptr) {
if (!strcmp(ptr, first_fit_key)) {
block_scan_policy = NANO_SCAN_FIRST_FIT;
@@ -895,7 +743,7 @@
}
}
}
-
+
if (!failed) {
nanov2_policy_config.block_scan_policy = block_scan_policy;
nanov2_policy_config.block_scan_min_capacity = scan_min_capacity;
@@ -1059,137 +907,69 @@
// determine whether the pointer is for a Nano V2 allocation and, if not,
// delegates to the helper zone. Returns 0 if the pointer is not to memory
// allocated by Nano V2 or attributable to the helper zone.
-MALLOC_NOEXPORT size_t
+size_t
nanov2_size(nanozonev2_t *nanozone, const void *ptr)
{
- size_t size = nanov2_pointer_size_inline(nanozone, (void *)ptr, FALSE,
- NULL, NULL);
+ size_t size = nanov2_pointer_size(nanozone, (void *)ptr, FALSE);
return size ? size : nanozone->helper_zone->size(nanozone->helper_zone, ptr);
}
-MALLOC_NOEXPORT void *
+void *
nanov2_malloc(nanozonev2_t *nanozone, size_t size)
{
size_t rounded_size = _nano_common_good_size(size);
if (rounded_size <= NANO_MAX_SIZE) {
- nanov2_block_meta_t *madvise_block_metap = NULL;
- nanov2_size_class_t size_class = nanov2_size_class_from_size(rounded_size);
-
- // Get the index of the pointer to the block from which we are should be
- // allocating. This currently depends on the physical CPU number.
- int allocation_index = nanov2_get_allocation_block_index();
-
- // Get the current allocation block meta data pointer. If this is NULL,
- // we need to find a new allocation block.
- nanov2_block_meta_t **block_metapp =
- &nanozone->current_block[size_class][allocation_index];
- nanov2_block_meta_t *block_metap = os_atomic_load(block_metapp, relaxed);
- bool corruption = false;
- void *ptr = NULL;
- if (block_metap) {
- // Fast path: we have a block -- try to allocate from it.
- ptr = nanov2_allocate_from_block_inline(nanozone, block_metap,
- size_class, &madvise_block_metap, &corruption);
- if (ptr && !corruption) {
- // Always clear the double-free guard so that we can recognize
- // that this block is not on the free list.
- nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
- os_atomic_store(&slotp->double_free_guard, 0, relaxed);
-
- // We know the body of the allocation is already clear, so we just
- // need to clean up the next_slot word to get to all-zero. Do so in
- // all cases, even if a cleared allocation is not requested, to
- // prevent any leakage through the next_slot bits.
- os_atomic_store(&slotp->next_slot, 0, relaxed);
- return ptr;
- }
- }
-
- return nanov2_allocate_outlined(nanozone, block_metapp, rounded_size,
- size_class, allocation_index, madvise_block_metap, ptr, false);
- }
-
- // Too big for nano, so delegate to the helper zone.
+ void *ptr = nanov2_allocate(nanozone, rounded_size, FALSE);
+ if (ptr) {
+ if (os_unlikely(size && (nanozone->debug_flags & MALLOC_DO_SCRIBBLE))) {
+ memset(ptr, SCRIBBLE_BYTE, size);
+ }
+ return ptr;
+ }
+ }
+
+ // If we reach this point, we couldn't allocate, so delegate to the
+ // helper zone.
return nanozone->helper_zone->malloc(nanozone->helper_zone, size);
}
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
void
-nanov2_bzero(void *ptr, size_t size)
-{
- // TODO: inline bzero from libplatform
- bzero(ptr, size);
-}
-
-MALLOC_NOEXPORT void
nanov2_free_definite_size(nanozonev2_t *nanozone, void *ptr, size_t size)
{
+ // Check whether it's a Nano pointer and get the size. We should only get
+ // here if it is and furthermore we already know that "size" is the actual
+ // rounded size, so don't waste time rechecking that. This is just a
+ // sanity check.
if (ptr && nanov2_has_valid_signature(ptr)) {
- nanov2_size_class_t size_class = nanov2_size_class_from_size(size);
-
- if (malloc_zero_on_free) {
- if (size_class != 0) {
- nanov2_bzero((char *)ptr + sizeof(nanov2_free_slot_t),
- size - sizeof(nanov2_free_slot_t));
- }
- }
-
- nanov2_block_meta_t *madvise_block_metap = nanov2_free_to_block_inline(
- nanozone, ptr, size_class, NULL);
- if (madvise_block_metap) {
- nanov2_madvise_block(nanozone, madvise_block_metap, size_class,
- SLOT_CAN_MADVISE);
- }
+ if (os_unlikely(nanozone->debug_flags & MALLOC_DO_SCRIBBLE)) {
+ memset(ptr, SCRABBLE_BYTE, size);
+ }
+ nanov2_free_to_block(nanozone, ptr, nanov2_size_class_from_size(size));
return;
}
return nanozone->helper_zone->free_definite_size(nanozone->helper_zone, ptr,
size);
}
-static void
-_nanov2_free(nanozonev2_t *nanozone, void *ptr, bool try)
-{
- if (ptr) {
+void
+nanov2_free(nanozonev2_t *nanozone, void *ptr)
+{
+ if (ptr && nanov2_has_valid_signature(ptr)) {
// Check whether it's a Nano pointer and get the size. If it's not
// Nano, pass it to the helper zone.
- nanov2_size_class_t size_class;
- nanov2_block_meta_t *block_metap;
- size_t size = nanov2_pointer_size_inline(nanozone, ptr, FALSE,
- &size_class, &block_metap);
+ size_t size = nanov2_pointer_size(nanozone, ptr, FALSE);
if (size) {
- if (malloc_zero_on_free) {
- if (size > sizeof(nanov2_free_slot_t)) {
- nanov2_bzero((char *)ptr + sizeof(nanov2_free_slot_t),
- size - sizeof(nanov2_free_slot_t));
- }
- }
-
- nanov2_block_meta_t *madvise_block_metap = nanov2_free_to_block_inline(
- nanozone, ptr, size_class, block_metap);
- if (madvise_block_metap) {
- nanov2_madvise_block(nanozone, madvise_block_metap, size_class,
- SLOT_CAN_MADVISE);
- }
+ if (os_unlikely(nanozone->debug_flags & MALLOC_DO_SCRIBBLE)) {
+ memset(ptr, SCRABBLE_BYTE, size);
+ }
+ nanov2_free_to_block(nanozone, ptr, nanov2_size_class_from_size(size));
return;
}
}
- return try ? nanozone->helper_zone->try_free_default(nanozone->helper_zone, ptr) :
- nanozone->helper_zone->free(nanozone->helper_zone, ptr);
-}
-
-MALLOC_NOEXPORT void
-nanov2_free(nanozonev2_t *nanozone, void *ptr)
-{
- _nanov2_free(nanozone, ptr, false);
-}
-
-MALLOC_NOEXPORT void
-nanov2_try_free_default(nanozonev2_t *nanozone, void *ptr)
-{
- _nanov2_free(nanozone, ptr, true);
-}
-
-MALLOC_NOEXPORT void *
+ return nanozone->helper_zone->free(nanozone->helper_zone, ptr);
+}
+
+void *
nanov2_calloc(nanozonev2_t *nanozone, size_t num_items, size_t size)
{
size_t total_bytes;
@@ -1198,48 +978,14 @@
}
size_t rounded_size = _nano_common_good_size(total_bytes);
if (total_bytes <= NANO_MAX_SIZE) {
- nanov2_block_meta_t *madvise_block_metap = NULL;
- nanov2_size_class_t size_class = nanov2_size_class_from_size(rounded_size);
-
- // Get the index of the pointer to the block from which we are should be
- // allocating. This currently depends on the physical CPU number.
- int allocation_index = nanov2_get_allocation_block_index();
-
- // Get the current allocation block meta data pointer. If this is NULL,
- // we need to find a new allocation block.
- nanov2_block_meta_t **block_metapp =
- &nanozone->current_block[size_class][allocation_index];
- nanov2_block_meta_t *block_metap = os_atomic_load(block_metapp, relaxed);
- bool corruption = false;
- void *ptr = NULL;
- if (block_metap) {
- // Fast path: we have a block -- try to allocate from it.
- ptr = nanov2_allocate_from_block_inline(nanozone, block_metap,
- size_class, &madvise_block_metap, &corruption);
- if (ptr && !corruption) {
- if (malloc_zero_on_free) {
- // Always clear the double-free guard so that we can recognize that
- // this block is not on the free list.
- nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
- os_atomic_store(&slotp->double_free_guard, 0, relaxed);
-
- // We know the body of the allocation is already clear, so we just
- // need to clean up the next_slot word to get to all-zero. Do so in
- // all cases, even if a cleared allocation is not requested, to
- // prevent any leakage through the next_slot bits.
- os_atomic_store(&slotp->next_slot, 0, relaxed);
- } else {
- nanov2_bzero(ptr, rounded_size);
- }
- return ptr;
- }
- }
-
- return nanov2_allocate_outlined(nanozone, block_metapp, rounded_size,
- size_class, allocation_index, madvise_block_metap, ptr, true);
- }
-
- // Too big for nano, so delegate to the helper zone.
+ void *ptr = nanov2_allocate(nanozone, rounded_size, TRUE);
+ if (ptr) {
+ return ptr;
+ }
+ }
+
+ // If we reach this point, we couldn't allocate, so delegate to the
+ // helper zone.
return nanozone->helper_zone->calloc(nanozone->helper_zone, 1, total_bytes);
}
#endif // OS_VARIANT_RESOLVED
@@ -1254,7 +1000,7 @@
#endif // OS_VARIANT_NOTRESOLVED
#if OS_VARIANT_RESOLVED
-MALLOC_NOEXPORT void *
+void *
nanov2_realloc(nanozonev2_t *nanozone, void *ptr, size_t new_size)
{
// If we are given a NULL pointer, just allocate memory of the requested
@@ -1294,9 +1040,7 @@
}
} else {
// Same size or shrinking by less than half size. Keep the same
- // allocation and scribble the area that's being released. Nothing
- // to do for zero-on-free yet; that will be taken care of when the
- // shrunk allocation is freed.
+ // allocation and clear the area that's being released.
if (new_size != old_size) {
MALLOC_ASSERT(new_size < old_size);
if (os_unlikely(nanozone->debug_flags & MALLOC_DO_SCRIBBLE)) {
@@ -1306,7 +1050,7 @@
return ptr;
}
}
-
+
// If we reach this point, we allocated new memory. Copy the existing
// content to the new location and release the old allocation.
MALLOC_ASSERT(new_ptr);
@@ -1328,14 +1072,13 @@
#endif // OS_VARIANT_NOTRESOLVED
#if OS_VARIANT_RESOLVED
-MALLOC_NOEXPORT boolean_t
+boolean_t
nanov2_claimed_address(nanozonev2_t *nanozone, void *ptr)
{
- return nanov2_pointer_size(nanozone, ptr, TRUE)
- || malloc_zone_claimed_address(nanozone->helper_zone, ptr);
-}
-
-MALLOC_NOEXPORT unsigned
+ return nanov2_pointer_size(nanozone, ptr, TRUE) != 0;
+}
+
+unsigned
nanov2_batch_malloc(nanozonev2_t *nanozone, size_t size, void **results,
unsigned count)
{
@@ -1343,8 +1086,7 @@
size_t rounded_size = _nano_common_good_size(size);
if (rounded_size <= NANO_MAX_SIZE) {
while (allocated < count) {
- // TODO: nanov2_malloc will redo _nano_common_good_size
- void *ptr = nanov2_malloc(nanozone, rounded_size);
+ void *ptr = nanov2_allocate(nanozone, rounded_size, FALSE);
if (!ptr) {
break;
}
@@ -1363,7 +1105,7 @@
nanozone->helper_zone, size, results, count - allocated);
}
-MALLOC_NOEXPORT void
+void
nanov2_batch_free(nanozonev2_t *nanozone, void **to_be_freed, unsigned count)
{
if (count) {
@@ -1375,20 +1117,19 @@
}
}
}
-
-MALLOC_NOEXPORT void *
+#endif // OS_VARIANT_RESOLVED
+
+#if OS_VARIANT_NOTRESOLVED
+static void *
nanov2_memalign(nanozonev2_t *nanozone, size_t alignment, size_t size)
{
- // Serve directly if the requested alignment is trivially satisfied by our
- // baseline alignment (16 bytes)
- if (alignment <= NANO_REGIME_QUANTA_SIZE) {
- return nanov2_malloc(nanozone, size);
- }
-
- // Otherwise delegate to the helper zone
+ // Always delegate this to the helper zone.
return nanozone->helper_zone->memalign(nanozone->helper_zone, alignment,
size);
}
+#endif // OS_VARIANT_NOTRESOLVED
+
+#if OS_VARIANT_RESOLVED
size_t
nanov2_pressure_relief(nanozonev2_t *nanozone, size_t goal)
@@ -1410,12 +1151,9 @@
// until we reach our goal.
nanov2_region_t *region = nanozone->first_region_base;
nanov2_meta_index_t metablock_meta_index = nanov2_metablock_meta_index(nanozone);
- nanov2_arena_t *current_region_next_arena = os_atomic_load(
- &nanozone->current_region_next_arena, acquire);
while (region) {
nanov2_arena_t *arena = nanov2_first_arena_for_region(region);
- nanov2_arena_t *arena_after_region = nanov2_limit_arena_for_region(
- nanozone, region, current_region_next_arena);
+ nanov2_arena_t *arena_after_region = nanov2_limit_arena_for_region(nanozone, region);
while (arena < arena_after_region) {
// Scan all of the blocks in the arena, skipping the metadata block.
nanov2_arena_metablock_t *meta_blockp =
@@ -1434,9 +1172,8 @@
if (meta.next_slot == SLOT_CAN_MADVISE) {
nanov2_block_t *blockp = nanov2_block_address_from_meta_index(
nanozone, arena, i);
- if (nanov2_madvise_block_locked(nanozone, block_metap,
- blockp, nanov2_size_class_for_ptr(nanozone, blockp),
- SLOT_CAN_MADVISE)) {
+ if (nanov2_madvise_block(nanozone, block_metap,
+ blockp, nanov2_size_class_for_ptr(nanozone, blockp))) {
total += NANOV2_BLOCK_SIZE;
}
}
@@ -1448,10 +1185,9 @@
}
arena++;
}
- region = nanov2_next_region_for_region(nanozone, region,
- current_region_next_arena);
- }
-
+ region = nanov2_next_region_for_region(nanozone, region);
+ }
+
done:
MAGMALLOC_PRESSURERELIEFEND((void *)nanozone, name, (int)goal, (int)total);
MALLOC_TRACE(TRACE_nano_memory_pressure | DBG_FUNC_END,
@@ -1506,7 +1242,7 @@
if (kr) {
return kr;
}
- boolean_t self_zone = mach_task_is_self(task) && (nanozonev2_t *)zone_address == nanozone;
+ boolean_t self_zone = (nanozonev2_t *)zone_address == nanozone;
memcpy(&zone_copy, nanozone, sizeof(zone_copy));
nanozone = &zone_copy;
nanov2_meta_index_t metablock_meta_index = nanov2_metablock_meta_index(nanozone);
@@ -1514,8 +1250,6 @@
// Process the zone one region at a time. Report each in-use block as a
// pointer range and each in-use slot as a pointer.
nanov2_region_t *region = nanozone->first_region_base;
- nanov2_arena_t *current_region_next_arena = os_atomic_load(
- &nanozone->current_region_next_arena, acquire);
while (region) {
mach_vm_address_t vm_addr = (mach_vm_address_t)NULL;
kern_return_t kr = reader(task, (vm_address_t)region, NANOV2_REGION_SIZE, (void **)&vm_addr);
@@ -1527,8 +1261,7 @@
// and its mapped address in this process.
mach_vm_offset_t ptr_offset = (mach_vm_address_t)region - vm_addr;
nanov2_arena_t *arena = nanov2_first_arena_for_region(region);
- nanov2_arena_t *limit_arena = nanov2_limit_arena_for_region(nanozone, region,
- current_region_next_arena);
+ nanov2_arena_t *limit_arena = nanov2_limit_arena_for_region(nanozone, region);
vm_range_t ptr_range;
while (arena < limit_arena) {
// Find the metadata block and process every entry, apart from the
@@ -1642,8 +1375,7 @@
nanov2_region_linkage_t *mapped_region_linkagep =
NANOV2_ZONE_PTR_TO_MAPPED_PTR(nanov2_region_linkage_t *,
region_linkagep, ptr_offset);
- int offset = os_atomic_load(&mapped_region_linkagep->next_region_offset,
- relaxed);
+ int offset = mapped_region_linkagep->next_region_offset;
region = offset ? region + offset : NULL;
}
return 0;
@@ -1662,94 +1394,72 @@
static boolean_t
nanov2_check(nanozonev2_t *nanozone)
{
- // Does nothing
+ // Does nothing, just like Nano V1.
return 1;
}
static void
-nanov2_print(task_t task, unsigned level, vm_address_t zone_address,
- memory_reader_t reader, print_task_printer_t printer)
-{
- // Ensure that we have configured enough of the allocator to be able to
- // examine its data structures. In tools that do not directly use Nano, we
- // won't have done this yet. nanov2_configure() runs the initialization
- // only once.
- nanov2_configure();
-
- nanozonev2_t *mapped_nanozone;
- if (reader(task, (vm_address_t)zone_address, sizeof(nanozonev2_t),
- (void **)&mapped_nanozone)) {
- printer("Failed to map nanozonev2_s at %p\n", zone_address);
- return;
- }
-
+nanov2_print(nanozonev2_t *nanozone, boolean_t verbose)
+{
// Zone-wide statistics
malloc_statistics_t stats;
- nanov2_statistics_task_printer(task, zone_address, reader, printer, &stats);
- nanov2_statistics_t *nano_stats = &mapped_nanozone->statistics;
- printer("Nanozonev2 %p: blocks in use: %llu, size in use: %llu "
- "allocated size: %llu, allocated regions: %d, region holes: %d\n",
- zone_address, (uint64_t)stats.blocks_in_use,
- (uint64_t)stats.size_in_use, (uint64_t)stats.size_allocated,
- nano_stats->allocated_regions, nano_stats->region_address_clashes);
+ nanov2_statistics_t *nano_stats = &nanozone->statistics;
+ nanov2_statistics(nanozone, &stats);
+ malloc_report(MALLOC_REPORT_NOLOG | MALLOC_REPORT_NOPREFIX,
+ "Nanozonev2 %p: blocks in use: %llu, size in use: %llu allocated size: %llu, "
+ "allocated regions: %d, region holes: %d\n",
+ nanozone, (uint64_t)stats.blocks_in_use, (uint64_t)stats.size_in_use,
+ (uint64_t)stats.size_allocated, nano_stats->allocated_regions,
+ nano_stats->region_address_clashes);
#if DEBUG_MALLOC
// Per-size class statistics
- printer("\nPer size-class statistics:\n");
+ malloc_report(MALLOC_REPORT_NOLOG | MALLOC_REPORT_NOPREFIX,
+ "\nPer size-class statistics:\n");
for (int i = 0; i < NANO_SIZE_CLASSES; i++) {
nanov2_size_class_statistics *cs = &nano_stats->size_class_statistics[i];
- printer(" Class %d: ", i);
- printer("total alloc: %llu, total frees: %llu, madvised blocks: %llu, "
- "madvise races: %llu",
- cs->total_allocations, cs->total_frees, cs->madvised_blocks,
- cs->madvise_races);
- printer("\n");
+ malloc_report(MALLOC_REPORT_NOLOG | MALLOC_REPORT_NOPREFIX,
+ " Class %d: ", i);
+ malloc_report(MALLOC_REPORT_NOLOG | MALLOC_REPORT_NOPREFIX,
+ "total alloc: %llu, total frees: %llu, madvised blocks: %llu, madvise races: %llu",
+ cs->total_allocations, cs->total_frees, cs->madvised_blocks,
+ cs->madvise_races);
+ malloc_report(MALLOC_REPORT_NOLOG | MALLOC_REPORT_NOPREFIX, "\n");
}
#endif // DEBUG_MALLOC
// Per-context block pointers.
- printer("Current Allocation Blocks By Size Class/Context [CPU]\n");
+ malloc_report(MALLOC_REPORT_NOLOG | MALLOC_REPORT_NOPREFIX,
+ "Current Allocation Blocks By Size Class/Context [CPU]\n");
for (int i = 0; i < NANO_SIZE_CLASSES; i++) {
- printer(" Class %d: ", i);
+ malloc_report(MALLOC_REPORT_NOLOG | MALLOC_REPORT_NOPREFIX,
+ " Class %d: ", i);
for (int j = 0; j < MAX_CURRENT_BLOCKS; j++) {
- if (mapped_nanozone->current_block[i][j]) {
- printer("%d: %p; ", j, mapped_nanozone->current_block[i][j]);
- }
- }
- printer("\n");
- }
-
- nanov2_meta_index_t metablock_meta_index =
- nanov2_metablock_meta_index(mapped_nanozone);
- nanov2_region_t *region = mapped_nanozone->first_region_base;
- // Use a single, consistent snapshot of current_region_next_arena throughout
- // iteration, ignoring any arenas or regions allocated after it.
- nanov2_arena_t *current_region_next_arena = os_atomic_load(
- &mapped_nanozone->current_region_next_arena, acquire);
+ if (nanozone->current_block[i][j]) {
+ malloc_report(MALLOC_REPORT_NOLOG | MALLOC_REPORT_NOPREFIX,
+ "%d: %p; ", j, nanozone->current_block[i][j]);
+ }
+ }
+ malloc_report(MALLOC_REPORT_NOLOG | MALLOC_REPORT_NOPREFIX, "\n");
+ }
+
+ nanov2_meta_index_t metablock_meta_index = nanov2_metablock_meta_index(nanozone);
+ nanov2_region_t *region = nanozone->first_region_base;
int region_index = 0;
while (region) {
- printer("\nRegion %d: base address %p\n", region_index, region);
- nanov2_region_t *mapped_region;
- if (reader(task, (vm_address_t)region, sizeof(nanov2_region_t),
- (void **)&mapped_region)) {
- printer("Failed to map nanov2 region at %p\n", region);
- return;
- }
- off_t region_offset = (uintptr_t)mapped_region - (uintptr_t)region;
+ malloc_report(MALLOC_REPORT_NOLOG | MALLOC_REPORT_NOPREFIX,
+ "\nRegion %d: base address %p\n", region_index, region);
nanov2_arena_t *arena = nanov2_first_arena_for_region(region);
- nanov2_arena_t *limit_arena = nanov2_limit_arena_for_region(
- mapped_nanozone, region, current_region_next_arena);
+ nanov2_arena_t *limit_arena = nanov2_limit_arena_for_region(nanozone, region);
int arena_index = 0;
while (arena < limit_arena) {
// Find the metadata block and process every entry, apart from the
// one for the metadata block itself.
nanov2_arena_metablock_t *arena_meta_blockp =
- nanov2_metablock_address_for_ptr(mapped_nanozone, arena);
- nanov2_arena_metablock_t *mapped_arena_meta_blockp =
- (nanov2_arena_metablock_t *)((uintptr_t)arena_meta_blockp + region_offset);
+ nanov2_metablock_address_for_ptr(nanozone, arena);
+
nanov2_block_meta_t *block_metap = &arena_meta_blockp->arena_block_meta[0];
- nanov2_block_meta_t *mapped_block_metap = &mapped_arena_meta_blockp->arena_block_meta[0];
int active_blocks = 0;
int madvisable_blocks = 0;
@@ -1761,7 +1471,7 @@
// Skip the metadata block.
continue;
}
- nanov2_block_meta_t meta = mapped_block_metap[i];
+ nanov2_block_meta_t meta = block_metap[i];
switch (meta.next_slot) {
case SLOT_NULL:
unused_blocks++;
@@ -1780,7 +1490,8 @@
break;
}
}
- printer("Arena #%d: base address %p. Blocks - active: %d, "
+ malloc_report(MALLOC_REPORT_NOLOG | MALLOC_REPORT_NOPREFIX,
+ "Arena #%d: base address %p. Blocks - active: %d, "
"madvisable: %d, madvising: %d, madvised: %d, unused: %d\n",
arena_index, arena, active_blocks, madvisable_blocks,
madvising_blocks, madvised_blocks, unused_blocks);
@@ -1795,9 +1506,9 @@
// Skip the metadata block.
continue;
}
- nanov2_block_meta_t meta = mapped_block_metap[i];
+ nanov2_block_meta_t meta = block_metap[i];
nanov2_size_class_t size_class =
- nanov2_size_class_for_meta_index(mapped_nanozone, i);
+ nanov2_size_class_for_meta_index(nanozone, i);
switch (meta.next_slot) {
case SLOT_FULL:
case SLOT_BUMP:
@@ -1812,97 +1523,85 @@
break;
}
}
- printer("Size classes with allocated blocks: ");
+ malloc_report(MALLOC_REPORT_NOLOG | MALLOC_REPORT_NOPREFIX,
+ "Size classes with allocated blocks: ");
for (int i = 0; i < NANO_SIZE_CLASSES; i++) {
if (non_empty_size_classes[i]) {
- printer("%d ", i);
+ malloc_report(MALLOC_REPORT_NOLOG | MALLOC_REPORT_NOPREFIX,
+ "%d ", i);
}
}
- printer("\n");
-
- if (level >= MALLOC_VERBOSE_PRINT_LEVEL) {
- for (nanov2_meta_index_t i = 0; i < NANOV2_BLOCKS_PER_ARENA; i++) {
- if (i == metablock_meta_index) {
- // Skip the metadata block.
- continue;
+ malloc_report(MALLOC_REPORT_NOLOG | MALLOC_REPORT_NOPREFIX, "\n");
+
+ for (nanov2_meta_index_t i = 0; i < NANOV2_BLOCKS_PER_ARENA; i++) {
+ if (i == metablock_meta_index) {
+ // Skip the metadata block.
+ continue;
+ }
+ nanov2_block_meta_t meta = block_metap[i];
+ if (!nanov2_is_block_active(meta) && !verbose) {
+ continue;
+ }
+ nanov2_size_class_t size_class =
+ nanov2_size_class_for_meta_index(nanozone, i);
+ char *slot_text;
+ switch (meta.next_slot) {
+ case SLOT_NULL:
+ slot_text = "NOT USED";
+ break;
+ case SLOT_FULL:
+ slot_text = "FULL";
+ break;
+ case SLOT_CAN_MADVISE:
+ slot_text = "CAN MADVISE";
+ break;
+ case SLOT_MADVISING:
+ slot_text = "MADVISING";
+ break;
+ case SLOT_MADVISED:
+ slot_text = "MADVISED";
+ break;
+ default:
+ slot_text = NULL;
+ break;
+ }
+ malloc_report(MALLOC_REPORT_NOLOG | MALLOC_REPORT_NOPREFIX,
+ " Block %d: base %p; metadata: %p, size %d (class %d) in-use: %d ",
+ i, nanov2_block_address_from_meta_index(nanozone, arena, i),
+ &block_metap[i], nanov2_size_from_size_class(size_class),
+ size_class, meta.in_use);
+ if (slot_text) {
+ malloc_report(MALLOC_REPORT_NOLOG | MALLOC_REPORT_NOPREFIX,
+ "%s\n", slot_text);
+ } else {
+ int allocated = slots_by_size_class[size_class] - meta.free_count - 1;
+ if (meta.next_slot == SLOT_BUMP) {
+ malloc_report(MALLOC_REPORT_NOLOG | MALLOC_REPORT_NOPREFIX,
+ "BUMP (free list empty)");
+ } else {
+ malloc_report(MALLOC_REPORT_NOLOG | MALLOC_REPORT_NOPREFIX,
+ "next_slot (1-based) = %d", meta.next_slot);
+
}
- nanov2_block_meta_t meta = mapped_block_metap[i];
- if (!nanov2_is_block_active(meta)) {
- continue;
- }
- nanov2_size_class_t size_class =
- nanov2_size_class_for_meta_index(mapped_nanozone, i);
- char *slot_text;
- switch (meta.next_slot) {
- case SLOT_NULL:
- slot_text = "NOT USED";
- break;
- case SLOT_FULL:
- slot_text = "FULL";
- break;
- case SLOT_CAN_MADVISE:
- slot_text = "CAN MADVISE";
- break;
- case SLOT_MADVISING:
- slot_text = "MADVISING";
- break;
- case SLOT_MADVISED:
- slot_text = "MADVISED";
- break;
- default:
- slot_text = NULL;
- break;
- }
- printer(" Block %d: base %p; metadata: %p, size %d "
- "(class %d) in-use: %d ",
- i, nanov2_block_address_from_meta_index(mapped_nanozone, arena, i),
- &block_metap[i], nanov2_size_from_size_class(size_class),
- size_class, meta.in_use);
- if (slot_text) {
- printer("%s\n", slot_text);
- } else {
- int allocated = slots_by_size_class[size_class] - meta.free_count - 1;
- if (meta.next_slot == SLOT_BUMP) {
- printer("BUMP (free list empty)");
- } else {
- printer("next_slot (1-based) = %d", meta.next_slot);
-
- }
- printer(", allocated slots: %d, free slots = %d, "
- "occupancy: %d%%\n",
- allocated, meta.free_count + 1,
- (100 * allocated)/slots_by_size_class[size_class]);
- }
+ malloc_report(MALLOC_REPORT_NOLOG | MALLOC_REPORT_NOPREFIX,
+ ", allocated slots: %d, free slots = %d, occupancy: %d%%\n",
+ allocated, meta.free_count + 1,
+ (100 * allocated)/slots_by_size_class[size_class]);
}
}
arena++;
arena_index++;
}
- region = nanov2_next_region_for_region_offset(mapped_nanozone, region,
- region_offset, current_region_next_arena);
+ region = nanov2_next_region_for_region(nanozone, region);
region_index++;
}
-}
-
-static void
-nanov2_print_self(nanozonev2_t *nanozone, boolean_t verbose)
-{
- nanov2_print(mach_task_self(), verbose ? MALLOC_VERBOSE_PRINT_LEVEL : 0,
- (vm_address_t)nanozone, _malloc_default_reader, malloc_report_simple);
-}
-
-static void
-nanov2_print_task(task_t task, unsigned level, vm_address_t zone_address,
- memory_reader_t reader, print_task_printer_t printer)
-{
- nanov2_print(task, level, zone_address, reader, printer);
}
static void
nanov2_log(malloc_zone_t *zone, void *log_address)
{
- // Does nothing
+ // Does nothing, just like Nano V1.
}
static void
@@ -1931,74 +1630,35 @@
}
static void
-nanov2_null_printer(const char __unused *fmt, ...)
-{
-}
-
-static kern_return_t
-nanov2_statistics(task_t task, vm_address_t zone_address,
- memory_reader_t reader, print_task_printer_t printer,
- malloc_statistics_t *stats)
-{
- printer = printer ? printer : nanov2_null_printer;
- reader = !reader && task == mach_task_self() ? _malloc_default_reader : reader;
-
- kern_return_t err;
-
- // Ensure that we have configured enough of the allocator to be able to
- // examine its data structures. In tools that do not directly use Nano, we
- // won't have done this yet. nanov2_configure() runs the initialization
- // only once.
- nanov2_configure();
-
+nanov2_statistics(nanozonev2_t *nanozone, malloc_statistics_t *stats)
+{
memset(stats, '\0', sizeof(*stats));
- nanozonev2_t *mapped_nanozone;
- err = reader(task, (vm_address_t)zone_address, sizeof(nanozonev2_t),
- (void **)&mapped_nanozone);
- if (err) {
- printer("Failed to map nanozonev2_s at %p\n", zone_address);
- return err;
- }
-
nanov2_region_t *region;
- nanov2_arena_t *arena;
- nanov2_meta_index_t metadata_block_index =
- nanov2_metablock_meta_index(mapped_nanozone);
+ nanov2_arena_t * arena;
+ nanov2_meta_index_t metadata_block_index = nanov2_metablock_meta_index(nanozone);
// Iterate over each arena in each region. Within each region, add
// statistics for each slot in each block, excluding the meta data block.
- nanov2_arena_t *current_region_next_arena = os_atomic_load(
- &mapped_nanozone->current_region_next_arena, acquire);
- for (region = mapped_nanozone->first_region_base; region;) {
- nanov2_region_t *mapped_region;
- err = reader(task, (vm_address_t)region, sizeof(nanov2_region_t), (void **)&mapped_region);
- if (err) {
- printer("Failed to map nanov2 region at %p\n", region);
- return err;
- }
- off_t region_offset = (uintptr_t)mapped_region - (uintptr_t)region;
+ for (region = nanozone->first_region_base; region;
+ region = nanov2_next_region_for_region(nanozone, region)) {
for (arena = nanov2_first_arena_for_region(region);
- arena < nanov2_limit_arena_for_region(mapped_nanozone, region,
- current_region_next_arena);
+ arena < nanov2_limit_arena_for_region(nanozone, region);
arena++) {
nanov2_arena_metablock_t *meta_block =
- nanov2_metablock_address_for_ptr(mapped_nanozone, arena);
- nanov2_arena_metablock_t *mapped_meta_block =
- (nanov2_arena_metablock_t *)((uintptr_t)meta_block + region_offset);
+ nanov2_metablock_address_for_ptr(nanozone, arena);
for (nanov2_meta_index_t i = 0; i < NANOV2_BLOCKS_PER_ARENA; i++) {
if (i == metadata_block_index) {
// Skip the metadata block.
continue;
}
- nanov2_block_meta_t *mapped_block_metap = &mapped_meta_block->arena_block_meta[i];
+ nanov2_block_meta_t *block_metap = &meta_block->arena_block_meta[i];
nanov2_size_class_t size_class =
- nanov2_size_class_for_meta_index(mapped_nanozone, i);
+ nanov2_size_class_for_meta_index(nanozone, i);
int slot_size = nanov2_size_from_size_class(size_class);
- nanov2_block_meta_t meta =
- os_atomic_load(mapped_block_metap, relaxed);
+ nanov2_block_meta_t meta = os_atomic_load(block_metap, relaxed);
int slots_in_use = 0;
switch (meta.next_slot) {
case SLOT_NULL:
@@ -2008,8 +1668,6 @@
case SLOT_MADVISING:
// FALLTHRU
case SLOT_MADVISED:
- // FALLTHRU
- case SLOT_GUARD:
// These blocks have no active content.
break;
case SLOT_FULL:
@@ -2031,43 +1689,18 @@
}
}
}
- region = nanov2_next_region_for_region_offset(mapped_nanozone,
- region, region_offset, current_region_next_arena);
- }
- return KERN_SUCCESS;
-}
-
-static void
-nanov2_statistics_self(nanozonev2_t *nanozone, malloc_statistics_t *stats)
-{
- nanov2_statistics(mach_task_self(), (vm_address_t)nanozone,
- _malloc_default_reader, malloc_report_simple, stats);
-}
-
-static kern_return_t
-nanov2_statistics_task_printer(task_t task, vm_address_t zone_address,
- memory_reader_t reader, print_task_printer_t printer,
- malloc_statistics_t *stats)
-{
- return nanov2_statistics(task, zone_address, reader, printer, stats);
-}
-
-static kern_return_t
-nanov2_statistics_task(task_t task, vm_address_t zone_address, memory_reader_t reader, malloc_statistics_t *stats)
-{
- return nanov2_statistics(task, zone_address, reader, NULL, stats);
-}
-
+ }
+}
static const struct malloc_introspection_t nanov2_introspect = {
.enumerator = (void *)nanov2_ptr_in_use_enumerator,
.good_size = (void *)nanov2_good_size,
.check = (void *)nanov2_check,
- .print = (void *)nanov2_print_self,
+ .print = (void *)nanov2_print,
.log = (void *)nanov2_log,
.force_lock = (void *)nanov2_force_lock,
.force_unlock = (void *)nanov2_force_unlock,
- .statistics = (void *)nanov2_statistics_self,
+ .statistics = (void *)nanov2_statistics,
.zone_locked = (void *)nanov2_locked,
.enable_discharge_checking = NULL,
.disable_discharge_checking = NULL,
@@ -2077,8 +1710,6 @@
.enumerate_unavailable_without_blocks = NULL,
#endif // __BLOCKS__
.reinit_lock = (void *)nanov2_reinit_lock,
- .print_task = (void *)nanov2_print_task,
- .task_statistics = (void*)nanov2_statistics_task,
};
#endif // OS_VARIANT_NOTRESOLVED
@@ -2092,10 +1723,8 @@
// the allocation, or 0 if the pointer does not correspond to an active
// allocation. If allow_inner is true, the pointer need not point to the start
// of the allocation.
-MALLOC_ALWAYS_INLINE MALLOC_INLINE size_t
-nanov2_pointer_size_inline(nanozonev2_t *nanozone, void *ptr,
- boolean_t allow_inner, nanov2_size_class_t *size_class_out,
- nanov2_block_meta_t **block_metap_out)
+size_t
+nanov2_pointer_size(nanozonev2_t *nanozone, void *ptr, boolean_t allow_inner)
{
// First check the address signature.
if (!nanov2_has_valid_signature((void *)ptr)) {
@@ -2107,40 +1736,25 @@
return 0;
}
- // Atomically load the value of current_region_next_arena. No thread is
- // allowed to allocate from an arena until it observes a greater value of
- // current_region_next_arena, which must have happened before now if we're
- // being called in the context of a deallocation, so we can safely use it as
- // the upper bound for an overall address range check.
- nanov2_arena_t *current_region_next_arena = os_atomic_load(
- &nanozone->current_region_next_arena, relaxed);
-
// Bounds check against the active address space.
if (ptr < (void *)nanozone->first_region_base ||
- ptr > (void *)current_region_next_arena) {
+ ptr > (void *)nanozone->current_region_next_arena) {
return 0;
}
#if NANOV2_MULTIPLE_REGIONS
// Need to check that the region part is valid because there could be holes.
// Do this only if we know there is a hole.
- //
- // If we're looking at a legitimately-allocated nano pointer, a load-acquire
- // of current_region_next_arena must have already happened when its
- // containing arena was first allocated from, so any region_address_clashes
- // increment that preceded the store-release of current_region_next_arena
- // should be visible.
- //
- // TODO: use a hashed structure to make this more efficient.
- if (os_atomic_load(&nanozone->statistics.region_address_clashes, relaxed)) {
+ // NOTE: in M2 convergence, use a hashed structure to make this more
+ // efficient.
+ if (nanozone->statistics.region_address_clashes) {
nanov2_region_t *ptr_region = nanov2_region_address_for_ptr(ptr);
nanov2_region_t *region = nanozone->first_region_base;
while (region) {
if (ptr_region == region) {
break;
}
- region = nanov2_next_region_for_region(nanozone, region,
- current_region_next_arena);
+ region = nanov2_next_region_for_region(nanozone, region);
}
if (!region) {
// Reached the end of the region list without matching - not a
@@ -2177,42 +1791,27 @@
return 0;
}
- if (size_class_out) {
- *size_class_out = size_class;
- }
- if (block_metap_out) {
- *block_metap_out = block_metap;
- }
return size;
-}
-
-size_t
-nanov2_pointer_size(nanozonev2_t *nanozone, void *ptr, boolean_t allow_inner)
-{
- return nanov2_pointer_size_inline(nanozone, ptr, allow_inner, NULL, NULL);
}
#pragma mark -
#pragma mark Madvise Management
-// Given a pointer to a block and its metadata, calls madvise() on that block if
-// it is still in the state we expect, either SLOT_CAN_MADVISE or SLOT_MADVISED
-// (the latter expected when we need to pessimistically re-madvise a block we
-// may have touched while racing to allocate against a transition to
-// SLOT_CAN_MADVISE). Returns true on success, false if the block is not in the
-// correct state or if the state changed during the operation.
+// Given a pointer to a block and its metadata, calls madvise() on that block
+// if it is in state SLOT_CAN_MADVISE. Returns true on success, false if the
+// block is not in the correct state or if the state changed during the
+// operation.
//
// This function must be called with the zone's madvise_lock held
-static boolean_t
-nanov2_madvise_block_locked(nanozonev2_t *nanozone,
- nanov2_block_meta_t *block_metap, nanov2_block_t *blockp,
- nanov2_size_class_t size_class, uint32_t expected_state)
+boolean_t
+nanov2_madvise_block(nanozonev2_t *nanozone, nanov2_block_meta_t *block_metap,
+ nanov2_block_t *blockp, nanov2_size_class_t size_class)
{
_malloc_lock_assert_owner(&nanozone->madvise_lock);
boolean_t madvised = FALSE;
nanov2_block_meta_t old_meta = os_atomic_load(block_metap, relaxed);
- if (old_meta.next_slot == expected_state) {
+ if (old_meta.next_slot == SLOT_CAN_MADVISE) {
// Nobody raced with us. We can safely madvise this block. First change
// the state to SLOT_MADVISING so that other threads don't try to
// grab the block for new allocations.
@@ -2254,18 +1853,6 @@
return madvised;
}
-static void
-nanov2_madvise_block(nanozonev2_t *nanozone, nanov2_block_meta_t *block_metap,
- nanov2_size_class_t size_class, uint32_t expected_state)
-{
- nanov2_block_t *blockp = nanov2_block_address_from_meta_ptr(nanozone,
- block_metap);
- _malloc_lock_lock(&nanozone->madvise_lock);
- nanov2_madvise_block_locked(nanozone, block_metap, blockp, size_class,
- expected_state);
- _malloc_lock_unlock(&nanozone->madvise_lock);
-}
-
#endif // OS_VARIANT_RESOLVED
#pragma mark -
@@ -2296,62 +1883,45 @@
// Allocates a new region adjacent to the current one. If the allocation fails,
// keep sliding up by the size of a region until we either succeed or run out of
-// address space. The caller must own the Nanozone regions lock. Returns the
-// first arena of the newly-allocated region if successful, or NULL otherwise.
-MALLOC_NOEXPORT nanov2_arena_t *
+// address space. The caller must own the Nanozone regions lock.
+boolean_t
nanov2_allocate_new_region(nanozonev2_t *nanozone)
{
#if NANOV2_MULTIPLE_REGIONS
- bool allocated = false;
+ boolean_t result = FALSE;
_malloc_lock_assert_owner(&nanozone->regions_lock);
- nanov2_region_t *current_region = nanov2_current_region_base(
- os_atomic_load(&nanozone->current_region_next_arena, relaxed));
- nanov2_region_t *next_region = current_region + 1;
+ nanov2_region_t *current_region = nanozone->current_region_base;
+ nanov2_region_t *next_region = (nanov2_region_t *)nanozone->current_region_limit;
while ((void *)next_region <= nanov2_max_region_base.addr) {
if (nanov2_allocate_region(next_region)) {
+ nanozone->current_region_base = next_region;
+ nanozone->current_region_next_arena = (nanov2_arena_t *)next_region;
+ nanozone->current_region_limit = next_region + 1;
nanozone->statistics.allocated_regions++;
- allocated = true;
+ result = TRUE;
break;
}
next_region++;
-
- // Loaded atomically in nanov2_pointer_size() to determine whether or
- // not it's necessary to walk the region list, so we need to increment
- // atomically here. Published by the store-release of
- // current_region_next_arena.
- os_atomic_inc(&nanozone->statistics.region_address_clashes, relaxed);
- }
-
- if (!allocated) {
- return NULL;
- }
-
- // Link this region to the previous one.
- nanov2_region_linkage_t *current_region_linkage =
- nanov2_region_linkage_for_region(nanozone, current_region);
-
- // The linkage of the next region is in pristine memory, so already zero -
- // don't touch it.
-
- // Store-release the linkage update so any dependent loads through it
- // observe the (implicit zero-)initialization of the next region.
- uint16_t offset = next_region - current_region;
- os_atomic_store(¤t_region_linkage->next_region_offset, offset,
- release);
-
- // Store-release the update to current_region_next_arena to publish the
- // linkage update. Pairs with load-acquires of current_region_next_arena
- // followed by walks of the region list.
- nanov2_arena_t *first_arena = nanov2_first_arena_for_region(next_region);
- os_atomic_store(&nanozone->current_region_next_arena, first_arena + 1,
- release);
-
- return first_arena;
+ nanozone->statistics.region_address_clashes++;
+ }
+
+ if (result) {
+ // Link this region to the previous one.
+ nanov2_region_linkage_t *current_region_linkage =
+ nanov2_region_linkage_for_region(nanozone, current_region);
+ nanov2_region_linkage_t *next_region_linkage =
+ nanov2_region_linkage_for_region(nanozone, next_region);
+ uint16_t offset = next_region - current_region;
+ current_region_linkage->next_region_offset = offset;
+ next_region_linkage->next_region_offset = 0;
+ }
+
+ return result;
#else // NANOV2_MULTIPLE_REGIONS
// On iOS, only one region is supported, so we fail since the first
// region is allocated separately.
- return NULL;
+ return FALSE;
#endif // CONFIG_NANOV2_MULTIPLE_REGIONS
}
#endif // OS_VARIANT_NOTRESOLVED
@@ -2360,39 +1930,18 @@
#pragma mark Allocation
#if OS_VARIANT_RESOLVED
-
-MALLOC_NOINLINE MALLOC_NORETURN
-static void
-nanov2_guard_corruption_detected(void *corrupt_slot)
-{
- uint64_t guard = *(uint64_t *)corrupt_slot;
- malloc_zone_error(MALLOC_ABORT_ON_CORRUPTION, true,
- "Heap corruption detected, free list is damaged at %p\n"
- "*** Incorrect guard value: %lu\n", corrupt_slot, guard);
- __builtin_unreachable();
-}
// Allocates memory from the block that corresponds to a given block meta data
// pointer. The memory is taken from the free list if possible, or from the
// unused region of the block if not. If the block is no longer in use or is
// full, NULL is returned and the caller is expected to find another block to
// allocate from.
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
void *
-nanov2_allocate_from_block_inline(nanozonev2_t *nanozone,
- nanov2_block_meta_t *block_metap, nanov2_size_class_t size_class,
- nanov2_block_meta_t **madvise_block_metap_out, bool *corruption)
+nanov2_allocate_from_block(nanozonev2_t *nanozone,
+ nanov2_block_meta_t *block_metap, nanov2_size_class_t size_class)
{
nanov2_block_meta_view_t old_meta_view;
-
- // Our loads of the block metadata use dependency ordering, which guarantees
- // that any loads we do from a slot pointer derived from the metadata value
- // as we do below will observe all of the stores preceding the store-release
- // of that value we observed. This allows us to safely rely on the contents
- // of the slot updated when it was last freed, including the double-free
- // guard and zeroing done by zero-on-free (which is required for correctness
- // in the case of calloc).
- old_meta_view.meta = os_atomic_load(block_metap, dependency);
+ old_meta_view.meta = os_atomic_load(block_metap, relaxed);
// Calculating blockp and ptr is relatively expensive. Do both lazily to
// minimize the time in the block starting with "again:" and ending with the
@@ -2442,8 +1991,21 @@
if (old_meta_view.meta.next_slot == SLOT_CAN_MADVISE ||
old_meta_view.meta.next_slot == SLOT_MADVISING ||
old_meta_view.meta.next_slot == SLOT_MADVISED) {
- *madvise_block_metap_out = block_metap;
- return NULL;
+ _malloc_lock_lock(&nanozone->madvise_lock);
+ if (old_meta_view.meta.next_slot == SLOT_MADVISED) {
+ // We raced against another thread madvising this block. We need
+ // to redo the madvise because we may have touched it when
+ // reading the next pointer in the freelist.
+ if (!blockp) {
+ blockp = nanov2_block_address_from_meta_ptr(nanozone, block_metap);
+ }
+ if (mvm_madvise_free(nanozone, nanov2_region_address_for_ptr(blockp),
+ (uintptr_t)blockp, (uintptr_t)(blockp + 1), NULL, FALSE)) {
+ malloc_zone_error(0, false,
+ "Failed to remadvise block at blockp: %p, error: %d\n", blockp, errno);
+ }
+ }
+ _malloc_lock_unlock(&nanozone->madvise_lock);
}
goto again;
}
@@ -2455,40 +2017,30 @@
ptr = nanov2_slot_in_block_ptr(blockp, size_class, slot);
}
+ nanov2_free_slot_t *slotp =
+ (nanov2_free_slot_t *)os_atomic_force_dependency_on(ptr,
+ (unsigned long)old_meta_view.bits);
if (from_free_list) {
// We grabbed the item from the free list. Check the free list canary
// and crash if it's not valid. We can't do this check before the
// cmpxchgv because another thread may race with us, claim the slot and
// write to it.
- nanov2_free_slot_t *slotp = ptr;
uintptr_t guard = os_atomic_load(&slotp->double_free_guard, relaxed);
- if (os_unlikely((guard ^ nanozone->slot_freelist_cookie) != (uintptr_t)ptr)) {
- *corruption = true;
- }
- }
-
+ if ((guard ^ nanozone->slot_freelist_cookie) != (uintptr_t)ptr) {
+ malloc_zone_error(MALLOC_ABORT_ON_CORRUPTION, false,
+ "Heap corruption detected, free list is damaged at %p\n"
+ "*** Incorrect guard value: %lu\n", ptr, guard);
+ __builtin_unreachable();
+ }
+ }
+
+ // Reset the canary value so that the slot no longer looks free.
+ os_atomic_store(&slotp->double_free_guard, 0, relaxed);
+
#if DEBUG_MALLOC
nanozone->statistics.size_class_statistics[size_class].total_allocations++;
#endif // DEBUG_MALLOC
- return ptr;
-}
-
-static void *
-nanov2_allocate_from_block(nanozonev2_t *nanozone,
- nanov2_block_meta_t *block_metap, nanov2_size_class_t size_class)
-{
- nanov2_block_meta_t *madvise_block_metap = NULL;
- bool corruption = false;
- void *ptr = nanov2_allocate_from_block_inline(nanozone, block_metap,
- size_class, &madvise_block_metap, &corruption);
- if (os_unlikely(corruption)) {
- nanov2_guard_corruption_detected(ptr);
- }
- if (madvise_block_metap) {
- nanov2_madvise_block(nanozone, madvise_block_metap, size_class,
- SLOT_MADVISED);
- }
return ptr;
}
@@ -2703,7 +2255,7 @@
//
// In order to avoid races, this function must be called with the
// current_block_lock for the calling context [CPU] and size class locked.
-MALLOC_NOEXPORT MALLOC_NOINLINE void *
+MALLOC_NOINLINE void *
nanov2_find_block_and_allocate(nanozonev2_t *nanozone,
nanov2_size_class_t size_class, nanov2_block_meta_t **block_metapp)
{
@@ -2723,13 +2275,8 @@
start_region = nanov2_region_address_for_ptr(arena);
nanov2_arena_t *start_arena = arena;
nanov2_region_t *region = start_region;
- // The load-acquire pairs with store-release in nanov2_allocate_new_region()
- // to make the most recent region linkage update visible when we load it in
- // nanov2_next_region_for_region() below.
- nanov2_arena_t *initial_region_next_arena = os_atomic_load(
- &nanozone->current_region_next_arena, acquire);
- nanov2_arena_t *limit_arena = nanov2_limit_arena_for_region(nanozone,
- start_region, initial_region_next_arena);
+ nanov2_arena_t *limit_arena = nanov2_limit_arena_for_region(nanozone, start_region);
+ nanov2_arena_t *initial_region_next_arena = nanozone->current_region_next_arena;
do {
nanov2_block_meta_t *block_metap = nanov2_find_block_in_arena(nanozone,
arena, size_class, start_block);
@@ -2765,15 +2312,13 @@
start_block = NULL;
arena++;
if (arena >= limit_arena) {
- region = nanov2_next_region_for_region(nanozone, region,
- initial_region_next_arena);
+ region = nanov2_next_region_for_region(nanozone, region);
if (!region) {
// Reached the last region -- loop back to the first.
region = nanozone->first_region_base;
}
arena = nanov2_first_arena_for_region(region);
- limit_arena = nanov2_limit_arena_for_region(nanozone, region,
- initial_region_next_arena);
+ limit_arena = nanov2_limit_arena_for_region(nanozone, region);
}
} while (arena != start_arena);
@@ -2784,42 +2329,24 @@
}
// Allocate a new arena and maybe a new region. To do either of those
- // things, we need to take the regions_lock. After doing so, check that the
- // state is unchanged. If it has, just assume that we might have some new
- // space to allocate into and try again.
-
+ // things, we need to take the regions_lock. After doing so, check that
+ // the state is unchanged. If it has, just assume that we might have some
+ // new space to allocate into and try again.
boolean_t failed = FALSE;
-
+ arena = initial_region_next_arena;
_malloc_lock_lock(&nanozone->regions_lock);
- nanov2_arena_t *current_region_next_arena = os_atomic_load(
- &nanozone->current_region_next_arena, relaxed);
- if (current_region_next_arena == initial_region_next_arena) {
- if (nanov2_current_region_next_arena_is_limit(
- current_region_next_arena)) {
+ if (nanozone->current_region_next_arena == arena) {
+ if ((void *)arena >= nanozone->current_region_limit) {
// Reached the end of the region. Allocate a new one, if we can.
- arena = nanov2_allocate_new_region(nanozone);
- if (!arena) {
+ if (nanov2_allocate_new_region(nanozone)) {
+ arena = nanozone->current_region_next_arena++;
+ } else {
failed = TRUE;
}
} else {
- // Assign the new arena, in the current region.
- arena = current_region_next_arena;
-
- // Bump current_region_next_arena by 1. No need for an atomic add
- // because we're under the regions_lock.
- os_atomic_store(&nanozone->current_region_next_arena,
- current_region_next_arena + 1, relaxed);
- }
-
- // Set up the guard blocks for the new arena, if requested
- if (!failed) {
- nanov2_init_guard_blocks(nanozone, arena);
- }
- } else {
- // The arena just before current_region_next_arena is always the most
- // recently allocated arena. Let's retry from that arena, which was
- // allocated in the time since we started our last try.
- arena = current_region_next_arena - 1;
+ // Assign the new arena, in the same region.
+ nanozone->current_region_next_arena = arena + 1;
+ }
}
_malloc_lock_unlock(&nanozone->regions_lock);
@@ -2836,36 +2363,39 @@
return NULL;
}
-// This function is called when a fast-path allocation from a given (size_class,
-// allocation_index) has been tried and failed, and we need to act on
-// observations from that attempt and/or retry the allocation. Its rather
-// tortured calling contract is designed to allow the caller to avoid pushing a
-// frame and pass along as much of what it has already computed as possible.
+// Allocates memory of a given size (which must be a multiple of the Nano
+// quantum size) and optionally clears it (for calloc).
//
-// If @corrupt_slot is non-NULL it means we detected corruption of the slot's
-// guard on the fast path, and we need to report that corruption.
-//
-// If @madvise_block_metap is non-NULL it means we raced with another thread
-// madvising the block we tried to allocate from and need to re-madvise it.
+// Allocation is attempted first from the block last used for the caller's
+// context (which is initially the physical CPU by default). If there is no
+// last block, or the block is full or now out of use, find another one, if
+// possible. See the comments for nanov2_get_allocation_block() for the details.
//
// If the allocation fails, NULL is returned.
-static void *
-nanov2_allocate_outlined(nanozonev2_t *nanozone, nanov2_block_meta_t **block_metapp,
- size_t rounded_size, nanov2_size_class_t size_class,
- int allocation_index, nanov2_block_meta_t *madvise_block_metap,
- void *corrupt_slot, bool clear)
+void *
+nanov2_allocate(nanozonev2_t *nanozone, size_t rounded_size, boolean_t clear)
{
void *ptr = NULL;
-
- if (os_unlikely(corrupt_slot)) {
- nanov2_guard_corruption_detected(corrupt_slot);
- }
-
- // If we need to re-madvise the old block that we might have accidentally
- // touched out of turn, do so now.
- if (madvise_block_metap) {
- nanov2_madvise_block(nanozone, madvise_block_metap, size_class,
- SLOT_MADVISED);
+ nanov2_size_class_t size_class = nanov2_size_class_from_size(rounded_size);
+ MALLOC_ASSERT(size_class < NANO_SIZE_CLASSES);
+ MALLOC_ASSERT(rounded_size != 0);
+ nanov2_block_meta_t *block_metap;
+ nanov2_block_meta_t **block_metapp;
+
+ // Get the index of the pointer to the block from which we are should be
+ // allocating. This currently depends on the physical CPU number.
+ int allocation_index = nanov2_get_allocation_block_index() & MAX_CURRENT_BLOCKS_MASK;
+
+ // Get the current allocation block meta data pointer. If this is NULL,
+ // we need to find a new allocation block.
+ block_metapp = &nanozone->current_block[size_class][allocation_index];
+ block_metap = os_atomic_load(block_metapp, relaxed);
+ if (block_metap) {
+ // Fast path: we have a block -- try to allocate from it.
+ ptr = nanov2_allocate_from_block(nanozone, block_metap, size_class);
+ if (ptr) {
+ goto done;
+ }
}
// No current allocation block, or we were unable to allocate. We need to
@@ -2882,7 +2412,7 @@
_malloc_lock_s *lock = &nanozone->current_block_lock[size_class][allocation_index];
_malloc_lock_lock(lock);
- nanov2_block_meta_t *block_metap = os_atomic_load(block_metapp, relaxed);
+ block_metap = os_atomic_load(block_metapp, relaxed);
if (block_metap) {
ptr = nanov2_allocate_from_block(nanozone, block_metap, size_class);
if (ptr) {
@@ -2904,37 +2434,21 @@
// We could not find a block to allocate from -- make future
// allocations for this size class go to the helper zone until
// we have enough free space.
- os_atomic_or(&nanozone->delegate_allocations,
- (uint16_t)(1 << size_class), relaxed);
-
- ptr = nanozone->helper_zone->malloc(nanozone->helper_zone, rounded_size);
+ _malloc_lock_lock(&nanozone->delegate_allocations_lock);
+ nanozone->delegate_allocations |= 1 << size_class;
+ _malloc_lock_unlock(&nanozone->delegate_allocations_lock);
}
done:
- if (os_likely(ptr)) {
- if (malloc_zero_on_free) {
+ if (ptr) {
+ if (clear) {
+ memset(ptr, '\0', rounded_size);
+ } else {
// Always clear the double-free guard so that we can recognize that
// this block is not on the free list.
nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
os_atomic_store(&slotp->double_free_guard, 0, relaxed);
-
- // We know the body of the allocation is already clear, so we just
- // need to clean up the next_slot word to get to all-zero. Do so in
- // all cases, even if a cleared allocation is not requested, to
- // prevent any leakage through the next_slot bits.
- os_atomic_store(&slotp->next_slot, 0, relaxed);
- } else {
- if (clear) {
- memset(ptr, '\0', rounded_size);
- } else {
- // Always clear the double-free guard so that we can recognize that
- // this block is not on the free list.
- nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
- os_atomic_store(&slotp->double_free_guard, 0, relaxed);
- }
- }
- } else {
- malloc_set_errno_fast(MZ_POSIX, ENOMEM);
+ }
}
return ptr;
}
@@ -2943,32 +2457,20 @@
#pragma mark Freeing
// Frees an allocation to its owning block and updates the block's state.
-//
-// If the block becomes empty, it is marked as SLOT_CAN_MADVISE and we return
-// the block to the caller to madvise if dictated by policy.
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-nanov2_block_meta_t *
-nanov2_free_to_block_inline(nanozonev2_t *nanozone, void *ptr,
- nanov2_size_class_t size_class, nanov2_block_meta_t *block_metap)
+// If the block becomes empty, it is marked as SLOT_CAN_MADVISE and is
+// madvised immediately if the policy is NANO_MADVISE_IMMEDIATE.
+void
+nanov2_free_to_block(nanozonev2_t *nanozone, void *ptr,
+ nanov2_size_class_t size_class)
{
nanov2_block_t *blockp = nanov2_block_address_for_ptr(ptr);
- if (!block_metap) {
- block_metap = nanov2_meta_ptr_for_ptr(nanozone, ptr);
- }
+ nanov2_block_meta_t *block_metap = nanov2_meta_ptr_for_ptr(nanozone, ptr);
// Release the slot memory onto the block's freelist.
nanov2_block_meta_t old_meta = os_atomic_load(block_metap, relaxed);
int slot_count = slots_by_size_class[size_class];
nanov2_block_meta_t new_meta;
boolean_t was_full;
-
- nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
- // All of the free slot content (double_free_guard, next_slot word and the
- // zeroed remainder of the slot) must be visible when the os_atomic_cmpxchgv
- // completes, so the metadata updates on either path below need a release
- // barrier.
- os_atomic_store(&slotp->double_free_guard,
- nanozone->slot_freelist_cookie ^ (uintptr_t)ptr, relaxed);
again:
was_full = old_meta.next_slot == SLOT_FULL;
@@ -2978,14 +2480,12 @@
boolean_t freeing_last_active_slot = !was_full &&
new_meta.free_count == slots_by_size_class[size_class] - 1;
if (freeing_last_active_slot) {
- os_atomic_store(&slotp->next_slot, SLOT_NULL, relaxed);
-
// Releasing the last active slot onto the free list. Mark the block as
// ready to be madvised if it's not in use, otherwise reset next_slot
// to SLOT_BUMP.
new_meta.next_slot = new_meta.in_use ? SLOT_BUMP : SLOT_CAN_MADVISE;
// Write the updated meta data; try again if we raced with another thread.
- if (!os_atomic_cmpxchgv(block_metap, old_meta, new_meta, &old_meta, release)) {
+ if (!os_atomic_cmpxchgv(block_metap, old_meta, new_meta, &old_meta, relaxed)) {
goto again;
}
@@ -2993,14 +2493,20 @@
// is to do so immediately.
if (new_meta.next_slot == SLOT_CAN_MADVISE &&
nanov2_madvise_policy == NANO_MADVISE_IMMEDIATE) {
- return block_metap;
+ _malloc_lock_lock(&nanozone->madvise_lock);
+ nanov2_madvise_block(nanozone, block_metap, blockp, size_class);
+ _malloc_lock_unlock(&nanozone->madvise_lock);
}
} else {
int slot_index = nanov2_slot_index_in_block(blockp, size_class, ptr);
new_meta.next_slot = slot_index + 1; // meta.next_slot is 1-based
- os_atomic_store(&slotp->next_slot,
- was_full ? SLOT_BUMP : old_meta.next_slot, relaxed);
-
+ nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
+ slotp->next_slot = was_full ? SLOT_BUMP : old_meta.next_slot;
+ os_atomic_store(&slotp->double_free_guard,
+ nanozone->slot_freelist_cookie ^ (uintptr_t)ptr, relaxed);
+
+ // The double_free_guard change must be visible when the os_atomic_cmpxchgv
+ // completes.
// Write the updated meta data; try again if we raced with another thread.
if (!os_atomic_cmpxchgv(block_metap, old_meta, new_meta, &old_meta, release)) {
goto again;
@@ -3014,14 +2520,14 @@
uint16_t class_mask = 1 << size_class;
if (!new_meta.in_use && (nanozone->delegate_allocations & class_mask) &&
(new_meta.free_count >= 0.75 * slot_count)) {
- os_atomic_and(&nanozone->delegate_allocations, ~class_mask, relaxed);
+ _malloc_lock_lock(&nanozone->delegate_allocations_lock);
+ nanozone->delegate_allocations &= ~class_mask;
+ _malloc_lock_unlock(&nanozone->delegate_allocations_lock);
}
#if DEBUG_MALLOC
nanozone->statistics.size_class_statistics[size_class].total_frees++;
#endif // DEBUG_MALLOC
-
- return NULL;
}
#endif // OS_VARIANT_RESOLVED
@@ -3035,10 +2541,10 @@
nanov2_create_zone(malloc_zone_t *helper_zone, unsigned debug_flags)
{
// Note: It is important that nanov2_create_zone resets _malloc_engaged_nano
- // if it is unable to enable the nanozone (and chooses not to abort). As
+ // if it is unable to enable the nanozone (and chooses not to abort). As
// several functions rely on _malloc_engaged_nano to determine if they
// should manipulate the nanozone, and these should not run if we failed
- // to create the zone.
+ // to create the zone.
MALLOC_ASSERT(_malloc_engaged_nano == NANO_V2);
// Get memory for the zone and disable Nano if we fail.
@@ -3050,7 +2556,7 @@
}
// Set up the basic_zone portion of the nanozonev2 structure
- nanozone->basic_zone.version = 13;
+ nanozone->basic_zone.version = 10;
nanozone->basic_zone.size = OS_RESOLVED_VARIANT_ADDR(nanov2_size);
nanozone->basic_zone.malloc = OS_RESOLVED_VARIANT_ADDR(nanov2_malloc);
nanozone->basic_zone.calloc = OS_RESOLVED_VARIANT_ADDR(nanov2_calloc);
@@ -3062,11 +2568,10 @@
nanozone->basic_zone.batch_free = OS_RESOLVED_VARIANT_ADDR(nanov2_batch_free);
nanozone->basic_zone.introspect =
(struct malloc_introspection_t *)&nanov2_introspect;
- nanozone->basic_zone.memalign = OS_RESOLVED_VARIANT_ADDR(nanov2_memalign);
+ nanozone->basic_zone.memalign = (void *)nanov2_memalign;
nanozone->basic_zone.free_definite_size = OS_RESOLVED_VARIANT_ADDR(nanov2_free_definite_size);
nanozone->basic_zone.pressure_relief = OS_RESOLVED_VARIANT_ADDR(nanov2_pressure_relief);
nanozone->basic_zone.claimed_address = OS_RESOLVED_VARIANT_ADDR(nanov2_claimed_address);
- nanozone->basic_zone.try_free_default = OS_RESOLVED_VARIANT_ADDR(nanov2_try_free_default);
// Set these both to zero as required by CFAllocator.
nanozone->basic_zone.reserved1 = 0;
@@ -3074,6 +2579,12 @@
// Prevent overwriting the function pointers in basic_zone.
mprotect(nanozone, sizeof(nanozone->basic_zone), PROT_READ);
+
+ // Nano V2 zone does not support MALLOC_ADD_GUARD_PAGES
+ if (debug_flags & MALLOC_ADD_GUARD_PAGES) {
+ malloc_report(ASL_LEVEL_INFO, "nano does not support guard pages\n");
+ debug_flags &= ~MALLOC_ADD_GUARD_PAGES;
+ }
// Set up the remainder of the nanozonev2 structure
nanozone->debug_flags = debug_flags;
@@ -3099,7 +2610,7 @@
// align it to the block field of a Nano address.
nanozone->aslr_cookie = malloc_entropy[1] >> (64 - NANOV2_BLOCK_BITS);
nanozone->aslr_cookie_aligned = nanozone->aslr_cookie << NANOV2_OFFSET_BITS;
-
+
_malloc_lock_init(&nanozone->blocks_lock);
_malloc_lock_init(&nanozone->regions_lock);
_malloc_lock_init(&nanozone->madvise_lock);
@@ -3117,16 +2628,14 @@
}
nanov2_region_linkage_t *region_linkage =
nanov2_region_linkage_for_region(nanozone, region);
- os_atomic_store(®ion_linkage->next_region_offset, 0, relaxed);
+ region_linkage->next_region_offset = 0;
// Install the first region and pre-allocate the first arena.
nanozone->first_region_base = region;
- os_atomic_store(&nanozone->current_region_next_arena,
- ((nanov2_arena_t *)region) + 1, release);
+ nanozone->current_region_base = region;
+ nanozone->current_region_next_arena = ((nanov2_arena_t *)region) + 1;
+ nanozone->current_region_limit = region + 1;
nanozone->statistics.allocated_regions = 1;
-
- // Set up the guard blocks for the initial arena, if requested
- nanov2_init_guard_blocks(nanozone, (nanov2_arena_t *)region);
return (malloc_zone_t *)nanozone;
}
@@ -3143,7 +2652,7 @@
// leak, but this is better than possibly crashing.
#if OS_VARIANT_RESOLVED
-MALLOC_NOEXPORT void *
+void *
nanov2_forked_malloc(nanozonev2_t *nanozone, size_t size)
{
// Just hand to the helper zone.
@@ -3161,19 +2670,11 @@
size);
}
-static void *
-nanov2_forked_memalign(nanozonev2_t *nanozone, size_t alignment, size_t size)
-{
- // Just hand to the helper zone.
- return nanozone->helper_zone->memalign(nanozone->helper_zone, alignment,
- size);
-}
-
#endif // OS_VARIANT_NOTRESOLVED
#if OS_VARIANT_RESOLVED
-MALLOC_NOEXPORT void
+void
nanov2_forked_free(nanozonev2_t *nanozone, void *ptr)
{
if (!ptr) {
@@ -3195,13 +2696,13 @@
/* NOTREACHED */
}
-MALLOC_NOEXPORT void
+void
nanov2_forked_free_definite_size(nanozonev2_t *nanozone, void *ptr, size_t size)
{
nanov2_forked_free(nanozone, ptr);
}
-MALLOC_NOEXPORT void *
+void *
nanov2_forked_realloc(nanozonev2_t *nanozone, void *ptr, size_t new_size)
{
// could occur through malloc_zone_realloc() path
@@ -3259,7 +2760,7 @@
#if OS_VARIANT_RESOLVED
-MALLOC_NOEXPORT void
+void
nanov2_forked_batch_free(nanozonev2_t *nanozone, void **to_be_freed,
unsigned count)
{
@@ -3307,11 +2808,10 @@
nanozone->basic_zone.batch_free = OS_RESOLVED_VARIANT_ADDR(nanov2_forked_batch_free);
nanozone->basic_zone.introspect =
(struct malloc_introspection_t *)&nanov2_introspect;// Unchanged
- nanozone->basic_zone.memalign = (void *)nanov2_forked_memalign;
+ nanozone->basic_zone.memalign = (void *)nanov2_memalign; // Unchanged
nanozone->basic_zone.free_definite_size =
OS_RESOLVED_VARIANT_ADDR(nanov2_forked_free_definite_size);
nanozone->basic_zone.claimed_address = nanov2_forked_claimed_address;
- nanozone->basic_zone.try_free_default = NULL; // Fall back to old protocol
mprotect(nanozone, sizeof(nanozone->basic_zone), PROT_READ);
}