Loading...
--- /dev/null
+++ libmalloc/libmalloc-474.0.13/src/nanov2_malloc.c
@@ -0,0 +1,3405 @@
+/*
+ * Copyright (c) 2018 Apple Inc. All rights reserved.
+ *
+ * @APPLE_LICENSE_HEADER_START@
+ *
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this
+ * file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_LICENSE_HEADER_END@
+ */
+#include "resolver.h"
+#include "internal.h"
+
+#if CONFIG_NANOZONE
+
+#pragma mark -
+#pragma mark Forward Declarations
+
+#if OS_VARIANT_NOTRESOLVED
+static kern_return_t
+nanov2_statistics_task_printer(task_t task, vm_address_t zone_address,
+ memory_reader_t reader, print_task_printer_t printer,
+ malloc_statistics_t *stats);
+static kern_return_t
+nanov2_statistics_task(task_t task, vm_address_t zone_address,
+ memory_reader_t reader, malloc_statistics_t *stats);
+#endif // OS_VARIANT_NOTRESOLVED
+
+#pragma mark -
+#pragma mark Externals for resolved functions
+
+MALLOC_NOEXPORT extern size_t nanov2_pointer_size(nanozonev2_t *nanozone, void *ptr,
+ boolean_t allow_inner);
+MALLOC_NOEXPORT extern size_t nanov2_pressure_relief(nanozonev2_t *nanozone, size_t goal);
+
+#if OS_VARIANT_RESOLVED
+MALLOC_ALWAYS_INLINE MALLOC_INLINE size_t
+nanov2_pointer_size_inline(nanozonev2_t *nanozone, void *ptr, boolean_t allow_inner,
+ nanov2_size_class_t *size_class_out, nanov2_block_meta_t **block_metap_out);
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE void *
+nanov2_allocate_from_block_inline(nanozonev2_t *nanozone,
+ nanov2_block_meta_t *block_metap, nanov2_size_class_t size_class,
+ nanov2_block_meta_t **madvise_block_metapp_out, bool *corruption);
+
+static void *
+nanov2_allocate_outlined(nanozonev2_t *nanozone,
+ nanov2_block_meta_t **block_metapp, size_t rounded_size,
+ nanov2_size_class_t size_class, int allocation_index,
+ nanov2_block_meta_t *madvise_block_metap, void *corrupt_slot,
+ bool clear);
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_meta_t *
+nanov2_free_to_block_inline(nanozonev2_t *nanozone, void *ptr,
+ nanov2_size_class_t size_class, nanov2_block_meta_t *block_metap);
+
+static boolean_t nanov2_madvise_block_locked(
+ nanozonev2_t *nanozone, nanov2_block_meta_t *block_metap,
+ nanov2_block_t *blockp, nanov2_size_class_t size_class, uint32_t expected_state);
+static void nanov2_madvise_block(nanozonev2_t *nanozone,
+ nanov2_block_meta_t *block_metap, nanov2_size_class_t size_class,
+ uint32_t expected_state);
+
+MALLOC_NOEXPORT extern nanov2_arena_t *nanov2_allocate_new_region(nanozonev2_t *nanozone);
+#endif // OS_VARIANT_RESOLVED
+
+#pragma mark -
+#pragma mark Global Allocator State
+
+// -- Block scanning
+typedef enum {
+ NANO_SCAN_FIRST_FIT = 0,
+ NANO_SCAN_CAPACITY_BASED,
+} nanov2_block_scan_policy_t;
+
+// Minimum occupancy percentage for an ideal block.
+#define DEFAULT_SCAN_MIN_CAPACITY 20
+
+// Maximum occupancy percentage for an ideal block.
+#define DEFAULT_SCAN_MAX_CAPACITY 80
+
+// Maximum number of blocks to scan while looking for a best fit once a
+// candidate block has been found. Value 0 means no limit.
+#define DEFAULT_SCAN_LIMIT 10
+
+// -- Madvise policy
+typedef enum {
+ NANO_MADVISE_IMMEDIATE = 0,
+ NANO_MADVISE_WARNING_PRESSURE,
+ NANO_MADVISE_CRITICAL_PRESSURE,
+} nanov2_madvise_policy_t;
+
+typedef struct nanov2_policy_config_s {
+ // Determines the algorithm for scanning for the next allocation block.
+ // Used in conjunction with nanov2_block_scan_capacity_min,
+ // nanov2_block_scan_capacity_max and nanov2_block_scan_limit. Set from the
+ // MallocNanoScanPolicy environment variable.
+ nanov2_block_scan_policy_t block_scan_policy;
+
+ // Minimum occupancy percentage for an ideal block.
+ int block_scan_min_capacity;
+
+ // Maximum occupancy percentage for an ideal block.
+ int block_scan_max_capacity;
+
+ // Maximum number of blocks to scan while looking for a best fit once a
+ // candidate block has been found. Value 0 means no limit.
+ int block_scan_limit;
+
+ // Bitmask for size classes that are only allowed a single arena. Set from
+ // the MallocNanoSingleArena environment variable.
+ uint16_t single_arena_size_classes;
+
+ // Madvise policy. Set from the MallocNanoMadvisePolicy environment variable
+ nanov2_madvise_policy_t madvise_policy;
+} nanov2_policy_config_t;
+
+#if OS_VARIANT_NOTRESOLVED
+
+// Madvise policy. Set from the MallocNanoMadvisePolicy environment variable.
+MALLOC_NOEXPORT nanov2_madvise_policy_t nanov2_madvise_policy;
+
+MALLOC_NOEXPORT nanov2_policy_config_t nanov2_policy_config = {
+ .block_scan_policy = NANO_SCAN_CAPACITY_BASED,
+ .block_scan_min_capacity = DEFAULT_SCAN_MIN_CAPACITY,
+ .block_scan_max_capacity = DEFAULT_SCAN_MAX_CAPACITY,
+ .block_scan_limit = DEFAULT_SCAN_LIMIT,
+ .single_arena_size_classes = 0,
+ .madvise_policy = NANO_MADVISE_IMMEDIATE,
+};
+
+#else // OS_VARIANT_NOTRESOLVED
+
+MALLOC_NOEXPORT extern nanov2_policy_config_t nanov2_policy_config;
+MALLOC_NOEXPORT extern nanov2_madvise_policy_t nanov2_madvise_policy;
+
+#endif // OS_VARIANT_NOTRESOLVED
+
+// BLOCKS_PER_UNIT must be a power of two to make it possible to get the size
+// class from a pointer reasonably cheaply. Do not change the value without
+// fixing the code that depends on it.
+#define BLOCKS_PER_UNIT_SHIFT 6
+#define BLOCKS_PER_UNIT (1 << BLOCKS_PER_UNIT_SHIFT)
+
+#if OS_VARIANT_NOTRESOLVED
+// Number of units of each size class in an arena. The numbers here must add
+// up to 64. One unit corresponds to BLOCKS_PER_UNIT blocks in the corresponding
+// size class, so 64 units maps to a total of 64 * 64 = 4096 blocks and each
+// block is 16K, making a total of 64MB, which is the size of an arena.
+static int block_units_by_size_class[] = {
+ 2, // 16-byte allocations (less 1 for the metadata block)
+ 10, // 32-byte allocations
+ 11, // 48-byte allocations
+ 10, // 64-byte allocations
+ 5, // 80-byte allocations
+ 3, // 96-byte allocations
+ 3, // 112-byte allocations
+ 4, // 128-byte allocations
+ 3, // 144-byte allocations
+ 2, // 160-byte allocations
+ 2, // 176-byte allocations
+ 2, // 192-byte allocations
+ 2, // 208-byte allocations
+ 2, // 224-byte allocations
+ 1, // 240-byte allocations
+ 2, // 256-byte allocations
+};
+
+MALLOC_STATIC_ASSERT(
+ sizeof(block_units_by_size_class)/sizeof(block_units_by_size_class[0])
+ == NANO_SIZE_CLASSES,
+ "Size of block_units_by_size_class is incorrect");
+
+// Total of the number of blocks in all size classes. Currently this is 64.
+#define TOTAL_BLOCK_UNITS (NANOV2_BLOCKS_PER_ARENA/BLOCKS_PER_UNIT)
+
+// Offsets to the first and last blocks for each size class within an arena, in
+// the logical address space. These tables are constructed from the values in
+// the block_units_by_size_class table.
+MALLOC_NOEXPORT int first_block_offset_by_size_class[NANO_SIZE_CLASSES];
+MALLOC_NOEXPORT int last_block_offset_by_size_class[NANO_SIZE_CLASSES];
+
+// Table mapping the part of a logical address that depends on size class to
+// the size class. Also built from the block_units_by_size_class table.
+MALLOC_NOEXPORT int ptr_offset_to_size_class[TOTAL_BLOCK_UNITS];
+
+// Number of slots in a block, indexed by size class. Note that there is a small
+// amount of wastage in some size classes because the block size is not always
+// exactly divisible by the allocation size. The number of wasted bytes is shown
+// in parentheses in the comments below.
+MALLOC_NOEXPORT const int slots_by_size_class[] = {
+ NANOV2_BLOCK_SIZE/(1 * NANO_REGIME_QUANTA_SIZE), // 16 bytes: 1024 (0)
+ NANOV2_BLOCK_SIZE/(2 * NANO_REGIME_QUANTA_SIZE), // 32 bytes: 512 (0)
+ NANOV2_BLOCK_SIZE/(3 * NANO_REGIME_QUANTA_SIZE), // 48 bytes: 341 (16)
+ NANOV2_BLOCK_SIZE/(4 * NANO_REGIME_QUANTA_SIZE), // 64 bytes: 256 (0)
+ NANOV2_BLOCK_SIZE/(5 * NANO_REGIME_QUANTA_SIZE), // 80 bytes: 204 (64)
+ NANOV2_BLOCK_SIZE/(6 * NANO_REGIME_QUANTA_SIZE), // 96 bytes: 170 (64)
+ NANOV2_BLOCK_SIZE/(7 * NANO_REGIME_QUANTA_SIZE), // 112 bytes: 146 (32)
+ NANOV2_BLOCK_SIZE/(8 * NANO_REGIME_QUANTA_SIZE), // 128 bytes: 128 (0)
+ NANOV2_BLOCK_SIZE/(9 * NANO_REGIME_QUANTA_SIZE), // 144 bytes: 113 (112)
+ NANOV2_BLOCK_SIZE/(10 * NANO_REGIME_QUANTA_SIZE), // 160 bytes: 102 (64)
+ NANOV2_BLOCK_SIZE/(11 * NANO_REGIME_QUANTA_SIZE), // 176 bytes: 93 (16)
+ NANOV2_BLOCK_SIZE/(12 * NANO_REGIME_QUANTA_SIZE), // 192 bytes: 85 (64)
+ NANOV2_BLOCK_SIZE/(13 * NANO_REGIME_QUANTA_SIZE), // 208 bytes: 78 (160)
+ NANOV2_BLOCK_SIZE/(14 * NANO_REGIME_QUANTA_SIZE), // 224 bytes: 73 (32)
+ NANOV2_BLOCK_SIZE/(15 * NANO_REGIME_QUANTA_SIZE), // 240 bytes: 68 (64)
+ NANOV2_BLOCK_SIZE/(16 * NANO_REGIME_QUANTA_SIZE), // 256 bytes: 64 (0)
+};
+#else // OS_VARIANT_NOTRESOLVED
+
+extern int block_units_by_size_class[];
+extern int ptr_offset_to_size_class[];
+extern int first_block_offset_by_size_class[];
+extern int last_block_offset_by_size_class[];
+extern const int slots_by_size_class[];
+
+#endif // OS_VARIANT_NOTRESOLVED
+
+#pragma mark -
+#pragma mark Conversion and Mapping Inlines
+
+// nanov2_block_index_to_meta_index() and nanov2_meta_index_to_block_index()
+// map between the index of a block in its arena and the index of the meta data
+// header for that block in the metadata block. The mapping is not direct
+// to avoid false sharing caused by CPUs that are using adjacent blocks
+// writing to what would otherwise be adjacent meta data headers. The effect of
+// these functions is to separate the meta data headers for adjacent blocks by
+// at least the size of a cache line (assumed to be 64 bytes).
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_meta_index_t
+nanov2_block_index_to_meta_index(nanov2_block_index_t block_index)
+{
+ return ((block_index >> 6) | (block_index << 6)) & 0xFFF;
+}
+
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_index_t
+nanov2_meta_index_to_block_index(nanov2_meta_index_t block_meta_index)
+{
+ return ((block_meta_index >> 6) | (block_meta_index << 6)) & 0xFFF;
+}
+
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_meta_index_t
+nanov2_metablock_meta_index(nanozonev2_t *nanozone)
+{
+ return nanov2_block_index_to_meta_index((nanov2_block_index_t)
+ nanozone->aslr_cookie);
+}
+
+// Given a block metadata pointer, returns whether the block is active (that is,
+// it is being used for allocations, it has allocations that have not been freed,
+// or is waiting to be madvised and is not a guard block).
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE boolean_t
+nanov2_is_block_active(nanov2_block_meta_t block_meta)
+{
+ return block_meta.next_slot != SLOT_NULL
+ && block_meta.next_slot != SLOT_MADVISING
+ && block_meta.next_slot != SLOT_MADVISED
+ && block_meta.next_slot != SLOT_GUARD;
+}
+
+#if OS_VARIANT_RESOLVED
+// Given a block metadata pointer, returns whether an allocation could be
+// attempted from it. Allocations are not allowed from blocks that have not yet
+// been used (since such a block has not been assigned), is full or has been
+// madvised.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE boolean_t
+nanov2_can_allocate_from_block(nanov2_block_meta_t block_meta)
+{
+ return block_meta.in_use && block_meta.next_slot != SLOT_FULL;
+}
+
+// Given a pointer, returns whether it has the correct signature to be a
+// Nano V2 address.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE boolean_t
+nanov2_has_valid_signature(void *ptr)
+{
+ return (((uintptr_t)ptr) >> SHIFT_NANO_SIGNATURE) == NANOZONE_SIGNATURE;
+}
+#endif // OS_VARIANT_RESOLVED
+
+// Converts a Nano V2 logical address to the corresponding real address.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE void *
+nanov2_logical_address_to_ptr(nanozonev2_t *nanozone, void *laddr)
+{
+ return (void *)(((uintptr_t)laddr) ^ nanozone->aslr_cookie_aligned);
+}
+
+// Gets the maximum allocation size for a given size class.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE int
+nanov2_size_from_size_class(nanov2_size_class_t size_class)
+{
+ return (size_class + 1) * NANO_REGIME_QUANTA_SIZE;
+}
+
+#if OS_VARIANT_RESOLVED
+// Given an allocation size, returns the corresponding size class. It is the
+// responsibility of the caller to ensure that the size is valid. Returned
+// value is zero-based.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_size_class_t
+nanov2_size_class_from_size(size_t size)
+{
+ return (nanov2_size_class_t)howmany(size, NANO_REGIME_QUANTA_SIZE) - 1;
+}
+#endif // OS_VARIANT_RESOLVED
+
+#if OS_VARIANT_RESOLVED
+// Given a pointer that is assumed to be in the Nano zone, returns the address
+// of its containing block. Works for both real and logical pointers and returns
+// a pointer of the same type.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_t *
+nanov2_block_address_for_ptr(void *ptr)
+{
+ return (void *)(((uintptr_t)ptr) & NANOV2_BLOCK_ADDRESS_MASK);
+}
+#endif // OS_VARIANT_RESOLVED
+
+// Given a pointer that is assumed to be in the Nano zone, returns the address
+// of its containing arena. Works for both real and logical pointers.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_arena_t *
+nanov2_arena_address_for_ptr(void *ptr)
+{
+ return (void *)(((uintptr_t)ptr) & NANOV2_ARENA_ADDRESS_MASK);
+}
+
+// Given a pointer that is assumed to be in the Nano zone, returns the address
+// of its containing region. Works for both real and logical pointers.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_region_t *
+nanov2_region_address_for_ptr(void *ptr)
+{
+ return (nanov2_region_t *)(((uintptr_t)ptr) & NANOV2_REGION_ADDRESS_MASK);
+}
+
+// Given a pointer that is assumed to be in the Nano zone, returns the real
+// address of its metadata block. Works for both real and logical pointers.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_arena_metablock_t *
+nanov2_metablock_address_for_ptr(nanozonev2_t *nanozone, void *ptr)
+{
+ // The metadata block is the first logical block in the arena, so its
+ // logical address is that of the arena. To get a real pointer, we map it
+ // through nanov2_logical_address_to_ptr().
+ return (nanov2_arena_metablock_t *)nanov2_logical_address_to_ptr(nanozone,
+ nanov2_arena_address_for_ptr(ptr));
+}
+
+#if OS_VARIANT_RESOLVED
+// Given a pointer to a block_metap_t for a block, returns a pointer to the
+// block itself.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_t *
+nanov2_block_address_from_meta_ptr(nanozonev2_t *nanozone,
+ nanov2_block_meta_t *block_metap)
+{
+ nanov2_block_t *meta_block = nanov2_block_address_for_ptr(block_metap);
+ nanov2_arena_t *arena = nanov2_arena_address_for_ptr(block_metap);
+
+ // Get the block's index and use that to get the address of the block.
+ nanov2_meta_index_t meta_index =
+ (nanov2_meta_index_t)(block_metap - (nanov2_block_meta_t *)meta_block);
+ nanov2_block_index_t block_index = nanov2_meta_index_to_block_index(meta_index);
+ return &arena->blocks[block_index];
+}
+#endif // OS_VARIANT_RESOLVED
+
+// Given the index of a block_metap_t for a block, returns a pointer to the
+// block itself.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_t *
+nanov2_block_address_from_meta_index(nanozonev2_t *nanozone,
+ nanov2_arena_t *arena, nanov2_meta_index_t meta_index)
+{
+ nanov2_block_index_t block_index = nanov2_meta_index_to_block_index(meta_index);
+ return &arena->blocks[block_index];
+}
+
+// Given a pointer that is assumed to be in the nanozone, returns the index
+// of its containing block within its hosting arena. Works for both logical and
+// real pointers and returns an index in the corresponding address space.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_index_t
+nanov2_block_index_for_ptr(void *ptr)
+{
+ return (nanov2_block_index_t)(((uintptr_t)ptr) >> NANOV2_OFFSET_BITS)
+ & ((1 << NANOV2_BLOCK_BITS) - 1);
+}
+
+#if OS_VARIANT_RESOLVED
+// Given a pointer that is assumed to be in the nanozone, returns a pointer to
+// the meta data for its containing block. Expects ptr be a real address.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_meta_t *
+nanov2_meta_ptr_for_ptr(nanozonev2_t *nanozone, void *ptr)
+{
+ nanov2_arena_metablock_t *meta_block = nanov2_metablock_address_for_ptr(
+ nanozone, ptr);
+ nanov2_block_index_t block_index = nanov2_block_index_for_ptr(ptr);
+ nanov2_meta_index_t meta_index = nanov2_block_index_to_meta_index(block_index);
+ return &meta_block->arena_block_meta[meta_index];
+}
+#endif // OS_VARIANT_RESOLVED
+
+// Given a region pointer, returns the address of the first arena in the region.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_arena_t *
+nanov2_first_arena_for_region(nanov2_region_t *region)
+{
+ // The first arena is colocated with the region itself.
+ return (nanov2_arena_t *)region;
+}
+
+#if OS_VARIANT_RESOLVED
+// Given an atomically-observed current_region_next_arena pointer, returns
+// whether or not it's a usable arena or a limit arena (indicating exhaustion of
+// the current region).
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE bool
+nanov2_current_region_next_arena_is_limit(
+ nanov2_arena_t *current_region_next_arena)
+{
+ // The first arena of a region is never stored in current_region_next_arena,
+ // so a value at the beginning of a region must be a limit arena.
+ return current_region_next_arena == (nanov2_arena_t *)(
+ nanov2_region_address_for_ptr(current_region_next_arena));
+}
+#endif // OS_VARIANT_RESOLVED
+
+// Given an atomically-observed current_region_next_arena pointer, returns the
+// base of the current region at the time of the observation.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_region_t *
+nanov2_current_region_base(nanov2_arena_t *current_region_next_arena)
+{
+ return nanov2_region_address_for_ptr(
+ (void *)(((uintptr_t)current_region_next_arena) - 1));
+}
+
+// Given a region pointer, returns a pointer to the arena after the last
+// active arena in the region.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_arena_t *
+nanov2_limit_arena_for_region(nanozonev2_t __unused *nanozone,
+ nanov2_region_t *region, nanov2_arena_t *current_region_next_arena)
+{
+ // The first arena is colocated with the region itself.
+ nanov2_arena_t *limit_arena;
+ if (region == nanov2_current_region_base(current_region_next_arena)) {
+ limit_arena = current_region_next_arena;
+ } else {
+ limit_arena = nanov2_first_arena_for_region(region + 1);
+ }
+ return limit_arena;
+}
+
+// Given a region pointer, returns the address of the linkage structure for
+// that region. The linkage structure is stored in the first entry of the
+// metadata block of the first arena in the region.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_region_linkage_t *
+nanov2_region_linkage_for_region(nanozonev2_t *nanozone, nanov2_region_t *region)
+{
+ nanov2_arena_metablock_t *first_metadata_block =
+ nanov2_metablock_address_for_ptr(nanozone, region);
+ return (nanov2_region_linkage_t *)&first_metadata_block->arena_block_meta[
+ nanov2_metablock_meta_index(nanozone)];
+}
+
+#if OS_VARIANT_RESOLVED
+// Given a pointer to a region, returns a pointer to the region that follows it,
+// or NULL if there isn't one. We may observe linkage to a new region that
+// hasn't yet actually been installed into current_region_next_arena; ignore the
+// linkage in this case.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_region_t *
+nanov2_next_region_for_region(nanozonev2_t *nanozone, nanov2_region_t *region,
+ nanov2_arena_t *current_region_next_arena)
+{
+ nanov2_region_linkage_t *linkage =
+ nanov2_region_linkage_for_region(nanozone, region);
+ int offset = os_atomic_load(&linkage->next_region_offset, relaxed);
+ if (!offset) {
+ return NULL;
+ }
+
+ nanov2_region_t *next_region = region + offset;
+ return (nanov2_arena_t *)next_region < current_region_next_arena ?
+ next_region : NULL;
+}
+#endif // OS_VARIANT_RESOLVED
+
+#if OS_VARIANT_NOTRESOLVED
+// Given a pointer to a region, returns a pointer to the region that follows it,
+// or NULL if there isn't one. This variant is used when mapping the nanozone
+// for another process.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_region_t *
+nanov2_next_region_for_region_offset(nanozonev2_t *nanozone,
+ nanov2_region_t *region, off_t region_offset,
+ nanov2_arena_t *current_region_next_arena)
+{
+ nanov2_region_linkage_t *linkage =
+ nanov2_region_linkage_for_region(nanozone, region);
+ nanov2_region_linkage_t *mapped_linkage = (nanov2_region_linkage_t *)(
+ ((uintptr_t)linkage + region_offset));
+ int offset = os_atomic_load(&mapped_linkage->next_region_offset, relaxed);
+ if (!offset) {
+ return NULL;
+ }
+
+ nanov2_region_t *next_region = region + offset;
+ return (nanov2_arena_t *)next_region < current_region_next_arena ?
+ next_region : NULL;
+}
+#endif // OS_VARIANT_NOTRESOLVED
+
+// Given the index of a slot in a block of a given size and the base address of
+// the block, returns a pointer to the start of the slot. This works for both
+// real and logical block pointers and returns a pointer of the same type.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE void *
+nanov2_slot_in_block_ptr(nanov2_block_t *block, nanov2_size_class_t size_class,
+ int slot_index)
+{
+ return (void *)((uintptr_t)block +
+ nanov2_size_from_size_class(size_class) * slot_index);
+}
+
+#if OS_VARIANT_RESOLVED
+// Given the base address of a block, the size class for the block and a pointer,
+// returns the index of the slot represented by the pointer. It is assumed that
+// the pointer is slot-aligned and is within the bounds of the block.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE int
+nanov2_slot_index_in_block(nanov2_block_t *block, nanov2_size_class_t size_class,
+ void *ptr)
+{
+ return (int)((uintptr_t)ptr - (uintptr_t)block)/
+ (nanov2_size_from_size_class(size_class));
+}
+#endif // OS_VARIANT_RESOLVED
+
+// Given a (real) pointer, gets the size class of its containing block. Assumes
+// that the pointer is in a valid region, arena and block.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_size_class_t
+nanov2_size_class_for_ptr(nanozonev2_t *nanozone, void *ptr)
+{
+ // To get the size class, we need to convert the block number from
+ // physical to logical, since the ptr_offset_to_size_class table is
+ // indexed by logical block.
+ nanov2_block_index_t block =
+ (int)(nanov2_block_index_for_ptr(ptr) ^ nanozone->aslr_cookie);
+ return ptr_offset_to_size_class[block >> BLOCKS_PER_UNIT_SHIFT];
+}
+
+#if OS_VARIANT_NOTRESOLVED
+
+// Given a meta data index, gets the size class of the corresponding block.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_size_class_t
+nanov2_size_class_for_meta_index(nanozonev2_t *nanozone, nanov2_meta_index_t meta_index)
+{
+ // To get the size class, we need to get the block index from meta index
+ // and then convert it from real to logical, since the
+ // ptr_offset_to_size_class table is indexed by logical block.
+ nanov2_block_index_t block_index = nanov2_meta_index_to_block_index(meta_index);
+ int logical_block_index = (int)(block_index ^ nanozone->aslr_cookie);
+ return ptr_offset_to_size_class[logical_block_index >> BLOCKS_PER_UNIT_SHIFT];
+}
+#endif // OS_VARIANT_NOTRESOLVED
+
+#if OS_VARIANT_RESOLVED
+// Given a size class and an arena, returns a pointer to the metadata for the
+// first block for that size class in the arena, ignoring the metadata block.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_meta_t *
+nanov2_first_block_for_size_class_in_arena(nanozonev2_t *nanozone,
+ nanov2_size_class_t size_class, nanov2_arena_t *arena)
+{
+ int block_offset = first_block_offset_by_size_class[size_class];
+ nanov2_arena_metablock_t *meta_blockp =
+ nanov2_metablock_address_for_ptr(nanozone, arena);
+ nanov2_block_index_t block_index =
+ (nanov2_block_index_t)(block_offset ^ nanozone->aslr_cookie);
+ nanov2_meta_index_t meta_index = nanov2_block_index_to_meta_index(block_index);
+ return &meta_blockp->arena_block_meta[meta_index];
+}
+
+// Given a pointer to the metadata for a block in a given size class, returns
+// a pointer to the metadata for the next block, wrapping from the last block
+// to the first if necessary.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_meta_t *
+nanov2_next_block_for_size_class(nanozonev2_t *nanozone,
+ nanov2_size_class_t size_class, nanov2_block_meta_t *meta_blockp,
+ boolean_t *wrapped)
+{
+ // To find the next block, get the index of the current block, which is in
+ // the real address space, unscramble it to get a logical block number,
+ // add 1 to it, wrapping if necessary, then scramble the result.
+ nanov2_block_meta_t *base_meta_blockp =
+ (nanov2_block_meta_t *)(((uintptr_t)meta_blockp) & (NANOV2_BLOCK_ADDRESS_MASK));
+ nanov2_meta_index_t meta_index = (int)(meta_blockp - base_meta_blockp);
+ nanov2_block_index_t block_index = nanov2_meta_index_to_block_index(meta_index);
+ block_index ^= nanozone->aslr_cookie; // Unscramble
+ int last_offset = last_block_offset_by_size_class[size_class];
+ if (wrapped) *wrapped = block_index == last_offset;
+ block_index = block_index == last_offset ?
+ first_block_offset_by_size_class[size_class] : block_index + 1;
+ block_index = (nanov2_block_index_t)(block_index ^ nanozone->aslr_cookie);
+ meta_index = nanov2_block_index_to_meta_index(block_index);
+ return &base_meta_blockp[meta_index];
+}
+
+// Given a pointer to the metadata for a block in a given size class, returns
+// a pointer to the metadata for the previous block, wrapping from the first
+// block to the last if necessary.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_meta_t *
+nanov2_previous_block_for_size_class(nanozonev2_t *nanozone,
+ nanov2_size_class_t size_class, nanov2_block_meta_t *meta_blockp,
+ boolean_t *wrapped)
+{
+ // To find the previous block, get the index of the current block, which is
+ // in the real address space, unscramble it to get a logical block number,
+ // subtract 1 from it, wrapping if necessary, then scramble the result.
+ nanov2_block_meta_t *base_meta_blockp =
+ (nanov2_block_meta_t *)(((uintptr_t)meta_blockp) & (NANOV2_BLOCK_ADDRESS_MASK));
+ nanov2_meta_index_t meta_index = (int)(meta_blockp - base_meta_blockp);
+ nanov2_block_index_t block_index = nanov2_meta_index_to_block_index(meta_index);
+ block_index ^= nanozone->aslr_cookie; // Unscramble
+ int first_offset = first_block_offset_by_size_class[size_class];
+ if (wrapped) *wrapped = block_index == first_offset;
+ block_index = block_index == first_offset ?
+ last_block_offset_by_size_class[size_class] : block_index - 1;
+ block_index = (nanov2_block_index_t)(block_index ^ nanozone->aslr_cookie);
+ meta_index = nanov2_block_index_to_meta_index(block_index);
+ return &base_meta_blockp[meta_index];
+}
+
+// Turns off the in-use bit in the meta data for a given block.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE void
+nanov2_turn_off_in_use(nanov2_block_meta_t *block_metap)
+{
+ // TODO: find a way to do this that is more efficient and readable.
+ static nanov2_block_meta_t mask = {
+ .in_use = 0,
+ .next_slot = ~0,
+ .free_count = ~0,
+ .gen_count = ~0,
+ };
+ os_atomic_and((uint32_t *)block_metap, *(uint32_t *)&mask, relaxed);
+}
+
+#pragma mark -
+#pragma mark Policy Functions
+
+// Gets the index of the block in the zone's current_block array from which
+// allocations should be made. This function should be replaced to implement
+// a different strategy (e.g. for E- vs P-cores).
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE int
+nanov2_get_allocation_block_index(void)
+{
+#if CONFIG_NANO_USES_HYPER_SHIFT
+ if (os_likely(nano_common_max_magazines_is_ncpu)) {
+ // Default case is max magazines == physical number of CPUs, which
+ // must be > _malloc_cpu_number() >> hyper_shift, so the modulo
+ // operation is not required.
+ return (_malloc_cpu_number() >> hyper_shift) & MAX_CURRENT_BLOCKS_MASK;
+ }
+#else // CONFIG_NANO_USES_HYPER_SHIFT
+ if (os_likely(nano_common_max_magazines_is_ncpu)) {
+ // Default case is max magazines == logical number of CPUs, which
+ // must be > _malloc_cpu_number() so the modulo operation is not required.
+ return _malloc_cpu_number() & MAX_CURRENT_BLOCKS_MASK;
+ }
+#endif // CONFIG_NANO_USES_HYPER_SHIFT
+
+ unsigned int shift = 0;
+#if CONFIG_NANO_USES_HYPER_SHIFT
+ shift = hyper_shift;
+#endif // CONFIG_NANO_USES_HYPER_SHIFT
+
+ if (os_likely(_os_cpu_number_override == -1)) {
+ return ((_malloc_cpu_number() >> shift) % nano_common_max_magazines) &
+ MAX_CURRENT_BLOCKS_MASK;
+ }
+ return ((_os_cpu_number_override >> shift) % nano_common_max_magazines) &
+ MAX_CURRENT_BLOCKS_MASK;
+}
+#endif // OS_VARIANT_RESOLVED
+
+#pragma mark -
+#pragma mark Guard Blocks
+
+// Converts a given block (specified by absolute block number) in an arena into
+// a guard block. The block will be marked as in-use so that it is not available
+// for allocations and its permissions are set to PROT_READ. Note that
+// PROT_READ is used instead of PROT_NONE because the latter breaks the
+// enumerator, which tries to map the whole region and fails if there are
+// PROT_NONE pages in the range. We can't fix that in the allocator because the
+// code that does the mapping is part of the sampling tools and is simply
+// invoked as a callback from the enumerator.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE void
+nanov2_create_guard_block(nanozonev2_t *nanozone, nanov2_arena_t *arena,
+ nanov2_block_index_t block_index) {
+ // Mark the block as in-use in the meta data
+ static nanov2_block_meta_t in_use_block = {
+ .in_use = 1,
+ .next_slot = SLOT_GUARD
+ };
+ nanov2_meta_index_t block_meta_index =
+ nanov2_block_index_to_meta_index(block_index);
+ nanov2_arena_metablock_t *block_metap = nanov2_metablock_address_for_ptr(
+ nanozone, arena);
+ block_metap->arena_block_meta[block_meta_index] = in_use_block;
+ void *block_ptr = &arena->blocks[block_index];
+
+ // Apply PROT_NONE to the block itself.
+ kern_return_t err = mprotect(block_ptr, NANOV2_BLOCK_SIZE, PROT_READ);
+ if (err != KERN_SUCCESS) {
+ malloc_report(ASL_LEVEL_ERR, "Failed to create guard block at %p (%d)\n",
+ block_ptr, err);
+ }
+}
+
+// Creates the guard blocks for an arena, if required. The guard blocks are
+// the first and last physical blocks in the arena that are not the metadata
+// block.
+static MALLOC_ALWAYS_INLINE MALLOC_INLINE void
+nanov2_init_guard_blocks(nanozonev2_t *nanozone, nanov2_arena_t *arena)
+{
+ if (nanozone->debug_flags & MALLOC_ALL_GUARD_PAGE_FLAGS) {
+ // Use the first and last blocks in the arena as guard regions,
+ // avoiding the metadata block.
+ nanov2_meta_index_t meta_index = nanov2_metablock_meta_index(nanozone);
+ nanov2_create_guard_block(nanozone, arena, meta_index == 0 ? 1 : 0);
+ nanov2_create_guard_block(nanozone, arena,
+ meta_index == NANOV2_BLOCKS_PER_ARENA - 1 ?
+ NANOV2_BLOCKS_PER_ARENA - 2 : NANOV2_BLOCKS_PER_ARENA - 1);
+ }
+}
+
+#pragma mark -
+#pragma mark Allocator Initialization
+
+#if OS_VARIANT_NOTRESOLVED
+
+static const char madvise_policy_env[] = "MallocNanoMadvisePolicy";
+static const char madvise_policy_bootarg[] = "nanov2_madvise_policy";
+static const char madvise_immediate[] = "immediate";
+static const char madvise_warning[] = "warning";
+static const char madvise_critical[] = "critical";
+
+static const char single_arena_env[] = "MallocNanoSingleArena";
+static const char single_arena_bootarg[] = "nanov2_single_arena";
+
+static const char scan_policy_env[] = "MallocNanoScanPolicy";
+static const char scan_policy_bootarg[] = "nanov2_scan_policy";
+
+static const char size_class_blocks_env[] = "MallocNanoSizeClassBlocks";
+static const char size_class_blocks_bootarg[] = "nanov2_size_class_blocks";
+
+// Parse and set the madvise policy setting. If ptr is NULL, sets the default
+// policy.
+static void
+nanov2_set_madvise_policy(const char *name, const char *ptr)
+{
+ nanov2_madvise_policy_t madvise_policy = NANO_MADVISE_IMMEDIATE;
+ if (ptr) {
+ if (!strncmp(ptr, madvise_immediate, sizeof(madvise_immediate) - 1)) {
+ madvise_policy = NANO_MADVISE_IMMEDIATE;
+ } else if (!strncmp(ptr, madvise_warning, sizeof(madvise_warning) - 1)) {
+ madvise_policy = NANO_MADVISE_WARNING_PRESSURE;
+ } else if (!strncmp(ptr, madvise_critical, sizeof(madvise_critical) - 1)) {
+ madvise_policy = NANO_MADVISE_CRITICAL_PRESSURE;
+ } else {
+ malloc_report(ASL_LEVEL_ERR,
+ "%s value (%s) invalid - ignored.\n", name, ptr);
+ }
+ }
+ nanov2_madvise_policy = madvise_policy;
+}
+
+// Parse and set the list of size classes that are allowed only one arena. If
+// ptr is NULL, no size classes are restricted to a single arena,
+// Format is a list of sizes separated by colons (e.g. 16:240). Each size must
+// be a multiple of NANO_REGIME_QUANTA_SIZE and must be between 16 and 256.
+static void
+nanov2_set_single_arena_size_classes(const char *name, const char *ptr)
+{
+ uint16_t single_arena_size_classes = 0;
+ if (ptr) {
+ const char *value = ptr;
+ const char *endp;
+ boolean_t failed = FALSE;
+ while (*ptr) {
+ long size = malloc_common_convert_to_long(ptr, &endp);
+ if (endp != ptr) {
+ if (*endp && *endp != ':') {
+ failed = TRUE;
+ break;
+ }
+ if (size > NANO_MAX_SIZE || size < NANO_REGIME_QUANTA_SIZE ||
+ (size % NANO_REGIME_QUANTA_SIZE) != 0) {
+ failed = TRUE;
+ break;
+ }
+ single_arena_size_classes |=
+ 1 << ((size/NANO_REGIME_QUANTA_SIZE) - 1);
+ } else {
+ failed = true;
+ break;
+ }
+ if (!*endp) {
+ break;
+ }
+ ptr = endp + 1;
+ }
+ if (failed) {
+ malloc_report(ASL_LEVEL_ERR,
+ "%s value (%s) invalid - ignored.\n", name, value);
+ single_arena_size_classes = 0;
+ }
+ }
+ nanov2_policy_config.single_arena_size_classes = single_arena_size_classes;
+}
+
+// Parse and set the block scan policy setting. If ptr is NULL, the default
+// policy is used. Format is either "firstfit" or "minXX:maxYY:limZZ", where
+// XX, YY and ZZ are numbers, XX and YY must be between 0 and 100 and XX must
+// not be greater than YY. min, max and lim may appear in any order or may be
+// omitted to get default values.
+static void
+nanov2_set_block_scan_policy(const char *name, const char *ptr)
+{
+ static char first_fit_key[] = "firstfit";
+ static char min_key[] = "min";
+ static char max_key[] = "max";
+ static char lim_key[] = "lim";
+
+ nanov2_block_scan_policy_t block_scan_policy = NANO_SCAN_CAPACITY_BASED;
+ int scan_min_capacity = DEFAULT_SCAN_MIN_CAPACITY;
+ int scan_max_capacity = DEFAULT_SCAN_MAX_CAPACITY;
+ int scan_limit = DEFAULT_SCAN_LIMIT;
+ const char *endp;
+ boolean_t failed = FALSE;
+ boolean_t min_found = FALSE;
+ boolean_t max_found = FALSE;
+ boolean_t lim_found = FALSE;
+ const char *value = ptr;
+
+ if (ptr) {
+ if (!strcmp(ptr, first_fit_key)) {
+ block_scan_policy = NANO_SCAN_FIRST_FIT;
+ } else {
+ while (!failed && ptr && *ptr) {
+ if (!strncmp(ptr, min_key, sizeof(min_key) - 1) && !min_found) {
+ min_found = TRUE;
+ ptr += sizeof(min_key) - 1;
+ long value = malloc_common_convert_to_long(ptr, &endp);
+ if (ptr != endp && value >= 0 && value <= 100) {
+ scan_min_capacity = (int)value;
+ ptr = endp;
+ } else {
+ failed = TRUE;
+ }
+ } else if (!strncmp(ptr, max_key, sizeof(max_key) - 1)
+ && !max_found) {
+ max_found = TRUE;
+ ptr += sizeof(max_key) - 1;
+ long value = malloc_common_convert_to_long(ptr, &endp);
+ if (ptr != endp && value >= 0 && value <= 100) {
+ scan_max_capacity = (int)value;
+ ptr = endp;
+ } else {
+ failed = TRUE;
+ }
+ } else if (!strncmp(ptr, lim_key, sizeof(lim_key) - 1)
+ && !lim_found) {
+ lim_found = TRUE;
+ ptr += sizeof(lim_key) - 1;
+ long value = malloc_common_convert_to_long(ptr, &endp);
+ if (ptr != endp && value >= 0) {
+ scan_limit = (int)value;
+ ptr = endp;
+ } else {
+ failed = TRUE;
+ }
+ } else {
+ failed = TRUE;
+ }
+ if (*ptr) {
+ if (*ptr == ':') {
+ ptr++;
+ } else {
+ failed = TRUE;
+ }
+ }
+ }
+
+ if (!failed && scan_min_capacity > scan_max_capacity) {
+ failed = TRUE;
+ }
+ }
+ }
+
+ if (!failed) {
+ nanov2_policy_config.block_scan_policy = block_scan_policy;
+ nanov2_policy_config.block_scan_min_capacity = scan_min_capacity;
+ nanov2_policy_config.block_scan_max_capacity = scan_max_capacity;
+ nanov2_policy_config.block_scan_limit = scan_limit;
+ } else {
+ malloc_report(ASL_LEVEL_ERR, "%s value (%s) invalid - ignored.\n",
+ name, value);
+ }
+}
+
+// Configures the nanov2_blocks_by_size_class array. If ptr is not NULL and
+// *ptr is not empty, it is expected to be a list of 16 positive integers
+// separated by commas that sum to TOTAL_BLOCK_UNITS (which is currently 64).
+// For example, as an environment variable:
+// MallocNanoSizeClassBlocks=2,7,6,6,6,5,5,5,5,2,2,2,2,2,6,1
+// or as a boot argument:
+// nanov2_size_class_blocks=2,7,6,6,6,5,5,5,5,2,2,2,2,2,6,1
+static void
+nanov2_set_blocks_by_size_class(const char *name, const char *ptr)
+{
+ int new_total_block_units = 0;
+ int new_blocks_by_size_class[NANO_SIZE_CLASSES];
+ MALLOC_STATIC_ASSERT(
+ sizeof(new_blocks_by_size_class) == sizeof(block_units_by_size_class),
+ "Size mismatch in nanov2_set_blocks_by_size_class()");
+ const char *endp;
+ const char *sptr = ptr;
+ for (int i = 0; i < NANO_SIZE_CLASSES; i++) {
+ int count = (int)malloc_common_convert_to_long(ptr, &endp);
+ char separator = i == NANO_SIZE_CLASSES - 1 ? '\0' : ',';
+ if (endp == ptr || *endp != separator || count > TOTAL_BLOCK_UNITS) {
+ malloc_report(ASL_LEVEL_ERR,
+ "%s value invalid: [%s] - ignored.\n", name, sptr);
+ return;
+ }
+ new_blocks_by_size_class[i] = count;
+ new_total_block_units += count;
+ ptr = endp + 1;
+ }
+
+ if (new_total_block_units != TOTAL_BLOCK_UNITS) {
+ malloc_report(ASL_LEVEL_ERR,
+ "%s value invalid - values must sum to %d, not %d - ignored.\n",
+ name, TOTAL_BLOCK_UNITS, new_total_block_units);
+ } else {
+ memcpy(block_units_by_size_class, new_blocks_by_size_class,
+ sizeof(block_units_by_size_class));
+ }
+}
+
+// First stage initialization. Called during libSystem initialization.
+// Reads environment variables and boot arguments and sets the madvise policy,
+// single arena list and the block scan policy. Environment variables override
+// boot arguments.
+void
+nanov2_init(const char *envp[], const char *apple[], const char *bootargs)
+{
+ // Get and process the boot args and environment variables.
+ char value_buf[256];
+ const char *value = _simple_getenv(envp, madvise_policy_env);
+ const char *name = madvise_policy_env;
+ if (!value) {
+ value = malloc_common_value_for_key(bootargs, madvise_policy_bootarg);
+ if (value) {
+ name = madvise_policy_bootarg;
+ }
+ }
+ nanov2_set_madvise_policy(name, value);
+
+ name = single_arena_env;
+ value = _simple_getenv(envp, single_arena_env);
+ if (!value) {
+ value = malloc_common_value_for_key_copy(bootargs, single_arena_bootarg,
+ value_buf, sizeof(value_buf));
+ if (value) {
+ name = single_arena_bootarg;
+ }
+ }
+ nanov2_set_single_arena_size_classes(name, value);
+
+ name = scan_policy_env;
+ value = _simple_getenv(envp, scan_policy_env);
+ if (!value) {
+ value = malloc_common_value_for_key_copy(bootargs, scan_policy_bootarg,
+ value_buf, sizeof(value_buf));
+ if (value) {
+ name = scan_policy_bootarg;
+ }
+ }
+ nanov2_set_block_scan_policy(name, value);
+
+ name = size_class_blocks_env;
+ value = _simple_getenv(envp, size_class_blocks_env);
+ if (!value) {
+ value = malloc_common_value_for_key_copy(bootargs, size_class_blocks_bootarg,
+ value_buf, sizeof(value_buf));
+ if (value) {
+ name = size_class_blocks_bootarg;
+ }
+ }
+ if (value) {
+ nanov2_set_blocks_by_size_class(name, value);
+ }
+}
+
+static void
+nanov2_configure_once(void *context MALLOC_UNUSED)
+{
+ // Check that the block_units_by_size_class table is consistent.
+ int total_blocks = 0;
+ for (int i = 0; i < NANO_SIZE_CLASSES; i++) {
+ total_blocks += block_units_by_size_class[i] * BLOCKS_PER_UNIT;
+ }
+ MALLOC_ASSERT(total_blocks == NANOV2_BLOCKS_PER_ARENA);
+
+ // Build the first_block_offset_by_size_class and
+ // last_block_offset_by_size_class tables. The first entry is special
+ // because block 0 is reserved for the metadata block, so the first offset
+ // is 1 and the number of blocks allocated is reduced by 1.
+ int next_offset = 1;
+ first_block_offset_by_size_class[0] = next_offset;
+ next_offset = block_units_by_size_class[0] * BLOCKS_PER_UNIT;
+ last_block_offset_by_size_class[0] = next_offset - 1;
+
+ for (int i = 1; i < NANO_SIZE_CLASSES; i++) {
+ first_block_offset_by_size_class[i] = next_offset;
+ next_offset += block_units_by_size_class[i] * BLOCKS_PER_UNIT;
+ last_block_offset_by_size_class[i] = next_offset - 1;
+ }
+ MALLOC_ASSERT(next_offset == NANOV2_BLOCKS_PER_ARENA);
+
+ // Construct the ptr_offset_to_size_class map, which maps the part of the
+ // logical address that depends on size class to the corresponding size
+ // class. This would be a simple mask operation if all size classes were of
+ // equal size, but sadly they are not.
+ int next_index = 0;
+ for (int i = 0; i < NANO_SIZE_CLASSES; i++) {
+ int block_units = block_units_by_size_class[i];
+ for (int j = 0; j < block_units; j++) {
+ ptr_offset_to_size_class[next_index++] = i;
+ }
+ }
+ MALLOC_ASSERT(next_index == NANOV2_BLOCKS_PER_ARENA/BLOCKS_PER_UNIT);
+}
+
+static os_once_t nanov2_config_predicate;
+
+void
+nanov2_configure(void)
+{
+ os_once(&nanov2_config_predicate, NULL, nanov2_configure_once);
+}
+#endif // OS_VARIANT_NOTRESOLVED
+
+#pragma mark -
+#pragma mark Zone Functions
+
+#if OS_VARIANT_RESOLVED
+// Returns the allocation size for a pointer. Uses nanov2_pointer_size() to
+// determine whether the pointer is for a Nano V2 allocation and, if not,
+// delegates to the helper zone. Returns 0 if the pointer is not to memory
+// allocated by Nano V2 or attributable to the helper zone.
+MALLOC_NOEXPORT size_t
+nanov2_size(nanozonev2_t *nanozone, const void *ptr)
+{
+ size_t size = nanov2_pointer_size_inline(nanozone, (void *)ptr, FALSE,
+ NULL, NULL);
+ return size ? size : nanozone->helper_zone->size(nanozone->helper_zone, ptr);
+}
+
+MALLOC_NOEXPORT void *
+nanov2_malloc(nanozonev2_t *nanozone, size_t size)
+{
+ size_t rounded_size = _nano_common_good_size(size);
+ if (rounded_size <= NANO_MAX_SIZE) {
+ nanov2_block_meta_t *madvise_block_metap = NULL;
+ nanov2_size_class_t size_class = nanov2_size_class_from_size(rounded_size);
+
+ // Get the index of the pointer to the block from which we are should be
+ // allocating. This currently depends on the physical CPU number.
+ int allocation_index = nanov2_get_allocation_block_index();
+
+ // Get the current allocation block meta data pointer. If this is NULL,
+ // we need to find a new allocation block.
+ nanov2_block_meta_t **block_metapp =
+ &nanozone->current_block[size_class][allocation_index];
+ nanov2_block_meta_t *block_metap = os_atomic_load(block_metapp, relaxed);
+ bool corruption = false;
+ void *ptr = NULL;
+ if (block_metap) {
+ // Fast path: we have a block -- try to allocate from it.
+ ptr = nanov2_allocate_from_block_inline(nanozone, block_metap,
+ size_class, &madvise_block_metap, &corruption);
+ if (ptr && !corruption) {
+ // Always clear the double-free guard so that we can recognize
+ // that this block is not on the free list.
+ nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
+ os_atomic_store(&slotp->double_free_guard, 0, relaxed);
+
+ // We know the body of the allocation is already clear, so we just
+ // need to clean up the next_slot word to get to all-zero. Do so in
+ // all cases, even if a cleared allocation is not requested, to
+ // prevent any leakage through the next_slot bits.
+ os_atomic_store(&slotp->next_slot, 0, relaxed);
+ return ptr;
+ }
+ }
+
+ return nanov2_allocate_outlined(nanozone, block_metapp, rounded_size,
+ size_class, allocation_index, madvise_block_metap, ptr, false);
+ }
+
+ // Too big for nano, so delegate to the helper zone.
+ return nanozone->helper_zone->malloc(nanozone->helper_zone, size);
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+void
+nanov2_bzero(void *ptr, size_t size)
+{
+ // TODO: inline bzero from libplatform
+ bzero(ptr, size);
+}
+
+MALLOC_NOEXPORT void
+nanov2_free_definite_size(nanozonev2_t *nanozone, void *ptr, size_t size)
+{
+ if (ptr && nanov2_has_valid_signature(ptr)) {
+ nanov2_size_class_t size_class = nanov2_size_class_from_size(size);
+
+ if (malloc_zero_policy == MALLOC_ZERO_ON_FREE) {
+ if (size_class != 0) {
+ nanov2_bzero((char *)ptr + sizeof(nanov2_free_slot_t),
+ size - sizeof(nanov2_free_slot_t));
+ }
+ }
+
+ nanov2_block_meta_t *madvise_block_metap = nanov2_free_to_block_inline(
+ nanozone, ptr, size_class, NULL);
+ if (madvise_block_metap) {
+ nanov2_madvise_block(nanozone, madvise_block_metap, size_class,
+ SLOT_CAN_MADVISE);
+ }
+ return;
+ }
+ return nanozone->helper_zone->free_definite_size(nanozone->helper_zone, ptr,
+ size);
+}
+
+static void
+_nanov2_free(nanozonev2_t *nanozone, void *ptr, bool try)
+{
+ if (ptr) {
+ // Check whether it's a Nano pointer and get the size. If it's not
+ // Nano, pass it to the helper zone.
+ nanov2_size_class_t size_class;
+ nanov2_block_meta_t *block_metap;
+ size_t size = nanov2_pointer_size_inline(nanozone, ptr, FALSE,
+ &size_class, &block_metap);
+ if (size) {
+ if (malloc_zero_policy == MALLOC_ZERO_ON_FREE) {
+ if (size > sizeof(nanov2_free_slot_t)) {
+ nanov2_bzero((char *)ptr + sizeof(nanov2_free_slot_t),
+ size - sizeof(nanov2_free_slot_t));
+ }
+ }
+
+ nanov2_block_meta_t *madvise_block_metap = nanov2_free_to_block_inline(
+ nanozone, ptr, size_class, block_metap);
+ if (madvise_block_metap) {
+ nanov2_madvise_block(nanozone, madvise_block_metap, size_class,
+ SLOT_CAN_MADVISE);
+ }
+ return;
+ }
+ }
+ return try ? nanozone->helper_zone->try_free_default(nanozone->helper_zone, ptr) :
+ nanozone->helper_zone->free(nanozone->helper_zone, ptr);
+}
+
+MALLOC_NOEXPORT void
+nanov2_free(nanozonev2_t *nanozone, void *ptr)
+{
+ _nanov2_free(nanozone, ptr, false);
+}
+
+MALLOC_NOEXPORT void
+nanov2_try_free_default(nanozonev2_t *nanozone, void *ptr)
+{
+ _nanov2_free(nanozone, ptr, true);
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+void *
+nanov2_malloc_zero(nanozonev2_t *nanozone, size_t rounded_size)
+{
+ nanov2_block_meta_t *madvise_block_metap = NULL;
+ nanov2_size_class_t size_class = nanov2_size_class_from_size(rounded_size);
+
+ // Get the index of the pointer to the block from which we are should be
+ // allocating. This currently depends on the physical CPU number.
+ int allocation_index = nanov2_get_allocation_block_index();
+
+ // Get the current allocation block meta data pointer. If this is NULL,
+ // we need to find a new allocation block.
+ nanov2_block_meta_t **block_metapp =
+ &nanozone->current_block[size_class][allocation_index];
+ nanov2_block_meta_t *block_metap = os_atomic_load(block_metapp, relaxed);
+ bool corruption = false;
+ void *ptr = NULL;
+ if (block_metap) {
+ // Fast path: we have a block -- try to allocate from it.
+ ptr = nanov2_allocate_from_block_inline(nanozone, block_metap,
+ size_class, &madvise_block_metap, &corruption);
+ if (ptr && !corruption) {
+ if (malloc_zero_policy == MALLOC_ZERO_ON_FREE) {
+ // Always clear the double-free guard so that we can recognize that
+ // this block is not on the free list.
+ nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
+ os_atomic_store(&slotp->double_free_guard, 0, relaxed);
+
+ // We know the body of the allocation is already clear, so we just
+ // need to clean up the next_slot word to get to all-zero. Do so in
+ // all cases, even if a cleared allocation is not requested, to
+ // prevent any leakage through the next_slot bits.
+ os_atomic_store(&slotp->next_slot, 0, relaxed);
+ } else {
+ nanov2_bzero(ptr, rounded_size);
+ }
+ return ptr;
+ }
+ }
+
+ return nanov2_allocate_outlined(nanozone, block_metapp, rounded_size,
+ size_class, allocation_index, madvise_block_metap, ptr, true);
+}
+
+MALLOC_NOEXPORT void *
+nanov2_calloc(nanozonev2_t *nanozone, size_t num_items, size_t size)
+{
+ size_t total_bytes;
+ if (calloc_get_size(num_items, size, 0, &total_bytes)) {
+ return NULL;
+ }
+ size_t rounded_size = _nano_common_good_size(total_bytes);
+ if (total_bytes <= NANO_MAX_SIZE) {
+ return nanov2_malloc_zero(nanozone, rounded_size);
+ }
+
+ // Too big for nano, so delegate to the helper zone.
+ return nanozone->helper_zone->calloc(nanozone->helper_zone, 1, total_bytes);
+}
+
+MALLOC_NOEXPORT void *
+nanov2_malloc_zero_on_alloc(nanozonev2_t *nanozone, size_t size)
+{
+ size_t rounded_size = _nano_common_good_size(size);
+ if (rounded_size <= NANO_MAX_SIZE) {
+ return nanov2_malloc_zero(nanozone, rounded_size);
+ }
+
+ // Too big for nano, so delegate to the helper zone.
+ return nanozone->helper_zone->malloc(nanozone->helper_zone, size);
+}
+#endif // OS_VARIANT_RESOLVED
+
+#if OS_VARIANT_NOTRESOLVED
+static void *
+nanov2_valloc(nanozonev2_t *nanozone, size_t size)
+{
+ // Always delegate this to the helper zone.
+ return nanozone->helper_zone->valloc(nanozone->helper_zone, size);
+}
+#endif // OS_VARIANT_NOTRESOLVED
+
+#if OS_VARIANT_RESOLVED
+MALLOC_NOEXPORT void *
+nanov2_realloc(nanozonev2_t *nanozone, void *ptr, size_t new_size)
+{
+ // If we are given a NULL pointer, just allocate memory of the requested
+ // size.
+ if (ptr == NULL) {
+ return nanov2_malloc(nanozone, new_size);
+ }
+
+ size_t old_size = nanov2_pointer_size(nanozone, ptr, FALSE);
+ if (!old_size) {
+ // Not a Nano pointer - let the helper deal with it
+ return nanozone->helper_zone->realloc(nanozone->helper_zone, ptr, new_size);
+ }
+
+ void *new_ptr;
+ if (new_size > NANO_MAX_SIZE) {
+ // Too large for Nano. Try to allocate from the helper zone.
+ new_ptr = nanozone->helper_zone->malloc(nanozone->helper_zone, new_size);
+ if (!new_ptr) {
+ // Failed to allocate - leave the existing allocation alone.
+ return NULL;
+ }
+ } else if (!new_size) {
+ // Resizing to zero. Free the existing memory and explicitly allocate
+ // zero bytes.
+ nanov2_free(nanozone, ptr);
+ return nanov2_malloc(nanozone, 0);
+ } else {
+ size_t new_good_size = _nano_common_good_size(new_size);
+ if (new_good_size > old_size || new_good_size <= old_size/2) {
+ // Growing or shrinking to less than half size - we need to
+ // reallocate.
+ new_ptr = nanov2_malloc(nanozone, new_good_size);
+ if (!new_ptr) {
+ // Failed to allocate - leave the existing allocation alone.
+ return NULL;
+ }
+ } else {
+ // Same size or shrinking by less than half size. Keep the same
+ // allocation and scribble the area that's being released. Nothing
+ // to do for zero-on-free yet; that will be taken care of when the
+ // shrunk allocation is freed.
+ if (new_size != old_size) {
+ MALLOC_ASSERT(new_size < old_size);
+ if (os_unlikely(nanozone->debug_flags & MALLOC_DO_SCRIBBLE)) {
+ memset(ptr + new_size, SCRABBLE_BYTE, old_size - new_size);
+ }
+ }
+ return ptr;
+ }
+ }
+
+ // If we reach this point, we allocated new memory. Copy the existing
+ // content to the new location and release the old allocation.
+ MALLOC_ASSERT(new_ptr);
+ memcpy(new_ptr, ptr, MIN(old_size, new_size));
+ nanov2_free(nanozone, ptr);
+
+ return new_ptr;
+}
+#endif // OS_VARIANT_RESOLVED
+
+#if OS_VARIANT_NOTRESOLVED
+static void
+nanov2_destroy(nanozonev2_t *nanozone)
+{
+ nanozone->helper_zone->destroy(nanozone->helper_zone);
+ nano_common_deallocate_pages((void *)nanozone, NANOZONEV2_ZONE_PAGED_SIZE,
+ nanozone->debug_flags);
+}
+#endif // OS_VARIANT_NOTRESOLVED
+
+#if OS_VARIANT_RESOLVED
+MALLOC_NOEXPORT boolean_t
+nanov2_claimed_address(nanozonev2_t *nanozone, void *ptr)
+{
+ return nanov2_pointer_size(nanozone, ptr, TRUE)
+ || malloc_zone_claimed_address(nanozone->helper_zone, ptr);
+}
+
+MALLOC_NOEXPORT unsigned
+nanov2_batch_malloc(nanozonev2_t *nanozone, size_t size, void **results,
+ unsigned count)
+{
+ unsigned allocated = 0;
+ size_t rounded_size = _nano_common_good_size(size);
+ if (rounded_size <= NANO_MAX_SIZE) {
+ while (allocated < count) {
+ // TODO: nanov2_malloc will redo _nano_common_good_size
+ void *ptr = nanov2_malloc(nanozone, rounded_size);
+ if (!ptr) {
+ break;
+ }
+
+ *results++ = ptr;
+ allocated++;
+ }
+ if (allocated == count) {
+ // Allocated everything.
+ return allocated;
+ }
+ }
+
+ // We could not allocate everything. Let the helper zone do the rest.
+ return allocated + nanozone->helper_zone->batch_malloc(
+ nanozone->helper_zone, size, results, count - allocated);
+}
+
+MALLOC_NOEXPORT void
+nanov2_batch_free(nanozonev2_t *nanozone, void **to_be_freed, unsigned count)
+{
+ if (count) {
+ while (count--) {
+ void *ptr = to_be_freed[count];
+ if (ptr) {
+ nanov2_free(nanozone, ptr);
+ }
+ }
+ }
+}
+
+MALLOC_NOEXPORT void *
+nanov2_memalign(nanozonev2_t *nanozone, size_t alignment, size_t size)
+{
+ // Serve directly if the requested alignment is trivially satisfied by our
+ // baseline alignment (16 bytes)
+ if (alignment <= NANO_REGIME_QUANTA_SIZE) {
+ return nanov2_malloc(nanozone, size);
+ }
+
+ // Otherwise delegate to the helper zone
+ return nanozone->helper_zone->memalign(nanozone->helper_zone, alignment,
+ size);
+}
+
+size_t
+nanov2_pressure_relief(nanozonev2_t *nanozone, size_t goal)
+{
+ if (nanov2_madvise_policy != NANO_MADVISE_WARNING_PRESSURE
+ && nanov2_madvise_policy != NANO_MADVISE_CRITICAL_PRESSURE) {
+ // In the current implementation, we only get called on warning, so
+ // act if the policy is either warning or critical. We would need to
+ // add a new zone entry point to respond to critical.
+ return 0;
+ }
+ const char *name = nanozone->basic_zone.zone_name;
+ MAGMALLOC_PRESSURERELIEFBEGIN((void *)nanozone, name, (int)goal);
+ MALLOC_TRACE(TRACE_nano_memory_pressure | DBG_FUNC_START,
+ (uint64_t)nanozone, goal, 0, 0);
+ size_t total = 0;
+
+ // Loop over all arenas madvising blocks that are marked as madvisable,
+ // until we reach our goal.
+ nanov2_region_t *region = nanozone->first_region_base;
+ nanov2_meta_index_t metablock_meta_index = nanov2_metablock_meta_index(nanozone);
+ nanov2_arena_t *current_region_next_arena = os_atomic_load(
+ &nanozone->current_region_next_arena, acquire);
+ while (region) {
+ nanov2_arena_t *arena = nanov2_first_arena_for_region(region);
+ nanov2_arena_t *arena_after_region = nanov2_limit_arena_for_region(
+ nanozone, region, current_region_next_arena);
+ while (arena < arena_after_region) {
+ // Scan all of the blocks in the arena, skipping the metadata block.
+ nanov2_arena_metablock_t *meta_blockp =
+ nanov2_metablock_address_for_ptr(nanozone, arena);
+ nanov2_block_meta_t *block_metap = &meta_blockp->arena_block_meta[0];
+
+ // We need to hold the zone madvise lock to madvise. We could take
+ // it for the duration of this function, but that might hold up
+ // ongoing allocation and free operations for too long. So just
+ // lock and unlock for each arena.
+ _malloc_lock_lock(&nanozone->madvise_lock);
+ for (nanov2_meta_index_t i = 0; i < NANOV2_BLOCKS_PER_ARENA;
+ i++, block_metap++) {
+ if (i != metablock_meta_index) {
+ nanov2_block_meta_t meta = os_atomic_load(block_metap, relaxed);
+ if (meta.next_slot == SLOT_CAN_MADVISE) {
+ nanov2_block_t *blockp = nanov2_block_address_from_meta_index(
+ nanozone, arena, i);
+ if (nanov2_madvise_block_locked(nanozone, block_metap,
+ blockp, nanov2_size_class_for_ptr(nanozone, blockp),
+ SLOT_CAN_MADVISE)) {
+ total += NANOV2_BLOCK_SIZE;
+ }
+ }
+ }
+ }
+ _malloc_lock_unlock(&nanozone->madvise_lock);
+ if (goal && total >= goal) {
+ goto done;
+ }
+ arena++;
+ }
+ region = nanov2_next_region_for_region(nanozone, region,
+ current_region_next_arena);
+ }
+
+done:
+ MAGMALLOC_PRESSURERELIEFEND((void *)nanozone, name, (int)goal, (int)total);
+ MALLOC_TRACE(TRACE_nano_memory_pressure | DBG_FUNC_END,
+ (uint64_t)nanozone, goal, total, 0);
+
+ return total;
+}
+#endif // OS_VARIANT_RESOLVED
+
+#pragma mark -
+#pragma mark Zone Introspection
+
+#if OS_VARIANT_NOTRESOLVED
+
+// NOTE: in the code that follows, address that we obtain from the Nano
+// structures are relative to the target process. They need to be translated
+// before they can be used to read the mapping in this process.
+
+#define NANOV2_ZONE_PTR_TO_MAPPED_PTR(type, zone_ptr, offset) \
+ (type)((mach_vm_address_t)zone_ptr - (mach_vm_offset_t)offset)
+#define NANOV2_MAPPED_PTR_TO_ZONE_PTR(type, mapped_ptr, offset) \
+ (type)((mach_vm_address_t)mapped_ptr + (mach_vm_offset_t)offset)
+
+static kern_return_t
+nanov2_ptr_in_use_enumerator(task_t task, void *context, unsigned type_mask,
+ vm_address_t zone_address, memory_reader_t reader,
+ vm_range_recorder_t recorder)
+{
+ // Ensure that we have configured enough of the allocator to be able to
+ // examine its data structures. In tools that do not directly use Nano, we
+ // won't have done this yet. nanov2_configure() runs the initialization
+ // only once.
+ nanov2_configure();
+
+ // Only MALLOC_PTR_IN_USE_RANGE_TYPE and MALLOC_PTR_REGION_RANGE_TYPE have
+ // meaning for Nano. Anything else returns immediately.
+ if (!(type_mask & (MALLOC_PTR_IN_USE_RANGE_TYPE|MALLOC_PTR_REGION_RANGE_TYPE))) {
+ return 0;
+ }
+
+ // Read the zone data.
+ nanozonev2_t *nanozone;
+ nanozonev2_t zone_copy;
+ kern_return_t kr;
+ bitarray_t slots;
+
+ if (!reader) {
+ reader = nano_common_default_reader;
+ }
+
+ kr = reader(task, zone_address, sizeof(nanozonev2_t), (void **)&nanozone);
+ if (kr) {
+ return kr;
+ }
+ boolean_t self_zone = mach_task_is_self(task) && (nanozonev2_t *)zone_address == nanozone;
+ memcpy(&zone_copy, nanozone, sizeof(zone_copy));
+ nanozone = &zone_copy;
+ nanov2_meta_index_t metablock_meta_index = nanov2_metablock_meta_index(nanozone);
+
+ // Process the zone one region at a time. Report each in-use block as a
+ // pointer range and each in-use slot as a pointer.
+ nanov2_region_t *region = nanozone->first_region_base;
+ nanov2_arena_t *current_region_next_arena = os_atomic_load(
+ &nanozone->current_region_next_arena, acquire);
+ while (region) {
+ mach_vm_address_t vm_addr = (mach_vm_address_t)NULL;
+ kern_return_t kr = reader(task, (vm_address_t)region, NANOV2_REGION_SIZE, (void **)&vm_addr);
+ if (kr) {
+ return kr;
+ }
+
+ // ptr_offset is the difference between an address in the target process
+ // and its mapped address in this process.
+ mach_vm_offset_t ptr_offset = (mach_vm_address_t)region - vm_addr;
+ nanov2_arena_t *arena = nanov2_first_arena_for_region(region);
+ nanov2_arena_t *limit_arena = nanov2_limit_arena_for_region(nanozone, region,
+ current_region_next_arena);
+ vm_range_t ptr_range;
+ while (arena < limit_arena) {
+ // Find the metadata block and process every entry, apart from the
+ // one for the metadata block itself.
+ nanov2_arena_metablock_t *arena_meta_blockp =
+ NANOV2_ZONE_PTR_TO_MAPPED_PTR(nanov2_arena_metablock_t *,
+ nanov2_metablock_address_for_ptr(nanozone, arena),
+ ptr_offset);
+ nanov2_block_meta_t *block_metap = &arena_meta_blockp->arena_block_meta[0];
+
+ for (nanov2_meta_index_t i = 0; i < NANOV2_BLOCKS_PER_ARENA; i++, block_metap++) {
+ if (i == metablock_meta_index) {
+ // Skip the metadata block.
+ continue;
+ }
+ nanov2_block_meta_t meta = os_atomic_load(block_metap, relaxed);
+ if (!nanov2_is_block_active(meta)) {
+ continue;
+ }
+
+ nanov2_block_t *blockp = nanov2_block_address_from_meta_index(
+ nanozone, arena, i);
+ if (type_mask & MALLOC_PTR_REGION_RANGE_TYPE) {
+ // Report this block as an in-use range.
+ ptr_range.address = (vm_address_t)blockp;
+ ptr_range.size = NANOV2_BLOCK_SIZE;
+ recorder(task, context, MALLOC_PTR_REGION_RANGE_TYPE, &ptr_range, 1);
+ }
+ if (type_mask & MALLOC_PTR_IN_USE_RANGE_TYPE) {
+ // Report all of the pointers in the block that are not on
+ // the free list.
+ nanov2_size_class_t size_class = nanov2_size_class_for_ptr(
+ nanozone, blockp);
+ int slot_size = nanov2_size_from_size_class(size_class);
+ int slot_count = slots_by_size_class[size_class];
+ vm_range_t ranges[NANOV2_MAX_SLOTS_PER_BLOCK];
+ int range_count = 0;
+ if (meta.next_slot == SLOT_BUMP || meta.next_slot == SLOT_FULL) {
+ // Either the block is full or the freelist is empty. If
+ // it's full, everything is in use. If the free list is
+ // empty, everything up to slot_count - meta.free_count - 1
+ // is in use.
+ range_count = meta.next_slot == SLOT_BUMP ?
+ slot_count - meta.free_count - 1 : slot_count;
+ for (int i = 0; i < range_count; i++) {
+ ranges[i].address = (vm_address_t)nanov2_slot_in_block_ptr(blockp, size_class, i);
+ ranges[i].size = slot_size;
+ }
+ } else {
+ // We need to scan the freelist to see what's in use.
+ int log_size = 64 - __builtin_clzl(slot_count);
+ if (self_zone) {
+ // Don't allocate from ourselves!
+ slots = nanozone->helper_zone->calloc(nanozone->helper_zone,
+ 1, bitarray_size(log_size));
+ } else {
+ slots = bitarray_create(log_size);
+ }
+ for (int i = 0; i < slot_count; i++) {
+ bitarray_set(slots, log_size, i);
+ }
+
+ int next_slot = meta.next_slot;
+ int free_list_count = 0;
+ while (next_slot != SLOT_BUMP) {
+ next_slot--; // meta.next_slot is 1-based.
+ if (next_slot < 0 || next_slot >= slot_count ||
+ !bitarray_get(slots, log_size, next_slot)) {
+ // Out of range or already seen?? We may have
+ // snapshotted the block while it was updating.
+ // Don't go any further to avoid an infinite loop.
+ break;
+ }
+ bitarray_zap(slots, log_size, next_slot);
+ void *ptr = nanov2_slot_in_block_ptr(blockp, size_class, next_slot);
+ nanov2_free_slot_t *slotp = NANOV2_ZONE_PTR_TO_MAPPED_PTR(nanov2_free_slot_t *, ptr, ptr_offset);
+ next_slot = (uint16_t)slotp->next_slot;
+ free_list_count++;
+ }
+ // Add a range for each slot that is not on the freelist,
+ // unless that slot has never been allocated.
+ int block_free_count = meta.free_count + 1; // actual free count.
+ int in_use_count = slot_count - block_free_count;
+ int slots_used_count = in_use_count + free_list_count;
+ index_t index;
+ while (bitarray_zap_first_set(slots, log_size, &index)) {
+ if (index >= slots_used_count) {
+ // Reached the end of the slots that have been
+ // allocated at some point.
+ break;
+ }
+ ranges[range_count].address = (vm_address_t)nanov2_slot_in_block_ptr(blockp, size_class, index);
+ ranges[range_count].size = slot_size;
+ range_count++;
+ }
+ free(slots);
+ }
+ if (range_count) {
+ // Notify the in-use pointers that we found.
+ recorder(task, context, MALLOC_PTR_IN_USE_RANGE_TYPE, ranges, range_count);
+ }
+ }
+ }
+ arena++;
+ }
+
+ // We have to manually handle the linkage to the next region because
+ // of the address slide between this process and the target.
+ nanov2_region_linkage_t *region_linkagep =
+ nanov2_region_linkage_for_region(nanozone, region);
+ nanov2_region_linkage_t *mapped_region_linkagep =
+ NANOV2_ZONE_PTR_TO_MAPPED_PTR(nanov2_region_linkage_t *,
+ region_linkagep, ptr_offset);
+ int offset = os_atomic_load(&mapped_region_linkagep->next_region_offset,
+ relaxed);
+ region = offset ? region + offset : NULL;
+ }
+ return 0;
+}
+
+static size_t
+nanov2_good_size(nanozonev2_t *nanozone, size_t size)
+{
+ if (size <= NANO_MAX_SIZE) {
+ return _nano_common_good_size(size);
+ }
+ return nanozone->helper_zone->introspect->good_size(nanozone->helper_zone,
+ size);
+}
+
+static boolean_t
+nanov2_check(nanozonev2_t *nanozone)
+{
+ // Does nothing
+ return 1;
+}
+
+static void
+nanov2_print(task_t task, unsigned level, vm_address_t zone_address,
+ memory_reader_t reader, print_task_printer_t printer)
+{
+ // Ensure that we have configured enough of the allocator to be able to
+ // examine its data structures. In tools that do not directly use Nano, we
+ // won't have done this yet. nanov2_configure() runs the initialization
+ // only once.
+ nanov2_configure();
+
+ nanozonev2_t *mapped_nanozone;
+ if (reader(task, (vm_address_t)zone_address, sizeof(nanozonev2_t),
+ (void **)&mapped_nanozone)) {
+ printer("Failed to map nanozonev2_s at %p\n", zone_address);
+ return;
+ }
+
+ // Zone-wide statistics
+ malloc_statistics_t stats;
+ nanov2_statistics_task_printer(task, zone_address, reader, printer, &stats);
+ nanov2_statistics_t *nano_stats = &mapped_nanozone->statistics;
+ printer("Nanozonev2 %p: blocks in use: %llu, size in use: %llu "
+ "allocated size: %llu, allocated regions: %d, region holes: %d\n",
+ zone_address, (uint64_t)stats.blocks_in_use,
+ (uint64_t)stats.size_in_use, (uint64_t)stats.size_allocated,
+ nano_stats->allocated_regions, nano_stats->region_address_clashes);
+
+#if DEBUG_MALLOC
+ // Per-size class statistics
+ printer("\nPer size-class statistics:\n");
+ for (int i = 0; i < NANO_SIZE_CLASSES; i++) {
+ nanov2_size_class_statistics *cs = &nano_stats->size_class_statistics[i];
+ printer(" Class %d: ", i);
+ printer("total alloc: %llu, total frees: %llu, madvised blocks: %llu, "
+ "madvise races: %llu",
+ cs->total_allocations, cs->total_frees, cs->madvised_blocks,
+ cs->madvise_races);
+ printer("\n");
+ }
+#endif // DEBUG_MALLOC
+
+ // Per-context block pointers.
+ printer("Current Allocation Blocks By Size Class/Context [CPU]\n");
+ for (int i = 0; i < NANO_SIZE_CLASSES; i++) {
+ printer(" Class %d: ", i);
+ for (int j = 0; j < MAX_CURRENT_BLOCKS; j++) {
+ if (mapped_nanozone->current_block[i][j]) {
+ printer("%d: %p; ", j, mapped_nanozone->current_block[i][j]);
+ }
+ }
+ printer("\n");
+ }
+
+ nanov2_meta_index_t metablock_meta_index =
+ nanov2_metablock_meta_index(mapped_nanozone);
+ nanov2_region_t *region = mapped_nanozone->first_region_base;
+ // Use a single, consistent snapshot of current_region_next_arena throughout
+ // iteration, ignoring any arenas or regions allocated after it.
+ nanov2_arena_t *current_region_next_arena = os_atomic_load(
+ &mapped_nanozone->current_region_next_arena, acquire);
+ int region_index = 0;
+ while (region) {
+ printer("\nRegion %d: base address %p\n", region_index, region);
+ nanov2_region_t *mapped_region;
+ if (reader(task, (vm_address_t)region, sizeof(nanov2_region_t),
+ (void **)&mapped_region)) {
+ printer("Failed to map nanov2 region at %p\n", region);
+ return;
+ }
+ off_t region_offset = (uintptr_t)mapped_region - (uintptr_t)region;
+
+ nanov2_arena_t *arena = nanov2_first_arena_for_region(region);
+ nanov2_arena_t *limit_arena = nanov2_limit_arena_for_region(
+ mapped_nanozone, region, current_region_next_arena);
+ int arena_index = 0;
+ while (arena < limit_arena) {
+ // Find the metadata block and process every entry, apart from the
+ // one for the metadata block itself.
+ nanov2_arena_metablock_t *arena_meta_blockp =
+ nanov2_metablock_address_for_ptr(mapped_nanozone, arena);
+ nanov2_arena_metablock_t *mapped_arena_meta_blockp =
+ (nanov2_arena_metablock_t *)((uintptr_t)arena_meta_blockp + region_offset);
+ nanov2_block_meta_t *block_metap = &arena_meta_blockp->arena_block_meta[0];
+ nanov2_block_meta_t *mapped_block_metap = &mapped_arena_meta_blockp->arena_block_meta[0];
+
+ int active_blocks = 0;
+ int madvisable_blocks = 0;
+ int unused_blocks = 0;
+ int madvised_blocks = 0;
+ int madvising_blocks = 0;
+ for (nanov2_meta_index_t i = 0; i < NANOV2_BLOCKS_PER_ARENA; i++) {
+ if (i == metablock_meta_index) {
+ // Skip the metadata block.
+ continue;
+ }
+ nanov2_block_meta_t meta = mapped_block_metap[i];
+ switch (meta.next_slot) {
+ case SLOT_NULL:
+ unused_blocks++;
+ break;
+ case SLOT_MADVISED:
+ madvised_blocks++;
+ break;
+ case SLOT_MADVISING:
+ madvising_blocks++;
+ break;
+ case SLOT_CAN_MADVISE:
+ madvisable_blocks++;
+ break;
+ default:
+ active_blocks++;
+ break;
+ }
+ }
+ printer("Arena #%d: base address %p. Blocks - active: %d, "
+ "madvisable: %d, madvising: %d, madvised: %d, unused: %d\n",
+ arena_index, arena, active_blocks, madvisable_blocks,
+ madvising_blocks, madvised_blocks, unused_blocks);
+
+ // Print which size classes have blocks allocated in this arena.
+ int non_empty_size_classes[NANO_SIZE_CLASSES];
+ for (int i = 0; i < NANO_SIZE_CLASSES; i++) {
+ non_empty_size_classes[i] = 0;
+ }
+ for (nanov2_meta_index_t i = 0; i < NANOV2_BLOCKS_PER_ARENA; i++) {
+ if (i == metablock_meta_index) {
+ // Skip the metadata block.
+ continue;
+ }
+ nanov2_block_meta_t meta = mapped_block_metap[i];
+ nanov2_size_class_t size_class =
+ nanov2_size_class_for_meta_index(mapped_nanozone, i);
+ switch (meta.next_slot) {
+ case SLOT_FULL:
+ case SLOT_BUMP:
+ default:
+ non_empty_size_classes[size_class]++;
+ break;
+ case SLOT_NULL:
+ case SLOT_CAN_MADVISE:
+ case SLOT_MADVISING:
+ case SLOT_MADVISED:
+ // Do not count these.
+ break;
+ }
+ }
+ printer("Size classes with allocated blocks: ");
+ for (int i = 0; i < NANO_SIZE_CLASSES; i++) {
+ if (non_empty_size_classes[i]) {
+ printer("%d ", i);
+ }
+ }
+ printer("\n");
+
+ if (level >= MALLOC_VERBOSE_PRINT_LEVEL) {
+ for (nanov2_meta_index_t i = 0; i < NANOV2_BLOCKS_PER_ARENA; i++) {
+ if (i == metablock_meta_index) {
+ // Skip the metadata block.
+ continue;
+ }
+ nanov2_block_meta_t meta = mapped_block_metap[i];
+ if (!nanov2_is_block_active(meta)) {
+ continue;
+ }
+ nanov2_size_class_t size_class =
+ nanov2_size_class_for_meta_index(mapped_nanozone, i);
+ char *slot_text;
+ switch (meta.next_slot) {
+ case SLOT_NULL:
+ slot_text = "NOT USED";
+ break;
+ case SLOT_FULL:
+ slot_text = "FULL";
+ break;
+ case SLOT_CAN_MADVISE:
+ slot_text = "CAN MADVISE";
+ break;
+ case SLOT_MADVISING:
+ slot_text = "MADVISING";
+ break;
+ case SLOT_MADVISED:
+ slot_text = "MADVISED";
+ break;
+ default:
+ slot_text = NULL;
+ break;
+ }
+ printer(" Block %d: base %p; metadata: %p, size %d "
+ "(class %d) in-use: %d ",
+ i, nanov2_block_address_from_meta_index(mapped_nanozone, arena, i),
+ &block_metap[i], nanov2_size_from_size_class(size_class),
+ size_class, meta.in_use);
+ if (slot_text) {
+ printer("%s\n", slot_text);
+ } else {
+ int allocated = slots_by_size_class[size_class] - meta.free_count - 1;
+ if (meta.next_slot == SLOT_BUMP) {
+ printer("BUMP (free list empty)");
+ } else {
+ printer("next_slot (1-based) = %d", meta.next_slot);
+
+ }
+ printer(", allocated slots: %d, free slots = %d, "
+ "occupancy: %d%%\n",
+ allocated, meta.free_count + 1,
+ (100 * allocated)/slots_by_size_class[size_class]);
+ }
+ }
+ }
+ arena++;
+ arena_index++;
+ }
+
+ region = nanov2_next_region_for_region_offset(mapped_nanozone, region,
+ region_offset, current_region_next_arena);
+ region_index++;
+ }
+}
+
+static void
+nanov2_print_self(nanozonev2_t *nanozone, boolean_t verbose)
+{
+ nanov2_print(mach_task_self(), verbose ? MALLOC_VERBOSE_PRINT_LEVEL : 0,
+ (vm_address_t)nanozone, _malloc_default_reader, malloc_report_simple);
+}
+
+static void
+nanov2_print_task(task_t task, unsigned level, vm_address_t zone_address,
+ memory_reader_t reader, print_task_printer_t printer)
+{
+ nanov2_print(task, level, zone_address, reader, printer);
+}
+
+static void
+nanov2_log(malloc_zone_t *zone, void *log_address)
+{
+ // Does nothing
+}
+
+static void
+nanov2_force_lock(nanozonev2_t *nanozone)
+{
+ // Nothing to do - Nano V2 does not have a zone lock.
+}
+
+static void
+nanov2_force_unlock(nanozonev2_t *nanozone)
+{
+ // Nothing to do - Nano V2 does not have a zone lock.
+}
+
+static void
+nanov2_reinit_lock(nanozonev2_t *nanozone)
+{
+ // Nothing to do - Nano V2 does not have a zone lock.
+}
+
+static boolean_t
+nanov2_locked(nanozonev2_t *nanozone)
+{
+ // Nothing to do - Nano V2 does not have a zone lock.
+ return FALSE;
+}
+
+static void
+nanov2_null_printer(const char __unused *fmt, ...)
+{
+}
+
+static kern_return_t
+nanov2_statistics(task_t task, vm_address_t zone_address,
+ memory_reader_t reader, print_task_printer_t printer,
+ malloc_statistics_t *stats)
+{
+ printer = printer ? printer : nanov2_null_printer;
+ reader = !reader && task == mach_task_self() ? _malloc_default_reader : reader;
+
+ kern_return_t err;
+
+ // Ensure that we have configured enough of the allocator to be able to
+ // examine its data structures. In tools that do not directly use Nano, we
+ // won't have done this yet. nanov2_configure() runs the initialization
+ // only once.
+ nanov2_configure();
+
+ memset(stats, '\0', sizeof(*stats));
+
+ nanozonev2_t *mapped_nanozone;
+ err = reader(task, (vm_address_t)zone_address, sizeof(nanozonev2_t),
+ (void **)&mapped_nanozone);
+ if (err) {
+ printer("Failed to map nanozonev2_s at %p\n", zone_address);
+ return err;
+ }
+
+ nanov2_region_t *region;
+ nanov2_arena_t *arena;
+ nanov2_meta_index_t metadata_block_index =
+ nanov2_metablock_meta_index(mapped_nanozone);
+
+ // Iterate over each arena in each region. Within each region, add
+ // statistics for each slot in each block, excluding the meta data block.
+ nanov2_arena_t *current_region_next_arena = os_atomic_load(
+ &mapped_nanozone->current_region_next_arena, acquire);
+ for (region = mapped_nanozone->first_region_base; region;) {
+ nanov2_region_t *mapped_region;
+ err = reader(task, (vm_address_t)region, sizeof(nanov2_region_t), (void **)&mapped_region);
+ if (err) {
+ printer("Failed to map nanov2 region at %p\n", region);
+ return err;
+ }
+ off_t region_offset = (uintptr_t)mapped_region - (uintptr_t)region;
+ for (arena = nanov2_first_arena_for_region(region);
+ arena < nanov2_limit_arena_for_region(mapped_nanozone, region,
+ current_region_next_arena);
+ arena++) {
+ nanov2_arena_metablock_t *meta_block =
+ nanov2_metablock_address_for_ptr(mapped_nanozone, arena);
+ nanov2_arena_metablock_t *mapped_meta_block =
+ (nanov2_arena_metablock_t *)((uintptr_t)meta_block + region_offset);
+ for (nanov2_meta_index_t i = 0; i < NANOV2_BLOCKS_PER_ARENA; i++) {
+ if (i == metadata_block_index) {
+ // Skip the metadata block.
+ continue;
+ }
+
+ nanov2_block_meta_t *mapped_block_metap = &mapped_meta_block->arena_block_meta[i];
+ nanov2_size_class_t size_class =
+ nanov2_size_class_for_meta_index(mapped_nanozone, i);
+ int slot_size = nanov2_size_from_size_class(size_class);
+
+ nanov2_block_meta_t meta =
+ os_atomic_load(mapped_block_metap, relaxed);
+ int slots_in_use = 0;
+ switch (meta.next_slot) {
+ case SLOT_NULL:
+ // FALLTHRU
+ case SLOT_CAN_MADVISE:
+ // FALLTHRU
+ case SLOT_MADVISING:
+ // FALLTHRU
+ case SLOT_MADVISED:
+ // FALLTHRU
+ case SLOT_GUARD:
+ // These blocks have no active content.
+ break;
+ case SLOT_FULL:
+ slots_in_use = slots_by_size_class[size_class];
+ break;
+ case SLOT_BUMP:
+ // FALLTHRU
+ default:
+ slots_in_use = slots_by_size_class[size_class] - meta.free_count - 1;
+ break;
+ }
+
+ // We can't report max_size_in_use because we don't have the
+ // metadata to do so.
+ if (slots_in_use) {
+ stats->blocks_in_use += slots_in_use;
+ stats->size_in_use += slots_in_use * slot_size;
+ stats->size_allocated += NANOV2_BLOCK_SIZE;
+ }
+ }
+ }
+ region = nanov2_next_region_for_region_offset(mapped_nanozone,
+ region, region_offset, current_region_next_arena);
+ }
+ return KERN_SUCCESS;
+}
+
+static void
+nanov2_statistics_self(nanozonev2_t *nanozone, malloc_statistics_t *stats)
+{
+ nanov2_statistics(mach_task_self(), (vm_address_t)nanozone,
+ _malloc_default_reader, malloc_report_simple, stats);
+}
+
+static kern_return_t
+nanov2_statistics_task_printer(task_t task, vm_address_t zone_address,
+ memory_reader_t reader, print_task_printer_t printer,
+ malloc_statistics_t *stats)
+{
+ return nanov2_statistics(task, zone_address, reader, printer, stats);
+}
+
+static kern_return_t
+nanov2_statistics_task(task_t task, vm_address_t zone_address, memory_reader_t reader, malloc_statistics_t *stats)
+{
+ return nanov2_statistics(task, zone_address, reader, NULL, stats);
+}
+
+
+static const struct malloc_introspection_t nanov2_introspect = {
+ .enumerator = (void *)nanov2_ptr_in_use_enumerator,
+ .good_size = (void *)nanov2_good_size,
+ .check = (void *)nanov2_check,
+ .print = (void *)nanov2_print_self,
+ .log = (void *)nanov2_log,
+ .force_lock = (void *)nanov2_force_lock,
+ .force_unlock = (void *)nanov2_force_unlock,
+ .statistics = (void *)nanov2_statistics_self,
+ .zone_locked = (void *)nanov2_locked,
+ .enable_discharge_checking = NULL,
+ .disable_discharge_checking = NULL,
+#ifdef __BLOCKS__
+ .enumerate_discharged_pointers = NULL,
+#else // __BLOCKS__
+ .enumerate_unavailable_without_blocks = NULL,
+#endif // __BLOCKS__
+ .reinit_lock = (void *)nanov2_reinit_lock,
+ .print_task = (void *)nanov2_print_task,
+ .task_statistics = (void*)nanov2_statistics_task,
+};
+
+#endif // OS_VARIANT_NOTRESOLVED
+
+#pragma mark -
+#pragma mark Utility Functions
+
+#if OS_VARIANT_RESOLVED
+
+// Given a pointer that may be to Nano V2-allocated memory, returns the size of
+// the allocation, or 0 if the pointer does not correspond to an active
+// allocation. If allow_inner is true, the pointer need not point to the start
+// of the allocation.
+MALLOC_ALWAYS_INLINE MALLOC_INLINE size_t
+nanov2_pointer_size_inline(nanozonev2_t *nanozone, void *ptr,
+ boolean_t allow_inner, nanov2_size_class_t *size_class_out,
+ nanov2_block_meta_t **block_metap_out)
+{
+ // First check the address signature.
+ if (!nanov2_has_valid_signature((void *)ptr)) {
+ return 0;
+ }
+
+ // Check for proper alignment, unless we could have an inner pointer.
+ if (!allow_inner && ((uintptr_t)ptr) & NANO_QUANTA_MASK) {
+ return 0;
+ }
+
+ // Atomically load the value of current_region_next_arena. No thread is
+ // allowed to allocate from an arena until it observes a greater value of
+ // current_region_next_arena, which must have happened before now if we're
+ // being called in the context of a deallocation, so we can safely use it as
+ // the upper bound for an overall address range check.
+ nanov2_arena_t *current_region_next_arena = os_atomic_load(
+ &nanozone->current_region_next_arena, relaxed);
+
+ // Bounds check against the active address space.
+ if (ptr < (void *)nanozone->first_region_base ||
+ ptr > (void *)current_region_next_arena) {
+ return 0;
+ }
+
+#if NANOV2_MULTIPLE_REGIONS
+ // Need to check that the region part is valid because there could be holes.
+ // Do this only if we know there is a hole.
+ //
+ // If we're looking at a legitimately-allocated nano pointer, a load-acquire
+ // of current_region_next_arena must have already happened when its
+ // containing arena was first allocated from, so any region_address_clashes
+ // increment that preceded the store-release of current_region_next_arena
+ // should be visible.
+ //
+ // TODO: use a hashed structure to make this more efficient.
+ if (os_atomic_load(&nanozone->statistics.region_address_clashes, relaxed)) {
+ nanov2_region_t *ptr_region = nanov2_region_address_for_ptr(ptr);
+ nanov2_region_t *region = nanozone->first_region_base;
+ while (region) {
+ if (ptr_region == region) {
+ break;
+ }
+ region = nanov2_next_region_for_region(nanozone, region,
+ current_region_next_arena);
+ }
+ if (!region) {
+ // Reached the end of the region list without matching - not a
+ // valid Nano V2 pointer.
+ return 0;
+ }
+ }
+#endif // NANOV2_MULTIPLE_REGIONS
+
+ // Get the size class for the pointer and the address of its meta block
+ // header.
+ nanov2_size_class_t size_class = nanov2_size_class_for_ptr(nanozone, ptr);
+ nanov2_block_meta_t *block_metap = nanov2_meta_ptr_for_ptr(nanozone, ptr);
+
+ // Reject if the block is not active, or it doesn't have any allocations.
+ nanov2_block_meta_t meta = os_atomic_load(block_metap, relaxed);
+ if (!nanov2_is_block_active(meta) || (meta.next_slot != SLOT_FULL &&
+ meta.free_count == slots_by_size_class[size_class] - 1)) {
+ return 0;
+ }
+
+ size_t size = nanov2_size_from_size_class(size_class);
+ nanov2_addr_t addr = { .addr = ptr };
+ if (!allow_inner && (addr.fields.nano_offset % size)) {
+ return 0;
+ }
+
+ // The only reasonable way to check whether the pointer is free is to
+ // inspect the canary value at the start of the slot, since we cannot take
+ // a huge hit for walking the free list.
+ nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
+ uintptr_t guard = os_atomic_load(&slotp->double_free_guard, relaxed);
+ if ((guard ^ nanozone->slot_freelist_cookie) == (uintptr_t)ptr) {
+ return 0;
+ }
+
+ if (size_class_out) {
+ *size_class_out = size_class;
+ }
+ if (block_metap_out) {
+ *block_metap_out = block_metap;
+ }
+ return size;
+}
+
+size_t
+nanov2_pointer_size(nanozonev2_t *nanozone, void *ptr, boolean_t allow_inner)
+{
+ return nanov2_pointer_size_inline(nanozone, ptr, allow_inner, NULL, NULL);
+}
+
+#pragma mark -
+#pragma mark Madvise Management
+
+// Given a pointer to a block and its metadata, calls madvise() on that block if
+// it is still in the state we expect, either SLOT_CAN_MADVISE or SLOT_MADVISED
+// (the latter expected when we need to pessimistically re-madvise a block we
+// may have touched while racing to allocate against a transition to
+// SLOT_CAN_MADVISE). Returns true on success, false if the block is not in the
+// correct state or if the state changed during the operation.
+//
+// This function must be called with the zone's madvise_lock held
+static boolean_t
+nanov2_madvise_block_locked(nanozonev2_t *nanozone,
+ nanov2_block_meta_t *block_metap, nanov2_block_t *blockp,
+ nanov2_size_class_t size_class, uint32_t expected_state)
+{
+ _malloc_lock_assert_owner(&nanozone->madvise_lock);
+
+ boolean_t madvised = FALSE;
+ nanov2_block_meta_t old_meta = os_atomic_load(block_metap, relaxed);
+ if (old_meta.next_slot == expected_state) {
+ // Nobody raced with us. We can safely madvise this block. First change
+ // the state to SLOT_MADVISING so that other threads don't try to
+ // grab the block for new allocations.
+ nanov2_block_meta_t new_meta = {
+ .next_slot = SLOT_MADVISING,
+ .gen_count = old_meta.gen_count + 1,
+ };
+ if (!os_atomic_cmpxchgv(block_metap, old_meta, new_meta, &old_meta,
+ relaxed)) {
+ // Somebody else tampered with this block. This can happen if
+ // another thread raced with us to allocate in this block. Count
+ // the contended access.
+ nanozone->statistics.size_class_statistics[size_class].madvise_races++;
+ return false;
+ }
+
+ if (mvm_madvise_free(nanozone, nanov2_region_address_for_ptr(blockp),
+ (uintptr_t)blockp, (uintptr_t)(blockp + 1), NULL, FALSE)) {
+ malloc_zone_error(0, false, "Failed to madvise block at blockp: %p, error: %d\n", blockp, errno);
+ } else {
+ nanozone->statistics.size_class_statistics[size_class].madvised_blocks++;
+ madvised = TRUE;
+ }
+
+ nanov2_block_meta_t final_meta = {
+ .next_slot = SLOT_MADVISED,
+ .gen_count = new_meta.gen_count + 1,
+ };
+
+ if (!os_atomic_cmpxchgv(block_metap, new_meta, final_meta, &old_meta,
+ relaxed)) {
+ // This should not happen since we should have exclusive interest
+ // in this block.
+ malloc_zone_error(nanozone->debug_flags, false,
+ "Failed when changing state from MADVISING to MADVISED, "
+ "block_metap = %p, blockp = %p\n", block_metap, blockp);
+ }
+ }
+ return madvised;
+}
+
+static void
+nanov2_madvise_block(nanozonev2_t *nanozone, nanov2_block_meta_t *block_metap,
+ nanov2_size_class_t size_class, uint32_t expected_state)
+{
+ nanov2_block_t *blockp = nanov2_block_address_from_meta_ptr(nanozone,
+ block_metap);
+ _malloc_lock_lock(&nanozone->madvise_lock);
+ nanov2_madvise_block_locked(nanozone, block_metap, blockp, size_class,
+ expected_state);
+ _malloc_lock_unlock(&nanozone->madvise_lock);
+}
+
+#endif // OS_VARIANT_RESOLVED
+
+#pragma mark -
+#pragma mark Region Management
+
+#if OS_VARIANT_NOTRESOLVED
+
+#if CONFIG_NANO_RESERVE_REGIONS
+// Update protection for region to DEFAULT
+static bool
+nanov2_unprotect_region(nanov2_region_t *region)
+{
+ MALLOC_TRACE(TRACE_nanov2_region_protection | DBG_FUNC_START,
+ (uint64_t)region, 0, 0, 0);
+ bool result = nano_common_unprotect_vm_space((mach_vm_address_t)region,
+ NANOV2_REGION_SIZE);
+ MALLOC_TRACE(TRACE_nanov2_region_protection | DBG_FUNC_END,
+ (uint64_t)region, result, 0, 0);
+ return result;
+}
+
+// Reserve VA at [base, base+num_regions*REGION_SIZE].
+// Note: permissions must still be granted on reserved region with `nanov2_unprotect_region`
+static bool
+nanov2_reserve_regions(nanov2_region_t *base, unsigned int num_regions)
+{
+ MALLOC_TRACE(TRACE_nanov2_region_reservation | DBG_FUNC_START,
+ (uint64_t)base, num_regions, 0, 0);
+ bool result = nano_common_reserve_vm_space((mach_vm_address_t)base,
+ (NANOV2_REGION_SIZE * (mach_vm_size_t)num_regions));
+ MALLOC_TRACE(TRACE_nanov2_region_reservation | DBG_FUNC_END,
+ (uint64_t)base, num_regions, result, 0);
+
+ return result;
+}
+#else
+// Attempts to allocate VM space for a region at a given address and returns
+// whether the allocation succeeded.
+static bool
+nanov2_allocate_region(nanov2_region_t *region)
+{
+ MALLOC_TRACE(TRACE_nanov2_region_allocation | DBG_FUNC_START,
+ (uint64_t)region, 0, 0, 0);
+ bool result = nano_common_allocate_vm_space((mach_vm_address_t)region,
+ NANOV2_REGION_SIZE);
+ MALLOC_TRACE(TRACE_nanov2_region_allocation | DBG_FUNC_END,
+ (uint64_t)region, result, 0, 0);
+ return result;
+}
+#endif // CONFIG_NANO_RESERVE_REGIONS
+
+// Allocates a new region adjacent to the current one. If the allocation fails,
+// keep sliding up by the size of a region until we either succeed or run out of
+// address space. The caller must own the Nanozone regions lock. Returns the
+// first arena of the newly-allocated region if successful, or NULL otherwise.
+MALLOC_NOEXPORT nanov2_arena_t *
+nanov2_allocate_new_region(nanozonev2_t *nanozone)
+{
+#if NANOV2_MULTIPLE_REGIONS
+ bool allocated = false;
+
+ nanov2_addr_t nanov2_max_region_base = {
+ .fields.nano_signature = NANOZONE_SIGNATURE,
+ .fields.nano_region = nano_max_region,
+ };
+
+ _malloc_lock_assert_owner(&nanozone->regions_lock);
+ nanov2_region_t *current_region = nanov2_current_region_base(
+ os_atomic_load(&nanozone->current_region_next_arena, relaxed));
+ nanov2_region_t *next_region = current_region + 1;
+
+ while ((void *)next_region <= nanov2_max_region_base.addr) {
+#if CONFIG_NANO_RESERVE_REGIONS
+ if (!nanov2_unprotect_region(next_region)) {
+ MALLOC_REPORT_FATAL_ERROR(next_region,
+ "Nano: Unable to raise protection on pre-allocated region");
+ }
+ nanozone->statistics.allocated_regions++;
+ allocated = true;
+ break;
+#else // CONFIG_NANO_RESERVE_REGIONS
+ if (nanov2_allocate_region(next_region)) {
+ nanozone->statistics.allocated_regions++;
+ allocated = true;
+ break;
+ }
+ next_region++;
+
+ // Loaded atomically in nanov2_pointer_size() to determine whether or
+ // not it's necessary to walk the region list, so we need to increment
+ // atomically here. Published by the store-release of
+ // current_region_next_arena.
+ os_atomic_inc(&nanozone->statistics.region_address_clashes, relaxed);
+#endif // CONFIG_NANO_RESERVE_REGIONS
+ }
+
+ if (!allocated) {
+ return NULL;
+ }
+
+ // Link this region to the previous one.
+ nanov2_region_linkage_t *current_region_linkage =
+ nanov2_region_linkage_for_region(nanozone, current_region);
+
+ // The linkage of the next region is in pristine memory, so already zero -
+ // don't touch it.
+
+ // Store-release the linkage update so any dependent loads through it
+ // observe the (implicit zero-)initialization of the next region.
+ uint16_t offset = next_region - current_region;
+ os_atomic_store(¤t_region_linkage->next_region_offset, offset,
+ release);
+
+ // Store-release the update to current_region_next_arena to publish the
+ // linkage update. Pairs with load-acquires of current_region_next_arena
+ // followed by walks of the region list.
+ nanov2_arena_t *first_arena = nanov2_first_arena_for_region(next_region);
+ os_atomic_store(&nanozone->current_region_next_arena, first_arena + 1,
+ release);
+
+ return first_arena;
+#else // NANOV2_MULTIPLE_REGIONS
+ // On iOS, only one region is supported, so we fail since the first
+ // region is allocated separately.
+ return NULL;
+#endif // CONFIG_NANOV2_MULTIPLE_REGIONS
+}
+#endif // OS_VARIANT_NOTRESOLVED
+
+#pragma mark -
+#pragma mark Allocation
+
+#if OS_VARIANT_RESOLVED
+
+MALLOC_NOINLINE MALLOC_NORETURN
+static void
+nanov2_guard_corruption_detected(void *corrupt_slot)
+{
+ uint64_t guard = *(uint64_t *)corrupt_slot;
+ malloc_zone_error(MALLOC_ABORT_ON_CORRUPTION, true,
+ "Heap corruption detected, free list is damaged at %p\n"
+ "*** Incorrect guard value: %llu\n", corrupt_slot,
+ (unsigned long long)guard);
+ __builtin_unreachable();
+}
+
+// Allocates memory from the block that corresponds to a given block meta data
+// pointer. The memory is taken from the free list if possible, or from the
+// unused region of the block if not. If the block is no longer in use or is
+// full, NULL is returned and the caller is expected to find another block to
+// allocate from.
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+void *
+nanov2_allocate_from_block_inline(nanozonev2_t *nanozone,
+ nanov2_block_meta_t *block_metap, nanov2_size_class_t size_class,
+ nanov2_block_meta_t **madvise_block_metap_out, bool *corruption)
+{
+ nanov2_block_meta_view_t old_meta_view;
+
+ // Our loads of the block metadata use dependency ordering, which guarantees
+ // that any loads we do from a slot pointer derived from the metadata value
+ // as we do below will observe all of the stores preceding the store-release
+ // of that value we observed. This allows us to safely rely on the contents
+ // of the slot updated when it was last freed, including the double-free
+ // guard and zeroing done by zero-on-free (which is required for correctness
+ // in the case of calloc).
+ old_meta_view.meta = os_atomic_load(block_metap, dependency);
+
+ // Calculating blockp and ptr is relatively expensive. Do both lazily to
+ // minimize the time in the block starting with "again:" and ending with the
+ // atomic update so that we lose at little time as possible if we have to
+ // repeat that loop due to contention. This should also reduce the risk of
+ // contention.
+ nanov2_block_t *blockp = NULL;
+
+again:
+ if (!nanov2_can_allocate_from_block(old_meta_view.meta)) {
+ // Move along, nothing to allocate here...
+ return NULL;
+ }
+
+ int slot;
+ void *ptr = NULL;
+ boolean_t from_free_list = FALSE;
+ nanov2_block_meta_t new_meta = {
+ .in_use = 1,
+ .free_count = old_meta_view.meta.free_count - 1,
+ .gen_count = old_meta_view.meta.gen_count + 1
+ };
+
+ // Grab a slot from the free list or get the next unused slot. We know there
+ // should be one because the block is not full.
+ boolean_t slot_full = old_meta_view.meta.free_count == 0;
+ if (old_meta_view.meta.next_slot == SLOT_BUMP
+ || old_meta_view.meta.next_slot == SLOT_CAN_MADVISE) {
+ // Free list empty, grab the next unused slot.
+ new_meta.next_slot = slot_full ? SLOT_FULL : SLOT_BUMP;
+ slot = slots_by_size_class[size_class] - old_meta_view.meta.free_count - 1;
+ } else {
+ // Grab the first item from the free list.
+ from_free_list = TRUE;
+ if (!blockp) {
+ blockp = nanov2_block_address_from_meta_ptr(nanozone, block_metap);
+ }
+ slot = old_meta_view.meta.next_slot - 1; // meta.next_slot is 1-based.
+ ptr = nanov2_slot_in_block_ptr(blockp, size_class, slot);
+ nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
+ new_meta.next_slot = slot_full ? SLOT_FULL : (uint16_t)slotp->next_slot;
+ }
+
+ // Write the updated meta data; try again if we raced with another thread.
+ if (!os_atomic_cmpxchgv(block_metap, old_meta_view.meta, new_meta,
+ &old_meta_view.meta, dependency)) {
+ if (old_meta_view.meta.next_slot == SLOT_CAN_MADVISE ||
+ old_meta_view.meta.next_slot == SLOT_MADVISING ||
+ old_meta_view.meta.next_slot == SLOT_MADVISED) {
+ *madvise_block_metap_out = block_metap;
+ return NULL;
+ }
+ goto again;
+ }
+
+ if (!ptr) {
+ if (!blockp) {
+ blockp = nanov2_block_address_from_meta_ptr(nanozone, block_metap);
+ }
+ ptr = nanov2_slot_in_block_ptr(blockp, size_class, slot);
+ }
+
+ if (from_free_list) {
+ // We grabbed the item from the free list. Check the free list canary
+ // and crash if it's not valid. We can't do this check before the
+ // cmpxchgv because another thread may race with us, claim the slot and
+ // write to it.
+ nanov2_free_slot_t *slotp = ptr;
+ uintptr_t guard = os_atomic_load(&slotp->double_free_guard, relaxed);
+ if (os_unlikely((guard ^ nanozone->slot_freelist_cookie) != (uintptr_t)ptr)) {
+ *corruption = true;
+ }
+ }
+
+#if DEBUG_MALLOC
+ nanozone->statistics.size_class_statistics[size_class].total_allocations++;
+#endif // DEBUG_MALLOC
+
+ return ptr;
+}
+
+static void *
+nanov2_allocate_from_block(nanozonev2_t *nanozone,
+ nanov2_block_meta_t *block_metap, nanov2_size_class_t size_class)
+{
+ nanov2_block_meta_t *madvise_block_metap = NULL;
+ bool corruption = false;
+ void *ptr = nanov2_allocate_from_block_inline(nanozone, block_metap,
+ size_class, &madvise_block_metap, &corruption);
+ if (os_unlikely(corruption)) {
+ nanov2_guard_corruption_detected(ptr);
+ }
+ if (madvise_block_metap) {
+ nanov2_madvise_block(nanozone, madvise_block_metap, size_class,
+ SLOT_MADVISED);
+ }
+ return ptr;
+}
+
+// Finds a block for allocation in an arena and returns a pointer to its
+// metadata header. The search begins from the block with metadata pointer
+// start_block (which must not be NULL). If no acceptable block was found,
+// NULL is returned and it is expected that the caller will take appropriate
+// action (typically allocate a new arena).
+//
+// The search starts with start_block. If this is in-use and not full, that
+// block is returned. Otherwise, a scan for a usable block is initiated. The
+// search starts from start_block and initially works backward towards the
+// start of the arena. If this does not succeed, a forward search from
+// start_block is made.
+//
+// A block is considered a candidate if it is not in use. As the scan proceeds,
+// we remember blocks which have been madvisable, blocks which have been
+// madvised or never used and those blocks which still have allocated slots
+// but which fall within the reuse criteria (i.e. their occupancy is within the
+// max/min occupancy range).
+//
+// If the scan policy is NANO_SCAN_FIRST_FIT, we just return the first block
+// from the above list that we find. This is the fastest option, but likely
+// maximises fragmentation.
+//
+// Otherwise, the scan policy is NANO_SCAN_CAPACITY_BASED. If we find a block
+// that fits the reuse criteria, we return it immediately. Otherwise, we
+// continue to scan until we find such a block, or we find a less ideal block
+// and we reach the scan limit or exhaust the arena. At that point, we return
+// one of the candidate blocks that we found, choosing based on the state of
+// that block:
+// - blocks that have allocations that are greater than the minimum capacity
+// are preferred.
+// - failing that, return an unused or madvise'd block.
+// - failing that, return a block that is waiting to be madvised.
+//
+// In order to avoid races, this function must be called with the
+// current_block_lock for the calling context [CPU] and size class locked.
+// On return, the selected block has been marked as in-use, so the caller must
+// either assign it as the active allocation block for the calling context or
+// clear the in-use bit.
+//
+MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_meta_t *
+nanov2_find_block_in_arena(nanozonev2_t *nanozone,
+ nanov2_arena_t *arena, nanov2_size_class_t size_class,
+ nanov2_block_meta_t *start_block)
+{
+ // If we don't have a starting point, start with the first block in the
+ // arena for the given size class. This is the case where we are looking for
+ // the first allocation block for a new context (i.e probably a new CPU, so
+ // take the first fit to avoid having to scan the whole size class for this
+ // very common start up case.)
+ boolean_t use_first_fit = !start_block ||
+ nanov2_policy_config.block_scan_policy == NANO_SCAN_FIRST_FIT;
+ nanov2_block_meta_t *first_block = nanov2_first_block_for_size_class_in_arena(
+ nanozone, size_class, arena);
+ boolean_t scanning_backwards;
+ if (!start_block) {
+ start_block = first_block;
+ }
+ int slots_in_block = slots_by_size_class[size_class];
+ nanov2_block_meta_t old_meta;
+ nanov2_block_meta_t *this_block;
+ nanov2_block_meta_t *found_block;
+ nanov2_block_meta_t *madvisable_block;
+ nanov2_block_meta_t *free_block;
+ nanov2_block_meta_t *fallback_block;
+ boolean_t fallback_below_max;
+ int scan_limit;
+
+ // Check all of the blocks in the size class until we find one that we can
+ // use, based on nanov2_block_scan_policy.
+retry:
+ this_block = start_block;
+ found_block = NULL;
+ madvisable_block = NULL;
+ free_block = NULL;
+ fallback_block = NULL;
+ fallback_below_max = FALSE;
+ scan_limit = nanov2_policy_config.block_scan_limit;
+ scanning_backwards = TRUE;
+
+ do {
+ old_meta = os_atomic_load(this_block, relaxed);
+ if (!old_meta.in_use && old_meta.next_slot != SLOT_FULL
+ && old_meta.next_slot != SLOT_MADVISING) {
+ if (old_meta.next_slot == SLOT_CAN_MADVISE) {
+ if (!madvisable_block) {
+ // We can use this block as a last-ditch fallback.
+ madvisable_block = this_block;
+ }
+ } else if (old_meta.next_slot == SLOT_NULL
+ || old_meta.next_slot == SLOT_MADVISED) {
+ if (!free_block) {
+ free_block = this_block;
+ }
+ } else if (use_first_fit) {
+ found_block = this_block;
+ } else {
+ MALLOC_ASSERT(nanov2_policy_config.block_scan_policy == NANO_SCAN_CAPACITY_BASED);
+ int percent_used = (100 * old_meta.free_count)/slots_in_block;
+ if (percent_used >= nanov2_policy_config.block_scan_min_capacity
+ && percent_used <= nanov2_policy_config.block_scan_max_capacity) {
+ // Within specified limits -- take this one.
+ found_block = this_block;
+ } else if (percent_used >= nanov2_policy_config.block_scan_min_capacity) {
+ if (!fallback_block || fallback_below_max) {
+ // More full than we want, but still acceptable as a
+ // fallback.
+ fallback_block = this_block;
+ }
+ } else if (!fallback_block
+ && percent_used < nanov2_policy_config.block_scan_min_capacity) {
+ // Less full than we want. Keep it as a backup, but set
+ // fallback_below_max to allow a block that's above max to
+ // be preferred. The rationale behind this is to allow
+ // blocks that have low occupancy to drain so that they can
+ // be madvised.
+ fallback_block = this_block;
+ fallback_below_max = TRUE;
+ } else if (!free_block) {
+ // Not ideal, but we could use it.
+ free_block = this_block;
+ }
+ }
+ if (use_first_fit && (found_block || fallback_block || free_block)) {
+ // Take whatever we got.
+ break;
+ }
+ }
+
+ if (scan_limit > 0) {
+ // Only enforce the scan limit once we have a candidate.
+ if ((fallback_block || free_block) && --scan_limit == 0) {
+ break;
+ }
+ }
+
+ if (scanning_backwards) {
+ boolean_t wrapped;
+ nanov2_block_meta_t *prev_block = nanov2_previous_block_for_size_class(
+ nanozone, size_class, this_block, &wrapped);
+ if (wrapped) {
+ // We wrapped. Scan forward from the start block instead.
+ scan_limit = nanov2_policy_config.block_scan_limit;
+ scanning_backwards = FALSE;
+ this_block = start_block;
+ } else {
+ this_block = prev_block;
+ }
+ } else {
+ // Move to the next block, wrapping when we reach the last one for
+ // this size class. Stop once we get to the block where we started.
+ this_block = nanov2_next_block_for_size_class(nanozone, size_class,
+ this_block, NULL);
+ if (this_block == start_block) {
+ break;
+ }
+ }
+ } while (!found_block);
+
+ if (!found_block) {
+ if (fallback_block) {
+ found_block = fallback_block;
+ } else if (free_block) {
+ found_block = free_block;
+ } else if (madvisable_block) {
+ found_block = madvisable_block;
+ }
+ }
+
+ if (found_block) {
+ // Now we need to activate the block. If this fails, we look for
+ // another block.
+ // If we are bringing a block that is draining back into use, we
+ // just need to set in_use to 1. Otherwise, we fully initialize it.
+ old_meta = os_atomic_load(found_block, relaxed);
+ if (old_meta.next_slot == SLOT_MADVISING) {
+ goto retry;
+ }
+ boolean_t reset_slot = old_meta.next_slot == SLOT_NULL
+ || old_meta.next_slot == SLOT_CAN_MADVISE
+ || old_meta.next_slot == SLOT_MADVISED;
+ nanov2_block_meta_t new_meta = {
+ .in_use = 1,
+ .free_count = reset_slot ? slots_in_block - 1 : old_meta.free_count,
+ .next_slot = reset_slot ? SLOT_BUMP : old_meta.next_slot,
+ .gen_count = reset_slot ? 0 : old_meta.gen_count + 1,
+ };
+ if (!os_atomic_cmpxchgv(found_block, old_meta, new_meta, &old_meta,
+ relaxed)) {
+ goto retry;
+ }
+ }
+
+ return found_block;
+}
+
+// Finds a block to allocate from and allocates memory from it. The search
+// for a block starts from *block_metapp if not NULL, otherwise from the first
+// arena in the first block (which is the case when the first block is allocated
+// for a size class for a CPU).
+// If none of the blocks for a size class in the current arena can be used, a
+// new arena is allocated and, if necessary, a new region is added.
+//
+// The address of the allocated memory is returned and its metadata pointer is
+// stored in *block_metapp. If a new region is required and it can't be
+// allocated, NULL is returned and *block_metapp is unmodified.
+//
+// On success, the returned block is marked as in-use and the block originally
+// pointed to by *block_metapp has its in-use bit cleared.
+//
+// In order to avoid races, this function must be called with the
+// current_block_lock for the calling context [CPU] and size class locked.
+MALLOC_NOEXPORT MALLOC_NOINLINE void *
+nanov2_find_block_and_allocate(nanozonev2_t *nanozone,
+ nanov2_size_class_t size_class, nanov2_block_meta_t **block_metapp)
+{
+ nanov2_arena_t *arena;
+ nanov2_block_meta_t *start_block = os_atomic_load(block_metapp, relaxed);
+ nanov2_block_meta_t *orig_block = start_block;
+ if (start_block) {
+ // Use the arena for the starting block.
+ arena = nanov2_arena_address_for_ptr(start_block);
+ } else {
+ // Start from the first arena.
+ arena = nanov2_arena_address_for_ptr(nanozone->first_region_base);
+ }
+
+ nanov2_region_t *start_region;
+retry:
+ start_region = nanov2_region_address_for_ptr(arena);
+ nanov2_arena_t *start_arena = arena;
+ nanov2_region_t *region = start_region;
+ // The load-acquire pairs with store-release in nanov2_allocate_new_region()
+ // to make the most recent region linkage update visible when we load it in
+ // nanov2_next_region_for_region() below.
+ nanov2_arena_t *initial_region_next_arena = os_atomic_load(
+ &nanozone->current_region_next_arena, acquire);
+ nanov2_arena_t *limit_arena = nanov2_limit_arena_for_region(nanozone,
+ start_region, initial_region_next_arena);
+ do {
+ nanov2_block_meta_t *block_metap = nanov2_find_block_in_arena(nanozone,
+ arena, size_class, start_block);
+ if (block_metap) {
+ // Try to allocate from this block and return if it succeeds. Note
+ // that the block is now marked as in-use, so effectively belongs
+ // to the calling context.
+ void *ptr = nanov2_allocate_from_block(nanozone, block_metap, size_class);
+ if (ptr) {
+ // Make the new block the current one for the calling context.
+ os_atomic_store(block_metapp, block_metap, relaxed);
+
+ // Turn off in-use in old block_metap, if there is one.
+ if (orig_block) {
+ // Turn off in-use in the original current block.
+ nanov2_turn_off_in_use(orig_block);
+ }
+ return ptr;
+ }
+
+ // We found a block but failed to allocate from it, probably because
+ // it became full. Look for a new block, using the one that we just
+ // failed with as the starting point. First, we need to turn off the
+ // in-use bit for the block that we just failed to allocate from.
+ nanov2_turn_off_in_use(block_metap);
+
+ start_block = block_metap;
+ goto retry;
+ }
+
+ // Try the next arena. If this is the last arena in the region, try the
+ // next region.
+ start_block = NULL;
+ arena++;
+ if (arena >= limit_arena) {
+ region = nanov2_next_region_for_region(nanozone, region,
+ initial_region_next_arena);
+ if (!region) {
+ // Reached the last region -- loop back to the first.
+ region = nanozone->first_region_base;
+ }
+ arena = nanov2_first_arena_for_region(region);
+ limit_arena = nanov2_limit_arena_for_region(nanozone, region,
+ initial_region_next_arena);
+ }
+ } while (arena != start_arena);
+
+ // If we get to this point, we need to allocate a new arena and possibly
+ // a new region. If we are not permitted to do so by policy, return NULL.
+ if (nanov2_policy_config.single_arena_size_classes & (1 << size_class)) {
+ return NULL;
+ }
+
+ // Allocate a new arena and maybe a new region. To do either of those
+ // things, we need to take the regions_lock. After doing so, check that the
+ // state is unchanged. If it has, just assume that we might have some new
+ // space to allocate into and try again.
+
+ boolean_t failed = FALSE;
+
+ _malloc_lock_lock(&nanozone->regions_lock);
+ nanov2_arena_t *current_region_next_arena = os_atomic_load(
+ &nanozone->current_region_next_arena, relaxed);
+ if (current_region_next_arena == initial_region_next_arena) {
+ if (nanov2_current_region_next_arena_is_limit(
+ current_region_next_arena)) {
+ // Reached the end of the region. Allocate a new one, if we can.
+ arena = nanov2_allocate_new_region(nanozone);
+ if (!arena) {
+ failed = TRUE;
+ }
+ } else {
+ // Assign the new arena, in the current region.
+ arena = current_region_next_arena;
+
+ // Bump current_region_next_arena by 1. No need for an atomic add
+ // because we're under the regions_lock.
+ os_atomic_store(&nanozone->current_region_next_arena,
+ current_region_next_arena + 1, relaxed);
+ }
+
+ // Set up the guard blocks for the new arena, if requested
+ if (!failed) {
+ nanov2_init_guard_blocks(nanozone, arena);
+ }
+ } else {
+ // The arena just before current_region_next_arena is always the most
+ // recently allocated arena. Let's retry from that arena, which was
+ // allocated in the time since we started our last try.
+ arena = current_region_next_arena - 1;
+ }
+ _malloc_lock_unlock(&nanozone->regions_lock);
+
+ if (!failed) {
+ // Now allocate from the new arena. Since we updated the nanozone, it's
+ // possible that some other thread has already raced with us to allocate
+ // some space from it, so just use the normal allocation path to avoid
+ // assumptions. It's a little more expensive, but this path is rare.
+ start_block = NULL;
+ goto retry;
+ }
+
+ // We need more space and we can't get it. We'll delegate to the helper.
+ return NULL;
+}
+
+// This function is called when a fast-path allocation from a given (size_class,
+// allocation_index) has been tried and failed, and we need to act on
+// observations from that attempt and/or retry the allocation. Its rather
+// tortured calling contract is designed to allow the caller to avoid pushing a
+// frame and pass along as much of what it has already computed as possible.
+//
+// If @corrupt_slot is non-NULL it means we detected corruption of the slot's
+// guard on the fast path, and we need to report that corruption.
+//
+// If @madvise_block_metap is non-NULL it means we raced with another thread
+// madvising the block we tried to allocate from and need to re-madvise it.
+//
+// If the allocation fails, NULL is returned.
+static void *
+nanov2_allocate_outlined(nanozonev2_t *nanozone, nanov2_block_meta_t **block_metapp,
+ size_t rounded_size, nanov2_size_class_t size_class,
+ int allocation_index, nanov2_block_meta_t *madvise_block_metap,
+ void *corrupt_slot, bool clear)
+{
+ void *ptr = NULL;
+
+ if (os_unlikely(corrupt_slot)) {
+ nanov2_guard_corruption_detected(corrupt_slot);
+ }
+
+ // If we need to re-madvise the old block that we might have accidentally
+ // touched out of turn, do so now.
+ if (madvise_block_metap) {
+ nanov2_madvise_block(nanozone, madvise_block_metap, size_class,
+ SLOT_MADVISED);
+ }
+
+ // No current allocation block, or we were unable to allocate. We need to
+ // get a new block. Before doing so, delegate to the helper allocator if
+ // the size class was full and has not released enough memory yet.
+ if (nanozone->delegate_allocations & (1 << size_class)) {
+ ptr = nanozone->helper_zone->malloc(nanozone->helper_zone, rounded_size);
+ goto done;
+ }
+
+ // Before we try to get another block, lock and try another allocation,
+ // which may succeed because another thread may have beaten us to it, or
+ // some space may have freed up in the current block.
+ _malloc_lock_s *lock = &nanozone->current_block_lock[size_class][allocation_index];
+ _malloc_lock_lock(lock);
+
+ nanov2_block_meta_t *block_metap = os_atomic_load(block_metapp, relaxed);
+ if (block_metap) {
+ ptr = nanov2_allocate_from_block(nanozone, block_metap, size_class);
+ if (ptr) {
+ // Good to go - keep the current block.
+ goto unlock;
+ }
+ }
+
+ // At this point, we do not have a current allocation block and the old one,
+ // if there was one, has been marked as not in use. We need to find and
+ // assign a new block. Since we have the lock, nobody else can change the
+ // current_block pointer.
+ ptr = nanov2_find_block_and_allocate(nanozone, size_class, block_metapp);
+
+unlock:
+ _malloc_lock_unlock(lock);
+
+ if (!ptr) {
+ // We could not find a block to allocate from -- make future
+ // allocations for this size class go to the helper zone until
+ // we have enough free space.
+ os_atomic_or(&nanozone->delegate_allocations,
+ (uint16_t)(1 << size_class), relaxed);
+
+ ptr = nanozone->helper_zone->malloc(nanozone->helper_zone, rounded_size);
+ }
+
+done:
+ if (os_likely(ptr)) {
+ switch (malloc_zero_policy) {
+ case MALLOC_ZERO_ON_FREE: {
+ // Always clear the double-free guard so that we can recognize that
+ // this block is not on the free list.
+ nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
+ os_atomic_store(&slotp->double_free_guard, 0, relaxed);
+
+ // We know the body of the allocation is already clear, so we just
+ // need to clean up the next_slot word to get to all-zero. Do so in
+ // all cases, even if a cleared allocation is not requested, to
+ // prevent any leakage through the next_slot bits.
+ os_atomic_store(&slotp->next_slot, 0, relaxed);
+ break;
+ }
+ case MALLOC_ZERO_NONE:
+ if (!clear) {
+ // Always clear the double-free guard so that we can recognize that
+ // this block is not on the free list.
+ nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
+ os_atomic_store(&slotp->double_free_guard, 0, relaxed);
+ break;
+ }
+ // fall through
+ case MALLOC_ZERO_ON_ALLOC:
+ memset(ptr, '\0', rounded_size);
+ break;
+ }
+ } else {
+ malloc_set_errno_fast(MZ_POSIX, ENOMEM);
+ }
+ return ptr;
+}
+
+#pragma mark -
+#pragma mark Freeing
+
+// Frees an allocation to its owning block and updates the block's state.
+//
+// If the block becomes empty, it is marked as SLOT_CAN_MADVISE and we return
+// the block to the caller to madvise if dictated by policy.
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+nanov2_block_meta_t *
+nanov2_free_to_block_inline(nanozonev2_t *nanozone, void *ptr,
+ nanov2_size_class_t size_class, nanov2_block_meta_t *block_metap)
+{
+ nanov2_block_t *blockp = nanov2_block_address_for_ptr(ptr);
+ if (!block_metap) {
+ block_metap = nanov2_meta_ptr_for_ptr(nanozone, ptr);
+ }
+
+ // Release the slot memory onto the block's freelist.
+ nanov2_block_meta_t old_meta = os_atomic_load(block_metap, relaxed);
+ int slot_count = slots_by_size_class[size_class];
+ nanov2_block_meta_t new_meta;
+ boolean_t was_full;
+
+ nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
+ // All of the free slot content (double_free_guard, next_slot word and the
+ // zeroed remainder of the slot) must be visible when the os_atomic_cmpxchgv
+ // completes, so the metadata updates on either path below need a release
+ // barrier.
+ os_atomic_store(&slotp->double_free_guard,
+ nanozone->slot_freelist_cookie ^ (uintptr_t)ptr, relaxed);
+
+again:
+ was_full = old_meta.next_slot == SLOT_FULL;
+ new_meta.free_count = old_meta.free_count + 1;
+ new_meta.in_use = old_meta.in_use;
+ new_meta.gen_count = old_meta.gen_count + 1;
+ boolean_t freeing_last_active_slot = !was_full &&
+ new_meta.free_count == slots_by_size_class[size_class] - 1;
+ if (freeing_last_active_slot) {
+ os_atomic_store(&slotp->next_slot, SLOT_NULL, relaxed);
+
+ // Releasing the last active slot onto the free list. Mark the block as
+ // ready to be madvised if it's not in use, otherwise reset next_slot
+ // to SLOT_BUMP.
+ new_meta.next_slot = new_meta.in_use ? SLOT_BUMP : SLOT_CAN_MADVISE;
+ // Write the updated meta data; try again if we raced with another thread.
+ if (!os_atomic_cmpxchgv(block_metap, old_meta, new_meta, &old_meta, release)) {
+ goto again;
+ }
+
+ // If the block is now empty and it's not in use, madvise it if the policy
+ // is to do so immediately.
+ if (new_meta.next_slot == SLOT_CAN_MADVISE &&
+ nanov2_madvise_policy == NANO_MADVISE_IMMEDIATE) {
+ return block_metap;
+ }
+ } else {
+ int slot_index = nanov2_slot_index_in_block(blockp, size_class, ptr);
+ new_meta.next_slot = slot_index + 1; // meta.next_slot is 1-based
+ os_atomic_store(&slotp->next_slot,
+ was_full ? SLOT_BUMP : old_meta.next_slot, relaxed);
+
+ // Write the updated meta data; try again if we raced with another thread.
+ if (!os_atomic_cmpxchgv(block_metap, old_meta, new_meta, &old_meta, release)) {
+ goto again;
+ }
+ }
+
+ // If this size class has been marked as full and this block is below an
+ // acceptable level of occupancy, turn off delegation to the helper. Do this
+ // only if the block is not in-use, because an in-use block cannot be a
+ // candidate when searching for a new block.
+ uint16_t class_mask = 1 << size_class;
+ if (!new_meta.in_use && (nanozone->delegate_allocations & class_mask) &&
+ (new_meta.free_count >= 0.75 * slot_count)) {
+ os_atomic_and(&nanozone->delegate_allocations, ~class_mask, relaxed);
+ }
+
+#if DEBUG_MALLOC
+ nanozone->statistics.size_class_statistics[size_class].total_frees++;
+#endif // DEBUG_MALLOC
+
+ return NULL;
+}
+
+#endif // OS_VARIANT_RESOLVED
+
+#if OS_VARIANT_NOTRESOLVED
+
+#pragma mark -
+#pragma mark Zone Operations
+
+malloc_zone_t *
+nanov2_create_zone(malloc_zone_t *helper_zone, unsigned debug_flags)
+{
+ // Note: It is important that nanov2_create_zone resets _malloc_engaged_nano
+ // if it is unable to enable the nanozone (and chooses not to abort). As
+ // several functions rely on _malloc_engaged_nano to determine if they
+ // should manipulate the nanozone, and these should not run if we failed
+ // to create the zone.
+ MALLOC_ASSERT(_malloc_engaged_nano == NANO_V2);
+
+ // Get memory for the zone and disable Nano if we fail.
+ nanozonev2_t *nanozone = nano_common_allocate_based_pages(
+ NANOZONEV2_ZONE_PAGED_SIZE, 0, 0, VM_MEMORY_MALLOC, 0);
+ if (!nanozone) {
+ _malloc_engaged_nano = NANO_NONE;
+ return NULL;
+ }
+
+ // Set up the basic_zone portion of the nanozonev2 structure
+ nanozone->basic_zone.version = 13;
+ nanozone->basic_zone.size = OS_RESOLVED_VARIANT_ADDR(nanov2_size);
+ if (malloc_zero_policy == MALLOC_ZERO_ON_ALLOC) {
+ nanozone->basic_zone.malloc =
+ OS_RESOLVED_VARIANT_ADDR(nanov2_malloc_zero_on_alloc);
+ } else {
+ nanozone->basic_zone.malloc = OS_RESOLVED_VARIANT_ADDR(nanov2_malloc);
+ }
+ nanozone->basic_zone.calloc = OS_RESOLVED_VARIANT_ADDR(nanov2_calloc);
+ nanozone->basic_zone.valloc = (void *)nanov2_valloc;
+ nanozone->basic_zone.free = OS_RESOLVED_VARIANT_ADDR(nanov2_free);
+ nanozone->basic_zone.realloc = OS_RESOLVED_VARIANT_ADDR(nanov2_realloc);
+ nanozone->basic_zone.destroy = (void *)nanov2_destroy;
+ nanozone->basic_zone.batch_malloc = OS_RESOLVED_VARIANT_ADDR(nanov2_batch_malloc);
+ nanozone->basic_zone.batch_free = OS_RESOLVED_VARIANT_ADDR(nanov2_batch_free);
+ nanozone->basic_zone.introspect =
+ (struct malloc_introspection_t *)&nanov2_introspect;
+ nanozone->basic_zone.memalign = OS_RESOLVED_VARIANT_ADDR(nanov2_memalign);
+ nanozone->basic_zone.free_definite_size = OS_RESOLVED_VARIANT_ADDR(nanov2_free_definite_size);
+ nanozone->basic_zone.pressure_relief = OS_RESOLVED_VARIANT_ADDR(nanov2_pressure_relief);
+ nanozone->basic_zone.claimed_address = OS_RESOLVED_VARIANT_ADDR(nanov2_claimed_address);
+ nanozone->basic_zone.try_free_default = OS_RESOLVED_VARIANT_ADDR(nanov2_try_free_default);
+
+ // Set these both to zero as required by CFAllocator.
+ nanozone->basic_zone.reserved1 = 0;
+ nanozone->basic_zone.reserved2 = 0;
+
+ // Prevent overwriting the function pointers in basic_zone.
+ mprotect(nanozone, sizeof(nanozone->basic_zone), PROT_READ);
+
+ // Set up the remainder of the nanozonev2 structure
+ nanozone->debug_flags = debug_flags;
+ nanozone->helper_zone = helper_zone;
+
+ // Initialize the cookies used to detect double freeing and for the ASLR
+ // scramble mapping.
+#define COOKIE_ENTROPY_MASK 0x0000ffffffff0000ULL
+#define DEFAULT_ENTROPY_BITS 0x0000DEADDEAD0000ULL
+ uintptr_t cookie = (uintptr_t)malloc_entropy[0] & COOKIE_ENTROPY_MASK;
+ if (!cookie) {
+ cookie = malloc_entropy[1] & COOKIE_ENTROPY_MASK;
+ if (!cookie) {
+ // The cookie can't be zero, because it's used to compute the guard
+ // value in free slots, so make sure we have a non-zero value. Using
+ // a fixed value allows us to recognize that it isn't real entropy.
+ cookie = DEFAULT_ENTROPY_BITS;
+ }
+ }
+ nanozone->slot_freelist_cookie = cookie;
+
+ // For the ASLR cookie, we take the top 12 bits of malloc_entropy[1] and
+ // align it to the block field of a Nano address.
+ nanozone->aslr_cookie = malloc_entropy[1] >> (64 - NANOV2_BLOCK_BITS);
+ nanozone->aslr_cookie_aligned = nanozone->aslr_cookie << NANOV2_OFFSET_BITS;
+
+ _malloc_lock_init(&nanozone->blocks_lock);
+ _malloc_lock_init(&nanozone->regions_lock);
+ _malloc_lock_init(&nanozone->madvise_lock);
+
+ // Allocate the initial region. If this does not succeed, we disable Nano.
+ nanov2_region_t *region = (nanov2_region_t *)NANOZONE_BASE_REGION_ADDRESS;
+
+ bool result;
+#if CONFIG_NANO_RESERVE_REGIONS
+ unsigned int num_regions = (nano_max_region + 1);
+ result = nanov2_reserve_regions(region, num_regions);
+ if (result) {
+ result = nanov2_unprotect_region(region);
+ if (!result) {
+ malloc_report(ASL_LEVEL_ERR,
+ "unable to protect initial region\n");
+ nano_common_deallocate_pages((void *)region,
+ num_regions * (size_t)NANOV2_REGION_SIZE, 0);
+ }
+ }
+#else // CONFIG_NANO_RESERVE_REGIONS
+ result = nanov2_allocate_region(region);
+#endif // CONFIG_NANO_RESERVE_REGIONS
+ if (!result) {
+ nano_common_deallocate_pages((void *)nanozone,
+ NANOZONEV2_ZONE_PAGED_SIZE, 0);
+ _malloc_engaged_nano = NANO_NONE;
+ malloc_report(ASL_LEVEL_NOTICE, "nano zone abandoned due to inability "
+ "to reserve vm space.\n");
+ return NULL;
+ }
+ nanov2_region_linkage_t *region_linkage =
+ nanov2_region_linkage_for_region(nanozone, region);
+ os_atomic_store(®ion_linkage->next_region_offset, 0, relaxed);
+
+ // Install the first region and pre-allocate the first arena.
+ nanozone->first_region_base = region;
+ os_atomic_store(&nanozone->current_region_next_arena,
+ ((nanov2_arena_t *)region) + 1, release);
+ nanozone->statistics.allocated_regions = 1;
+ // Set up the guard blocks for the initial arena, if requested
+ nanov2_init_guard_blocks(nanozone, (nanov2_arena_t *)region);
+
+ return (malloc_zone_t *)nanozone;
+}
+#endif // OS_VARIANT_NOTRESOLVED
+
+#pragma mark -
+#pragma mark Zone Fork Handling
+
+// Nanomalloc assumes that after a fork, it would be dangerous to rely on
+// the integrity of the zone data. During a fork, some of the zone handlers are
+// switched to the versions below, which do the following:
+// 1. Delegate all new allocation to the helper zone.
+// 2. Do nothing when asked to free memory that Nano allocated. There will be a
+// leak, but this is better than possibly crashing.
+
+#if OS_VARIANT_RESOLVED
+MALLOC_NOEXPORT void *
+nanov2_forked_malloc(nanozonev2_t *nanozone, size_t size)
+{
+ // Just hand to the helper zone.
+ return nanozone->helper_zone->malloc(nanozone->helper_zone, size);
+}
+#endif // OS_VARIANT_RESOLVED
+
+#if OS_VARIANT_NOTRESOLVED
+
+static void *
+nanov2_forked_calloc(nanozonev2_t *nanozone, size_t num_items, size_t size)
+{
+ // Just hand to the helper zone.
+ return nanozone->helper_zone->calloc(nanozone->helper_zone, num_items,
+ size);
+}
+
+static void *
+nanov2_forked_memalign(nanozonev2_t *nanozone, size_t alignment, size_t size)
+{
+ // Just hand to the helper zone.
+ return nanozone->helper_zone->memalign(nanozone->helper_zone, alignment,
+ size);
+}
+
+#endif // OS_VARIANT_NOTRESOLVED
+
+#if OS_VARIANT_RESOLVED
+
+MALLOC_NOEXPORT void
+nanov2_forked_free(nanozonev2_t *nanozone, void *ptr)
+{
+ if (!ptr) {
+ return; // Protect against malloc_zone_free() passing NULL.
+ }
+
+ // <rdar://problem/26481467> exhausting a slot may result in a pointer with
+ // the nanozone prefix being given to nano_free via malloc_zone_free. Calling
+ // vet_and_size here, instead of in _nano_free_check_scribble means we can
+ // early-out into the helper_zone if it turns out nano does not own this ptr.
+ size_t sz = nanov2_pointer_size(nanozone, ptr, FALSE);
+ if (sz || nanov2_has_valid_signature(ptr)) {
+ /* Drop it on the floor as nanozone metadata could be fouled by fork. */
+ return;
+ } else {
+ nanozone->helper_zone->free(nanozone->helper_zone, ptr);
+ return;
+ }
+ /* NOTREACHED */
+}
+
+MALLOC_NOEXPORT void
+nanov2_forked_free_definite_size(nanozonev2_t *nanozone, void *ptr, size_t size)
+{
+ nanov2_forked_free(nanozone, ptr);
+}
+
+MALLOC_NOEXPORT void *
+nanov2_forked_realloc(nanozonev2_t *nanozone, void *ptr, size_t new_size)
+{
+ // could occur through malloc_zone_realloc() path
+ if (!ptr) {
+ // If ptr is a null pointer, realloc() shall be equivalent to malloc()
+ // for the specified size.
+ return nanov2_forked_malloc(nanozone, new_size);
+ }
+
+ size_t old_size = nanov2_pointer_size(nanozone, ptr, FALSE);
+ if (!old_size) {
+ // not-nano pointer, hand down to helper zone
+ malloc_zone_t *zone = (malloc_zone_t *)(nanozone->helper_zone);
+ return zone->realloc(zone, ptr, new_size);
+ } else {
+ if (!new_size) {
+ // If size is 0 and ptr is not a null pointer, the object pointed to
+ // is freed. However as nanozone metadata could be fouled by fork,
+ // we'll intentionally leak it.
+
+ // If size is 0, either a null pointer or a unique pointer that can
+ // be successfully passed to free() shall be returned.
+ return nanov2_forked_malloc(nanozone, 1);
+ }
+
+ void *new_ptr = nanozone->helper_zone->malloc(nanozone->helper_zone,
+ new_size);
+ if (new_ptr) {
+ size_t valid_size = MIN(old_size, new_size);
+ memcpy(new_ptr, ptr, valid_size);
+ // Original pointer is intentionally leaked as nanozone metadata
+ // could be fouled by fork.
+ return new_ptr;
+ } else {
+ // Original ptr is left intact
+ return NULL;
+ }
+ /* NOTREACHED */
+ }
+ /* NOTREACHED */
+}
+#endif // OS_VARIANT_RESOLVED
+
+#if OS_VARIANT_NOTRESOLVED
+
+static unsigned
+nanov2_forked_batch_malloc(nanozonev2_t *nanozone, size_t size, void **results,
+ unsigned count)
+{
+ // Just pass this to the helper zone.
+ return nanozone->helper_zone->batch_malloc(nanozone->helper_zone, size,
+ results, count);
+}
+#endif // OS_VARIANT_NOTRESOLVED
+
+#if OS_VARIANT_RESOLVED
+
+MALLOC_NOEXPORT void
+nanov2_forked_batch_free(nanozonev2_t *nanozone, void **to_be_freed,
+ unsigned count)
+{
+ if (!count) {
+ return;
+ }
+
+ while (count--) {
+ void *ptr = to_be_freed[count];
+ if (ptr) {
+ nanov2_forked_free(nanozone, ptr);
+ }
+ }
+}
+#endif // OS_VARIANT_RESOLVED
+
+#if OS_VARIANT_NOTRESOLVED
+
+static boolean_t
+nanov2_forked_claimed_address(struct _malloc_zone_t *zone, void *ptr)
+{
+ // This does not operate after fork - default to true to avoid
+ // false negatives.
+ return true;
+}
+
+void
+nanov2_forked_zone(nanozonev2_t *nanozone)
+{
+ // Hobble the nano zone in the child of a fork prior to an exec since
+ // the state of the zone can be made inconsistent by a parent thread while
+ // the fork is underway. All new allocations will be referred to the helper
+ // zone (which is more stable.) All free()'s of existing nano objects will
+ // be leaked.
+ mprotect(nanozone, sizeof(nanozone->basic_zone), PROT_READ | PROT_WRITE);
+
+ nanozone->basic_zone.size = OS_RESOLVED_VARIANT_ADDR(nanov2_size); // Unchanged
+ nanozone->basic_zone.malloc = OS_RESOLVED_VARIANT_ADDR(nanov2_forked_malloc);
+ nanozone->basic_zone.calloc = (void *)nanov2_forked_calloc;
+ nanozone->basic_zone.valloc = (void *)nanov2_valloc; // Unchanged
+ nanozone->basic_zone.free = OS_RESOLVED_VARIANT_ADDR(nanov2_forked_free);
+ nanozone->basic_zone.realloc = OS_RESOLVED_VARIANT_ADDR(nanov2_forked_realloc);
+ nanozone->basic_zone.destroy = (void *)nanov2_destroy; // Unchanged
+ nanozone->basic_zone.batch_malloc = (void *)nanov2_forked_batch_malloc;
+ nanozone->basic_zone.batch_free = OS_RESOLVED_VARIANT_ADDR(nanov2_forked_batch_free);
+ nanozone->basic_zone.introspect =
+ (struct malloc_introspection_t *)&nanov2_introspect;// Unchanged
+ nanozone->basic_zone.memalign = (void *)nanov2_forked_memalign;
+ nanozone->basic_zone.free_definite_size =
+ OS_RESOLVED_VARIANT_ADDR(nanov2_forked_free_definite_size);
+ nanozone->basic_zone.claimed_address = nanov2_forked_claimed_address;
+ nanozone->basic_zone.try_free_default = NULL; // Fall back to old protocol
+ mprotect(nanozone, sizeof(nanozone->basic_zone), PROT_READ);
+}
+
+#endif // OS_VARIANT_NOTRESOLVED
+
+#endif // CONFIG_NANOZONE