/*
 * Copyright (c) 2018 Apple Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_LICENSE_HEADER_END@
 */
#include "resolver.h"
#include "internal.h"

#if CONFIG_NANOZONE

#pragma mark -
#pragma mark Forward Declarations

#if OS_VARIANT_NOTRESOLVED
static kern_return_t
nanov2_statistics_task_printer(task_t task, vm_address_t zone_address,
							   memory_reader_t reader, print_task_printer_t printer,
							   malloc_statistics_t *stats);
static kern_return_t
nanov2_statistics_task(task_t task, vm_address_t zone_address,
					   memory_reader_t reader, malloc_statistics_t *stats);
#endif // OS_VARIANT_NOTRESOLVED

#pragma mark -
#pragma mark Externals for resolved functions

MALLOC_NOEXPORT extern size_t nanov2_pointer_size(nanozonev2_t *nanozone, void *ptr,
		boolean_t allow_inner);
MALLOC_NOEXPORT extern size_t nanov2_pressure_relief(nanozonev2_t *nanozone, size_t goal);

#if OS_VARIANT_RESOLVED
MALLOC_ALWAYS_INLINE MALLOC_INLINE
static size_t
nanov2_pointer_size_inline(nanozonev2_t *nanozone, void *ptr, boolean_t allow_inner,
		nanov2_size_class_t *size_class_out, nanov2_block_meta_t **block_metap_out);

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static void *
nanov2_allocate_from_block_inline(nanozonev2_t *nanozone,
		nanov2_block_meta_t *block_metap, nanov2_size_class_t size_class,
		nanov2_block_meta_t **madvise_block_metapp_out, bool *corruption);

static void *
nanov2_allocate_outlined(nanozonev2_t *nanozone,
		nanov2_block_meta_t **block_metapp, size_t rounded_size,
		nanov2_size_class_t size_class, int allocation_index,
		nanov2_block_meta_t *madvise_block_metap, void *corrupt_slot,
		bool clear, bool typed, malloc_type_id_t type_id);

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static nanov2_block_meta_t *
nanov2_free_to_block_inline(nanozonev2_t *nanozone, void *ptr,
		nanov2_size_class_t size_class, nanov2_block_meta_t *block_metap);

static boolean_t nanov2_madvise_block_locked(
		nanozonev2_t *nanozone, nanov2_block_meta_t *block_metap,
		nanov2_block_t *blockp, nanov2_size_class_t size_class, uint32_t expected_state);
static void nanov2_madvise_block(nanozonev2_t *nanozone,
		nanov2_block_meta_t *block_metap, nanov2_size_class_t size_class,
		uint32_t expected_state);

MALLOC_NOEXPORT extern nanov2_arena_t *nanov2_allocate_new_region(nanozonev2_t *nanozone);
#endif // OS_VARIANT_RESOLVED

#pragma mark -
#pragma mark Global Allocator State

// -- Block scanning
typedef enum {
	NANO_SCAN_FIRST_FIT = 0,
	NANO_SCAN_CAPACITY_BASED,
} nanov2_block_scan_policy_t;

// Minimum occupancy percentage for an ideal block.
#define DEFAULT_SCAN_MIN_CAPACITY 20

// Maximum occupancy percentage for an ideal block.
#define DEFAULT_SCAN_MAX_CAPACITY 80

// Maximum number of blocks to scan while looking for a best fit once a
// candidate block has been found. Value 0 means no limit.
#define DEFAULT_SCAN_LIMIT 10

// -- Madvise policy
typedef enum {
	NANO_MADVISE_IMMEDIATE = 0,
	NANO_MADVISE_WARNING_PRESSURE,
	NANO_MADVISE_CRITICAL_PRESSURE,
} nanov2_madvise_policy_t;

typedef struct nanov2_policy_config_s {
	// Determines the algorithm for scanning for the next allocation block.
	// Used in conjunction with nanov2_block_scan_capacity_min,
	// nanov2_block_scan_capacity_max and nanov2_block_scan_limit. Set from the
	// MallocNanoScanPolicy environment variable.
	nanov2_block_scan_policy_t block_scan_policy;

	// Minimum occupancy percentage for an ideal block.
	int block_scan_min_capacity;

	// Maximum occupancy percentage for an ideal block.
	int block_scan_max_capacity;

	// Maximum number of blocks to scan while looking for a best fit once a
	// candidate block has been found. Value 0 means no limit.
	int block_scan_limit;

	// Bitmask for size classes that are only allowed a single arena. Set from
	// the MallocNanoSingleArena environment variable.
	uint16_t single_arena_size_classes;

	// Madvise policy. Set from the MallocNanoMadvisePolicy environment variable
	nanov2_madvise_policy_t madvise_policy;
} nanov2_policy_config_t;

#if OS_VARIANT_NOTRESOLVED

// Madvise policy. Set from the MallocNanoMadvisePolicy environment variable.
MALLOC_NOEXPORT nanov2_madvise_policy_t nanov2_madvise_policy;

MALLOC_NOEXPORT nanov2_policy_config_t nanov2_policy_config = {
	.block_scan_policy = NANO_SCAN_CAPACITY_BASED,
	.block_scan_min_capacity = DEFAULT_SCAN_MIN_CAPACITY,
	.block_scan_max_capacity = DEFAULT_SCAN_MAX_CAPACITY,
	.block_scan_limit = DEFAULT_SCAN_LIMIT,
	.single_arena_size_classes = 0,
	.madvise_policy = NANO_MADVISE_IMMEDIATE,
};

#else  // OS_VARIANT_NOTRESOLVED

MALLOC_NOEXPORT extern nanov2_policy_config_t nanov2_policy_config;
MALLOC_NOEXPORT extern nanov2_madvise_policy_t nanov2_madvise_policy;

#endif // OS_VARIANT_NOTRESOLVED

// BLOCKS_PER_UNIT must be a power of two to make it possible to get the size
// class from a pointer reasonably cheaply. Do not change the value without
// fixing the code that depends on it.
#define BLOCKS_PER_UNIT_SHIFT 6
#define BLOCKS_PER_UNIT (1 << BLOCKS_PER_UNIT_SHIFT)

#if OS_VARIANT_NOTRESOLVED
// Number of units of each size class in an arena. The numbers here must add
// up to 64. One unit corresponds to BLOCKS_PER_UNIT blocks in the corresponding
// size class, so 64 units maps to a total of 64 * 64 = 4096 blocks and each
// block is 16K, making a total of 64MB, which is the size of an arena.
static int block_units_by_size_class[] = {
	2,	// 16-byte allocations (less 1 for the metadata block)
	10,	// 32-byte allocations
	11,	// 48-byte allocations
	10,	// 64-byte allocations
	5,	// 80-byte allocations
	3,	// 96-byte allocations
	3,	// 112-byte allocations
	4,	// 128-byte allocations
	3,	// 144-byte allocations
	2,	// 160-byte allocations
	2,	// 176-byte allocations
	2,	// 192-byte allocations
	2,	// 208-byte allocations
	2,	// 224-byte allocations
	1,	// 240-byte allocations
	2,	// 256-byte allocations
};

MALLOC_STATIC_ASSERT(
		sizeof(block_units_by_size_class)/sizeof(block_units_by_size_class[0])
				== NANO_SIZE_CLASSES,
		"Size of block_units_by_size_class is incorrect");

// Total of the number of blocks in all size classes. Currently this is 64.
#define TOTAL_BLOCK_UNITS (NANOV2_BLOCKS_PER_ARENA/BLOCKS_PER_UNIT)

// Offsets to the first and last blocks for each size class within an arena, in
// the logical address space. These tables are constructed from the values in
// the block_units_by_size_class table.
MALLOC_NOEXPORT int first_block_offset_by_size_class[NANO_SIZE_CLASSES];
MALLOC_NOEXPORT int last_block_offset_by_size_class[NANO_SIZE_CLASSES];

// Table mapping the part of a logical address that depends on size class to
// the size class. Also built from the block_units_by_size_class table.
MALLOC_NOEXPORT int ptr_offset_to_size_class[TOTAL_BLOCK_UNITS];

// Number of slots in a block, indexed by size class. Note that there is a small
// amount of wastage in some size classes because the block size is not always
// exactly divisible by the allocation size. The number of wasted bytes is shown
// in parentheses in the comments below.
MALLOC_NOEXPORT const int slots_by_size_class[] = {
	NANOV2_BLOCK_SIZE/(1 * NANO_REGIME_QUANTA_SIZE),  	// 16 bytes: 1024	(0)
	NANOV2_BLOCK_SIZE/(2 * NANO_REGIME_QUANTA_SIZE),	// 32 bytes: 512	(0)
	NANOV2_BLOCK_SIZE/(3 * NANO_REGIME_QUANTA_SIZE),	// 48 bytes: 341	(16)
	NANOV2_BLOCK_SIZE/(4 * NANO_REGIME_QUANTA_SIZE),	// 64 bytes: 256	(0)
	NANOV2_BLOCK_SIZE/(5 * NANO_REGIME_QUANTA_SIZE),	// 80 bytes: 204	(64)
	NANOV2_BLOCK_SIZE/(6 * NANO_REGIME_QUANTA_SIZE),	// 96 bytes: 170	(64)
	NANOV2_BLOCK_SIZE/(7 * NANO_REGIME_QUANTA_SIZE),	// 112 bytes: 146	(32)
	NANOV2_BLOCK_SIZE/(8 * NANO_REGIME_QUANTA_SIZE),	// 128 bytes: 128	(0)
	NANOV2_BLOCK_SIZE/(9 * NANO_REGIME_QUANTA_SIZE),	// 144 bytes: 113	(112)
	NANOV2_BLOCK_SIZE/(10 * NANO_REGIME_QUANTA_SIZE),	// 160 bytes: 102	(64)
	NANOV2_BLOCK_SIZE/(11 * NANO_REGIME_QUANTA_SIZE),	// 176 bytes: 93	(16)
	NANOV2_BLOCK_SIZE/(12 * NANO_REGIME_QUANTA_SIZE),	// 192 bytes: 85	(64)
	NANOV2_BLOCK_SIZE/(13 * NANO_REGIME_QUANTA_SIZE),	// 208 bytes: 78	(160)
	NANOV2_BLOCK_SIZE/(14 * NANO_REGIME_QUANTA_SIZE),	// 224 bytes: 73	(32)
	NANOV2_BLOCK_SIZE/(15 * NANO_REGIME_QUANTA_SIZE),	// 240 bytes: 68	(64)
	NANOV2_BLOCK_SIZE/(16 * NANO_REGIME_QUANTA_SIZE),	// 256 bytes: 64	(0)
};
#else // OS_VARIANT_NOTRESOLVED

extern int block_units_by_size_class[];
extern int ptr_offset_to_size_class[];
extern int first_block_offset_by_size_class[];
extern int last_block_offset_by_size_class[];
extern const int slots_by_size_class[];

#endif // OS_VARIANT_NOTRESOLVED

#pragma mark -
#pragma mark Conversion and Mapping Inlines

// nanov2_block_index_to_meta_index() and nanov2_meta_index_to_block_index()
// map between the index of a block in its arena and the index of the meta data
// header for that block in the metadata block. The mapping is not direct
// to avoid false sharing caused by CPUs that are using adjacent blocks
// writing to what would otherwise be adjacent meta data headers. The effect of
// these functions is to separate the meta data headers for adjacent blocks by
// at least the size of a cache line (assumed to be 64 bytes).
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_meta_index_t
nanov2_block_index_to_meta_index(nanov2_block_index_t block_index)
{
	return ((block_index >> 6) | (block_index << 6)) & 0xFFF;
}

static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_index_t
nanov2_meta_index_to_block_index(nanov2_meta_index_t block_meta_index)
{
	return ((block_meta_index >> 6) | (block_meta_index << 6)) & 0xFFF;
}

static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_meta_index_t
nanov2_metablock_meta_index(nanozonev2_t *nanozone)
{
	return nanov2_block_index_to_meta_index((nanov2_block_index_t)
			nanozone->aslr_cookie);
}

// Given a block metadata pointer, returns whether the block is active (that is,
// it is being used for allocations, it has allocations that have not been freed,
// or is waiting to be madvised  and is not a guard block).
static MALLOC_ALWAYS_INLINE MALLOC_INLINE boolean_t
nanov2_is_block_active(nanov2_block_meta_t block_meta)
{
	return block_meta.next_slot != SLOT_NULL
			&& block_meta.next_slot != SLOT_MADVISING
			&& block_meta.next_slot != SLOT_MADVISED
			&& block_meta.next_slot != SLOT_GUARD;
}

#if OS_VARIANT_RESOLVED
// Given a block metadata pointer, returns whether an allocation could be
// attempted from it. Allocations are not allowed from blocks that have not yet
// been used (since such a block has not been assigned), is full or has been
// madvised.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE boolean_t
nanov2_can_allocate_from_block(nanov2_block_meta_t block_meta)
{
	return block_meta.in_use && block_meta.next_slot != SLOT_FULL;
}

// Given a pointer, returns whether it has the correct signature to be a
// Nano V2 address.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE boolean_t
nanov2_has_valid_signature(void *ptr)
{
	return (((uintptr_t)ptr) >> SHIFT_NANO_SIGNATURE) == NANOZONE_SIGNATURE;
}
#endif // OS_VARIANT_RESOLVED

// Converts a Nano V2 logical address to the corresponding real address.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE void *
nanov2_logical_address_to_ptr(nanozonev2_t *nanozone, void *laddr)
{
	return (void *)(((uintptr_t)laddr) ^ nanozone->aslr_cookie_aligned);
}

// Gets the maximum allocation size for a given size class.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE int
nanov2_size_from_size_class(nanov2_size_class_t size_class)
{
	return (size_class + 1) * NANO_REGIME_QUANTA_SIZE;
}

#if OS_VARIANT_RESOLVED
// Given an allocation size, returns the corresponding size class. It is the
// responsibility of the caller to ensure that the size is valid. Returned
// value is zero-based.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_size_class_t
nanov2_size_class_from_size(size_t size)
{
	return (nanov2_size_class_t)howmany(size, NANO_REGIME_QUANTA_SIZE) - 1;
}
#endif // OS_VARIANT_RESOLVED

#if OS_VARIANT_RESOLVED
// Given a pointer that is assumed to be in the Nano zone, returns the address
// of its containing block. Works for both real and logical pointers and returns
// a pointer of the same type.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_t *
nanov2_block_address_for_ptr(void *ptr)
{
	return (void *)(((uintptr_t)ptr) & NANOV2_BLOCK_ADDRESS_MASK);
}
#endif // OS_VARIANT_RESOLVED

// Given a pointer that is assumed to be in the Nano zone, returns the address
// of its containing arena. Works for both real and logical pointers.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_arena_t *
nanov2_arena_address_for_ptr(void *ptr)
{
	return (void *)(((uintptr_t)ptr) & NANOV2_ARENA_ADDRESS_MASK);
}

// Given a pointer that is assumed to be in the Nano zone, returns the address
// of its containing region. Works for both real and logical pointers.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_region_t *
nanov2_region_address_for_ptr(void *ptr)
{
	return (nanov2_region_t *)(((uintptr_t)ptr) & NANOV2_REGION_ADDRESS_MASK);
}

// Given a pointer that is assumed to be in the Nano zone, returns the real
// address of its metadata block. Works for both real and logical pointers.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_arena_metablock_t *
nanov2_metablock_address_for_ptr(nanozonev2_t *nanozone, void *ptr)
{
	// The metadata block is the first logical block in the arena, so its
	// logical address is that of the arena. To get a real pointer, we map it
	// through nanov2_logical_address_to_ptr().
	return (nanov2_arena_metablock_t *)nanov2_logical_address_to_ptr(nanozone,
			nanov2_arena_address_for_ptr(ptr));
}

#if OS_VARIANT_RESOLVED
// Given a pointer to a block_metap_t for a block, returns a pointer to the
// block itself.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_t *
nanov2_block_address_from_meta_ptr(nanozonev2_t *nanozone,
		nanov2_block_meta_t *block_metap)
{
	nanov2_block_t *meta_block = nanov2_block_address_for_ptr(block_metap);
	nanov2_arena_t *arena = nanov2_arena_address_for_ptr(block_metap);

	// Get the block's index and use that to get the address of the block.
	nanov2_meta_index_t meta_index =
		(nanov2_meta_index_t)(block_metap - (nanov2_block_meta_t *)meta_block);
	nanov2_block_index_t block_index = nanov2_meta_index_to_block_index(meta_index);
	return &arena->blocks[block_index];
}
#endif // OS_VARIANT_RESOLVED

// Given the index of a block_metap_t for a block, returns a pointer to the
// block itself.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_t *
nanov2_block_address_from_meta_index(nanozonev2_t *nanozone,
		nanov2_arena_t *arena, nanov2_meta_index_t meta_index)
{
	nanov2_block_index_t block_index = nanov2_meta_index_to_block_index(meta_index);
	return &arena->blocks[block_index];
}

// Given a pointer that is assumed to be in the nanozone, returns the index
// of its containing block within its hosting arena. Works for both logical and
// real pointers and returns an index in the corresponding address space.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_index_t
nanov2_block_index_for_ptr(void *ptr)
{
	return (nanov2_block_index_t)(((uintptr_t)ptr) >> NANOV2_OFFSET_BITS)
			& ((1 << NANOV2_BLOCK_BITS) - 1);
}

#if OS_VARIANT_RESOLVED
// Given a pointer that is assumed to be in the nanozone, returns a pointer to
// the meta data for its containing block. Expects ptr be a real address.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_meta_t *
nanov2_meta_ptr_for_ptr(nanozonev2_t *nanozone, void *ptr)
{
	nanov2_arena_metablock_t *meta_block = nanov2_metablock_address_for_ptr(
			nanozone, ptr);
	nanov2_block_index_t block_index = nanov2_block_index_for_ptr(ptr);
	nanov2_meta_index_t meta_index = nanov2_block_index_to_meta_index(block_index);
	return &meta_block->arena_block_meta[meta_index];
}
#endif // OS_VARIANT_RESOLVED

// Given a region pointer, returns the address of the first arena in the region.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_arena_t *
nanov2_first_arena_for_region(nanov2_region_t *region)
{
	// The first arena is colocated with the region itself.
	return (nanov2_arena_t *)region;
}

#if OS_VARIANT_RESOLVED
// Given an atomically-observed current_region_next_arena pointer, returns
// whether or not it's a usable arena or a limit arena (indicating exhaustion of
// the current region).
static MALLOC_ALWAYS_INLINE MALLOC_INLINE bool
nanov2_current_region_next_arena_is_limit(
		nanov2_arena_t *current_region_next_arena)
{
	// The first arena of a region is never stored in current_region_next_arena,
	// so a value at the beginning of a region must be a limit arena.
	return current_region_next_arena == (nanov2_arena_t *)(
			nanov2_region_address_for_ptr(current_region_next_arena));
}
#endif // OS_VARIANT_RESOLVED

// Given an atomically-observed current_region_next_arena pointer, returns the
// base of the current region at the time of the observation.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_region_t *
nanov2_current_region_base(nanov2_arena_t *current_region_next_arena)
{
	return nanov2_region_address_for_ptr(
			(void *)(((uintptr_t)current_region_next_arena) - 1));
}

// Given a region pointer, returns a pointer to the arena after the last
// active arena in the region.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_arena_t *
nanov2_limit_arena_for_region(nanozonev2_t __unused *nanozone,
		nanov2_region_t *region, nanov2_arena_t *current_region_next_arena)
{
	// The first arena is colocated with the region itself.
	nanov2_arena_t *limit_arena;
	if (region == nanov2_current_region_base(current_region_next_arena)) {
		limit_arena = current_region_next_arena;
	} else {
		limit_arena = nanov2_first_arena_for_region(region + 1);
	}
	return limit_arena;
}

// Given a region pointer, returns the address of the linkage structure for
// that region. The linkage structure is stored in the first entry of the
// metadata block of the first arena in the region.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_region_linkage_t *
nanov2_region_linkage_for_region(nanozonev2_t *nanozone, nanov2_region_t *region)
{
	nanov2_arena_metablock_t *first_metadata_block =
			nanov2_metablock_address_for_ptr(nanozone, region);
	return (nanov2_region_linkage_t *)&first_metadata_block->arena_block_meta[
			nanov2_metablock_meta_index(nanozone)];
}

#if OS_VARIANT_RESOLVED
// Given a pointer to a region, returns a pointer to the region that follows it,
// or NULL if there isn't one. We may observe linkage to a new region that
// hasn't yet actually been installed into current_region_next_arena; ignore the
// linkage in this case.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_region_t *
nanov2_next_region_for_region(nanozonev2_t *nanozone, nanov2_region_t *region,
		nanov2_arena_t *current_region_next_arena)
{
	nanov2_region_linkage_t *linkage =
			nanov2_region_linkage_for_region(nanozone, region);
	int offset = os_atomic_load(&linkage->next_region_offset, relaxed);
	if (!offset) {
		return NULL;
	}

	nanov2_region_t *next_region = region + offset;
	return (nanov2_arena_t *)next_region < current_region_next_arena ?
			next_region : NULL;
}
#endif // OS_VARIANT_RESOLVED

#if OS_VARIANT_NOTRESOLVED
// Given a pointer to a region, returns a pointer to the region that follows it,
// or NULL if there isn't one. This variant is used when mapping the nanozone
// for another process.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_region_t *
nanov2_next_region_for_region_offset(nanozonev2_t *nanozone,
		nanov2_region_t *region, off_t region_offset,
		nanov2_arena_t *current_region_next_arena)
{
	nanov2_region_linkage_t *linkage =
			nanov2_region_linkage_for_region(nanozone, region);
	nanov2_region_linkage_t *mapped_linkage = (nanov2_region_linkage_t *)(
			((uintptr_t)linkage + region_offset));
	int offset = os_atomic_load(&mapped_linkage->next_region_offset, relaxed);
	if (!offset) {
		return NULL;
	}

	nanov2_region_t *next_region = region + offset;
	return (nanov2_arena_t *)next_region < current_region_next_arena ?
			next_region : NULL;
}
#endif // OS_VARIANT_NOTRESOLVED

// Given the index of a slot in a block of a given size and the base address of
// the block, returns a pointer to the start of the slot. This works for both
// real and logical block pointers and returns a pointer of the same type.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE void *
nanov2_slot_in_block_ptr(nanov2_block_t *block, nanov2_size_class_t size_class,
		int slot_index)
{
	return (void *)((uintptr_t)block +
			nanov2_size_from_size_class(size_class) * slot_index);
}

#if OS_VARIANT_RESOLVED
// Given the base address of a block, the size class for the block and a pointer,
// returns the index of the slot represented by the pointer. It is assumed that
// the pointer is slot-aligned and is within the bounds of the block.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE int
nanov2_slot_index_in_block(nanov2_block_t *block, nanov2_size_class_t size_class,
		void *ptr)
{
	return (int)((uintptr_t)ptr - (uintptr_t)block)/
			(nanov2_size_from_size_class(size_class));
}
#endif // OS_VARIANT_RESOLVED

// Given a (real) pointer, gets the size class of its containing block. Assumes
// that the pointer is in a valid region, arena and block.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_size_class_t
nanov2_size_class_for_ptr(nanozonev2_t *nanozone, void *ptr)
{
	// To get the size class, we need to convert the block number from
	// physical to logical, since the ptr_offset_to_size_class table is
	// indexed by logical block.
	nanov2_block_index_t block =
			(int)(nanov2_block_index_for_ptr(ptr) ^ nanozone->aslr_cookie);
	return ptr_offset_to_size_class[block >> BLOCKS_PER_UNIT_SHIFT];
}

#if OS_VARIANT_NOTRESOLVED

// Given a meta data index, gets the size class of the corresponding block.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_size_class_t
nanov2_size_class_for_meta_index(nanozonev2_t *nanozone, nanov2_meta_index_t meta_index)
{
	// To get the size class, we need to get the block index from meta index
	// and then convert it from real to logical, since the
	// ptr_offset_to_size_class table is indexed by logical block.
	nanov2_block_index_t block_index = nanov2_meta_index_to_block_index(meta_index);
	int logical_block_index = (int)(block_index ^ nanozone->aslr_cookie);
	return ptr_offset_to_size_class[logical_block_index >> BLOCKS_PER_UNIT_SHIFT];
}
#endif // OS_VARIANT_NOTRESOLVED

#if OS_VARIANT_RESOLVED
// Given a size class and an arena, returns a pointer to the metadata for the
// first block for that size class in the arena, ignoring the metadata block.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_meta_t *
nanov2_first_block_for_size_class_in_arena(nanozonev2_t *nanozone,
		nanov2_size_class_t size_class, nanov2_arena_t *arena)
{
	int block_offset = first_block_offset_by_size_class[size_class];
	nanov2_arena_metablock_t *meta_blockp =
			nanov2_metablock_address_for_ptr(nanozone, arena);
	nanov2_block_index_t block_index =
			(nanov2_block_index_t)(block_offset ^ nanozone->aslr_cookie);
	nanov2_meta_index_t meta_index = nanov2_block_index_to_meta_index(block_index);
	return &meta_blockp->arena_block_meta[meta_index];
}

// Given a pointer to the metadata for a block in a given size class, returns
// a pointer to the metadata for the next block, wrapping from the last block
// to the first if necessary.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_meta_t *
nanov2_next_block_for_size_class(nanozonev2_t *nanozone,
		nanov2_size_class_t size_class, nanov2_block_meta_t *meta_blockp,
		boolean_t *wrapped)
{
	// To find the next block, get the index of the current block, which is in
	// the real address space, unscramble it to get a logical block number,
	// add 1 to it, wrapping if necessary, then scramble the result.
	nanov2_block_meta_t *base_meta_blockp =
			(nanov2_block_meta_t *)(((uintptr_t)meta_blockp) & (NANOV2_BLOCK_ADDRESS_MASK));
	nanov2_meta_index_t meta_index = (int)(meta_blockp - base_meta_blockp);
	nanov2_block_index_t block_index = nanov2_meta_index_to_block_index(meta_index);
	block_index ^= nanozone->aslr_cookie;  // Unscramble
	int last_offset = last_block_offset_by_size_class[size_class];
	if (wrapped) *wrapped = block_index == last_offset;
	block_index = block_index == last_offset ?
			first_block_offset_by_size_class[size_class] : block_index + 1;
	block_index = (nanov2_block_index_t)(block_index ^ nanozone->aslr_cookie);
	meta_index = nanov2_block_index_to_meta_index(block_index);
	return &base_meta_blockp[meta_index];
}

// Given a pointer to the metadata for a block in a given size class, returns
// a pointer to the metadata for the previous block, wrapping from the first
// block to the last if necessary.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE nanov2_block_meta_t *
nanov2_previous_block_for_size_class(nanozonev2_t *nanozone,
		nanov2_size_class_t size_class, nanov2_block_meta_t *meta_blockp,
		boolean_t *wrapped)
{
	// To find the previous block, get the index of the current block, which is
	// in the real address space, unscramble it to get a logical block number,
	// subtract 1 from it, wrapping if necessary, then scramble the result.
	nanov2_block_meta_t *base_meta_blockp =
			(nanov2_block_meta_t *)(((uintptr_t)meta_blockp) & (NANOV2_BLOCK_ADDRESS_MASK));
	nanov2_meta_index_t meta_index = (int)(meta_blockp - base_meta_blockp);
	nanov2_block_index_t block_index = nanov2_meta_index_to_block_index(meta_index);
	block_index ^= nanozone->aslr_cookie;  // Unscramble
	int first_offset = first_block_offset_by_size_class[size_class];
	if (wrapped) *wrapped = block_index == first_offset;
	block_index = block_index == first_offset ?
			last_block_offset_by_size_class[size_class] : block_index - 1;
	block_index = (nanov2_block_index_t)(block_index ^ nanozone->aslr_cookie);
	meta_index = nanov2_block_index_to_meta_index(block_index);
	return &base_meta_blockp[meta_index];
}

// Turns off the in-use bit in the meta data for a given block.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE void
nanov2_turn_off_in_use(nanov2_block_meta_t *block_metap)
{
	// TODO: find a way to do this that is more efficient and readable.
	static nanov2_block_meta_t mask = {
		.in_use = 0,
		.next_slot = ~0,
		.free_count = ~0,
		.gen_count = ~0,
	};
	os_atomic_and((uint32_t *)block_metap, *(uint32_t *)&mask, relaxed);
}

#pragma mark -
#pragma mark Policy Functions

// Gets the index of the block in the zone's current_block array from which
// allocations should be made. This function should be replaced to implement
// a different strategy (e.g. for E- vs P-cores).
static MALLOC_ALWAYS_INLINE MALLOC_INLINE int
nanov2_get_allocation_block_index(void)
{
#if CONFIG_NANO_USES_HYPER_SHIFT
	if (os_likely(nano_common_max_magazines_is_ncpu)) {
		// Default case is max magazines == physical number of CPUs, which
		// must be > _malloc_cpu_number() >> hyper_shift, so the modulo
		// operation is not required.
		return (_malloc_cpu_number() >> hyper_shift) & MAX_CURRENT_BLOCKS_MASK;
	}
#else // CONFIG_NANO_USES_HYPER_SHIFT
	if (os_likely(nano_common_max_magazines_is_ncpu)) {
		// Default case is max magazines == logical number of CPUs, which
		// must be > _malloc_cpu_number() so the modulo operation is not required.
		return _malloc_cpu_number() & MAX_CURRENT_BLOCKS_MASK;
	}
#endif // CONFIG_NANO_USES_HYPER_SHIFT

	unsigned int shift = 0;
#if CONFIG_NANO_USES_HYPER_SHIFT
	shift = hyper_shift;
#endif // CONFIG_NANO_USES_HYPER_SHIFT

	if (os_likely(_os_cpu_number_override == -1)) {
		return ((_malloc_cpu_number() >> shift) % nano_common_max_magazines) &
				MAX_CURRENT_BLOCKS_MASK;
	}
	return ((_os_cpu_number_override >> shift) % nano_common_max_magazines) &
			MAX_CURRENT_BLOCKS_MASK;
}
#endif // OS_VARIANT_RESOLVED

#pragma mark -
#pragma mark Guard Blocks

// Converts a given block (specified by absolute block number) in an arena into
// a guard block. The block will be marked as in-use so that it is not available
// for allocations and its permissions are set to PROT_READ. Note that
// PROT_READ is used instead of PROT_NONE because the latter breaks the
// enumerator, which tries to map the whole region and fails if there are
// PROT_NONE pages in the range. We can't fix that in the allocator because the
// code that does the mapping is part of the sampling tools and is simply
// invoked as a callback from the enumerator.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE void
nanov2_create_guard_block(nanozonev2_t *nanozone, nanov2_arena_t *arena,
		nanov2_block_index_t block_index) {
	// Mark the block as in-use in the meta data
	static nanov2_block_meta_t in_use_block = {
		.in_use = 1,
		.next_slot = SLOT_GUARD
	};
	nanov2_meta_index_t	block_meta_index =
			nanov2_block_index_to_meta_index(block_index);
	nanov2_arena_metablock_t *block_metap = nanov2_metablock_address_for_ptr(
			nanozone, arena);
	block_metap->arena_block_meta[block_meta_index] = in_use_block;
	void *block_ptr = &arena->blocks[block_index];

	// Apply PROT_NONE to the block itself.
	kern_return_t err = mprotect(block_ptr, NANOV2_BLOCK_SIZE, PROT_READ);
	if (err != KERN_SUCCESS) {
		malloc_report(ASL_LEVEL_ERR, "Failed to create guard block at %p (%d)\n",
				block_ptr, err);
	}
}

// Creates the guard blocks for an arena, if required. The guard blocks are
// the first and last physical blocks in the arena that are not the metadata
// block.
static MALLOC_ALWAYS_INLINE MALLOC_INLINE void
nanov2_init_guard_blocks(nanozonev2_t *nanozone, nanov2_arena_t *arena)
{
	if (nanozone->debug_flags & MALLOC_ALL_GUARD_PAGE_FLAGS) {
		// Use the first and last blocks in the arena as guard regions,
		// avoiding the metadata block.
		nanov2_meta_index_t meta_index = nanov2_metablock_meta_index(nanozone);
		nanov2_create_guard_block(nanozone, arena, meta_index == 0 ? 1 : 0);
		nanov2_create_guard_block(nanozone, arena,
				meta_index == NANOV2_BLOCKS_PER_ARENA - 1 ?
					NANOV2_BLOCKS_PER_ARENA - 2 : NANOV2_BLOCKS_PER_ARENA - 1);
	}
}

#pragma mark -
#pragma mark Allocator Initialization

#if OS_VARIANT_NOTRESOLVED

static const char madvise_policy_env[] = "MallocNanoMadvisePolicy";
static const char madvise_policy_bootarg[] = "nanov2_madvise_policy";
static const char madvise_immediate[] = "immediate";
static const char madvise_warning[] = "warning";
static const char madvise_critical[] = "critical";

static const char single_arena_env[] = "MallocNanoSingleArena";
static const char single_arena_bootarg[] = "nanov2_single_arena";

static const char scan_policy_env[] = "MallocNanoScanPolicy";
static const char scan_policy_bootarg[] = "nanov2_scan_policy";

static const char size_class_blocks_env[] = "MallocNanoSizeClassBlocks";
static const char size_class_blocks_bootarg[] = "nanov2_size_class_blocks";

// Parse and set the madvise policy setting. If ptr is NULL, sets the default
// policy.
static void
nanov2_set_madvise_policy(const char *name, const char *ptr)
{
	nanov2_madvise_policy_t madvise_policy = NANO_MADVISE_IMMEDIATE;
	if (ptr) {
		if (!strncmp(ptr, madvise_immediate, sizeof(madvise_immediate) - 1)) {
			madvise_policy = NANO_MADVISE_IMMEDIATE;
		} else if (!strncmp(ptr, madvise_warning, sizeof(madvise_warning) - 1)) {
			madvise_policy = NANO_MADVISE_WARNING_PRESSURE;
		} else if (!strncmp(ptr, madvise_critical, sizeof(madvise_critical) - 1)) {
			madvise_policy = NANO_MADVISE_CRITICAL_PRESSURE;
		} else {
			malloc_report(ASL_LEVEL_ERR,
					"%s value (%s) invalid - ignored.\n", name, ptr);
		}
	}
	nanov2_madvise_policy = madvise_policy;
}

// Parse and set the list of size classes that are allowed only one arena. If
// ptr is NULL, no size classes are restricted to a single arena,
// Format is a list of sizes separated by colons (e.g. 16:240). Each size must
// be a multiple of NANO_REGIME_QUANTA_SIZE and must be between 16 and 256.
static void
nanov2_set_single_arena_size_classes(const char *name, const char *ptr)
{
	uint16_t single_arena_size_classes = 0;
	if (ptr) {
		const char *value = ptr;
		const char *endp;
		boolean_t failed = FALSE;
		while (*ptr) {
			long size = malloc_common_convert_to_long(ptr, &endp);
			if (endp != ptr) {
				if (*endp && *endp != ':') {
					failed = TRUE;
					break;
				}
				if (size > NANO_MAX_SIZE || size < NANO_REGIME_QUANTA_SIZE ||
						(size % NANO_REGIME_QUANTA_SIZE) != 0) {
					failed = TRUE;
					break;
				}
				single_arena_size_classes |=
						1 << ((size/NANO_REGIME_QUANTA_SIZE) - 1);
			} else {
				failed = true;
				break;
			}
			if (!*endp) {
				break;
			}
			ptr = endp + 1;
		}
		if (failed) {
			malloc_report(ASL_LEVEL_ERR,
					"%s value (%s) invalid - ignored.\n", name, value);
			single_arena_size_classes = 0;
		}
	}
	nanov2_policy_config.single_arena_size_classes = single_arena_size_classes;
}

// Parse and set the block scan policy setting. If ptr is NULL, the default
// policy is used. Format is either "firstfit" or "minXX:maxYY:limZZ", where
// XX, YY and ZZ are numbers, XX and YY must be between 0 and 100 and XX must
// not be greater than YY. min, max and lim may appear in any order or may be
// omitted to get default values.
static void
nanov2_set_block_scan_policy(const char *name, const char *ptr)
{
	static char first_fit_key[] = "firstfit";
	static char min_key[] = "min";
	static char max_key[] = "max";
	static char lim_key[] = "lim";

	nanov2_block_scan_policy_t block_scan_policy = NANO_SCAN_CAPACITY_BASED;
	int scan_min_capacity = DEFAULT_SCAN_MIN_CAPACITY;
	int scan_max_capacity = DEFAULT_SCAN_MAX_CAPACITY;
	int scan_limit = DEFAULT_SCAN_LIMIT;
	const char *endp;
	boolean_t failed = FALSE;
	boolean_t min_found = FALSE;
	boolean_t max_found = FALSE;
	boolean_t lim_found = FALSE;
	const char *value = ptr;

	if (ptr) {
		if (!strcmp(ptr, first_fit_key)) {
			block_scan_policy = NANO_SCAN_FIRST_FIT;
		} else {
			while (!failed && ptr && *ptr) {
				if (!strncmp(ptr, min_key, sizeof(min_key) - 1) && !min_found) {
					min_found = TRUE;
					ptr += sizeof(min_key) - 1;
					long value = malloc_common_convert_to_long(ptr, &endp);
					if (ptr != endp && value >= 0 && value <= 100) {
						scan_min_capacity = (int)value;
						ptr = endp;
					} else {
						failed = TRUE;
					}
				} else if (!strncmp(ptr, max_key, sizeof(max_key) - 1)
						&& !max_found) {
					max_found = TRUE;
					ptr += sizeof(max_key) - 1;
					long value = malloc_common_convert_to_long(ptr, &endp);
					if (ptr != endp && value >= 0 && value <= 100) {
						scan_max_capacity = (int)value;
						ptr = endp;
					} else {
						failed = TRUE;
					}
				} else if (!strncmp(ptr, lim_key, sizeof(lim_key) - 1)
						&& !lim_found) {
					lim_found = TRUE;
					ptr += sizeof(lim_key) - 1;
					long value = malloc_common_convert_to_long(ptr, &endp);
					if (ptr != endp && value >= 0) {
						scan_limit = (int)value;
						ptr = endp;
					} else {
						failed = TRUE;
					}
				} else {
					failed = TRUE;
				}
				if (*ptr) {
					if (*ptr == ':') {
						ptr++;
					} else {
						failed = TRUE;
					}
				}
			}

			if (!failed && scan_min_capacity > scan_max_capacity) {
				failed = TRUE;
			}
		}
	}

	if (!failed) {
		nanov2_policy_config.block_scan_policy = block_scan_policy;
		nanov2_policy_config.block_scan_min_capacity = scan_min_capacity;
		nanov2_policy_config.block_scan_max_capacity = scan_max_capacity;
		nanov2_policy_config.block_scan_limit = scan_limit;
	} else {
		malloc_report(ASL_LEVEL_ERR, "%s value (%s) invalid - ignored.\n",
				name, value);
	}
}

// Configures the nanov2_blocks_by_size_class array. If ptr is not NULL and
// *ptr is not empty, it is expected to be a list of 16 positive integers
// separated by commas that sum to TOTAL_BLOCK_UNITS (which is currently 64).
// For example, as an environment variable:
// 	MallocNanoSizeClassBlocks=2,7,6,6,6,5,5,5,5,2,2,2,2,2,6,1
// or as a boot argument:
//  nanov2_size_class_blocks=2,7,6,6,6,5,5,5,5,2,2,2,2,2,6,1
static void
nanov2_set_blocks_by_size_class(const char *name, const char *ptr)
{
	int new_total_block_units = 0;
	int new_blocks_by_size_class[NANO_SIZE_CLASSES];
	MALLOC_STATIC_ASSERT(
			sizeof(new_blocks_by_size_class) == sizeof(block_units_by_size_class),
			"Size mismatch in nanov2_set_blocks_by_size_class()");
	const char *endp;
	const char *sptr = ptr;
	for (int i = 0; i < NANO_SIZE_CLASSES; i++) {
		int count = (int)malloc_common_convert_to_long(ptr, &endp);
		char separator = i == NANO_SIZE_CLASSES - 1 ? '\0' : ',';
		if (endp == ptr || *endp != separator || count > TOTAL_BLOCK_UNITS) {
			malloc_report(ASL_LEVEL_ERR,
					"%s value invalid: [%s] - ignored.\n", name, sptr);
			return;
		}
		new_blocks_by_size_class[i] = count;
		new_total_block_units += count;
		ptr = endp + 1;
	}

	if (new_total_block_units != TOTAL_BLOCK_UNITS) {
		malloc_report(ASL_LEVEL_ERR,
				"%s value invalid - values must sum to %d, not %d - ignored.\n",
				name, TOTAL_BLOCK_UNITS, new_total_block_units);
	} else {
		memcpy(block_units_by_size_class, new_blocks_by_size_class,
				sizeof(block_units_by_size_class));
	}
}

// First stage initialization. Called during libSystem initialization.
// Reads environment variables and boot arguments and sets the madvise policy,
// single arena list and the block scan policy. Environment variables override
// boot arguments.
void
nanov2_init(const char *envp[], const char *apple[], const char *bootargs)
{
	// Get and process the boot args and environment variables.
	char value_buf[256];
	const char *value = _simple_getenv(envp, madvise_policy_env);
	const char *name = madvise_policy_env;
	if (!value) {
		value = malloc_common_value_for_key(bootargs, madvise_policy_bootarg);
		if (value) {
			name = madvise_policy_bootarg;
		}
	}
	nanov2_set_madvise_policy(name, value);

	name = single_arena_env;
	value = _simple_getenv(envp, single_arena_env);
	if (!value) {
		value = malloc_common_value_for_key_copy(bootargs, single_arena_bootarg,
				value_buf, sizeof(value_buf));
		if (value) {
			name = single_arena_bootarg;
		}
	}
	nanov2_set_single_arena_size_classes(name, value);

	name = scan_policy_env;
	value = _simple_getenv(envp, scan_policy_env);
	if (!value) {
		value = malloc_common_value_for_key_copy(bootargs, scan_policy_bootarg,
				value_buf, sizeof(value_buf));
		if (value) {
			name = scan_policy_bootarg;
		}
	}
	nanov2_set_block_scan_policy(name, value);

	name = size_class_blocks_env;
	value = _simple_getenv(envp, size_class_blocks_env);
	if (!value) {
		value = malloc_common_value_for_key_copy(bootargs, size_class_blocks_bootarg,
				value_buf, sizeof(value_buf));
		if (value) {
			name = size_class_blocks_bootarg;
		}
	}
	if (value) {
		nanov2_set_blocks_by_size_class(name, value);
	}
}

static void
nanov2_configure_once(void *context MALLOC_UNUSED)
{
	// Check that the block_units_by_size_class table is consistent.
	int total_blocks = 0;
	for (int i = 0; i < NANO_SIZE_CLASSES; i++) {
		total_blocks += block_units_by_size_class[i] * BLOCKS_PER_UNIT;
	}
	MALLOC_ASSERT(total_blocks == NANOV2_BLOCKS_PER_ARENA);

	// Build the first_block_offset_by_size_class and
	// last_block_offset_by_size_class tables. The first entry is special
	// because block 0 is reserved for the metadata block, so the first offset
	// is 1 and the number of blocks allocated is reduced by 1.
	int next_offset = 1;
	first_block_offset_by_size_class[0] = next_offset;
	next_offset = block_units_by_size_class[0] * BLOCKS_PER_UNIT;
	last_block_offset_by_size_class[0] = next_offset - 1;

	for (int i = 1; i < NANO_SIZE_CLASSES; i++) {
		first_block_offset_by_size_class[i] = next_offset;
		next_offset += block_units_by_size_class[i] * BLOCKS_PER_UNIT;
		last_block_offset_by_size_class[i] = next_offset - 1;
	}
	MALLOC_ASSERT(next_offset == NANOV2_BLOCKS_PER_ARENA);

	// Construct the ptr_offset_to_size_class map, which maps the part of the
	// logical address that depends on size class to the corresponding size
	// class. This would be a simple mask operation if all size classes were of
	// equal size, but sadly they are not.
	int next_index = 0;
	for (int i = 0; i < NANO_SIZE_CLASSES; i++) {
		int block_units = block_units_by_size_class[i];
		for (int j = 0; j < block_units; j++) {
			ptr_offset_to_size_class[next_index++] = i;
		}
	}
	MALLOC_ASSERT(next_index == NANOV2_BLOCKS_PER_ARENA/BLOCKS_PER_UNIT);
}

static os_once_t nanov2_config_predicate;

void
nanov2_configure(void)
{
	os_once(&nanov2_config_predicate, NULL, nanov2_configure_once);
}
#endif // OS_VARIANT_NOTRESOLVED

#pragma mark -
#pragma mark Zone Functions

#if OS_VARIANT_RESOLVED
// Returns the allocation size for a pointer. Uses nanov2_pointer_size() to
// determine whether the pointer is for a Nano V2 allocation and, if not,
// delegates to the helper zone. Returns 0 if the pointer is not to memory
// allocated by Nano V2 or attributable to the helper zone.
MALLOC_NOEXPORT size_t
nanov2_size(nanozonev2_t *nanozone, const void *ptr)
{
	size_t size = nanov2_pointer_size_inline(nanozone, (void *)ptr, FALSE,
			NULL, NULL);
	return size ? size : nanozone->helper_zone->size(nanozone->helper_zone, ptr);
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static void
nanov2_bzero(void *ptr, size_t size)
{
	// TODO: inline bzero from libplatform
	bzero(ptr, size);
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static void *
_nanov2_allocate(nanozonev2_t *nanozone, size_t rounded_size, bool clear,
		bool typed, malloc_type_id_t type_id)
{
	nanov2_block_meta_t *madvise_block_metap = NULL;
	nanov2_size_class_t size_class = nanov2_size_class_from_size(rounded_size);

	// Get the index of the pointer to the block from which we are should be
	// allocating. This currently depends on the physical CPU number.
	int allocation_index = nanov2_get_allocation_block_index();

	// Get the current allocation block meta data pointer. If this is NULL, we
	// need to find a new allocation block.
	nanov2_block_meta_t **block_metapp =
			&nanozone->current_block[size_class][allocation_index];
	nanov2_block_meta_t *block_metap = os_atomic_load(block_metapp, relaxed);
	bool corruption = false;
	void *ptr = NULL;
	if (block_metap) {
		// Fast path: we have a block -- try to allocate from it.
		ptr = nanov2_allocate_from_block_inline(nanozone, block_metap,
				size_class, &madvise_block_metap, &corruption);
		if (ptr && !corruption) {
			if (clear && malloc_zero_policy != MALLOC_ZERO_ON_FREE) {
				nanov2_bzero(ptr, rounded_size);
			} else {
				// Always clear the double-free guard so that we can recognize that
				// this block is not on the free list.
				nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
				os_atomic_store(&slotp->double_free_guard, 0, relaxed);

				// We know the body of the allocation is already clear, so we just
				// need to clean up the next_slot word to get to all-zero.  Do so in
				// all cases, even if a cleared allocation is not requested, to
				// prevent any leakage through the next_slot bits.
				os_atomic_store(&slotp->next_slot, 0, relaxed);
			}
			return ptr;
		}
	}

	return nanov2_allocate_outlined(nanozone, block_metapp, rounded_size,
			size_class, allocation_index, madvise_block_metap, ptr, clear,
			typed, type_id);
}

MALLOC_NOEXPORT void *
nanov2_malloc(nanozonev2_t *nanozone, size_t size)
{
	size_t rounded_size = _nano_common_good_size(size);
	if (rounded_size <= NANO_MAX_SIZE) {
		return _nanov2_allocate(nanozone, rounded_size, false, false,
				MALLOC_TYPE_ID_NONE);
	}

	// Too big for nano, so delegate to the helper zone.
	return nanozone->helper_zone->malloc(nanozone->helper_zone, size);
}

MALLOC_NOEXPORT void *
nanov2_malloc_type(nanozonev2_t *nanozone, size_t size,
		malloc_type_id_t type_id)
{
	size_t rounded_size = _nano_common_good_size(size);
	if (rounded_size <= NANO_MAX_SIZE) {
		return _nanov2_allocate(nanozone, rounded_size, false, true, type_id);
	}

	// Too big for nano, so delegate to the helper zone.
	malloc_zone_t *helper_zone = nanozone->helper_zone;
	return helper_zone->malloc_type_malloc(helper_zone, size, type_id);
}

MALLOC_NOEXPORT void
nanov2_free_definite_size(nanozonev2_t *nanozone, void *ptr, size_t size)
{
	if (ptr && nanov2_has_valid_signature(ptr)) {
		nanov2_size_class_t size_class = nanov2_size_class_from_size(size);

		if (malloc_zero_policy == MALLOC_ZERO_ON_FREE) {
			if (size_class != 0) {
				nanov2_bzero((char *)ptr + sizeof(nanov2_free_slot_t),
						size - sizeof(nanov2_free_slot_t));
			}
		}

		nanov2_block_meta_t *madvise_block_metap = nanov2_free_to_block_inline(
				nanozone, ptr, size_class, NULL);
		if (madvise_block_metap) {
			nanov2_madvise_block(nanozone, madvise_block_metap, size_class,
					SLOT_CAN_MADVISE);
		}
		return;
	}
	return nanozone->helper_zone->free_definite_size(nanozone->helper_zone, ptr,
			size);
}

static void
_nanov2_free(nanozonev2_t *nanozone, void *ptr, bool try)
{
	if (ptr) {
		// Check whether it's a Nano pointer and get the size. If it's not
		// Nano, pass it to the helper zone.
		nanov2_size_class_t size_class;
		nanov2_block_meta_t *block_metap;
		size_t size = nanov2_pointer_size_inline(nanozone, ptr, FALSE,
				&size_class, &block_metap);
		if (size) {
			if (malloc_zero_policy == MALLOC_ZERO_ON_FREE) {
				if (size > sizeof(nanov2_free_slot_t)) {
					nanov2_bzero((char *)ptr + sizeof(nanov2_free_slot_t),
							size - sizeof(nanov2_free_slot_t));
				}
			}

			nanov2_block_meta_t *madvise_block_metap = nanov2_free_to_block_inline(
					nanozone, ptr, size_class, block_metap);
			if (madvise_block_metap) {
				nanov2_madvise_block(nanozone, madvise_block_metap, size_class,
						SLOT_CAN_MADVISE);
			}
			return;
		}
	}
	return try ? nanozone->helper_zone->try_free_default(nanozone->helper_zone, ptr) :
			nanozone->helper_zone->free(nanozone->helper_zone, ptr);
}

MALLOC_NOEXPORT void
nanov2_free(nanozonev2_t *nanozone, void *ptr)
{
	_nanov2_free(nanozone, ptr, false);
}

MALLOC_NOEXPORT void
nanov2_try_free_default(nanozonev2_t *nanozone, void *ptr)
{
	_nanov2_free(nanozone, ptr, true);
}

MALLOC_NOEXPORT void *
nanov2_calloc(nanozonev2_t *nanozone, size_t num_items, size_t size)
{
	size_t total_bytes;
	if (calloc_get_size(num_items, size, 0, &total_bytes)) {
		return NULL;
	}
	size_t rounded_size = _nano_common_good_size(total_bytes);
	if (total_bytes <= NANO_MAX_SIZE) {
		return _nanov2_allocate(nanozone, rounded_size, true, false,
				MALLOC_TYPE_ID_NONE);
	}

	// Too big for nano, so delegate to the helper zone.
	return nanozone->helper_zone->calloc(nanozone->helper_zone, 1, total_bytes);
}

MALLOC_NOEXPORT void *
nanov2_calloc_type(nanozonev2_t *nanozone, size_t num_items, size_t size,
		malloc_type_id_t type_id)
{
	size_t total_bytes;
	if (calloc_get_size(num_items, size, 0, &total_bytes)) {
		return NULL;
	}
	size_t rounded_size = _nano_common_good_size(total_bytes);
	if (total_bytes <= NANO_MAX_SIZE) {
		return _nanov2_allocate(nanozone, rounded_size, true, true, type_id);
	}

	// Too big for nano, so delegate to the helper zone.
	malloc_zone_t *helper_zone = nanozone->helper_zone;
	return helper_zone->malloc_type_calloc(helper_zone, 1, total_bytes,
			type_id);
}

MALLOC_NOEXPORT void *
nanov2_malloc_zero_on_alloc(nanozonev2_t *nanozone, size_t size)
{
	size_t rounded_size = _nano_common_good_size(size);
	if (rounded_size <= NANO_MAX_SIZE) {
		return _nanov2_allocate(nanozone, rounded_size, true, false,
				MALLOC_TYPE_ID_NONE);
	}

	// Too big for nano, so delegate to the helper zone.
	return nanozone->helper_zone->malloc(nanozone->helper_zone, size);
}

MALLOC_NOEXPORT void *
nanov2_malloc_type_zero_on_alloc(nanozonev2_t *nanozone, size_t size,
		malloc_type_id_t type_id)
{
	size_t rounded_size = _nano_common_good_size(size);
	if (rounded_size <= NANO_MAX_SIZE) {
		return _nanov2_allocate(nanozone, rounded_size, true, true, type_id);
	}

	// Too big for nano, so delegate to the helper zone.
	malloc_zone_t *helper_zone = nanozone->helper_zone;
	return helper_zone->malloc_type_malloc(helper_zone, size, type_id);
}
#endif // OS_VARIANT_RESOLVED

#if OS_VARIANT_NOTRESOLVED
static void *
nanov2_valloc(nanozonev2_t *nanozone, size_t size)
{
	// Always delegate this to the helper zone.
	return nanozone->helper_zone->valloc(nanozone->helper_zone, size);
}
#endif // OS_VARIANT_NOTRESOLVED

#if OS_VARIANT_RESOLVED
MALLOC_ALWAYS_INLINE MALLOC_INLINE
static void *
_nanov2_realloc(nanozonev2_t *nanozone, void *ptr, size_t new_size, bool typed,
		malloc_type_id_t type_id)
{
	// If we are given a NULL pointer, just allocate memory of the requested
	// size.
	if (ptr == NULL) {
		if (typed) {
			return nanov2_malloc_type(nanozone, new_size, type_id);
		} else {
			return nanov2_malloc(nanozone, new_size);
		}
	}

	malloc_zone_t *helper_zone = nanozone->helper_zone;

	size_t old_size = nanov2_pointer_size(nanozone, ptr, FALSE);
	if (!old_size) {
		// Not a Nano pointer - let the helper deal with it
		if (typed) {
			return helper_zone->malloc_type_realloc(helper_zone, ptr, new_size,
					type_id);
		} else {
			return helper_zone->realloc(helper_zone, ptr, new_size);
		}
	}

	void *new_ptr;
	if (new_size > NANO_MAX_SIZE) {
		// Too large for Nano. Try to allocate from the helper zone.
		if (typed) {
			new_ptr = helper_zone->malloc_type_malloc(helper_zone, new_size,
					type_id);
		} else {
			new_ptr = helper_zone->malloc(helper_zone, new_size);
		}

		if (!new_ptr) {
			// Failed to allocate - leave the existing allocation alone.
			return NULL;
		}
	} else if (!new_size) {
		// Resizing to zero. Free the existing memory and explicitly allocate
		// zero bytes.
		nanov2_free(nanozone, ptr);
		return nanov2_malloc(nanozone, 0);
	} else {
		size_t new_good_size = _nano_common_good_size(new_size);
		if (new_good_size > old_size || new_good_size <= old_size / 2) {
			// Growing or shrinking to less than half size - we need to
			// reallocate.
			if (typed) {
				new_ptr = nanov2_malloc_type(nanozone, new_good_size, type_id);
			} else {
				new_ptr = nanov2_malloc(nanozone, new_good_size);
			}

			if (!new_ptr) {
				// Failed to allocate - leave the existing allocation alone.
				return NULL;
			}
		} else {
			// Same size or shrinking by less than half size. Keep the same
			// allocation and scribble the area that's being released. Nothing
			// to do for zero-on-free yet; that will be taken care of when the
			// shrunk allocation is freed.
			if (new_size != old_size) {
				MALLOC_ASSERT(new_size < old_size);
				if (os_unlikely(nanozone->debug_flags & MALLOC_DO_SCRIBBLE)) {
					memset(ptr + new_size, SCRABBLE_BYTE, old_size - new_size);
				}
			}
			return ptr;
		}
	}

	// If we reach this point, we allocated new memory. Copy the existing
	// content to the new location and release the old allocation.
	MALLOC_ASSERT(new_ptr);
	memcpy(new_ptr, ptr, MIN(old_size, new_size));
	nanov2_free(nanozone, ptr);

	return new_ptr;
}

MALLOC_NOEXPORT void *
nanov2_realloc(nanozonev2_t *nanozone, void *ptr, size_t new_size)
{
	return _nanov2_realloc(nanozone, ptr, new_size, false, MALLOC_TYPE_ID_NONE);
}

MALLOC_NOEXPORT void *
nanov2_realloc_type(nanozonev2_t *nanozone, void *ptr, size_t new_size,
		malloc_type_id_t type_id)
{
	return _nanov2_realloc(nanozone, ptr, new_size, true, type_id);
}
#endif // OS_VARIANT_RESOLVED

#if OS_VARIANT_NOTRESOLVED
static void
nanov2_destroy(nanozonev2_t *nanozone)
{
	nanozone->helper_zone->destroy(nanozone->helper_zone);
	nano_common_deallocate_pages((void *)nanozone, NANOZONEV2_ZONE_PAGED_SIZE,
			nanozone->debug_flags);
}
#endif // OS_VARIANT_NOTRESOLVED

#if OS_VARIANT_RESOLVED
MALLOC_NOEXPORT boolean_t
nanov2_claimed_address(nanozonev2_t *nanozone, void *ptr)
{
	return nanov2_pointer_size(nanozone, ptr, TRUE)
			|| malloc_zone_claimed_address(nanozone->helper_zone, ptr);
}

MALLOC_NOEXPORT unsigned
nanov2_batch_malloc(nanozonev2_t *nanozone, size_t size, void **results,
		unsigned count)
{
	unsigned allocated = 0;
	size_t rounded_size = _nano_common_good_size(size);
	if (rounded_size <= NANO_MAX_SIZE) {
		while (allocated < count) {
			// TODO: nanov2_malloc will redo _nano_common_good_size
			void *ptr = nanov2_malloc(nanozone, rounded_size);
			if (!ptr) {
				break;
			}

			*results++ = ptr;
			allocated++;
		}
		if (allocated == count) {
			// Allocated everything.
			return allocated;
		}
	}

	// We could not allocate everything. Let the helper zone do the rest.
	return allocated + nanozone->helper_zone->batch_malloc(
			nanozone->helper_zone, size, results, count - allocated);
}

MALLOC_NOEXPORT void
nanov2_batch_free(nanozonev2_t *nanozone, void **to_be_freed, unsigned count)
{
	if (count) {
		while (count--) {
			void *ptr = to_be_freed[count];
			if (ptr) {
				nanov2_free(nanozone, ptr);
			}
		}
	}
}

MALLOC_NOEXPORT void *
nanov2_memalign(nanozonev2_t *nanozone, size_t alignment, size_t size)
{
	// Serve directly if the requested alignment is trivially satisfied by our
	// baseline alignment (16 bytes)
	if (alignment <= NANO_REGIME_QUANTA_SIZE) {
		return nanov2_malloc(nanozone, size);
	}

	// Otherwise delegate to the helper zone
	return nanozone->helper_zone->memalign(nanozone->helper_zone, alignment,
			size);
}

MALLOC_NOEXPORT void *
nanov2_memalign_type(nanozonev2_t *nanozone, size_t alignment, size_t size,
		malloc_type_id_t type_id)
{
	// Serve directly if the requested alignment is trivially satisfied by our
	// baseline alignment (16 bytes)
	if (alignment <= NANO_REGIME_QUANTA_SIZE) {
		return nanov2_malloc_type(nanozone, size, type_id);
	}

	// Otherwise delegate to the helper zone
	malloc_zone_t *helper_zone = nanozone->helper_zone;
	return helper_zone->malloc_type_memalign(helper_zone, alignment, size,
			type_id);
}

size_t
nanov2_pressure_relief(nanozonev2_t *nanozone, size_t goal)
{
	if (nanov2_madvise_policy != NANO_MADVISE_WARNING_PRESSURE
			&& nanov2_madvise_policy != NANO_MADVISE_CRITICAL_PRESSURE) {
		// In the current implementation, we only get called on warning, so
		// act if the policy is either warning or critical. We would need to
		// add a new zone entry point to respond to critical.
		return 0;
	}
	const char *name = nanozone->basic_zone.zone_name;
	MAGMALLOC_PRESSURERELIEFBEGIN((void *)nanozone, name, (int)goal);
	MALLOC_TRACE(TRACE_nano_memory_pressure | DBG_FUNC_START,
			(uint64_t)nanozone, goal, 0, 0);
	size_t total = 0;

	// Loop over all arenas madvising blocks that are marked as madvisable,
	// until we reach our goal.
	nanov2_region_t *region = nanozone->first_region_base;
	nanov2_meta_index_t metablock_meta_index = nanov2_metablock_meta_index(nanozone);
	nanov2_arena_t *current_region_next_arena = os_atomic_load(
			&nanozone->current_region_next_arena, acquire);
	while (region) {
		nanov2_arena_t *arena = nanov2_first_arena_for_region(region);
		nanov2_arena_t *arena_after_region = nanov2_limit_arena_for_region(
				nanozone, region, current_region_next_arena);
		while (arena < arena_after_region) {
			// Scan all of the blocks in the arena, skipping the metadata block.
			nanov2_arena_metablock_t *meta_blockp =
					nanov2_metablock_address_for_ptr(nanozone, arena);
			nanov2_block_meta_t *block_metap = &meta_blockp->arena_block_meta[0];

			// We need to hold the zone madvise lock to madvise. We could take
			// it for the duration of this function, but that might hold up
			// ongoing allocation and free operations for too long. So just
			// lock and unlock for each arena.
			_malloc_lock_lock(&nanozone->madvise_lock);
			for (nanov2_meta_index_t i = 0; i < NANOV2_BLOCKS_PER_ARENA;
					i++, block_metap++) {
				if (i != metablock_meta_index) {
					nanov2_block_meta_t meta = os_atomic_load(block_metap, relaxed);
					if (meta.next_slot == SLOT_CAN_MADVISE) {
						nanov2_block_t *blockp = nanov2_block_address_from_meta_index(
								nanozone, arena, i);
						if (nanov2_madvise_block_locked(nanozone, block_metap,
								blockp, nanov2_size_class_for_ptr(nanozone, blockp),
								SLOT_CAN_MADVISE)) {
							total += NANOV2_BLOCK_SIZE;
						}
					}
				}
			}
			_malloc_lock_unlock(&nanozone->madvise_lock);
			if (goal && total >= goal) {
				goto done;
			}
			arena++;
		}
		region = nanov2_next_region_for_region(nanozone, region,
				current_region_next_arena);
	}

done:
	MAGMALLOC_PRESSURERELIEFEND((void *)nanozone, name, (int)goal, (int)total);
	MALLOC_TRACE(TRACE_nano_memory_pressure | DBG_FUNC_END,
			(uint64_t)nanozone, goal, total, 0);

	return total;
}
#endif // OS_VARIANT_RESOLVED

#pragma mark -
#pragma mark Zone Introspection

#if OS_VARIANT_NOTRESOLVED

// NOTE: in the code that follows, address that we obtain from the Nano
// structures are relative to the target process. They need to be translated
// before they can be used to read the mapping in this process.

#define NANOV2_ZONE_PTR_TO_MAPPED_PTR(type, zone_ptr, offset) \
		(type)((mach_vm_address_t)zone_ptr - (mach_vm_offset_t)offset)
#define NANOV2_MAPPED_PTR_TO_ZONE_PTR(type, mapped_ptr, offset) \
		(type)((mach_vm_address_t)mapped_ptr + (mach_vm_offset_t)offset)

static kern_return_t
nanov2_ptr_in_use_enumerator(task_t task, void *context, unsigned type_mask,
		vm_address_t zone_address, memory_reader_t reader,
		vm_range_recorder_t recorder)
{
	// Ensure that we have configured enough of the allocator to be able to
	// examine its data structures. In tools that do not directly use Nano, we
	// won't have done this yet. nanov2_configure() runs the initialization
	// only once.
	nanov2_configure();

	// Only MALLOC_PTR_IN_USE_RANGE_TYPE and MALLOC_PTR_REGION_RANGE_TYPE have
	// meaning for Nano. Anything else returns immediately.
	if (!(type_mask & (MALLOC_PTR_IN_USE_RANGE_TYPE|MALLOC_PTR_REGION_RANGE_TYPE))) {
		return 0;
	}

	// Read the zone data.
	nanozonev2_t *nanozone;
	nanozonev2_t zone_copy;
	kern_return_t kr;
	bitarray_t slots;

	reader = reader_or_in_memory_fallback(reader, task);

	kr = reader(task, zone_address, sizeof(nanozonev2_t), (void **)&nanozone);
	if (kr) {
		return kr;
	}
	boolean_t self_zone = mach_task_is_self(task) && (nanozonev2_t *)zone_address == nanozone;
	memcpy(&zone_copy, nanozone, sizeof(zone_copy));
	nanozone = &zone_copy;
	nanov2_meta_index_t metablock_meta_index = nanov2_metablock_meta_index(nanozone);

	// Process the zone one region at a time. Report each in-use block as a
	// pointer range and each in-use slot as a pointer.
	nanov2_region_t *region = nanozone->first_region_base;
	nanov2_arena_t *current_region_next_arena = os_atomic_load(
			&nanozone->current_region_next_arena, acquire);
	while (region) {
		mach_vm_address_t vm_addr = (mach_vm_address_t)NULL;
		kern_return_t kr = reader(task, (vm_address_t)region, NANOV2_REGION_SIZE, (void **)&vm_addr);
		if (kr) {
			return kr;
		}

		// ptr_offset is the difference between an address in the target process
		// and its mapped address in this process.
		mach_vm_offset_t ptr_offset = (mach_vm_address_t)region - vm_addr;
		nanov2_arena_t *arena = nanov2_first_arena_for_region(region);
		nanov2_arena_t *limit_arena = nanov2_limit_arena_for_region(nanozone, region,
				current_region_next_arena);
		vm_range_t ptr_range;
		while (arena < limit_arena) {
			// Find the metadata block and process every entry, apart from the
			// one for the metadata block itself.
			nanov2_arena_metablock_t *arena_meta_blockp =
					NANOV2_ZONE_PTR_TO_MAPPED_PTR(nanov2_arena_metablock_t *,
					nanov2_metablock_address_for_ptr(nanozone, arena),
					ptr_offset);
			nanov2_block_meta_t *block_metap = &arena_meta_blockp->arena_block_meta[0];

			for (nanov2_meta_index_t i = 0; i < NANOV2_BLOCKS_PER_ARENA; i++, block_metap++) {
				if (i == metablock_meta_index) {
					// Skip the metadata block.
					continue;
				}
				nanov2_block_meta_t meta = os_atomic_load(block_metap, relaxed);
				if (!nanov2_is_block_active(meta)) {
					continue;
				}

				nanov2_block_t *blockp = nanov2_block_address_from_meta_index(
						nanozone, arena, i);
				if (type_mask & MALLOC_PTR_REGION_RANGE_TYPE) {
					// Report this block as an in-use range.
					ptr_range.address = (vm_address_t)blockp;
					ptr_range.size = NANOV2_BLOCK_SIZE;
					recorder(task, context, MALLOC_PTR_REGION_RANGE_TYPE, &ptr_range, 1);
				}
				if (type_mask & MALLOC_PTR_IN_USE_RANGE_TYPE) {
					// Report all of the pointers in the block that are not on
					// the free list.
					nanov2_size_class_t size_class = nanov2_size_class_for_ptr(
							nanozone, blockp);
					int slot_size = nanov2_size_from_size_class(size_class);
					int slot_count = slots_by_size_class[size_class];
					vm_range_t ranges[NANOV2_MAX_SLOTS_PER_BLOCK];
					int range_count = 0;
					if (meta.next_slot == SLOT_BUMP || meta.next_slot == SLOT_FULL) {
						// Either the block is full or the freelist is empty. If
						// it's full, everything is in use. If the free list is
						// empty, everything up to slot_count - meta.free_count - 1
						// is in use.
						range_count = meta.next_slot == SLOT_BUMP ?
								slot_count - meta.free_count - 1 : slot_count;
						for (int i = 0; i < range_count; i++) {
							ranges[i].address = (vm_address_t)nanov2_slot_in_block_ptr(blockp, size_class, i);
							ranges[i].size = slot_size;
						}
					} else {
						// We need to scan the freelist to see what's in use.
						int log_size = 64 - __builtin_clzl(slot_count);
						if (self_zone) {
							// Don't allocate from ourselves!
							slots = nanozone->helper_zone->calloc(nanozone->helper_zone,
									1, bitarray_size(log_size));
						} else {
							slots = bitarray_create(log_size);
						}
						for (int i = 0; i < slot_count; i++) {
							bitarray_set(slots, log_size, i);
						}

						int next_slot = meta.next_slot;
						int free_list_count = 0;
						while (next_slot != SLOT_BUMP) {
							next_slot--;   // meta.next_slot is 1-based.
							if (next_slot < 0 || next_slot >= slot_count ||
									!bitarray_get(slots, log_size, next_slot)) {
								// Out of range or already seen?? We may have
								// snapshotted the block while it was updating.
								// Don't go any further to avoid an infinite loop.
								break;
							}
							bitarray_zap(slots, log_size, next_slot);
							void *ptr = nanov2_slot_in_block_ptr(blockp, size_class, next_slot);
							nanov2_free_slot_t *slotp = NANOV2_ZONE_PTR_TO_MAPPED_PTR(nanov2_free_slot_t *, ptr, ptr_offset);
							next_slot = (uint16_t)slotp->next_slot;
							free_list_count++;
						}
						// Add a range for each slot that is not on the freelist,
						// unless that slot has never been allocated.
						int block_free_count = meta.free_count + 1; // actual free count.
						int in_use_count = slot_count - block_free_count;
						int slots_used_count = in_use_count + free_list_count;
						index_t index;
						while (bitarray_zap_first_set(slots, log_size, &index)) {
							if (index >= slots_used_count) {
								// Reached the end of the slots that have been
								// allocated at some point.
								break;
							}
							ranges[range_count].address = (vm_address_t)nanov2_slot_in_block_ptr(blockp, size_class, index);
							ranges[range_count].size = slot_size;
							range_count++;
						}
						_free(slots);
					}
					if (range_count) {
						// Notify the in-use pointers that we found.
						recorder(task, context, MALLOC_PTR_IN_USE_RANGE_TYPE, ranges, range_count);
					}
				}
			}
			arena++;
		}

		// We have to manually handle the linkage to the next region because
		// of the address slide between this process and the target.
		nanov2_region_linkage_t *region_linkagep =
						nanov2_region_linkage_for_region(nanozone, region);
		nanov2_region_linkage_t *mapped_region_linkagep =
				NANOV2_ZONE_PTR_TO_MAPPED_PTR(nanov2_region_linkage_t *,
				region_linkagep, ptr_offset);
		int offset = os_atomic_load(&mapped_region_linkagep->next_region_offset,
				relaxed);
		region = offset ? region + offset : NULL;
	}
	return 0;
}

static size_t
nanov2_good_size(nanozonev2_t *nanozone, size_t size)
{
	if (size <= NANO_MAX_SIZE) {
		return _nano_common_good_size(size);
	}
	return nanozone->helper_zone->introspect->good_size(nanozone->helper_zone,
			size);
}

static boolean_t
nanov2_check(nanozonev2_t *nanozone)
{
	// Does nothing
	return 1;
}

static void
nanov2_print(task_t task, unsigned level, vm_address_t zone_address,
		memory_reader_t reader, print_task_printer_t printer)
{
    // Ensure that we have configured enough of the allocator to be able to
    // examine its data structures. In tools that do not directly use Nano, we
    // won't have done this yet. nanov2_configure() runs the initialization
    // only once.
    nanov2_configure();

	nanozonev2_t *mapped_nanozone;
	if (reader(task, (vm_address_t)zone_address, sizeof(nanozonev2_t),
            (void **)&mapped_nanozone)) {
        printer("Failed to map nanozonev2_s at %p\n", zone_address);
        return;
    }

	// Zone-wide statistics
	malloc_statistics_t stats;
	nanov2_statistics_task_printer(task, zone_address, reader, printer, &stats);
	nanov2_statistics_t *nano_stats = &mapped_nanozone->statistics;
	printer("Nanozonev2 %p: blocks in use: %llu, size in use: %llu "
			"allocated size: %llu, allocated regions: %d, region holes: %d\n",
			zone_address, (uint64_t)stats.blocks_in_use,
			(uint64_t)stats.size_in_use, (uint64_t)stats.size_allocated,
			nano_stats->allocated_regions, nano_stats->region_address_clashes);

#if DEBUG_MALLOC
	// Per-size class statistics
	printer("\nPer size-class statistics:\n");
	for (int i = 0; i < NANO_SIZE_CLASSES; i++) {
		nanov2_size_class_statistics *cs = &nano_stats->size_class_statistics[i];
		printer("  Class %d: ", i);
		printer("total alloc: %llu, total frees: %llu, madvised blocks: %llu, "
				"madvise races: %llu",
				cs->total_allocations, cs->total_frees, cs->madvised_blocks,
				cs->madvise_races);
		printer("\n");
	}
#endif // DEBUG_MALLOC

	// Per-context block pointers.
	printer("Current Allocation Blocks By Size Class/Context [CPU]\n");
	for (int i = 0; i < NANO_SIZE_CLASSES; i++) {
		printer("  Class %d: ", i);
		for (int j = 0; j < MAX_CURRENT_BLOCKS; j++) {
			if (mapped_nanozone->current_block[i][j]) {
				printer("%d: %p; ", j, mapped_nanozone->current_block[i][j]);
			}
		}
		printer("\n");
	}

	nanov2_meta_index_t metablock_meta_index =
			nanov2_metablock_meta_index(mapped_nanozone);
	nanov2_region_t *region = mapped_nanozone->first_region_base;
	// Use a single, consistent snapshot of current_region_next_arena throughout
	// iteration, ignoring any arenas or regions allocated after it.
	nanov2_arena_t *current_region_next_arena = os_atomic_load(
			&mapped_nanozone->current_region_next_arena, acquire);
	int region_index = 0;
	while (region) {
		printer("\nRegion %d: base address %p\n", region_index, region);
		nanov2_region_t *mapped_region;
		if (reader(task, (vm_address_t)region, sizeof(nanov2_region_t),
				(void **)&mapped_region)) {
			printer("Failed to map nanov2 region at %p\n", region);
			return;
		}
        off_t region_offset = (uintptr_t)mapped_region - (uintptr_t)region;

		nanov2_arena_t *arena = nanov2_first_arena_for_region(region);
		nanov2_arena_t *limit_arena = nanov2_limit_arena_for_region(
				mapped_nanozone, region, current_region_next_arena);
		int arena_index = 0;
		while (arena < limit_arena) {
			// Find the metadata block and process every entry, apart from the
			// one for the metadata block itself.
			nanov2_arena_metablock_t *arena_meta_blockp =
					nanov2_metablock_address_for_ptr(mapped_nanozone, arena);
			nanov2_arena_metablock_t *mapped_arena_meta_blockp =
				(nanov2_arena_metablock_t *)((uintptr_t)arena_meta_blockp + region_offset);
			nanov2_block_meta_t *block_metap = &arena_meta_blockp->arena_block_meta[0];
			nanov2_block_meta_t *mapped_block_metap = &mapped_arena_meta_blockp->arena_block_meta[0];

			int active_blocks = 0;
			int madvisable_blocks = 0;
			int unused_blocks = 0;
			int madvised_blocks = 0;
			int madvising_blocks = 0;
			for (nanov2_meta_index_t i = 0; i < NANOV2_BLOCKS_PER_ARENA; i++) {
				if (i == metablock_meta_index) {
					// Skip the metadata block.
					continue;
				}
				nanov2_block_meta_t meta = mapped_block_metap[i];
				switch (meta.next_slot) {
				case SLOT_NULL:
					unused_blocks++;
					break;
				case SLOT_MADVISED:
					madvised_blocks++;
					break;
				case SLOT_MADVISING:
					madvising_blocks++;
					break;
				case SLOT_CAN_MADVISE:
					madvisable_blocks++;
					break;
				default:
					active_blocks++;
					break;
				}
			}
			printer("Arena #%d: base address %p. Blocks - active: %d, "
					"madvisable: %d, madvising: %d, madvised: %d, unused: %d\n",
					arena_index, arena, active_blocks, madvisable_blocks,
					madvising_blocks, madvised_blocks, unused_blocks);

			// Print which size classes have blocks allocated in this arena.
			int non_empty_size_classes[NANO_SIZE_CLASSES];
			for (int i = 0; i < NANO_SIZE_CLASSES; i++) {
				non_empty_size_classes[i] = 0;
			}
			for (nanov2_meta_index_t i = 0; i < NANOV2_BLOCKS_PER_ARENA; i++) {
				if (i == metablock_meta_index) {
					// Skip the metadata block.
					continue;
				}
				nanov2_block_meta_t meta = mapped_block_metap[i];
				nanov2_size_class_t size_class =
						nanov2_size_class_for_meta_index(mapped_nanozone, i);
				switch (meta.next_slot) {
				case SLOT_FULL:
				case SLOT_BUMP:
				default:
					non_empty_size_classes[size_class]++;
					break;
				case SLOT_NULL:
				case SLOT_CAN_MADVISE:
				case SLOT_MADVISING:
				case SLOT_MADVISED:
					// Do not count these.
					break;
				}
			}
			printer("Size classes with allocated blocks: ");
			for (int i = 0; i < NANO_SIZE_CLASSES; i++) {
				if (non_empty_size_classes[i]) {
					printer("%d ", i);
				}
			}
			printer("\n");

			if (level >= MALLOC_VERBOSE_PRINT_LEVEL) {
				for (nanov2_meta_index_t i = 0; i < NANOV2_BLOCKS_PER_ARENA; i++) {
					if (i == metablock_meta_index) {
						// Skip the metadata block.
						continue;
					}
					nanov2_block_meta_t meta = mapped_block_metap[i];
					if (!nanov2_is_block_active(meta)) {
						continue;
					}
					nanov2_size_class_t size_class =
							nanov2_size_class_for_meta_index(mapped_nanozone, i);
					char *slot_text;
					switch (meta.next_slot) {
					case SLOT_NULL:
						slot_text = "NOT USED";
						break;
					case SLOT_FULL:
						slot_text = "FULL";
						break;
					case SLOT_CAN_MADVISE:
						slot_text = "CAN MADVISE";
						break;
					case SLOT_MADVISING:
						slot_text = "MADVISING";
						break;
					case SLOT_MADVISED:
						slot_text = "MADVISED";
						break;
					default:
						slot_text = NULL;
						break;
					}
					printer("    Block %d: base %p; metadata: %p, size %d "
							"(class %d) in-use: %d ",
							i, nanov2_block_address_from_meta_index(mapped_nanozone, arena, i),
							&block_metap[i], nanov2_size_from_size_class(size_class),
							size_class, meta.in_use);
					if (slot_text) {
						printer("%s\n", slot_text);
					} else {
						int allocated = slots_by_size_class[size_class] - meta.free_count - 1;
						if (meta.next_slot == SLOT_BUMP) {
							printer("BUMP (free list empty)");
						} else {
							printer("next_slot (1-based) = %d", meta.next_slot);

					}
						printer(", allocated slots: %d, free slots = %d, "
								"occupancy: %d%%\n",
								allocated, meta.free_count + 1,
								(100 * allocated)/slots_by_size_class[size_class]);
					}
				}
			}
			arena++;
			arena_index++;
		}

		region = nanov2_next_region_for_region_offset(mapped_nanozone, region,
                region_offset, current_region_next_arena);
		region_index++;
	}
}

static void
nanov2_print_self(nanozonev2_t *nanozone, boolean_t verbose)
{
	nanov2_print(mach_task_self(), verbose ? MALLOC_VERBOSE_PRINT_LEVEL : 0,
			(vm_address_t)nanozone, _malloc_default_reader, malloc_report_simple);
}

static void
nanov2_print_task(task_t task, unsigned level, vm_address_t zone_address,
		memory_reader_t reader, print_task_printer_t printer)
{
	nanov2_print(task, level, zone_address, reader, printer);
}

static void
nanov2_log(malloc_zone_t *zone, void *log_address)
{
	// Does nothing
}

static void
nanov2_force_lock(nanozonev2_t *nanozone)
{
	// Nothing to do - Nano V2 does not have a zone lock.
}

static void
nanov2_force_unlock(nanozonev2_t *nanozone)
{
	// Nothing to do - Nano V2 does not have a zone lock.
}

static void
nanov2_reinit_lock(nanozonev2_t *nanozone)
{
	// Nothing to do - Nano V2 does not have a zone lock.
}

static boolean_t
nanov2_locked(nanozonev2_t *nanozone)
{
	// Nothing to do - Nano V2 does not have a zone lock.
	return FALSE;
}

static void
nanov2_null_printer(const char __unused *fmt, ...)
{
}

static kern_return_t
nanov2_statistics(task_t task, vm_address_t zone_address,
		memory_reader_t reader, print_task_printer_t printer,
		malloc_statistics_t *stats)
{
	printer = printer ? printer : nanov2_null_printer;
	reader = reader_or_in_memory_fallback(reader, task);

	kern_return_t err;

    // Ensure that we have configured enough of the allocator to be able to
    // examine its data structures. In tools that do not directly use Nano, we
    // won't have done this yet. nanov2_configure() runs the initialization
    // only once.
    nanov2_configure();

	memset(stats, '\0', sizeof(*stats));

	nanozonev2_t *mapped_nanozone;
	err = reader(task, (vm_address_t)zone_address, sizeof(nanozonev2_t),
				 (void **)&mapped_nanozone);
	if (err) {
        printer("Failed to map nanozonev2_s at %p\n", zone_address);
        return err;
    }

	nanov2_region_t *region;
	nanov2_arena_t *arena;
	nanov2_meta_index_t metadata_block_index =
			nanov2_metablock_meta_index(mapped_nanozone);

	// Iterate over each arena in each region. Within each region, add
	// statistics for each slot in each block, excluding the meta data block.
	nanov2_arena_t *current_region_next_arena = os_atomic_load(
			&mapped_nanozone->current_region_next_arena, acquire);
	for (region = mapped_nanozone->first_region_base; region;) {
        nanov2_region_t *mapped_region;
		err = reader(task, (vm_address_t)region, sizeof(nanov2_region_t), (void **)&mapped_region);
        if (err) {
            printer("Failed to map nanov2 region at %p\n", region);
            return err;
        }
        off_t region_offset = (uintptr_t)mapped_region - (uintptr_t)region;
		for (arena = nanov2_first_arena_for_region(region);
				arena < nanov2_limit_arena_for_region(mapped_nanozone, region,
						current_region_next_arena);
				arena++) {
			nanov2_arena_metablock_t *meta_block =
					nanov2_metablock_address_for_ptr(mapped_nanozone, arena);
			nanov2_arena_metablock_t *mapped_meta_block =
				(nanov2_arena_metablock_t *)((uintptr_t)meta_block + region_offset);
			for (nanov2_meta_index_t i = 0; i < NANOV2_BLOCKS_PER_ARENA; i++) {
				if (i == metadata_block_index) {
					// Skip the metadata block.
					continue;
				}

				nanov2_block_meta_t *mapped_block_metap = &mapped_meta_block->arena_block_meta[i];
				nanov2_size_class_t size_class =
						nanov2_size_class_for_meta_index(mapped_nanozone, i);
				int slot_size = nanov2_size_from_size_class(size_class);

				nanov2_block_meta_t meta =
						os_atomic_load(mapped_block_metap, relaxed);
				int slots_in_use = 0;
				switch (meta.next_slot) {
				case SLOT_NULL:
					// FALLTHRU
				case SLOT_CAN_MADVISE:
					// FALLTHRU
				case SLOT_MADVISING:
					// FALLTHRU
				case SLOT_MADVISED:
					// FALLTHRU
				case SLOT_GUARD:
					// These blocks have no active content.
					break;
				case SLOT_FULL:
					slots_in_use = slots_by_size_class[size_class];
					break;
				case SLOT_BUMP:
					// FALLTHRU
				default:
					slots_in_use = slots_by_size_class[size_class] - meta.free_count - 1;
					break;
				}

				// We can't report max_size_in_use because we don't have the
				// metadata to do so.
				if (slots_in_use) {
					stats->blocks_in_use += slots_in_use;
					stats->size_in_use += slots_in_use * slot_size;
					stats->size_allocated += NANOV2_BLOCK_SIZE;
				}
			}
		}
        region = nanov2_next_region_for_region_offset(mapped_nanozone,
                region, region_offset, current_region_next_arena);
	}
	return KERN_SUCCESS;
}

static void
nanov2_statistics_self(nanozonev2_t *nanozone, malloc_statistics_t *stats)
{
	nanov2_statistics(mach_task_self(), (vm_address_t)nanozone,
			_malloc_default_reader, malloc_report_simple, stats);
}

static kern_return_t
nanov2_statistics_task_printer(task_t task, vm_address_t zone_address,
		memory_reader_t reader, print_task_printer_t printer,
		malloc_statistics_t *stats)
{
	return nanov2_statistics(task, zone_address, reader, printer, stats);
}

static kern_return_t
nanov2_statistics_task(task_t task, vm_address_t zone_address, memory_reader_t reader, malloc_statistics_t *stats)
{
	return nanov2_statistics(task, zone_address, reader, NULL, stats);
}


static const struct malloc_introspection_t nanov2_introspect = {
	.enumerator = 	(void *)nanov2_ptr_in_use_enumerator,
	.good_size =	(void *)nanov2_good_size,
	.check = 		(void *)nanov2_check,
	.print =		(void *)nanov2_print_self,
	.log = 			(void *)nanov2_log,
	.force_lock = 	(void *)nanov2_force_lock,
	.force_unlock =	(void *)nanov2_force_unlock,
	.statistics = 	(void *)nanov2_statistics_self,
	.zone_locked =	(void *)nanov2_locked,
	.enable_discharge_checking = NULL,
	.disable_discharge_checking = NULL,
#ifdef __BLOCKS__
	.enumerate_discharged_pointers = NULL,
#else // __BLOCKS__
	.enumerate_unavailable_without_blocks = NULL,
#endif // __BLOCKS__
	.reinit_lock = 	(void *)nanov2_reinit_lock,
	.print_task = 	(void *)nanov2_print_task,
	.task_statistics = (void*)nanov2_statistics_task,
};

#endif // OS_VARIANT_NOTRESOLVED

#pragma mark -
#pragma mark Utility Functions

#if OS_VARIANT_RESOLVED

// Given a pointer that may be to Nano V2-allocated memory, returns the size of
// the allocation, or 0 if the pointer does not correspond to an active
// allocation. If allow_inner is true, the pointer need not point to the start
// of the allocation.
MALLOC_ALWAYS_INLINE MALLOC_INLINE size_t
nanov2_pointer_size_inline(nanozonev2_t *nanozone, void *ptr,
		boolean_t allow_inner, nanov2_size_class_t *size_class_out,
		nanov2_block_meta_t **block_metap_out)
{
	// First check the address signature.
	if (!nanov2_has_valid_signature((void *)ptr)) {
		return 0;
	}

	// Check for proper alignment, unless we could have an inner pointer.
	if (!allow_inner && ((uintptr_t)ptr) & NANO_QUANTA_MASK) {
		return 0;
	}

	// Atomically load the value of current_region_next_arena. No thread is
	// allowed to allocate from an arena until it observes a greater value of
	// current_region_next_arena, which must have happened before now if we're
	// being called in the context of a deallocation, so we can safely use it as
	// the upper bound for an overall address range check.
	nanov2_arena_t *current_region_next_arena = os_atomic_load(
			&nanozone->current_region_next_arena, relaxed);

	// Bounds check against the active address space.
	if (ptr < (void *)nanozone->first_region_base ||
			ptr > (void *)current_region_next_arena) {
		return 0;
	}

#if NANOV2_MULTIPLE_REGIONS
	// Need to check that the region part is valid because there could be holes.
	// Do this only if we know there is a hole.
	//
	// If we're looking at a legitimately-allocated nano pointer, a load-acquire
	// of current_region_next_arena must have already happened when its
	// containing arena was first allocated from, so any region_address_clashes
	// increment that preceded the store-release of current_region_next_arena
	// should be visible.
	//
	// TODO: use a hashed structure to make this more efficient.
	if (os_atomic_load(&nanozone->statistics.region_address_clashes, relaxed)) {
		nanov2_region_t *ptr_region = nanov2_region_address_for_ptr(ptr);
		nanov2_region_t *region = nanozone->first_region_base;
		while (region) {
			if (ptr_region == region) {
				break;
			}
			region = nanov2_next_region_for_region(nanozone, region,
					current_region_next_arena);
		}
		if (!region) {
			// Reached the end of the region list without matching - not a
			// valid Nano V2 pointer.
			return 0;
		}
	}
#endif // NANOV2_MULTIPLE_REGIONS

	// Get the size class for the pointer and the address of its meta block
	// header.
	nanov2_size_class_t size_class = nanov2_size_class_for_ptr(nanozone, ptr);
	nanov2_block_meta_t *block_metap = nanov2_meta_ptr_for_ptr(nanozone, ptr);

	// Reject if the block is not active, or it doesn't have any allocations.
	nanov2_block_meta_t meta = os_atomic_load(block_metap, relaxed);
	if (!nanov2_is_block_active(meta) || (meta.next_slot != SLOT_FULL &&
			meta.free_count == slots_by_size_class[size_class] - 1)) {
		return 0;
	}

	size_t size = nanov2_size_from_size_class(size_class);
	nanov2_addr_t addr = { .addr = ptr };
	if (!allow_inner && (addr.fields.nano_offset % size)) {
		return 0;
	}

	// The only reasonable way to check whether the pointer is free is to
	// inspect the canary value at the start of the slot, since we cannot take
	// a huge hit for walking the free list.
	nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
	uintptr_t guard = os_atomic_load(&slotp->double_free_guard, relaxed);
	if ((guard ^ nanozone->slot_freelist_cookie) == (uintptr_t)ptr) {
		return 0;
	}

	if (size_class_out) {
		*size_class_out = size_class;
	}
	if (block_metap_out) {
		*block_metap_out = block_metap;
	}
	return size;
}

size_t
nanov2_pointer_size(nanozonev2_t *nanozone, void *ptr, boolean_t allow_inner)
{
	return nanov2_pointer_size_inline(nanozone, ptr, allow_inner, NULL, NULL);
}

#pragma mark -
#pragma mark Madvise Management

// Given a pointer to a block and its metadata, calls madvise() on that block if
// it is still in the state we expect, either SLOT_CAN_MADVISE or SLOT_MADVISED
// (the latter expected when we need to pessimistically re-madvise a block we
// may have touched while racing to allocate against a transition to
// SLOT_CAN_MADVISE). Returns true on success, false if the block is not in the
// correct state or if the state changed during the operation.
//
// This function must be called with the zone's madvise_lock held
static boolean_t
nanov2_madvise_block_locked(nanozonev2_t *nanozone,
		nanov2_block_meta_t *block_metap, nanov2_block_t *blockp,
		nanov2_size_class_t size_class, uint32_t expected_state)
{
	_malloc_lock_assert_owner(&nanozone->madvise_lock);

	boolean_t madvised = FALSE;
	nanov2_block_meta_t old_meta = os_atomic_load(block_metap, relaxed);
	if (old_meta.next_slot == expected_state) {
		// Nobody raced with us. We can safely madvise this block. First change
		// the state to SLOT_MADVISING so that other threads don't try to
		// grab the block for new allocations.
		nanov2_block_meta_t new_meta = {
			.next_slot = SLOT_MADVISING,
			.gen_count = old_meta.gen_count + 1,
		};
		if (!os_atomic_cmpxchgv(block_metap, old_meta, new_meta, &old_meta,
								relaxed)) {
			// Somebody else tampered with this block. This can happen if
			// another thread raced with us to allocate in this block. Count
			// the contended access.
			nanozone->statistics.size_class_statistics[size_class].madvise_races++;
			return false;
		}

		if (mvm_madvise_free(nanozone, nanov2_region_address_for_ptr(blockp),
				(uintptr_t)blockp, (uintptr_t)(blockp + 1), NULL, FALSE)) {
			malloc_zone_error(0, false, "Failed to madvise block at blockp: %p, error: %d\n", blockp, errno);
		} else {
			nanozone->statistics.size_class_statistics[size_class].madvised_blocks++;
			madvised = TRUE;
		}

		nanov2_block_meta_t final_meta = {
			.next_slot = SLOT_MADVISED,
			.gen_count = new_meta.gen_count + 1,
		};

		if (!os_atomic_cmpxchgv(block_metap, new_meta, final_meta, &old_meta,
				relaxed)) {
			// This should not happen since we should have exclusive interest
			// in this block.
			malloc_zone_error(nanozone->debug_flags, false,
					"Failed when changing state from MADVISING to MADVISED, "
					"block_metap = %p, blockp = %p\n", block_metap, blockp);
		}
	}
	return madvised;
}

static void
nanov2_madvise_block(nanozonev2_t *nanozone, nanov2_block_meta_t *block_metap,
		nanov2_size_class_t size_class, uint32_t expected_state)
{
	nanov2_block_t *blockp = nanov2_block_address_from_meta_ptr(nanozone,
			block_metap);
	_malloc_lock_lock(&nanozone->madvise_lock);
	nanov2_madvise_block_locked(nanozone, block_metap, blockp, size_class,
			expected_state);
	_malloc_lock_unlock(&nanozone->madvise_lock);
}

#endif // OS_VARIANT_RESOLVED

#pragma mark -
#pragma mark Region Management

#if OS_VARIANT_NOTRESOLVED

#if CONFIG_NANO_RESERVE_REGIONS
// Update protection for region to DEFAULT
static bool
nanov2_unprotect_region(nanov2_region_t *region)
{
	MALLOC_TRACE(TRACE_nanov2_region_protection | DBG_FUNC_START,
			(uint64_t)region, 0, 0, 0);
	bool result = nano_common_unprotect_vm_space((mach_vm_address_t)region,
			NANOV2_REGION_SIZE);
	MALLOC_TRACE(TRACE_nanov2_region_protection | DBG_FUNC_END,
			(uint64_t)region, result, 0, 0);
	return result;
}

// Reserve VA at [base, base+num_regions*REGION_SIZE].
// Note: permissions must still be granted on reserved region with `nanov2_unprotect_region`
static bool
nanov2_reserve_regions(nanov2_region_t *base, unsigned int num_regions)
{
	MALLOC_TRACE(TRACE_nanov2_region_reservation | DBG_FUNC_START,
			(uint64_t)base, num_regions, 0, 0);
	bool result = nano_common_reserve_vm_space((mach_vm_address_t)base,
			(NANOV2_REGION_SIZE * (mach_vm_size_t)num_regions));
	MALLOC_TRACE(TRACE_nanov2_region_reservation | DBG_FUNC_END,
			(uint64_t)base, num_regions, result, 0);

	return result;
}
#else
// Attempts to allocate VM space for a region at a given address and returns
// whether the allocation succeeded.
static bool
nanov2_allocate_region(nanov2_region_t *region)
{
	MALLOC_TRACE(TRACE_nanov2_region_allocation | DBG_FUNC_START,
			(uint64_t)region, 0, 0, 0);
	bool result = nano_common_allocate_vm_space((mach_vm_address_t)region,
			NANOV2_REGION_SIZE);
	MALLOC_TRACE(TRACE_nanov2_region_allocation | DBG_FUNC_END,
			(uint64_t)region, result, 0, 0);
	return result;
}
#endif // CONFIG_NANO_RESERVE_REGIONS

// Allocates a new region adjacent to the current one. If the allocation fails,
// keep sliding up by the size of a region until we either succeed or run out of
// address space. The caller must own the Nanozone regions lock. Returns the
// first arena of the newly-allocated region if successful, or NULL otherwise.
MALLOC_NOEXPORT nanov2_arena_t *
nanov2_allocate_new_region(nanozonev2_t *nanozone)
{
#if NANOV2_MULTIPLE_REGIONS
	bool allocated = false;

	nanov2_addr_t nanov2_max_region_base = {
		.fields.nano_signature = NANOZONE_SIGNATURE,
		.fields.nano_region = nano_max_region,
	};

	_malloc_lock_assert_owner(&nanozone->regions_lock);
	nanov2_region_t *current_region = nanov2_current_region_base(
			os_atomic_load(&nanozone->current_region_next_arena, relaxed));
	nanov2_region_t *next_region = current_region + 1;

	while ((void *)next_region <= nanov2_max_region_base.addr) {
#if CONFIG_NANO_RESERVE_REGIONS
		if (!nanov2_unprotect_region(next_region)) {
			MALLOC_REPORT_FATAL_ERROR(next_region,
					"Nano: Unable to raise protection on pre-allocated region");
		}
		nanozone->statistics.allocated_regions++;
		allocated = true;
		break;
#else // CONFIG_NANO_RESERVE_REGIONS
		if (nanov2_allocate_region(next_region)) {
			nanozone->statistics.allocated_regions++;
			allocated = true;
			break;
		}
		next_region++;

		// Loaded atomically in nanov2_pointer_size() to determine whether or
		// not it's necessary to walk the region list, so we need to increment
		// atomically here. Published by the store-release of
		// current_region_next_arena.
		os_atomic_inc(&nanozone->statistics.region_address_clashes, relaxed);
#endif // CONFIG_NANO_RESERVE_REGIONS
	}

	if (!allocated) {
		return NULL;
	}

	// Link this region to the previous one.
	nanov2_region_linkage_t *current_region_linkage =
			nanov2_region_linkage_for_region(nanozone, current_region);

	// The linkage of the next region is in pristine memory, so already zero -
	// don't touch it.

	// Store-release the linkage update so any dependent loads through it
	// observe the (implicit zero-)initialization of the next region.
	uint16_t offset = next_region - current_region;
	os_atomic_store(&current_region_linkage->next_region_offset, offset,
			release);

	// Store-release the update to current_region_next_arena to publish the
	// linkage update. Pairs with load-acquires of current_region_next_arena
	// followed by walks of the region list.
	nanov2_arena_t *first_arena = nanov2_first_arena_for_region(next_region);
	os_atomic_store(&nanozone->current_region_next_arena, first_arena + 1,
			release);

	return first_arena;
#else // NANOV2_MULTIPLE_REGIONS
	// On iOS, only one region is supported, so we fail since the first
	// region is allocated separately.
	return NULL;
#endif // CONFIG_NANOV2_MULTIPLE_REGIONS
}
#endif // OS_VARIANT_NOTRESOLVED

#pragma mark -
#pragma mark Allocation

#if OS_VARIANT_RESOLVED

MALLOC_NOINLINE MALLOC_NORETURN
static void
nanov2_guard_corruption_detected(void *corrupt_slot)
{
	uint64_t guard = *(uint64_t *)corrupt_slot;
	malloc_zone_error(MALLOC_ABORT_ON_CORRUPTION, true,
			"Heap corruption detected, free list is damaged at %p\n"
			"*** Incorrect guard value: %llu\n", corrupt_slot,
			(unsigned long long)guard);
	__builtin_unreachable();
}

// Allocates memory from the block that corresponds to a given block meta data
// pointer. The memory is taken from the free list if possible, or from the
// unused region of the block if not. If the block is no longer in use or is
// full, NULL is returned and the caller is expected to find another block to
// allocate from.
MALLOC_ALWAYS_INLINE MALLOC_INLINE
void *
nanov2_allocate_from_block_inline(nanozonev2_t *nanozone,
		nanov2_block_meta_t *block_metap, nanov2_size_class_t size_class,
		nanov2_block_meta_t **madvise_block_metap_out, bool *corruption)
{
	nanov2_block_meta_view_t old_meta_view;

	// Our loads of the block metadata use dependency ordering, which guarantees
	// that any loads we do from a slot pointer derived from the metadata value
	// as we do below will observe all of the stores preceding the store-release
	// of that value we observed.  This allows us to safely rely on the contents
	// of the slot updated when it was last freed, including the double-free
	// guard and zeroing done by zero-on-free (which is required for correctness
	// in the case of calloc).
	old_meta_view.meta = os_atomic_load(block_metap, dependency);

	// Calculating blockp and ptr is relatively expensive. Do both lazily to
	// minimize the time in the block starting with "again:" and ending with the
	// atomic update so that we lose at little time as possible if we have to
	// repeat that loop due to contention. This should also reduce the risk of
	// contention.
	nanov2_block_t *blockp = NULL;

again:
	if (!nanov2_can_allocate_from_block(old_meta_view.meta)) {
		// Move along, nothing to allocate here...
		return NULL;
	}

	int slot;
	void *ptr = NULL;
	boolean_t from_free_list = FALSE;
	nanov2_block_meta_t new_meta = {
		.in_use = 1,
		.free_count = old_meta_view.meta.free_count - 1,
		.gen_count = old_meta_view.meta.gen_count + 1
	};

	// Grab a slot from the free list or get the next unused slot. We know there
	// should be one because the block is not full.
	boolean_t slot_full = old_meta_view.meta.free_count == 0;
	if (old_meta_view.meta.next_slot == SLOT_BUMP
				|| old_meta_view.meta.next_slot == SLOT_CAN_MADVISE) {
		// Free list empty, grab the next unused slot.
		new_meta.next_slot = slot_full ? SLOT_FULL : SLOT_BUMP;
		slot = slots_by_size_class[size_class] - old_meta_view.meta.free_count - 1;
	} else {
		// Grab the first item from the free list.
		from_free_list = TRUE;
		if (!blockp) {
			blockp = nanov2_block_address_from_meta_ptr(nanozone, block_metap);
		}
		slot = old_meta_view.meta.next_slot - 1; // meta.next_slot is 1-based.
		ptr = nanov2_slot_in_block_ptr(blockp, size_class, slot);
		nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
		new_meta.next_slot = slot_full ? SLOT_FULL : (uint16_t)slotp->next_slot;
	}

	// Write the updated meta data; try again if we raced with another thread.
	if (!os_atomic_cmpxchgv(block_metap, old_meta_view.meta, new_meta,
				&old_meta_view.meta, dependency)) {
		if (old_meta_view.meta.next_slot == SLOT_CAN_MADVISE ||
				old_meta_view.meta.next_slot == SLOT_MADVISING ||
				old_meta_view.meta.next_slot == SLOT_MADVISED) {
			*madvise_block_metap_out = block_metap;
			return NULL;
		}
		goto again;
	}

	if (!ptr) {
		if (!blockp) {
			blockp = nanov2_block_address_from_meta_ptr(nanozone, block_metap);
		}
		ptr = nanov2_slot_in_block_ptr(blockp, size_class, slot);
	}

	if (from_free_list) {
		// We grabbed the item from the free list. Check the free list canary
		// and crash if it's not valid. We can't do this check before the
		// cmpxchgv because another thread may race with us, claim the slot and
		// write to it.
		nanov2_free_slot_t *slotp = ptr;
		uintptr_t guard = os_atomic_load(&slotp->double_free_guard, relaxed);
		if (os_unlikely((guard ^ nanozone->slot_freelist_cookie) != (uintptr_t)ptr)) {
			*corruption = true;
		}
	}

#if DEBUG_MALLOC
	nanozone->statistics.size_class_statistics[size_class].total_allocations++;
#endif // DEBUG_MALLOC

	return ptr;
}

static void *
nanov2_allocate_from_block(nanozonev2_t *nanozone,
		nanov2_block_meta_t *block_metap, nanov2_size_class_t size_class)
{
	nanov2_block_meta_t *madvise_block_metap = NULL;
	bool corruption = false;
	void *ptr = nanov2_allocate_from_block_inline(nanozone, block_metap,
			size_class, &madvise_block_metap, &corruption);
	if (os_unlikely(corruption)) {
		nanov2_guard_corruption_detected(ptr);
	}
	if (madvise_block_metap) {
		nanov2_madvise_block(nanozone, madvise_block_metap, size_class,
				SLOT_MADVISED);
	}
	return ptr;
}

// Finds a block for allocation in an arena and returns a pointer to its
// metadata header. The search begins from the block with metadata pointer
// start_block (which must not be NULL). If no acceptable block was found,
// NULL is returned and it is expected that the caller will take appropriate
// action (typically allocate a new arena).
//
// The search starts with start_block. If this is in-use and not full, that
// block is returned. Otherwise, a scan for a usable block is initiated. The
// search starts from start_block and initially works backward towards the
// start of the arena. If this does not succeed, a forward search from
// start_block is made.
//
// A block is considered a candidate if it is not in use. As the scan proceeds,
// we remember blocks which have been madvisable, blocks which have been
// madvised or never used and those blocks which still have allocated slots
// but which fall within the reuse criteria (i.e. their occupancy is within the
// max/min occupancy range).
//
// If the scan policy is NANO_SCAN_FIRST_FIT, we just return the first block
// from the above list that we find. This is the fastest option, but likely
// maximises fragmentation.
//
// Otherwise, the scan policy is NANO_SCAN_CAPACITY_BASED. If we find a block
// that fits the reuse criteria, we return it immediately. Otherwise, we
// continue to scan until we find such a block, or we find a less ideal block
// and we reach the scan limit or exhaust the arena. At that point, we return
// one of the candidate blocks that we found, choosing based on the state of
// that block:
// - blocks that have allocations that are greater than the minimum capacity
//   are preferred.
// - failing that, return an unused or madvise'd block.
// - failing that, return a block that is waiting to be madvised.
//
// In order to avoid races, this function must be called with the
// current_block_lock for the calling context [CPU] and size class locked.
// On return, the selected block has been marked as in-use, so the caller must
// either assign it as the active allocation block for the calling context or
// clear the in-use bit.
//
MALLOC_ALWAYS_INLINE MALLOC_INLINE
static nanov2_block_meta_t *
nanov2_find_block_in_arena(nanozonev2_t *nanozone,
		nanov2_arena_t *arena, nanov2_size_class_t size_class,
		nanov2_block_meta_t *start_block)
{
	// If we don't have a starting point, start with the first block in the
	// arena for the given size class. This is the case where we are looking for
	// the first allocation block for a new context (i.e probably a new CPU, so
	// take the first fit to avoid having to scan the whole size class for this
	// very common start up case.)
	boolean_t use_first_fit = !start_block ||
			nanov2_policy_config.block_scan_policy == NANO_SCAN_FIRST_FIT;
	nanov2_block_meta_t *first_block = nanov2_first_block_for_size_class_in_arena(
			nanozone, size_class, arena);
	boolean_t scanning_backwards;
	if (!start_block) {
		start_block = first_block;
	}
	int slots_in_block = slots_by_size_class[size_class];
	nanov2_block_meta_t old_meta;
	nanov2_block_meta_t *this_block;
	nanov2_block_meta_t *found_block;
	nanov2_block_meta_t *madvisable_block;
	nanov2_block_meta_t *free_block;
	nanov2_block_meta_t *fallback_block;
	boolean_t fallback_below_max;
	int scan_limit;

	// Check all of the blocks in the size class until we find one that we can
	// use, based on nanov2_block_scan_policy.
retry:
	this_block = start_block;
	found_block = NULL;
	madvisable_block = NULL;
	free_block = NULL;
	fallback_block = NULL;
	fallback_below_max = FALSE;
	scan_limit = nanov2_policy_config.block_scan_limit;
	scanning_backwards = TRUE;

	do {
		old_meta = os_atomic_load(this_block, relaxed);
		if (!old_meta.in_use && old_meta.next_slot != SLOT_FULL
				&& old_meta.next_slot != SLOT_MADVISING) {
			if (old_meta.next_slot == SLOT_CAN_MADVISE) {
				if (!madvisable_block) {
					// We can use this block as a last-ditch fallback.
					madvisable_block = this_block;
				}
			} else if (old_meta.next_slot == SLOT_NULL
					|| old_meta.next_slot == SLOT_MADVISED) {
				if (!free_block) {
					free_block = this_block;
				}
			} else if (use_first_fit) {
				found_block = this_block;
			} else {
				MALLOC_ASSERT(nanov2_policy_config.block_scan_policy == NANO_SCAN_CAPACITY_BASED);
				int percent_used = (100 * old_meta.free_count)/slots_in_block;
				if (percent_used >= nanov2_policy_config.block_scan_min_capacity
						&& percent_used <= nanov2_policy_config.block_scan_max_capacity) {
					// Within specified limits -- take this one.
					found_block = this_block;
				} else if (percent_used >= nanov2_policy_config.block_scan_min_capacity) {
					if (!fallback_block || fallback_below_max) {
						// More full than we want, but still acceptable as a
						// fallback.
						fallback_block = this_block;
					}
				} else if (!fallback_block
						&& percent_used < nanov2_policy_config.block_scan_min_capacity) {
					// Less full than we want. Keep it as a backup, but set
					// fallback_below_max to allow a block that's above max to
					// be preferred. The rationale behind this is to allow
					// blocks that have low occupancy to drain so that they can
					// be madvised.
					fallback_block = this_block;
					fallback_below_max = TRUE;
				} else if (!free_block) {
					// Not ideal, but we could use it.
					free_block = this_block;
				}
			}
			if (use_first_fit && (found_block || fallback_block || free_block)) {
				// Take whatever we got.
				break;
			}
		}

		if (scan_limit > 0) {
			// Only enforce the scan limit once we have a candidate.
			if ((fallback_block || free_block) && --scan_limit == 0) {
				break;
			}
		}

		if (scanning_backwards) {
			boolean_t wrapped;
			nanov2_block_meta_t *prev_block = nanov2_previous_block_for_size_class(
					nanozone, size_class, this_block, &wrapped);
			if (wrapped) {
				// We wrapped. Scan forward from the start block instead.
				scan_limit = nanov2_policy_config.block_scan_limit;
				scanning_backwards = FALSE;
				this_block = start_block;
			} else {
				this_block = prev_block;
			}
		} else {
			// Move to the next block, wrapping when we reach the last one for
			// this size class. Stop once we get to the block where we started.
			this_block = nanov2_next_block_for_size_class(nanozone, size_class,
					this_block, NULL);
			if (this_block == start_block) {
				break;
			}
		}
	} while (!found_block);

	if (!found_block) {
		if (fallback_block) {
			found_block = fallback_block;
		} else if (free_block) {
			found_block = free_block;
		} else if (madvisable_block) {
			found_block = madvisable_block;
		}
	}

	if (found_block) {
		// Now we need to activate the block. If this fails, we look for
		// another block.
		// If we are bringing a block that is draining back into use, we
		// just need to set in_use to 1. Otherwise, we fully initialize it.
		old_meta = os_atomic_load(found_block, relaxed);
		if (old_meta.next_slot == SLOT_MADVISING) {
			goto retry;
		}
		boolean_t reset_slot = old_meta.next_slot == SLOT_NULL
				|| old_meta.next_slot == SLOT_CAN_MADVISE
				|| old_meta.next_slot == SLOT_MADVISED;
		nanov2_block_meta_t new_meta = {
			.in_use = 1,
			.free_count = reset_slot ? slots_in_block - 1 : old_meta.free_count,
			.next_slot = reset_slot ? SLOT_BUMP : old_meta.next_slot,
			.gen_count = reset_slot ? 0 : old_meta.gen_count + 1,
		};
		if (!os_atomic_cmpxchgv(found_block, old_meta, new_meta, &old_meta,
				relaxed)) {
			goto retry;
		}
	}

	return found_block;
}

// Finds a block to allocate from and allocates memory from it. The search
// for a block starts from *block_metapp if not NULL, otherwise from the first
// arena in the first block (which is the case when the first block is allocated
// for a size class for a CPU).
// If none of the blocks for a size class in the current arena can be used, a
// new arena is allocated and, if necessary, a new region is added.
//
// The address of the allocated memory is returned and its metadata pointer is
// stored in *block_metapp. If a new region is required and it can't be
// allocated, NULL is returned and *block_metapp is unmodified.
//
// On success, the returned block is marked as in-use and the block originally
// pointed to by *block_metapp has its in-use bit cleared.
//
// In order to avoid races, this function must be called with the
// current_block_lock for the calling context [CPU] and size class locked.
MALLOC_NOEXPORT MALLOC_NOINLINE void *
nanov2_find_block_and_allocate(nanozonev2_t *nanozone,
		nanov2_size_class_t size_class, nanov2_block_meta_t **block_metapp)
{
	nanov2_arena_t *arena;
	nanov2_block_meta_t *start_block = os_atomic_load(block_metapp, relaxed);
	nanov2_block_meta_t *orig_block = start_block;
	if (start_block) {
		// Use the arena for the starting block.
		arena = nanov2_arena_address_for_ptr(start_block);
	} else {
		// Start from the first arena.
		arena = nanov2_arena_address_for_ptr(nanozone->first_region_base);
	}

	nanov2_region_t *start_region;
retry:
	start_region = nanov2_region_address_for_ptr(arena);
	nanov2_arena_t *start_arena = arena;
	nanov2_region_t *region = start_region;
	// The load-acquire pairs with store-release in nanov2_allocate_new_region()
	// to make the most recent region linkage update visible when we load it in
	// nanov2_next_region_for_region() below.
	nanov2_arena_t *initial_region_next_arena = os_atomic_load(
			&nanozone->current_region_next_arena, acquire);
	nanov2_arena_t *limit_arena = nanov2_limit_arena_for_region(nanozone,
			start_region, initial_region_next_arena);
	do {
		nanov2_block_meta_t *block_metap = nanov2_find_block_in_arena(nanozone,
				arena, size_class, start_block);
		if (block_metap) {
			// Try to allocate from this block and return if it succeeds. Note
			// that the block is now marked as in-use, so effectively belongs
			// to the calling context.
			void *ptr = nanov2_allocate_from_block(nanozone, block_metap, size_class);
			if (ptr) {
				// Make the new block the current one for the calling context.
				os_atomic_store(block_metapp, block_metap, relaxed);

				// Turn off in-use in old block_metap, if there is one.
				if (orig_block) {
					// Turn off in-use in the original current block.
					nanov2_turn_off_in_use(orig_block);
				}
				return ptr;
			}

			// We found a block but failed to allocate from it, probably because
			// it became full. Look for a new block, using the one that we just
			// failed with as the starting point. First, we need to turn off the
			// in-use bit for the block that we just failed to allocate from.
			nanov2_turn_off_in_use(block_metap);

			start_block = block_metap;
			goto retry;
		}

		// Try the next arena. If this is the last arena in the region, try the
		// next region.
		start_block = NULL;
		arena++;
		if (arena >= limit_arena) {
			region = nanov2_next_region_for_region(nanozone, region,
					initial_region_next_arena);
			if (!region) {
				// Reached the last region -- loop back to the first.
				region = nanozone->first_region_base;
			}
			arena = nanov2_first_arena_for_region(region);
			limit_arena = nanov2_limit_arena_for_region(nanozone, region,
					initial_region_next_arena);
		}
	} while (arena != start_arena);

	// If we get to this point, we need to allocate a new arena and possibly
	// a new region. If we are not permitted to do so by policy, return NULL.
	if (nanov2_policy_config.single_arena_size_classes & (1 << size_class)) {
		return NULL;
	}

	// Allocate a new arena and maybe a new region. To do either of those
	// things, we need to take the regions_lock. After doing so, check that the
	// state is unchanged. If it has, just assume that we might have some new
	// space to allocate into and try again.

	boolean_t failed = FALSE;

	_malloc_lock_lock(&nanozone->regions_lock);
	nanov2_arena_t *current_region_next_arena = os_atomic_load(
			&nanozone->current_region_next_arena, relaxed);
	if (current_region_next_arena == initial_region_next_arena) {
		if (nanov2_current_region_next_arena_is_limit(
				current_region_next_arena)) {
			// Reached the end of the region. Allocate a new one, if we can.
			arena = nanov2_allocate_new_region(nanozone);
			if (!arena) {
				failed = TRUE;
			}
		} else {
			// Assign the new arena, in the current region.
			arena = current_region_next_arena;

			// Bump current_region_next_arena by 1. No need for an atomic add
			// because we're under the regions_lock.
			os_atomic_store(&nanozone->current_region_next_arena,
					current_region_next_arena + 1, relaxed);
		}

		// Set up the guard blocks for the new arena, if requested
		if (!failed) {
			nanov2_init_guard_blocks(nanozone, arena);
		}
	} else {
		// The arena just before current_region_next_arena is always the most
		// recently allocated arena. Let's retry from that arena, which was
		// allocated in the time since we started our last try.
		arena = current_region_next_arena - 1;
	}
	_malloc_lock_unlock(&nanozone->regions_lock);

	if (!failed) {
		// Now allocate from the new arena. Since we updated the nanozone, it's
		// possible that some other thread has already raced with us to allocate
		// some space from it, so just use the normal allocation path to avoid
		// assumptions. It's a little more expensive, but this path is rare.
		start_block = NULL;
		goto retry;
	}

	// We need more space and we can't get it. We'll delegate to the helper.
	return NULL;
}

// This function is called when a fast-path allocation from a given (size_class,
// allocation_index) has been tried and failed, and we need to act on
// observations from that attempt and/or retry the allocation.  Its rather
// tortured calling contract is designed to allow the caller to avoid pushing a
// frame and pass along as much of what it has already computed as possible.
//
// If @corrupt_slot is non-NULL it means we detected corruption of the slot's
// guard on the fast path, and we need to report that corruption.
//
// If @madvise_block_metap is non-NULL it means we raced with another thread
// madvising the block we tried to allocate from and need to re-madvise it.
//
// If the allocation fails, NULL is returned.
static void *
nanov2_allocate_outlined(nanozonev2_t *nanozone, nanov2_block_meta_t **block_metapp,
		size_t rounded_size, nanov2_size_class_t size_class,
		int allocation_index, nanov2_block_meta_t *madvise_block_metap,
		void *corrupt_slot, bool clear, bool typed, malloc_type_id_t type_id)
{
	void *ptr = NULL;

	if (os_unlikely(corrupt_slot)) {
		nanov2_guard_corruption_detected(corrupt_slot);
	}

	// If we need to re-madvise the old block that we might have accidentally
	// touched out of turn, do so now.
	if (madvise_block_metap) {
		nanov2_madvise_block(nanozone, madvise_block_metap, size_class,
				SLOT_MADVISED);
	}

	// No current allocation block, or we were unable to allocate. We need to
	// get a new block. Before doing so, delegate to the helper allocator if
	// the size class was full and has not released enough memory yet.
	if (nanozone->delegate_allocations & (1 << size_class)) {
		malloc_zone_t *helper_zone = nanozone->helper_zone;
		if (typed) {
			ptr = helper_zone->malloc_type_malloc(helper_zone, rounded_size,
					type_id);
		} else {
			ptr = helper_zone->malloc(helper_zone, rounded_size);
		}
		goto done;
	}

	// Before we try to get another block, lock and try another allocation,
	// which may succeed because another thread may have beaten us to it, or
	// some space may have freed up in the current block.
	_malloc_lock_s *lock = &nanozone->current_block_lock[size_class][allocation_index];
	_malloc_lock_lock(lock);

	nanov2_block_meta_t *block_metap = os_atomic_load(block_metapp, relaxed);
	if (block_metap) {
		ptr = nanov2_allocate_from_block(nanozone, block_metap, size_class);
		if (ptr) {
			// Good to go - keep the current block.
			goto unlock;
		}
	}

	// At this point, we do not have a current allocation block and the old one,
	// if there was one, has been marked as not in use. We need to find and
	// assign a new block. Since we have the lock, nobody else can change the
	// current_block pointer.
	ptr = nanov2_find_block_and_allocate(nanozone, size_class, block_metapp);

unlock:
	_malloc_lock_unlock(lock);

	if (!ptr) {
		// We could not find a block to allocate from -- make future
		// allocations for this size class go to the helper zone until
		// we have enough free space.
		os_atomic_or(&nanozone->delegate_allocations,
				(uint16_t)(1 << size_class), relaxed);

		malloc_zone_t *helper_zone = nanozone->helper_zone;
		if (typed) {
			ptr = helper_zone->malloc_type_malloc(helper_zone, rounded_size,
					type_id);
		} else {
			ptr = helper_zone->malloc(helper_zone, rounded_size);
		}
	}

done:
	if (os_likely(ptr)) {
		switch (malloc_zero_policy) {
		case MALLOC_ZERO_ON_FREE: {
			// Always clear the double-free guard so that we can recognize that
			// this block is not on the free list.
			nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
			os_atomic_store(&slotp->double_free_guard, 0, relaxed);

			// We know the body of the allocation is already clear, so we just
			// need to clean up the next_slot word to get to all-zero.  Do so in
			// all cases, even if a cleared allocation is not requested, to
			// prevent any leakage through the next_slot bits.
			os_atomic_store(&slotp->next_slot, 0, relaxed);
			break;
		}
		case MALLOC_ZERO_NONE:
			if (!clear) {
				// Always clear the double-free guard so that we can recognize that
				// this block is not on the free list.
				nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
				os_atomic_store(&slotp->double_free_guard, 0, relaxed);
				break;
			}
			MALLOC_FALLTHROUGH;
		case MALLOC_ZERO_ON_ALLOC:
			memset(ptr, '\0', rounded_size);
			break;
		}
	} else {
		malloc_set_errno_fast(MZ_POSIX, ENOMEM);
	}
	return ptr;
}

#pragma mark -
#pragma mark Freeing

// Frees an allocation to its owning block and updates the block's state.
//
// If the block becomes empty, it is marked as SLOT_CAN_MADVISE and we return
// the block to the caller to madvise if dictated by policy.
MALLOC_ALWAYS_INLINE MALLOC_INLINE
nanov2_block_meta_t *
nanov2_free_to_block_inline(nanozonev2_t *nanozone, void *ptr,
		nanov2_size_class_t size_class, nanov2_block_meta_t *block_metap)
{
	nanov2_block_t *blockp = nanov2_block_address_for_ptr(ptr);
	if (!block_metap) {
		block_metap = nanov2_meta_ptr_for_ptr(nanozone, ptr);
	}

	// Release the slot memory onto the block's freelist.
	nanov2_block_meta_t old_meta = os_atomic_load(block_metap, relaxed);
	int slot_count = slots_by_size_class[size_class];
	nanov2_block_meta_t new_meta;
	boolean_t was_full;

	nanov2_free_slot_t *slotp = (nanov2_free_slot_t *)ptr;
	// All of the free slot content (double_free_guard, next_slot word and the
	// zeroed remainder of the slot) must be visible when the os_atomic_cmpxchgv
	// completes, so the metadata updates on either path below need a release
	// barrier.
	os_atomic_store(&slotp->double_free_guard,
			nanozone->slot_freelist_cookie ^ (uintptr_t)ptr, relaxed);

again:
	was_full = old_meta.next_slot == SLOT_FULL;
	new_meta.free_count = old_meta.free_count + 1;
	new_meta.in_use = old_meta.in_use;
	new_meta.gen_count = old_meta.gen_count + 1;
	boolean_t freeing_last_active_slot = !was_full &&
			new_meta.free_count == slots_by_size_class[size_class] - 1;
	if (freeing_last_active_slot) {
		os_atomic_store(&slotp->next_slot, SLOT_NULL, relaxed);

		// Releasing the last active slot onto the free list. Mark the block as
		// ready to be madvised if it's not in use, otherwise reset next_slot
		// to SLOT_BUMP.
		new_meta.next_slot = new_meta.in_use ? SLOT_BUMP : SLOT_CAN_MADVISE;
		// Write the updated meta data; try again if we raced with another thread.
		if (!os_atomic_cmpxchgv(block_metap, old_meta, new_meta, &old_meta, release)) {
			goto again;
		}

		// If the block is now empty and it's not in use, madvise it if the policy
		// is to do so immediately.
		if (new_meta.next_slot == SLOT_CAN_MADVISE &&
				nanov2_madvise_policy == NANO_MADVISE_IMMEDIATE) {
			return block_metap;
		}
	} else {
		int slot_index = nanov2_slot_index_in_block(blockp, size_class, ptr);
		new_meta.next_slot = slot_index + 1;  // meta.next_slot is 1-based
		os_atomic_store(&slotp->next_slot,
				was_full ? SLOT_BUMP : old_meta.next_slot, relaxed);

		// Write the updated meta data; try again if we raced with another thread.
		if (!os_atomic_cmpxchgv(block_metap, old_meta, new_meta, &old_meta, release)) {
			goto again;
		}
	}

	// If this size class has been marked as full and this block is below an
	// acceptable level of occupancy, turn off delegation to the helper. Do this
	// only if the block is not in-use, because an in-use block cannot be a
	// candidate when searching for a new block.
	uint16_t class_mask = 1 << size_class;
	if (!new_meta.in_use && (nanozone->delegate_allocations & class_mask) &&
			(new_meta.free_count >= 0.75 * slot_count)) {
		os_atomic_and(&nanozone->delegate_allocations, ~class_mask, relaxed);
	}

#if DEBUG_MALLOC
	nanozone->statistics.size_class_statistics[size_class].total_frees++;
#endif // DEBUG_MALLOC

	return NULL;
}

#endif // OS_VARIANT_RESOLVED

#if OS_VARIANT_NOTRESOLVED

#pragma mark -
#pragma mark Zone Operations

malloc_zone_t *
nanov2_create_zone(malloc_zone_t *helper_zone, unsigned debug_flags)
{
	// Note: It is not necessary that nanov2_create_zone resets _malloc_engaged_nano
	// if it is unable to enable the nanozone - functions that need to determine
	// whether the nanozone is preset should test initial_nano_zone.
	MALLOC_ASSERT(_malloc_engaged_nano == NANO_V2);

	// Get memory for the zone and disable Nano if we fail.
	nanozonev2_t *nanozone = nano_common_allocate_based_pages(
			NANOZONEV2_ZONE_PAGED_SIZE, 0, 0, VM_MEMORY_MALLOC, 0);
	if (!nanozone) {
		_malloc_engaged_nano = NANO_NONE;
		return NULL;
	}

	// Set up the basic_zone portion of the nanozonev2 structure
	nanozone->basic_zone.version = 16;
	nanozone->basic_zone.size = OS_RESOLVED_VARIANT_ADDR(nanov2_size);
	if (malloc_zero_policy == MALLOC_ZERO_ON_ALLOC) {
		nanozone->basic_zone.malloc =
				OS_RESOLVED_VARIANT_ADDR(nanov2_malloc_zero_on_alloc);
		nanozone->basic_zone.malloc_type_malloc =
				OS_RESOLVED_VARIANT_ADDR(nanov2_malloc_type_zero_on_alloc);
	} else {
		nanozone->basic_zone.malloc = OS_RESOLVED_VARIANT_ADDR(nanov2_malloc);
		nanozone->basic_zone.malloc_type_malloc =
				OS_RESOLVED_VARIANT_ADDR(nanov2_malloc_type);
	}
	nanozone->basic_zone.calloc = OS_RESOLVED_VARIANT_ADDR(nanov2_calloc);
	nanozone->basic_zone.valloc = (void *)nanov2_valloc;
	nanozone->basic_zone.free = OS_RESOLVED_VARIANT_ADDR(nanov2_free);
	nanozone->basic_zone.realloc = OS_RESOLVED_VARIANT_ADDR(nanov2_realloc);
	nanozone->basic_zone.destroy = (void *)nanov2_destroy;
	nanozone->basic_zone.batch_malloc = OS_RESOLVED_VARIANT_ADDR(nanov2_batch_malloc);
	nanozone->basic_zone.batch_free = OS_RESOLVED_VARIANT_ADDR(nanov2_batch_free);
	nanozone->basic_zone.introspect =
			(struct malloc_introspection_t *)&nanov2_introspect;
	nanozone->basic_zone.memalign = OS_RESOLVED_VARIANT_ADDR(nanov2_memalign);
	nanozone->basic_zone.free_definite_size = OS_RESOLVED_VARIANT_ADDR(nanov2_free_definite_size);
	nanozone->basic_zone.pressure_relief = OS_RESOLVED_VARIANT_ADDR(nanov2_pressure_relief);
	nanozone->basic_zone.claimed_address = OS_RESOLVED_VARIANT_ADDR(nanov2_claimed_address);
	nanozone->basic_zone.try_free_default = OS_RESOLVED_VARIANT_ADDR(nanov2_try_free_default);

	// zone_type == MALLOC_ZONE_TYPE_UNKNOWN
	// No malloc_with_options

	nanozone->basic_zone.malloc_type_calloc = OS_RESOLVED_VARIANT_ADDR(nanov2_calloc_type);
	nanozone->basic_zone.malloc_type_realloc = OS_RESOLVED_VARIANT_ADDR(nanov2_realloc_type);
	nanozone->basic_zone.malloc_type_memalign = OS_RESOLVED_VARIANT_ADDR(nanov2_memalign_type);

	// Set these both to zero as required by CFAllocator.
	nanozone->basic_zone.reserved1 = 0;
	nanozone->basic_zone.reserved2 = 0;

	// Prevent overwriting the function pointers in basic_zone.
	mprotect(nanozone, sizeof(nanozone->basic_zone), PROT_READ);

	// Set up the remainder of the nanozonev2 structure
	nanozone->debug_flags = debug_flags;
	nanozone->helper_zone = helper_zone;

	// Initialize the cookies used to detect double freeing and for the ASLR
	// scramble mapping.
#define COOKIE_ENTROPY_MASK		0x0000ffffffff0000ULL
#define DEFAULT_ENTROPY_BITS	0x0000DEADDEAD0000ULL
	uintptr_t cookie = (uintptr_t)malloc_entropy[0] & COOKIE_ENTROPY_MASK;
	if (!cookie) {
		cookie = malloc_entropy[1] & COOKIE_ENTROPY_MASK;
		if (!cookie) {
			// The cookie can't be zero, because it's used to compute the guard
			// value in free slots, so make sure we have a non-zero value. Using
			// a fixed value allows us to recognize that it isn't real entropy.
			cookie = DEFAULT_ENTROPY_BITS;
		}
	}
	nanozone->slot_freelist_cookie = cookie;

	// For the ASLR cookie, we take the top 12 bits of malloc_entropy[1] and
	// align it to the block field of a Nano address.
	nanozone->aslr_cookie = malloc_entropy[1] >> (64 - NANOV2_BLOCK_BITS);
	nanozone->aslr_cookie_aligned = nanozone->aslr_cookie << NANOV2_OFFSET_BITS;

	_malloc_lock_init(&nanozone->blocks_lock);
	_malloc_lock_init(&nanozone->regions_lock);
	_malloc_lock_init(&nanozone->madvise_lock);

	// Allocate the initial region. If this does not succeed, we disable Nano.
	nanov2_region_t *region = (nanov2_region_t *)NANOZONE_BASE_REGION_ADDRESS;

	bool result;
#if CONFIG_NANO_RESERVE_REGIONS
	unsigned int num_regions = (nano_max_region + 1);
	result = nanov2_reserve_regions(region, num_regions);
	if (result) {
		result = nanov2_unprotect_region(region);
		if (!result) {
			malloc_report(ASL_LEVEL_ERR,
					"unable to protect initial region\n");
			nano_common_deallocate_pages((void *)region,
					num_regions * (size_t)NANOV2_REGION_SIZE, 0);
		}
	}
#else // CONFIG_NANO_RESERVE_REGIONS
	result = nanov2_allocate_region(region);
#endif // CONFIG_NANO_RESERVE_REGIONS
	if (!result) {
		nano_common_deallocate_pages((void *)nanozone,
				NANOZONEV2_ZONE_PAGED_SIZE, 0);
		_malloc_engaged_nano = NANO_NONE;
		malloc_report(ASL_LEVEL_NOTICE, "nano zone abandoned due to inability "
				"to reserve vm space.\n");
		return NULL;
	}
	nanov2_region_linkage_t *region_linkage =
			nanov2_region_linkage_for_region(nanozone, region);
	os_atomic_store(&region_linkage->next_region_offset, 0, relaxed);

	// Install the first region and pre-allocate the first arena.
	nanozone->first_region_base = region;
	os_atomic_store(&nanozone->current_region_next_arena,
			((nanov2_arena_t *)region) + 1, release);
	nanozone->statistics.allocated_regions = 1;
	// Set up the guard blocks for the initial arena, if requested
	nanov2_init_guard_blocks(nanozone, (nanov2_arena_t *)region);

	return (malloc_zone_t *)nanozone;
}
#endif // OS_VARIANT_NOTRESOLVED

#pragma mark -
#pragma mark Zone Fork Handling

// Nanomalloc assumes that after a fork, it would be dangerous to rely on
// the integrity of the zone data. During a fork, some of the zone handlers are
// switched to the versions below, which do the following:
// 1. Delegate all new allocation to the helper zone.
// 2. Do nothing when asked to free memory that Nano allocated. There will be a
// leak, but this is better than possibly crashing.

#if OS_VARIANT_RESOLVED
MALLOC_NOEXPORT void *
nanov2_forked_malloc(nanozonev2_t *nanozone, size_t size)
{
	// Just hand to the helper zone.
	return nanozone->helper_zone->malloc(nanozone->helper_zone, size);
}

MALLOC_NOEXPORT void *
nanov2_forked_malloc_type(nanozonev2_t *nanozone, size_t size,
		malloc_type_id_t type_id)
{
	// Just hand to the helper zone.
	malloc_zone_t *helper_zone = nanozone->helper_zone;
	return helper_zone->malloc_type_malloc(helper_zone, size, type_id);
}
#endif // OS_VARIANT_RESOLVED

#if OS_VARIANT_NOTRESOLVED

static void *
nanov2_forked_calloc(nanozonev2_t *nanozone, size_t num_items, size_t size)
{
	// Just hand to the helper zone.
	return nanozone->helper_zone->calloc(nanozone->helper_zone, num_items,
			size);
}

static void *
nanov2_forked_calloc_type(nanozonev2_t *nanozone, size_t num_items, size_t size,
		malloc_type_id_t type_id)
{
	// Just hand to the helper zone.
	malloc_zone_t *helper_zone = nanozone->helper_zone;
	return helper_zone->malloc_type_calloc(helper_zone, num_items, size, type_id);
}

static void *
nanov2_forked_memalign(nanozonev2_t *nanozone, size_t alignment, size_t size)
{
	// Just hand to the helper zone.
	return nanozone->helper_zone->memalign(nanozone->helper_zone, alignment,
			size);
}

static void *
nanov2_forked_memalign_type(nanozonev2_t *nanozone, size_t alignment,
		size_t size, malloc_type_id_t type_id)
{
	// Just hand to the helper zone.
	malloc_zone_t *helper_zone = nanozone->helper_zone;
	return helper_zone->malloc_type_memalign(helper_zone, alignment, size,
			type_id);
}

#endif // OS_VARIANT_NOTRESOLVED

#if OS_VARIANT_RESOLVED

MALLOC_NOEXPORT void
nanov2_forked_free(nanozonev2_t *nanozone, void *ptr)
{
	if (!ptr) {
		return; // Protect against malloc_zone_free() passing NULL.
	}

	// <rdar://problem/26481467> exhausting a slot may result in a pointer with
	// the nanozone prefix being given to nano_free via malloc_zone_free. Calling
	// vet_and_size here, instead of in _nano_free_check_scribble means we can
	// early-out into the helper_zone if it turns out nano does not own this ptr.
	size_t sz = nanov2_pointer_size(nanozone, ptr, FALSE);
	if (sz || nanov2_has_valid_signature(ptr)) {
		/* Drop it on the floor as nanozone metadata could be fouled by fork. */
		return;
	} else {
		nanozone->helper_zone->free(nanozone->helper_zone, ptr);
		return;
	}
	/* NOTREACHED */
}

MALLOC_NOEXPORT void
nanov2_forked_free_definite_size(nanozonev2_t *nanozone, void *ptr, size_t size)
{
	nanov2_forked_free(nanozone, ptr);
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static void *
_nanov2_forked_realloc(nanozonev2_t *nanozone, void *ptr, size_t new_size,
		bool typed, malloc_type_id_t type_id)
{
	// could occur through malloc_zone_realloc() path
	if (!ptr) {
		// If ptr is a null pointer, realloc() shall be equivalent to malloc()
		// for the specified size.
		if (typed) {
			return nanov2_forked_malloc_type(nanozone, new_size, type_id);
		} else {
			return nanov2_forked_malloc(nanozone, new_size);
		}
	}

	size_t old_size = nanov2_pointer_size(nanozone, ptr, FALSE);
	if (!old_size) {
		// not-nano pointer, hand down to helper zone
		malloc_zone_t *zone = (malloc_zone_t *)(nanozone->helper_zone);
		if (typed) {
			return zone->malloc_type_realloc(zone, ptr, new_size, type_id);
		} else {
			return zone->realloc(zone, ptr, new_size);
		}
	} else {
		if (!new_size) {
			// If size is 0 and ptr is not a null pointer, the object pointed to
			// is freed. However as nanozone metadata could be fouled by fork,
			// we'll intentionally leak it.

			// If size is 0, either a null pointer or a unique pointer that can
			// be successfully passed to free() shall be returned.
			return nanov2_forked_malloc(nanozone, 1);
		}

		void *new_ptr;
		if (typed) {
			malloc_zone_t *helper_zone = nanozone->helper_zone;
			new_ptr = helper_zone->malloc_type_malloc(helper_zone, new_size,
					type_id);
		} else {
			new_ptr = nanozone->helper_zone->malloc(nanozone->helper_zone,
				new_size);
		}
		if (new_ptr) {
			size_t valid_size = MIN(old_size, new_size);
			memcpy(new_ptr, ptr, valid_size);
			// Original pointer is intentionally leaked as nanozone metadata
			// could be fouled by fork.
			return new_ptr;
		} else {
			// Original ptr is left intact
			return NULL;
		}
		/* NOTREACHED */
	}
	/* NOTREACHED */
}

MALLOC_NOEXPORT void *
nanov2_forked_realloc(nanozonev2_t *nanozone, void *ptr, size_t new_size)
{
	return _nanov2_forked_realloc(nanozone, ptr, new_size, false,
			MALLOC_TYPE_ID_NONE);
}

MALLOC_NOEXPORT void *
nanov2_forked_realloc_type(nanozonev2_t *nanozone, void *ptr, size_t new_size,
		malloc_type_id_t type_id)
{
	return _nanov2_forked_realloc(nanozone, ptr, new_size, true, type_id);
}
#endif // OS_VARIANT_RESOLVED

#if OS_VARIANT_NOTRESOLVED

static unsigned
nanov2_forked_batch_malloc(nanozonev2_t *nanozone, size_t size, void **results,
		unsigned count)
{
	// Just pass this to the helper zone.
	return nanozone->helper_zone->batch_malloc(nanozone->helper_zone, size,
			results, count);
}
#endif // OS_VARIANT_NOTRESOLVED

#if OS_VARIANT_RESOLVED

MALLOC_NOEXPORT void
nanov2_forked_batch_free(nanozonev2_t *nanozone, void **to_be_freed,
		unsigned count)
{
	if (!count) {
		return;
	}

	while (count--) {
		void *ptr = to_be_freed[count];
		if (ptr) {
			nanov2_forked_free(nanozone, ptr);
		}
	}
}
#endif // OS_VARIANT_RESOLVED

#if OS_VARIANT_NOTRESOLVED

static boolean_t
nanov2_forked_claimed_address(struct _malloc_zone_t *zone, void *ptr)
{
	// This does not operate after fork - default to true to avoid
	// false negatives.
	return true;
}

void
nanov2_forked_zone(nanozonev2_t *nanozone)
{
	// Hobble the nano zone in the child of a fork prior to an exec since
	// the state of the zone can be made inconsistent by a parent thread while
	// the fork is underway. All new allocations will be referred to the helper
	// zone (which is more stable.) All free()'s of existing nano objects will
	// be leaked.
	mprotect(nanozone, sizeof(nanozone->basic_zone), PROT_READ | PROT_WRITE);

	nanozone->basic_zone.size = OS_RESOLVED_VARIANT_ADDR(nanov2_size);	// Unchanged
	nanozone->basic_zone.malloc = OS_RESOLVED_VARIANT_ADDR(nanov2_forked_malloc);
	nanozone->basic_zone.calloc = (void *)nanov2_forked_calloc;
	nanozone->basic_zone.valloc = (void *)nanov2_valloc;		// Unchanged
	nanozone->basic_zone.free = OS_RESOLVED_VARIANT_ADDR(nanov2_forked_free);
	nanozone->basic_zone.realloc = OS_RESOLVED_VARIANT_ADDR(nanov2_forked_realloc);
	nanozone->basic_zone.destroy = (void *)nanov2_destroy;		// Unchanged
	nanozone->basic_zone.batch_malloc = (void *)nanov2_forked_batch_malloc;
	nanozone->basic_zone.batch_free = OS_RESOLVED_VARIANT_ADDR(nanov2_forked_batch_free);
	nanozone->basic_zone.introspect =
			(struct malloc_introspection_t *)&nanov2_introspect;// Unchanged
	nanozone->basic_zone.memalign = (void *)nanov2_forked_memalign;
	nanozone->basic_zone.free_definite_size =
			OS_RESOLVED_VARIANT_ADDR(nanov2_forked_free_definite_size);
	nanozone->basic_zone.claimed_address = nanov2_forked_claimed_address;
	nanozone->basic_zone.try_free_default = NULL; // Fall back to old protocol
	nanozone->basic_zone.malloc_type_malloc =
			OS_RESOLVED_VARIANT_ADDR(nanov2_forked_malloc_type);
	nanozone->basic_zone.malloc_type_calloc = (void *)nanov2_forked_calloc_type;
	nanozone->basic_zone.malloc_type_realloc =
			OS_RESOLVED_VARIANT_ADDR(nanov2_forked_realloc_type);
	nanozone->basic_zone.malloc_type_memalign =
			(void *)nanov2_forked_memalign_type;
	mprotect(nanozone, sizeof(nanozone->basic_zone), PROT_READ);
}

#endif // OS_VARIANT_NOTRESOLVED

#endif // CONFIG_NANOZONE