Loading...
--- /dev/null
+++ libmalloc/libmalloc-166.251.2/src/nanov2_zone.h
@@ -0,0 +1,322 @@
+/*
+ * Copyright (c) 2018 Apple Inc. All rights reserved.
+ *
+ * @APPLE_LICENSE_HEADER_START@
+ *
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this
+ * file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_LICENSE_HEADER_END@
+ */
+
+#ifndef __NANOV2_ZONE_H
+#define __NANOV2_ZONE_H
+
+#if CONFIG_NANOZONE
+
+#pragma mark -
+#pragma mark Address Structure
+
+#if TARGET_OS_OSX || TARGET_OS_SIMULATOR
+
+#define NANOV2_REGION_BITS 15
+#define NANOV2_ARENA_BITS 3
+#define NANOV2_BLOCK_BITS 12
+#define NANOV2_OFFSET_BITS 14
+
+#else // TARGET_OS_OSX || TARGET_OS_SIMULATOR
+
+#define NANOV2_REGION_BITS 0
+#define NANOV2_ARENA_BITS 3
+#define NANOV2_BLOCK_BITS 12
+#define NANOV2_OFFSET_BITS 14
+
+#endif // TARGET_OS_OSX || TARGET_OS_SIMULATOR
+
+#if NANOV2_REGION_BITS > 0
+#define NANOV2_MULTIPLE_REGIONS 1
+#else // NANOV2_REGION_BITS > 0
+#define NANOV2_MULTIPLE_REGIONS 0
+#endif // NANOV2_REGION_BITS > 0
+
+// Size of a block (currently 16KB)
+#define NANOV2_BLOCK_SIZE (1 << NANOV2_OFFSET_BITS)
+
+// Size of an arena (currently 64MB)
+#define NANOV2_ARENA_SIZE (64 * 1024 * 1024)
+
+// Size of a region (currently 512MB)
+#define NANOV2_REGION_SIZE (512 * 1024 * 1024)
+
+// Number of blocks per arena (currently 4096)
+#define NANOV2_BLOCKS_PER_ARENA (NANOV2_ARENA_SIZE/NANOV2_BLOCK_SIZE)
+
+// Number of arenas per region (currently 8)
+#define NANOV2_ARENAS_PER_REGION (NANOV2_REGION_SIZE/NANOV2_ARENA_SIZE)
+
+// Maximum number of slots per block
+#define NANOV2_MAX_SLOTS_PER_BLOCK (NANOV2_BLOCK_SIZE/NANO_REGIME_QUANTA_SIZE)
+
+// Highest region number.
+#if NANOV2_MULTIPLE_REGIONS
+#define NANOV2_MAX_REGION_NUMBER ((1 << NANOV2_REGION_BITS) - 1)
+#else // NANOV2_MULTIPLE_REGIONS
+#define NANOV2_MAX_REGION_NUMBER 0
+#endif // NANOV2_MULTIPLE_REGIONS
+
+// clang-format really dislikes the bitfields here
+// clang-format off
+#if defined(__BIG_ENDIAN__)
+
+// Nano V2 address structure.
+struct nanov2_addr_s {
+ uintptr_t nano_signature : NANOZONE_SIGNATURE_BITS;
+#if NANOV2_MULTIPLE_REGIONS
+ uintptr_t nano_region: NANOV2_REGION_BITS;
+#endif // NANOV2_MULTIPLE_REGIONS
+ uintptr_t nano_arena : NANOV2_ARENA_BITS;
+ uintptr_t nano_block : NANOV2_BLOCK_BITS;
+ uintptr_t nano_offset : NANOV2_OFFSET_BITS;
+};
+MALLOC_STATIC_ASSERT(sizeof(struct nanov2_addr_s) == sizeof(uintptr_t),
+ "Wrong size for nanov2_addr_s");
+
+#else // defined(__BIG_ENDIAN__)
+
+// least significant bits declared first
+struct nanov2_addr_s {
+ uintptr_t nano_offset : NANOV2_OFFSET_BITS;
+ uintptr_t nano_block : NANOV2_BLOCK_BITS;
+ uintptr_t nano_arena : NANOV2_ARENA_BITS;
+#if NANOV2_MULTIPLE_REGIONS
+ uintptr_t nano_region: NANOV2_REGION_BITS;
+#endif // NANOV2_MULTIPLE_REGIONS
+ uintptr_t nano_signature : NANOZONE_SIGNATURE_BITS;
+};
+MALLOC_STATIC_ASSERT(sizeof(struct nanov2_addr_s) == sizeof(uintptr_t),
+ "Wrong size for nanov2_addr_s");
+
+#endif // defined(__BIG_ENDIAN__)
+
+// clang-format on
+
+// Union that allows easy extraction of the fields in a Nano V2 address.
+typedef union {
+ void *addr;
+ struct nanov2_addr_s fields;
+} nanov2_addr_t;
+
+// Typedef that tags a size class value. Range is 0 to NANO_SIZE_CLASSES - 1.
+typedef unsigned nanov2_size_class_t;
+
+#pragma mark -
+#pragma mark Block Definitions
+
+// A block is a chunk of NANOV2_BLOCK_SIZE bytes of memory.
+typedef struct {
+ unsigned char content[NANOV2_BLOCK_SIZE];
+} nanov2_block_t;
+
+MALLOC_STATIC_ASSERT(sizeof(nanov2_block_t) == NANOV2_BLOCK_SIZE,
+ "nanov2_block_t must be the same size as a block");
+
+#pragma mark -
+#pragma mark Arena and Block Definitions
+
+// An arena is an array of NANOV2_BLOCKS_PER_ARENA blocks.
+typedef struct {
+ nanov2_block_t blocks[NANOV2_BLOCKS_PER_ARENA];
+} nanov2_arena_t;
+
+MALLOC_STATIC_ASSERT(sizeof(nanov2_arena_t) == NANOV2_BLOCK_SIZE * NANOV2_BLOCKS_PER_ARENA,
+ "nanov2_arena_t must be the same size as its blocks");
+
+// Per-block header structure, embedded in the arena metadata block.
+typedef struct {
+ uint32_t next_slot : 11; // Next slot on free list, 1-based.
+ uint32_t free_count : 10; // Free slots in this block - 1
+ uint32_t gen_count : 10; // A-B-A count
+ uint32_t in_use : 1; // Being used for allocations.
+} nanov2_block_meta_t;
+MALLOC_STATIC_ASSERT(sizeof(nanov2_block_meta_t) == sizeof(uint32_t),
+ "Incorrect size for nanov2_block_meta_t");
+
+// Distinguished values of next_slot
+#define SLOT_NULL 0 // Slot has never been used.
+#define SLOT_BUMP 0x7fb // Marks the end of the free list
+#define SLOT_FULL 0x7fc // Slot is full (no free slots)
+#define SLOT_CAN_MADVISE 0x7fd // Block can be madvised (and in_use == 0)
+#define SLOT_MADVISING 0x7fe // Block is being madvised. Do not touch
+#define SLOT_MADVISED 0x7ff // Block has been madvised.
+
+// View of the per-block header structure that allows it to be used where a
+// primitive type is required.
+typedef union {
+ nanov2_block_meta_t meta;
+ uint32_t bits;
+} nanov2_block_meta_view_t;
+
+// Structure overlaid onto an arena's metadata block. This must be exactly
+// the same size as a block.
+typedef struct {
+ nanov2_block_meta_t arena_block_meta[NANOV2_BLOCKS_PER_ARENA];
+} nanov2_arena_metablock_t;
+
+MALLOC_STATIC_ASSERT(sizeof(nanov2_arena_metablock_t) == NANOV2_BLOCK_SIZE,
+ "nanov2_arena_metablock_t must be the same size as a block");
+
+// Structure overlaid on slots that are on the block freelist.
+typedef struct {
+ uint64_t double_free_guard;
+ uint16_t next_slot;
+} nanov2_free_slot_t;
+
+MALLOC_STATIC_ASSERT(
+ sizeof(nanov2_free_slot_t) <= NANO_REGIME_QUANTA_SIZE,
+ "nanov2_free_slot_t too large");
+
+// Type for the index of a block in its hosting arena.
+typedef unsigned nanov2_block_index_t;
+
+// Type for the index of a block meta structure in its hosting metadata block.
+typedef unsigned nanov2_meta_index_t;
+
+#pragma mark -
+#pragma mark Region Definitions
+
+// A region is an array of NANOV2_ARENAS_PER_REGION arenas.
+typedef struct {
+ nanov2_arena_t arenas[NANOV2_ARENAS_PER_REGION];
+} nanov2_region_t;
+
+// Linkage between regions. Overlays the nanov2_block_meta_t that corresponds
+// to the arena metadata block, so must be the same size as nanov2_block_meta_t.
+typedef struct {
+ uint16_t next_region_offset; // Offset to next region in 512MB blocks
+ uint16_t unused;
+} nanov2_region_linkage_t;
+
+MALLOC_STATIC_ASSERT(
+ sizeof (nanov2_block_meta_t) == sizeof(nanov2_region_linkage_t),
+ "nanov2_block_meta_t must be the same size as nanov2_region_linkage_t");
+
+#pragma mark -
+#pragma mark Statistics
+
+typedef struct {
+ uint64_t total_allocations;
+ uint64_t total_frees;
+ uint64_t madvised_blocks; // Does not reduce when reused.
+ uint64_t madvise_races; // Reused while being madvised.
+} nanov2_size_class_statistics;
+
+typedef struct {
+ // Number of allocated regions
+ unsigned allocated_regions;
+
+ // Number of times a region could not be placed at its preferred location
+ unsigned region_address_clashes;
+
+ // Statistics collected by size class
+ nanov2_size_class_statistics size_class_statistics[NANO_SIZE_CLASSES];
+} nanov2_statistics_t;
+
+#pragma mark -
+#pragma mark Zone Definitions
+
+// Maximum number of currently active allocation blocks per size class.
+// Initially, the default is for each physical CPU to have a dedicated block.
+#define MAX_CURRENT_BLOCKS 64
+#define MAX_CURRENT_BLOCKS_MASK (MAX_CURRENT_BLOCKS - 1)
+MALLOC_STATIC_ASSERT(MAX_CURRENT_BLOCKS > 1 &&
+ !(MAX_CURRENT_BLOCKS & MAX_CURRENT_BLOCKS_MASK),
+ "MAX_CURRENT_BLOCKS must be a power of 2");
+
+typedef struct nanozonev2_s {
+ // first page will be given read-only protection
+ malloc_zone_t basic_zone;
+ uint8_t pad[PAGE_MAX_SIZE - sizeof(malloc_zone_t)];
+
+ // Current allocation blocks. Indexed by a factor that may change in the
+ // future (and may be tuneable). Initially indexed by physical CPU number.
+ nanov2_block_meta_t *current_block[NANO_SIZE_CLASSES][MAX_CURRENT_BLOCKS];
+
+ // Locks for the current allocation blocks.
+ _malloc_lock_s current_block_lock[NANO_SIZE_CLASSES][MAX_CURRENT_BLOCKS];
+
+ // Lock for delegate_allocations.
+ _malloc_lock_s delegate_allocations_lock;
+
+ // Mask of size classes for which allocation should be delegated when a new
+ // block is needed and the class has become full.
+ uint16_t delegate_allocations;
+
+ // Zone debug flags.
+ unsigned debug_flags;
+
+ // Cookie used for ASLR within an arena.
+ uint64_t aslr_cookie;
+ uint64_t aslr_cookie_aligned;
+
+ // cookie used to protect linkage on the block freelist
+ uintptr_t slot_freelist_cookie;
+
+ // The zone to which allocations that cannot be satisfied by Nano V2
+ // will be handed off.
+ malloc_zone_t *helper_zone;
+
+ // Lock used to serialize access to current_block.
+ _malloc_lock_s blocks_lock;
+
+ // Lock used to protect current_region_base, current_region_next_arena and
+ // current_region_limit.
+ _malloc_lock_s regions_lock;
+
+ // Base address of the first region. Fixed once set.
+ nanov2_region_t *first_region_base;
+
+ // Base address of the current region. Always the most recently allocated
+ // region and therefore the one with the highest base address.
+ nanov2_region_t *current_region_base;
+
+ // Address to use for the next arena. Always between current_region_base
+ // and current_region_limit.
+ nanov2_arena_t *current_region_next_arena;
+
+ // Limit address of the current region (first byte after the region).
+ void *current_region_limit;
+
+ // Lock used when madvising.
+ _malloc_lock_s madvise_lock;
+
+ // Global and per-size class statistics
+ nanov2_statistics_t statistics;
+} nanozonev2_t;
+
+#define NANOZONEV2_ZONE_PAGED_SIZE mach_vm_round_page(sizeof(nanozonev2_t))
+
+#pragma mark -
+#pragma mark Address Manipulation
+
+#define NANOV2_BLOCK_ADDRESS_MASK ~((1ULL << (NANOV2_OFFSET_BITS)) - 1)
+#define NANOV2_ARENA_ADDRESS_MASK \
+ ~((1ULL << (NANOV2_BLOCK_BITS + NANOV2_OFFSET_BITS)) - 1)
+#define NANOV2_REGION_ADDRESS_MASK \
+ ~((1ULL << (NANOV2_ARENA_BITS + NANOV2_BLOCK_BITS + NANOV2_OFFSET_BITS)) - 1)
+
+#endif // CONFIG_NANOZONE
+
+#endif // __NANOV2_ZONE_H
+