Loading...
--- libmalloc/libmalloc-374.60.3/src/magazine_zone.h
+++ libmalloc/libmalloc-116/src/magazine_zone.h
@@ -70,14 +70,8 @@
inplace_linkage_s next;
} small_inplace_free_entry_s, *small_inplace_free_entry_t;
-typedef struct _medium_inplace_free_entry_s {
- inplace_linkage_s previous;
- inplace_linkage_s next;
-} medium_inplace_free_entry_s, *medium_inplace_free_entry_t;
-
typedef union {
small_inplace_free_entry_t small_inplace;
- medium_inplace_free_entry_t medium_inplace;
inplace_free_entry_t inplace;
oob_free_entry_t oob;
void *p;
@@ -90,9 +84,10 @@
typedef unsigned int grain_t; // N.B. wide enough to index all free slots
+typedef int mag_index_t;
+
#define CHECK_REGIONS (1 << 31)
#define DISABLE_ASLR (1 << 30)
-#define DISABLE_LARGE_ASLR (1 << 29)
#define MAX_RECORDER_BUFFER 256
@@ -102,11 +97,11 @@
* Memory in the Tiny range is allocated from regions (heaps) pointed to by the
* szone's hashed_regions pointer.
*
- * Each region is laid out as a metadata block followed by a heap, all within
- * a 1MB (2^20) block. This means there are 64504 16-byte blocks and the metadata
+ * Each region is laid out as a heap, followed by a header block, all within
+ * a 1MB (2^20) block. This means there are 64520 16-byte blocks and the header
* is 16138 bytes, making the total 1048458 bytes, leaving 118 bytes unused.
*
- * The metadata block is arranged as in struct tiny_region defined just below, and
+ * The header block is arranged as in struct tiny_region defined just below, and
* consists of two bitfields (or bit arrays) interleaved 32 bits by 32 bits.
*
* Each bitfield comprises NUM_TINY_BLOCKS bits, and refers to the corresponding
@@ -118,12 +113,7 @@
* bit is not set, the in-use bit is invalid.
*
* The szone maintains an array of NUM_TINY_SLOTS freelists, each of which is used to hold
- * free objects of the corresponding quantum size. In the tiny region, the free
- * objects for each region are arranged so that they are grouped together in their
- * per-slot freelists and the groups are ordered roughly in the order of regions
- * as they appear in the magazine's region list. This approach helps to reduce
- * fragmentation. Not guaranteeing strictly the same ordering as the regions
- * helps reduce the CPU time required to reduce fragmentation.
+ * free objects of the corresponding quantum size.
*
* A free block is laid out depending on its size, in order to fit all free
* blocks in 16 bytes, on both 32 and 64 bit platforms. One quantum blocks do
@@ -170,26 +160,21 @@
* plus rounding to the nearest page.
*/
#define CEIL_NUM_TINY_BLOCKS_WORDS (((NUM_TINY_BLOCKS + 31) & ~31) >> 5)
-
-#define TINY_HEAP_SIZE (NUM_TINY_BLOCKS * TINY_QUANTUM)
-#define TINY_METADATA_SIZE (sizeof(region_trailer_t) + sizeof(tiny_header_inuse_pair_t) * CEIL_NUM_TINY_BLOCKS_WORDS + (sizeof(region_free_blocks_t) * NUM_TINY_SLOTS))
-#define TINY_REGION_SIZE ((TINY_HEAP_SIZE + TINY_METADATA_SIZE + PAGE_MAX_SIZE - 1) & ~(PAGE_MAX_SIZE - 1))
-
-/*
- * Location of the metadata for a given tiny region.
- */
-#define TINY_REGION_METADATA(region) ((uintptr_t)&((tiny_region_t)region)->trailer)
+#define TINY_METADATA_SIZE (sizeof(region_trailer_t) + sizeof(tiny_header_inuse_pair_t) * CEIL_NUM_TINY_BLOCKS_WORDS)
+#define TINY_REGION_SIZE ((NUM_TINY_BLOCKS * TINY_QUANTUM + TINY_METADATA_SIZE + PAGE_MAX_SIZE - 1) & ~(PAGE_MAX_SIZE - 1))
+
+#define TINY_METADATA_START (NUM_TINY_BLOCKS * TINY_QUANTUM)
/*
* Beginning and end pointers for a region's heap.
*/
-#define TINY_REGION_HEAP_BASE(region) ((void *)(((tiny_region_t)region)->blocks))
-#define TINY_REGION_HEAP_END(region) ((void *)(((uintptr_t)TINY_REGION_HEAP_BASE(region)) + TINY_HEAP_SIZE))
-
-/*
- * Locate the region for a pointer known to be within a tiny region.
- */
-#define TINY_REGION_FOR_PTR(ptr) ((tiny_region_t)((uintptr_t)(ptr) & ~((1 << TINY_BLOCKS_ALIGN) - 1)))
+#define TINY_REGION_ADDRESS(region) ((void *)(region))
+#define TINY_REGION_END(region) ((void *)(((uintptr_t)(region)) + (NUM_TINY_BLOCKS * TINY_QUANTUM)))
+
+/*
+ * Locate the heap base for a pointer known to be within a tiny region.
+ */
+#define TINY_REGION_FOR_PTR(_p) ((void *)((uintptr_t)(_p) & ~((1 << TINY_BLOCKS_ALIGN) - 1)))
/*
* Convert between byte and msize units.
@@ -207,85 +192,45 @@
/*
* Layout of a tiny region
*/
-typedef uint32_t tiny_block_t[TINY_QUANTUM / sizeof(uint32_t)];
-MALLOC_STATIC_ASSERT(sizeof(tiny_block_t) == TINY_QUANTUM,
- "Incorrect size tiny_block_t");
-
-#define TINY_REGION_PAD (TINY_REGION_SIZE - TINY_HEAP_SIZE - TINY_METADATA_SIZE - sizeof(region_cookie_t))
+typedef uint32_t tiny_block_t[4]; // assert(TINY_QUANTUM == sizeof(tiny_block_t))
typedef struct tiny_header_inuse_pair {
uint32_t header;
uint32_t inuse;
} tiny_header_inuse_pair_t;
-typedef struct {
- // Block indices are +1 so that 0 represents no free block.
- uint16_t first_block;
- uint16_t last_block;
-} region_free_blocks_t;
-
-typedef uint32_t region_cookie_t;
-
-OS_ENUM(rack_dispose_flags, uint32_t,
- RACK_DISPOSE_DELAY = 0x1,
- RACK_DISPOSE_NEEDED = 0x2,
-);
-
typedef struct region_trailer {
struct region_trailer *prev;
struct region_trailer *next;
+ boolean_t recirc_suitable;
+ volatile int pinned_to_depot;
unsigned bytes_used;
- unsigned objects_in_use; // Used only by tiny allocator.
mag_index_t mag_index;
- volatile int32_t pinned_to_depot;
- bool recirc_suitable;
- // Locking: dispose_flags must be locked under the rack's region lock
- rack_dispose_flags_t dispose_flags;
} region_trailer_t;
typedef struct tiny_region {
- // This must be first (because TINY_REGION_METADATA assumes it).
+ tiny_block_t blocks[NUM_TINY_BLOCKS];
+
region_trailer_t trailer;
// The interleaved bit arrays comprising the header and inuse bitfields.
// The unused bits of each component in the last pair will be initialized to sentinel values.
tiny_header_inuse_pair_t pairs[CEIL_NUM_TINY_BLOCKS_WORDS];
- // Indices of the first and last free block in this region. Value is the
- // block index + 1 so that 0 indicates no free block in this region for the
- // corresponding slot.
- region_free_blocks_t free_blocks_by_slot[NUM_TINY_SLOTS];
-
- uint8_t pad[TINY_REGION_PAD];
-
- // Intended to catch backward overspills from the heap into this structure.
- region_cookie_t region_cookie;
-
- tiny_block_t blocks[NUM_TINY_BLOCKS];
+ uint8_t pad[TINY_REGION_SIZE - (NUM_TINY_BLOCKS * sizeof(tiny_block_t)) - TINY_METADATA_SIZE];
} * tiny_region_t;
-// The layout described above should result in a tiny_region_t being 1MB.
-MALLOC_STATIC_ASSERT(TINY_REGION_SIZE == (1024 * 1024), "incorrect TINY_REGION_SIZE");
-MALLOC_STATIC_ASSERT(sizeof(struct tiny_region) == TINY_REGION_SIZE, "incorrect tiny_region_size");
-
/*
* Per-region meta data for tiny allocator
*/
#define REGION_TRAILER_FOR_TINY_REGION(r) (&(((tiny_region_t)(r))->trailer))
-#define REGION_COOKIE_FOR_TINY_REGION(r) (((tiny_region_t)(r))->region_cookie)
#define MAGAZINE_INDEX_FOR_TINY_REGION(r) (REGION_TRAILER_FOR_TINY_REGION(r)->mag_index)
#define BYTES_USED_FOR_TINY_REGION(r) (REGION_TRAILER_FOR_TINY_REGION(r)->bytes_used)
-#define OBJECTS_IN_USE_FOR_TINY_REGION(r) (REGION_TRAILER_FOR_TINY_REGION(r)->objects_in_use)
/*
* Locate the block header for a pointer known to be within a tiny region.
*/
-#define TINY_BLOCK_HEADER_FOR_PTR(ptr) ((void *)&(((tiny_region_t)TINY_REGION_FOR_PTR(ptr))->pairs))
-
-/*
- * Locate the block header for a tiny region.
- */
-#define TINY_BLOCK_HEADER_FOR_REGION(region) ((void *)&(((tiny_region_t)region)->pairs))
+#define TINY_BLOCK_HEADER_FOR_PTR(_p) ((void *)&(((tiny_region_t)TINY_REGION_FOR_PTR(_p))->pairs))
/*
* Locate the inuse map for a given block header pointer.
@@ -293,49 +238,26 @@
#define TINY_INUSE_FOR_HEADER(_h) ((void *)&(((tiny_header_inuse_pair_t *)(_h))->inuse))
/*
- * Heap offset for a pointer known to be within a tiny region.
- */
-#define TINY_HEAP_OFFSET_FOR_PTR(ptr) ((uintptr_t)(ptr) - (uintptr_t)TINY_REGION_HEAP_BASE(TINY_REGION_FOR_PTR(ptr)))
-
-/*
* Compute the bitmap index for a pointer known to be within a tiny region.
*/
-#define TINY_INDEX_FOR_PTR(ptr) ((TINY_HEAP_OFFSET_FOR_PTR(ptr) >> SHIFT_TINY_QUANTUM) & (NUM_TINY_CEIL_BLOCKS - 1))
-
-/*
- * Get the pointer for a given index in a region.
- */
-#define TINY_PTR_FOR_INDEX(index, region) (void *)((uintptr_t)TINY_REGION_HEAP_BASE(region) + ((index) << SHIFT_TINY_QUANTUM))
-
-/*
- * Offset back to an szone_t given prior knowledge that this rack_t
- * is contained within an szone_t.
- *
- * Note: the only place this is used, the dtrace probes, only occurs
- * when the rack has been set up inside a scalable zone. Should
- * this ever be used somewhere that this does not hold true
- * (say, the test cases) then the pointer returned will be junk.
- */
-#define TINY_SZONE_FROM_RACK(_r) \
- (szone_t *)((uintptr_t)(_r) - offsetof(struct szone_s, tiny_rack))
-
+#define TINY_INDEX_FOR_PTR(_p) (((uintptr_t)(_p) >> SHIFT_TINY_QUANTUM) & (NUM_TINY_CEIL_BLOCKS - 1))
#if !CONFIG_TINY_CACHE
#warning CONFIG_TINY_CACHE turned off
#endif
+#define TINY_REGION_PAYLOAD_BYTES (NUM_TINY_BLOCKS * TINY_QUANTUM)
/********************* DEFINITIONS for small ************************/
/*
- * Memory in the small range is allocated from regions (heaps) pointed to by the szone's hashed_regions
+ * Memory in the Small range is allocated from regions (heaps) pointed to by the szone's hashed_regions
* pointer.
*
- * Each region is laid out as metadata followed by the heap, all within an 8MB (2^23) block.
- * The metadata block is arranged as in struct small_region defined just below.
- * The array is arranged as an array of shorts, one for each SMALL_QUANTUM in the heap. There are
- * 16319 512-byte blocks and the array is 16319*2 bytes, which totals 8387966, leaving 642 bytes unused.
- * Once the region trailer is accounted for, there is room for 61 out-of-band free list entries in
+ * Each region is laid out as a heap, followed by the metadata array, all within an 8MB (2^23) block.
+ * The array is arranged as an array of shorts, one for each SMALL_QUANTUM in the heap. There are
+ * 16319 512-blocks and the array is 16319*2 bytes, which totals 8387966, leaving 642 bytes unused.
+ * Once the region trailer is accounted for, there is room for 61 out-of-band free list entries in
* the remaining padding (or 6, if the region was split into 16320 blocks, not 16319).
*
* The 16-bit shorts in the region are used for allocation metadata. The MSB bit marks a block as
@@ -366,7 +288,7 @@
* 2. Out-of-band free list entries. These utilitise the remaining padding in the 8mb region that
* follows the blocks, metadata and region trailer. Out-of-band entries are used *iff* the
* freed address lies on a page boundary and the freed region spans more than a page. If we were
- * to store the free list entry in-line in that memory, it would keep the entire page dirty,
+ * to store the free list entry in-line in that memory, it would keep the entire page dirty,
* so an out-of-band entry is used.
*
* An out-of-band free list entry is laid out as:
@@ -380,7 +302,7 @@
#define FOLLOWING_SMALL_PTR(ptr, msize) (((unsigned char *)(ptr)) + ((msize) << SHIFT_SMALL_QUANTUM))
/*
- * SMALL_IS_OOB is used to mark the MSB of OOB free list entries to show that they are in use, and
+ * SMALL_IS_OOB is used mark to the MSB of OOB free list entries to show that they are in use, and
* distinguish them from their initial, empty, state.
*/
#define SMALL_IS_OOB (1 << 15)
@@ -396,333 +318,88 @@
#error Too many entropy bits for small region requested
#endif
-#define SMALL_HEAP_SIZE (NUM_SMALL_BLOCKS * SMALL_QUANTUM)
#define SMALL_METADATA_SIZE (sizeof(region_trailer_t) + NUM_SMALL_BLOCKS * sizeof(msize_t))
-#define SMALL_REGION_SIZE ((SMALL_HEAP_SIZE + SMALL_METADATA_SIZE + PAGE_MAX_SIZE - 1) & ~(PAGE_MAX_SIZE - 1))
-
-/*
- * Location of the metadata for a given small region.
- */
-#define SMALL_REGION_METADATA(region) ((uintptr_t)&((small_region_t)region)->trailer)
+#define SMALL_REGION_SIZE ((NUM_SMALL_BLOCKS * SMALL_QUANTUM + SMALL_METADATA_SIZE + PAGE_MAX_SIZE - 1) & ~(PAGE_MAX_SIZE - 1))
+
+#define SMALL_METADATA_START (NUM_SMALL_BLOCKS * SMALL_QUANTUM)
/*
* Beginning and end pointers for a region's heap.
*/
-#define SMALL_REGION_HEAP_BASE(region) ((void *)((small_region_t)region)->blocks)
-#define SMALL_REGION_HEAP_END(region) (SMALL_REGION_HEAP_BASE(region) + SMALL_HEAP_SIZE)
+#define SMALL_REGION_ADDRESS(region) ((unsigned char *)region)
+#define SMALL_REGION_END(region) (SMALL_REGION_ADDRESS(region) + (NUM_SMALL_BLOCKS * SMALL_QUANTUM))
/*
* Locate the heap base for a pointer known to be within a small region.
*/
-#define SMALL_REGION_FOR_PTR(ptr) ((small_region_t)((uintptr_t)(ptr) & ~((1 << SMALL_BLOCKS_ALIGN) - 1)))
-#define SMALL_REGION_OFFSET_FOR_PTR(ptr) ((uintptr_t)(ptr) & ((1 << SMALL_BLOCKS_ALIGN) - 1))
+#define SMALL_REGION_FOR_PTR(_p) ((void *)((uintptr_t)(_p) & ~((1 << SMALL_BLOCKS_ALIGN) - 1)))
+#define SMALL_OFFSET_FOR_PTR(_p) ((uintptr_t)(_p) & ((1 << SMALL_BLOCKS_ALIGN) - 1))
/*
* Convert between byte and msize units.
*/
-#define SMALL_BYTES_FOR_MSIZE(_m) ((uint32_t)(_m) << SHIFT_SMALL_QUANTUM)
+#define SMALL_BYTES_FOR_MSIZE(_m) ((_m) << SHIFT_SMALL_QUANTUM)
#define SMALL_MSIZE_FOR_BYTES(_b) ((_b) >> SHIFT_SMALL_QUANTUM)
#define SMALL_PREVIOUS_MSIZE(ptr) (*SMALL_METADATA_FOR_PTR(ptr - 1) & ~SMALL_IS_FREE)
/*
- * Convert from msize unit to free list slot.
- */
-#define SMALL_FREE_SLOT_COUNT(_r) \
- (NUM_SMALL_SLOTS + 1)
-#define SMALL_FREE_SLOT_FOR_MSIZE(_r, _m) \
- (((_m) <= SMALL_FREE_SLOT_COUNT(_r)) ? ((_m) - 1) : (SMALL_FREE_SLOT_COUNT(_r) - 1))
-/* compare with MAGAZINE_FREELIST_BITMAP_WORDS */
-#define SMALL_FREELIST_BITMAP_WORDS(_r) ((SMALL_FREE_SLOT_COUNT(_r) + 31) >> 5)
-
-/*
- * Offset back to an szone_t given prior knowledge that this rack_t
- * is contained within an szone_t.
- *
- * Note: the only place this is used, the dtrace probes, only occurs
- * when the rack has been set up inside a scalable zone. Should
- * this ever be used somewhere that this does not hold true
- * (say, the test cases) then the pointer returned will be junk.
- */
-#define SMALL_SZONE_FROM_RACK(_r) \
- (szone_t *)((uintptr_t)(_r) - offsetof(struct szone_s, small_rack))
-
-/*
* Layout of a small region
*/
typedef uint32_t small_block_t[SMALL_QUANTUM / sizeof(uint32_t)];
-MALLOC_STATIC_ASSERT(sizeof(small_block_t) == SMALL_QUANTUM, "Incorrect size for small_block_t");
-#define SMALL_OOB_COUNT ((SMALL_REGION_SIZE - SMALL_HEAP_SIZE - SMALL_METADATA_SIZE - sizeof(region_cookie_t)) / sizeof(oob_free_entry_s))
+#define SMALL_HEAP_SIZE (NUM_SMALL_BLOCKS * sizeof(small_block_t))
+#define SMALL_OOB_COUNT ((SMALL_REGION_SIZE - SMALL_HEAP_SIZE - SMALL_METADATA_SIZE) / sizeof(oob_free_entry_s))
#define SMALL_OOB_SIZE (SMALL_OOB_COUNT * sizeof(oob_free_entry_s))
-#define SMALL_REGION_PAD (SMALL_REGION_SIZE - SMALL_HEAP_SIZE - SMALL_METADATA_SIZE - SMALL_OOB_SIZE - sizeof(region_cookie_t))
+#define SMALL_REGION_PAD (SMALL_REGION_SIZE - SMALL_HEAP_SIZE - SMALL_METADATA_SIZE - SMALL_OOB_SIZE)
typedef struct small_region {
- // This must be first (because SMALL_REGION_METADATA assumes it).
+ small_block_t blocks[NUM_SMALL_BLOCKS];
region_trailer_t trailer;
msize_t small_meta_words[NUM_SMALL_BLOCKS];
oob_free_entry_s small_oob_free_entries[SMALL_OOB_COUNT];
uint8_t pad[SMALL_REGION_PAD];
- region_cookie_t region_cookie;
- small_block_t blocks[NUM_SMALL_BLOCKS];
} * small_region_t;
// The layout described above should result in a small_region_t being 8MB.
-MALLOC_STATIC_ASSERT(SMALL_REGION_SIZE == (8 * 1024 * 1024), "incorrect SMALL_REGION_SIZE");
-MALLOC_STATIC_ASSERT(sizeof(struct small_region) == SMALL_REGION_SIZE, "incorrect small_region_size");
+MALLOC_STATIC_ASSERT(sizeof(struct small_region) == 8388608, "incorrect small_region_size");
/*
* Per-region meta data for small allocator
*/
#define REGION_TRAILER_FOR_SMALL_REGION(r) (&(((small_region_t)(r))->trailer))
-#define REGION_COOKIE_FOR_SMALL_REGION(r) (((small_region_t)(r))->region_cookie)
#define MAGAZINE_INDEX_FOR_SMALL_REGION(r) (REGION_TRAILER_FOR_SMALL_REGION(r)->mag_index)
#define BYTES_USED_FOR_SMALL_REGION(r) (REGION_TRAILER_FOR_SMALL_REGION(r)->bytes_used)
/*
- * Locate the metadata base for a small region.
- */
-#define SMALL_META_HEADER_FOR_REGION(region) (((small_region_t)region)->small_meta_words)
-
-/*
* Locate the metadata base for a pointer known to be within a small region.
*/
-#define SMALL_META_HEADER_FOR_PTR(ptr) (((small_region_t)SMALL_REGION_FOR_PTR(ptr))->small_meta_words)
-
-/*
- * Heap offset for a pointer known to be within a small region.
- */
-#define SMALL_HEAP_OFFSET_FOR_PTR(ptr) ((uintptr_t)(ptr) - (uintptr_t)SMALL_REGION_HEAP_BASE(SMALL_REGION_FOR_PTR(ptr)))
+#define SMALL_META_HEADER_FOR_PTR(_p) (((small_region_t)SMALL_REGION_FOR_PTR(_p))->small_meta_words)
/*
* Compute the metadata index for a pointer known to be within a small region.
*/
-#define SMALL_META_INDEX_FOR_PTR(ptr) ((SMALL_HEAP_OFFSET_FOR_PTR(ptr) >> SHIFT_SMALL_QUANTUM) & (NUM_SMALL_CEIL_BLOCKS - 1))
+#define SMALL_META_INDEX_FOR_PTR(_p) (((uintptr_t)(_p) >> SHIFT_SMALL_QUANTUM) & (NUM_SMALL_CEIL_BLOCKS - 1))
/*
* Find the metadata word for a pointer known to be within a small region.
*/
-#define SMALL_METADATA_FOR_PTR(ptr) (SMALL_META_HEADER_FOR_PTR(ptr) + SMALL_META_INDEX_FOR_PTR(ptr))
+#define SMALL_METADATA_FOR_PTR(_p) (SMALL_META_HEADER_FOR_PTR(_p) + SMALL_META_INDEX_FOR_PTR(_p))
/*
* Determine whether a pointer known to be within a small region points to memory which is free.
*/
-#define SMALL_PTR_IS_FREE(ptr) (*SMALL_METADATA_FOR_PTR(ptr) & SMALL_IS_FREE)
+#define SMALL_PTR_IS_FREE(_p) (*SMALL_METADATA_FOR_PTR(_p) & SMALL_IS_FREE)
/*
* Extract the msize value for a pointer known to be within a small region.
*/
-#define SMALL_PTR_SIZE(ptr) (*SMALL_METADATA_FOR_PTR(ptr) & ~SMALL_IS_FREE)
+#define SMALL_PTR_SIZE(_p) (*SMALL_METADATA_FOR_PTR(_p) & ~SMALL_IS_FREE)
#if !CONFIG_SMALL_CACHE
#warning CONFIG_SMALL_CACHE turned off
#endif
-
-/********************* DEFINITIONS for medium ************************/
-
-/*
- * Memory in the medium range is allocated from regions (heaps) pointed to by the szone's hashed_regions
- * pointer.
- *
- * Each region is laid out as a metadata array, followed by the heap, all within an 512MB block.
- * The array is arranged as an array of shorts, one for each MEDIUM_QUANTUM in the heap. There are
- * 16382 32k-blocks and the array is 16382*2 bytes, which totals 8387966, leaving 32,772b unused.
- *
- * The 16-bit shorts in the region are used for allocation metadata. The MSB bit marks a block as
- * either free, or not. The remaining 15-bits give the size of the allocation, defined in "msize", the
- * quantum-shifted size of the allocation.
- *
- * The metadata table either:
- *
- * 1. Stores the allocation size in the first short for the block, with the MSB cleared to indicate
- * that the block is allocated and in-use, or,
- *
- * 2. Stores the free-allocation size in the first and last shorts for the block, with the MSB set
- * in both places to indicate that the block is freed. (Storing the range in last block allows
- * for coalescing of adjacent free entries).
- *
- * 3. Zero, or "middle", meaning that this block in the region is not the start or end of an
- * allocated block.
- *
- * The medium zone represents the free list in one of two ways:
- *
- * 1. In-line free list entries. These are stored at the starting address of the just-freed memory
- * and both the previous and next pointer are checksummed to attempt to detect use-after-free
- * writes.
- *
- * An in-line free list entry is laid out as:
- * |prev (uintptr_t)|checksum (uint8_t)|next (uintptr_t)|checksum (uint8_t)
- *
- * 2. Out-of-band free list entries. These utilitise the remaining padding in the 8mb region that
- * follows the blocks, metadata and region trailer. Out-of-band entries are used *iff* the
- * freed address lies on a page boundary and the freed region spans more than a page. If we were
- * to store the free list entry in-line in that memory, it would keep the entire page dirty,
- * so an out-of-band entry is used.
- *
- * An out-of-band free list entry is laid out as:
- * |prev (uintptr_t)|next (uintptr_t)|ptr (uint16_t)|
- *
- * The szone maintains an array of 256 freelists, each of which is used to hold free objects
- * of the corresponding quantum size.
- */
-
-#define MEDIUM_IS_FREE (1 << 15)
-#define MEDIUM_IS_ADVISED (1 << 15)
-#define FOLLOWING_MEDIUM_PTR(ptr, msize) (((unsigned char *)(ptr)) + ((msize) << SHIFT_MEDIUM_QUANTUM))
-#define MEDIUM_MAX_MSIZE ((uint16_t)(NUM_MEDIUM_BLOCKS >> SHIFT_MEDIUM_QUANTUM) \
- & ~(uint16_t)MEDIUM_IS_FREE)
-
-// Ensure that the we don't overflow the number of blocks that msize can
-// represent (without running into the free bit).
-MALLOC_STATIC_ASSERT(NUM_MEDIUM_BLOCKS <= (uint16_t)(~MEDIUM_IS_FREE),
- "NUM_MEDIUM_BLOCKS should fit into a msize_t");
-
-/*
- * MEDIUM_IS_OOB is used mark to the MSB of OOB free list entries to show that they are in use, and
- * distinguish them from their initial, empty, state.
- */
-#define MEDIUM_IS_OOB (1 << 15)
-
-#define MEDIUM_ENTROPY_BITS 11
-#define MEDIUM_ENTROPY_MASK ((1 << MEDIUM_ENTROPY_BITS) - 1)
-
-/*
- * Avoid having so much entropy that the end of a valid medium allocation
- * might overrun the end of the medium region.
- */
-#if MEDIUM_ENTROPY_MASK + NUM_MEDIUM_SLOTS > NUM_MEDIUM_BLOCKS
-#error Too many entropy bits for medium region requested
-#endif
-
-#define MEDIUM_HEAP_SIZE (NUM_MEDIUM_BLOCKS * MEDIUM_QUANTUM)
-#define MEDIUM_METADATA_SIZE (sizeof(region_trailer_t) + \
- (NUM_MEDIUM_BLOCKS * sizeof(msize_t)) + \
- (NUM_MEDIUM_BLOCKS * sizeof(msize_t)))
-// Note: The other instances of x_REGION_SIZE use PAGE_MAX_SIZE as the rounding
-// and truncating constant but because medium's quanta size is larger than a
-// page, it's used instead.
-#define MEDIUM_REGION_SIZE ((MEDIUM_HEAP_SIZE + \
- MEDIUM_METADATA_SIZE + MEDIUM_QUANTUM - 1) & ~(MEDIUM_QUANTUM - 1))
-
-/*
- * Location of the metadata for a given medium region.
- */
-#define MEDIUM_REGION_METADATA(region) ((uintptr_t)&((medium_region_t)region)->trailer)
-
-/*
- * Beginning and end pointers for a region's heap.
- */
-#define MEDIUM_REGION_HEAP_BASE(region) ((void *)((medium_region_t)region)->blocks)
-#define MEDIUM_REGION_HEAP_END(region) (MEDIUM_REGION_HEAP_BASE(region) + MEDIUM_HEAP_SIZE)
-
-/*
- * Locate the heap base for a pointer known to be within a medium region.
- */
-#define MEDIUM_REGION_FOR_PTR(ptr) ((void *)((uintptr_t)(ptr) & ~((1ull << MEDIUM_BLOCKS_ALIGN) - 1)))
-#define MEDIUM_REGION_OFFSET_FOR_PTR(ptr) ((uintptr_t)(ptr) & ((1ull << MEDIUM_BLOCKS_ALIGN) - 1))
-
-/*
- * Convert between byte and msize units.
- */
-#define MEDIUM_BYTES_FOR_MSIZE(_m) ((uint32_t)(_m) << SHIFT_MEDIUM_QUANTUM)
-#define MEDIUM_MSIZE_FOR_BYTES(_b) ((_b) >> SHIFT_MEDIUM_QUANTUM)
-
-#define MEDIUM_PREVIOUS_MSIZE(ptr) (*MEDIUM_METADATA_FOR_PTR(ptr - 1) & ~MEDIUM_IS_FREE)
-
-/*
- * Convert from msize unit to free list slot.
- */
-#define MEDIUM_FREE_SLOT_COUNT(_r) (NUM_MEDIUM_SLOTS + 1)
-#define MEDIUM_FREE_SLOT_FOR_MSIZE(_r, _m) \
- (((_m) <= MEDIUM_FREE_SLOT_COUNT(_r)) ? ((_m) - 1) : (MEDIUM_FREE_SLOT_COUNT(_r) - 1))
-/* compare with MAGAZINE_FREELIST_BITMAP_WORDS */
-#define MEDIUM_FREELIST_BITMAP_WORDS(_r) ((MEDIUM_FREE_SLOT_COUNT(_r) + 31) >> 5)
-
-/*
- * Offset back to an szone_t given prior knowledge that this rack_t
- * is contained within an szone_t.
- *
- * Note: the only place this is used, the dtrace probes, only occurs
- * when the rack has been set up inside a scalable zone. Should
- * this ever be used somewhere that this does not hold true
- * (say, the test cases) then the pointer returned will be junk.
- */
-#define MEDIUM_SZONE_FROM_RACK(_r) \
- (szone_t *)((uintptr_t)(_r) - offsetof(struct szone_s, medium_rack))
-
-/*
- * Layout of a medium region
- */
-typedef uint32_t medium_block_t[MEDIUM_QUANTUM / sizeof(uint32_t)];
-MALLOC_STATIC_ASSERT(sizeof(medium_block_t) == MEDIUM_QUANTUM,
- "Incorrect size medium_block_t");
-#define MEDIUM_OOB_COUNT ((MEDIUM_REGION_SIZE - MEDIUM_HEAP_SIZE - \
- MEDIUM_METADATA_SIZE - sizeof(region_cookie_t)) / sizeof(oob_free_entry_s))
-#define MEDIUM_OOB_SIZE (MEDIUM_OOB_COUNT * sizeof(oob_free_entry_s))
-#define MEDIUM_REGION_PAD (MEDIUM_REGION_SIZE - MEDIUM_HEAP_SIZE - \
- MEDIUM_METADATA_SIZE - MEDIUM_OOB_SIZE - sizeof(region_cookie_t))
-
-typedef struct medium_region {
- // This must be first (because MEDIUM_REGION_METADATA assumes it).
- region_trailer_t trailer;
- msize_t medium_meta_words[NUM_MEDIUM_BLOCKS];
- msize_t medium_madvise_words[NUM_MEDIUM_BLOCKS];
- oob_free_entry_s medium_oob_free_entries[MEDIUM_OOB_COUNT];
- uint8_t pad[MEDIUM_REGION_PAD];
- region_cookie_t region_cookie;
- medium_block_t blocks[NUM_MEDIUM_BLOCKS];
-} * medium_region_t;
-
-// The layout described above should result in a medium_region_t being 512MB.
-MALLOC_STATIC_ASSERT(sizeof(struct medium_region) == 128 * 1024 * 1024,
- "incorrect medium_region_size");
-
-/*
- * Per-region meta data for medium allocator
- */
-#define REGION_TRAILER_FOR_MEDIUM_REGION(r) (&(((medium_region_t)(r))->trailer))
-#define REGION_COOKIE_FOR_MEDIUM_REGION(r) (((medium_region_t)(r))->region_cookie)
-#define MAGAZINE_INDEX_FOR_MEDIUM_REGION(r) (REGION_TRAILER_FOR_MEDIUM_REGION(r)->mag_index)
-#define BYTES_USED_FOR_MEDIUM_REGION(r) (REGION_TRAILER_FOR_MEDIUM_REGION(r)->bytes_used)
-
-/*
- * Locate the metadata base for a pointer known to be within a medium region.
- */
-#define MEDIUM_META_HEADER_FOR_PTR(ptr) (((medium_region_t)MEDIUM_REGION_FOR_PTR(ptr))->medium_meta_words)
-#define MEDIUM_MADVISE_HEADER_FOR_PTR(ptr) (((medium_region_t)MEDIUM_REGION_FOR_PTR(ptr))->medium_madvise_words)
-#define MEDIUM_META_HEADER_FOR_REGION(region) (((medium_region_t)region)->medium_meta_words)
-
-/*
- * Heap offset for a pointer known to be within a medium region.
- */
-#define MEDIUM_HEAP_OFFSET_FOR_PTR(ptr) ((uintptr_t)(ptr) - (uintptr_t)MEDIUM_REGION_HEAP_BASE(MEDIUM_REGION_FOR_PTR(ptr)))
-
-/*
- * Compute the metadata index for a pointer known to be within a medium region.
- */
-#define MEDIUM_META_INDEX_FOR_PTR(ptr) ((MEDIUM_HEAP_OFFSET_FOR_PTR(ptr) >> SHIFT_MEDIUM_QUANTUM) & (NUM_MEDIUM_CEIL_BLOCKS - 1))
-#define MEDIUM_PTR_FOR_META_INDEX(region, i) (MEDIUM_REGION_HEAP_BASE(region) + MEDIUM_BYTES_FOR_MSIZE(i))
-
-/*
- * Find the metadata word for a pointer known to be within a medium region.
- */
-#define MEDIUM_METADATA_FOR_PTR(ptr) (MEDIUM_META_HEADER_FOR_PTR(ptr) + MEDIUM_META_INDEX_FOR_PTR(ptr))
-
-/*
- * Determine whether a pointer known to be within a medium region points to memory which is free.
- */
-#define MEDIUM_PTR_IS_FREE(ptr) (*MEDIUM_METADATA_FOR_PTR(ptr) & MEDIUM_IS_FREE)
-
-/*
- * Extract the msize value for a pointer known to be within a medium region.
- */
-#define MEDIUM_PTR_SIZE(ptr) (*MEDIUM_METADATA_FOR_PTR(ptr) & ~MEDIUM_IS_FREE)
-
-#if !CONFIG_MEDIUM_CACHE
-#warning CONFIG_MEDIUM_CACHE turned off
-#endif
-
-#define MEDIUM_REGION_PAYLOAD_BYTES (NUM_MEDIUM_BLOCKS * MEDIUM_QUANTUM)
+#define SMALL_REGION_PAYLOAD_BYTES (NUM_SMALL_BLOCKS * SMALL_QUANTUM)
/************************* DEFINITIONS for large ****************************/
@@ -737,36 +414,26 @@
#warning CONFIG_LARGE_CACHE turned off
#endif
-#if CONFIG_MEDIUM_ALLOCATOR
-#define LARGE_THRESHOLD(szone) ((szone)->is_medium_engaged ? \
- (MEDIUM_LIMIT_THRESHOLD) : (SMALL_LIMIT_THRESHOLD))
-#else // CONFIG_MEDIUM_ALLOCATOR
-#define LARGE_THRESHOLD(szone) (SMALL_LIMIT_THRESHOLD)
-#endif // CONFIG_MEDIUM_ALLOCATOR
-
-// Gets the correct guard page flags for tiny/small/medium allocators.
-// The rules are:
-// 1. If MallocGuardEdges == "all" (which is indicated by MALLOC_GUARD_ALL being
-// set), we need to allocate just a postlude guard page in tiny/small/medium.
-// 2. If MallocGuardEdges is defined and has any value other than "all"
-// (indicated by MALLOC_GUARD_ALL being unset), we don't add any guard pages for
-// these allocators.
-//
-// This macro returns a copy of "flags" in which either the prelude guard page
-// bit or both guard page bits are turned off, depending on the value of the
-// MALLOC_GUARD_ALL bit. We can't simply keep the correct set of flags in the
-// zone or rack debug_flags field because the large allocator has different
-// rules (it allocates both guard pages when MallocGuardEdges is defined, and no
-// guard pages if it is not.)
-#define MALLOC_FIX_GUARD_PAGE_FLAGS(flags) \
- ((flags) & MALLOC_GUARD_ALL) ? \
- ((flags) & ~MALLOC_ADD_PRELUDE_GUARD_PAGE) \
- : (((flags) & ~MALLOC_ADD_GUARD_PAGE_FLAGS))
-
-// rdar://50715272 - allow us to have an escape hatch to disable ASLR sliding
-// on large allocatins for bincompat
-#define MALLOC_APPLY_LARGE_ASLR(flags) \
- (((flags) & DISABLE_LARGE_ASLR) ? ((flags) | DISABLE_ASLR) : (flags))
+
+/*******************************************************************************
+ * Definitions for region hash
+ ******************************************************************************/
+
+typedef void *region_t;
+typedef region_t *rgnhdl_t; /* A pointer into hashed_regions array. */
+
+#define INITIAL_NUM_REGIONS_SHIFT 6 // log2(INITIAL_NUM_REGIONS)
+#define INITIAL_NUM_REGIONS (1 << INITIAL_NUM_REGIONS_SHIFT) // Must be a power of 2!
+#define HASHRING_OPEN_ENTRY ((region_t)0) // Initial value and sentinel marking end of collision chain
+#define HASHRING_REGION_DEALLOCATED ((region_t)-1) // Region at this slot reclaimed by OS
+#define HASH_BLOCKS_ALIGN TINY_BLOCKS_ALIGN // MIN( TINY_BLOCKS_ALIGN, SMALL_BLOCKS_ALIGN, ... )
+
+typedef struct region_hash_generation {
+ size_t num_regions_allocated;
+ size_t num_regions_allocated_shift; // log2(num_regions_allocated)
+ region_t *hashed_regions; // hashed by location
+ struct region_hash_generation *nextgen;
+} region_hash_generation_t;
/*******************************************************************************
* Per-processor magazine for tiny and small allocators
@@ -779,15 +446,11 @@
volatile boolean_t alloc_underway;
// One element deep "death row", optimizes malloc/free/malloc for identical size.
- void *mag_last_free;
- msize_t mag_last_free_msize; // msize for mag_last_free
-#if MALLOC_TARGET_64BIT
- uint32_t _pad;
-#endif
+ void *mag_last_free; // low SHIFT_{TINY,SMALL}_QUANTUM bits indicate the msize
region_t mag_last_free_rgn; // holds the region for mag_last_free
- free_list_t mag_free_list[MAGAZINE_FREELIST_SLOTS];
- uint32_t mag_bitmap[MAGAZINE_FREELIST_BITMAP_WORDS];
+ free_list_t mag_free_list[256]; // assert( 256 >= MAX( NUM_TINY_SLOTS, NUM_SMALL_SLOTS_LARGEMEM ))
+ unsigned mag_bitmap[8]; // assert( sizeof(mag_bitmap) << 3 >= sizeof(mag_free_list)/sizeof(free_list_t) )
// the first and last free region in the last block are treated as big blocks in use that are not accounted for
size_t mag_bytes_free_at_end;
@@ -795,9 +458,9 @@
region_t mag_last_region; // Valid iff mag_bytes_free_at_end || mag_bytes_free_at_start > 0
// bean counting ...
+ unsigned mag_num_objects;
size_t mag_num_bytes_in_objects;
size_t num_bytes_in_magazine;
- unsigned mag_num_objects;
// recirculation list -- invariant: all regions owned by this magazine that meet the emptiness criteria
// are located nearer to the head of the list than any region that doesn't satisfy that criteria.
@@ -806,14 +469,7 @@
region_trailer_t *firstNode;
region_trailer_t *lastNode;
-#if MALLOC_TARGET_64BIT
- uintptr_t pad[320 - 14 - MAGAZINE_FREELIST_SLOTS -
- (MAGAZINE_FREELIST_BITMAP_WORDS + 1) / 2];
-#else
- uintptr_t pad[320 - 16 - MAGAZINE_FREELIST_SLOTS -
- MAGAZINE_FREELIST_BITMAP_WORDS];
-#endif
-
+ uintptr_t pad[50 - MALLOC_CACHE_LINE / sizeof(uintptr_t)];
} magazine_t;
#if MALLOC_TARGET_64BIT
@@ -822,12 +478,12 @@
MALLOC_STATIC_ASSERT(sizeof(magazine_t) == 1280, "Incorrect padding in magazine_t");
#endif
-#define TINY_MAX_MAGAZINES 64 /* MUST BE A POWER OF 2! */
+#define TINY_MAX_MAGAZINES 32 /* MUST BE A POWER OF 2! */
#define TINY_MAGAZINE_PAGED_SIZE \
(((sizeof(magazine_t) * (TINY_MAX_MAGAZINES + 1)) + vm_page_quanta_size - 1) & \
~(vm_page_quanta_size - 1)) /* + 1 for the Depot */
-#define SMALL_MAX_MAGAZINES 64 /* MUST BE A POWER OF 2! */
+#define SMALL_MAX_MAGAZINES 32 /* MUST BE A POWER OF 2! */
#define SMALL_MAGAZINE_PAGED_SIZE \
(((sizeof(magazine_t) * (SMALL_MAX_MAGAZINES + 1)) + vm_page_quanta_size - 1) & \
~(vm_page_quanta_size - 1)) /* + 1 for the Depot */
@@ -851,10 +507,35 @@
unsigned debug_flags;
void *log_address;
- /* Allocation racks per allocator type. */
- struct rack_s tiny_rack;
- struct rack_s small_rack;
- struct rack_s medium_rack;
+ /* Regions for tiny objects */
+ _malloc_lock_s tiny_regions_lock MALLOC_CACHE_ALIGN;
+ size_t num_tiny_regions;
+ size_t num_tiny_regions_dealloc;
+ region_hash_generation_t *tiny_region_generation;
+ region_hash_generation_t trg[2];
+
+ int num_tiny_magazines;
+ unsigned num_tiny_magazines_mask;
+ int num_tiny_magazines_mask_shift;
+ magazine_t *tiny_magazines; // array of per-processor magazines
+
+ uintptr_t last_tiny_advise;
+
+ /* Regions for small objects */
+ _malloc_lock_s small_regions_lock MALLOC_CACHE_ALIGN;
+ size_t num_small_regions;
+ size_t num_small_regions_dealloc;
+ region_hash_generation_t *small_region_generation;
+ region_hash_generation_t srg[2];
+
+ unsigned num_small_slots; // determined by physmem size
+
+ int num_small_magazines;
+ unsigned num_small_magazines_mask;
+ int num_small_magazines_mask_shift;
+ magazine_t *small_magazines; // array of per-processor magazines
+
+ uintptr_t last_small_advise;
/* large objects: all the rest */
_malloc_lock_s large_szone_lock MALLOC_CACHE_ALIGN; // One customer at a time for large
@@ -866,9 +547,7 @@
#if CONFIG_LARGE_CACHE
int large_entry_cache_oldest;
int large_entry_cache_newest;
- large_entry_t large_entry_cache[LARGE_ENTRY_CACHE_SIZE_HIGH]; // "death row" for large malloc/free
- int large_cache_depth;
- size_t large_cache_entry_limit;
+ large_entry_t large_entry_cache[LARGE_ENTRY_CACHE_SIZE]; // "death row" for large malloc/free
boolean_t large_legacy_reset_mprotect;
size_t large_entry_cache_reserve_bytes;
size_t large_entry_cache_reserve_limit;
@@ -877,10 +556,16 @@
/* flag and limits pertaining to altered malloc behavior for systems with
* large amounts of physical memory */
- bool is_medium_engaged;
+ unsigned is_largemem;
+ unsigned large_threshold;
+ unsigned vm_copy_threshold;
/* security cookie */
uintptr_t cookie;
+
+ /* Initial region list */
+ region_t initial_tiny_regions[INITIAL_NUM_REGIONS];
+ region_t initial_small_regions[INITIAL_NUM_REGIONS];
/* The purgeable zone constructed by create_purgeable_zone() would like to hand off tiny and small
* allocations to the default scalable zone. Record the latter as the "helper" zone here. */