Loading...
--- libmalloc/libmalloc-166.251.2/src/magazine_zone.h
+++ libmalloc/libmalloc-166.200.60/src/magazine_zone.h
@@ -70,14 +70,8 @@
inplace_linkage_s next;
} small_inplace_free_entry_s, *small_inplace_free_entry_t;
-typedef struct _medium_inplace_free_entry_s {
- inplace_linkage_s previous;
- inplace_linkage_s next;
-} medium_inplace_free_entry_s, *medium_inplace_free_entry_t;
-
typedef union {
small_inplace_free_entry_t small_inplace;
- medium_inplace_free_entry_t medium_inplace;
inplace_free_entry_t inplace;
oob_free_entry_t oob;
void *p;
@@ -270,13 +264,13 @@
/********************* DEFINITIONS for small ************************/
/*
- * Memory in the small range is allocated from regions (heaps) pointed to by the szone's hashed_regions
+ * Memory in the Small range is allocated from regions (heaps) pointed to by the szone's hashed_regions
* pointer.
*
* Each region is laid out as a heap, followed by the metadata array, all within an 8MB (2^23) block.
- * The array is arranged as an array of shorts, one for each SMALL_QUANTUM in the heap. There are
- * 16319 512-blocks and the array is 16319*2 bytes, which totals 8387966, leaving 642 bytes unused.
- * Once the region trailer is accounted for, there is room for 61 out-of-band free list entries in
+ * The array is arranged as an array of shorts, one for each SMALL_QUANTUM in the heap. There are
+ * 16319 512-blocks and the array is 16319*2 bytes, which totals 8387966, leaving 642 bytes unused.
+ * Once the region trailer is accounted for, there is room for 61 out-of-band free list entries in
* the remaining padding (or 6, if the region was split into 16320 blocks, not 16319).
*
* The 16-bit shorts in the region are used for allocation metadata. The MSB bit marks a block as
@@ -307,7 +301,7 @@
* 2. Out-of-band free list entries. These utilitise the remaining padding in the 8mb region that
* follows the blocks, metadata and region trailer. Out-of-band entries are used *iff* the
* freed address lies on a page boundary and the freed region spans more than a page. If we were
- * to store the free list entry in-line in that memory, it would keep the entire page dirty,
+ * to store the free list entry in-line in that memory, it would keep the entire page dirty,
* so an out-of-band entry is used.
*
* An out-of-band free list entry is laid out as:
@@ -321,7 +315,7 @@
#define FOLLOWING_SMALL_PTR(ptr, msize) (((unsigned char *)(ptr)) + ((msize) << SHIFT_SMALL_QUANTUM))
/*
- * SMALL_IS_OOB is used mark to the MSB of OOB free list entries to show that they are in use, and
+ * SMALL_IS_OOB is used mark to the MSB of OOB free list entries to show that they are in use, and
* distinguish them from their initial, empty, state.
*/
#define SMALL_IS_OOB (1 << 15)
@@ -366,7 +360,8 @@
* Convert from msize unit to free list slot.
*/
#define SMALL_FREE_SLOT_COUNT(_r) \
- (NUM_SMALL_SLOTS + 1)
+ (((_r)->debug_flags & MALLOC_EXTENDED_SMALL_SLOTS) ? \
+ NUM_SMALL_SLOTS_LARGEMEM + 1 : NUM_SMALL_SLOTS + 1)
#define SMALL_FREE_SLOT_FOR_MSIZE(_r, _m) \
(((_m) <= SMALL_FREE_SLOT_COUNT(_r)) ? ((_m) - 1) : (SMALL_FREE_SLOT_COUNT(_r) - 1))
/* compare with MAGAZINE_FREELIST_BITMAP_WORDS */
@@ -442,198 +437,6 @@
#define SMALL_REGION_PAYLOAD_BYTES (NUM_SMALL_BLOCKS * SMALL_QUANTUM)
-/********************* DEFINITIONS for medium ************************/
-
-/*
- * Memory in the medium range is allocated from regions (heaps) pointed to by the szone's hashed_regions
- * pointer.
- *
- * Each region is laid out as a heap, followed by the metadata array, all within an 512MB block.
- * The array is arranged as an array of shorts, one for each MEDIUM_QUANTUM in the heap. There are
- * 16382 32k-blocks and the array is 16382*2 bytes, which totals 8387966, leaving 32,772b unused.
- *
- * The 16-bit shorts in the region are used for allocation metadata. The MSB bit marks a block as
- * either free, or not. The remaining 15-bits give the size of the allocation, defined in "msize", the
- * quantum-shifted size of the allocation.
- *
- * The metadata table either:
- *
- * 1. Stores the allocation size in the first short for the block, with the MSB cleared to indicate
- * that the block is allocated and in-use, or,
- *
- * 2. Stores the free-allocation size in the first and last shorts for the block, with the MSB set
- * in both places to indicate that the block is freed. (Storing the range in last block allows
- * for coalescing of adjacent free entries).
- *
- * 3. Zero, or "middle", meaning that this block in the region is not the start or end of an
- * allocated block.
- *
- * The medium zone represents the free list in one of two ways:
- *
- * 1. In-line free list entries. These are stored at the starting address of the just-freed memory
- * and both the previous and next pointer are checksummed to attempt to detect use-after-free
- * writes.
- *
- * An in-line free list entry is laid out as:
- * |prev (uintptr_t)|checksum (uint8_t)|next (uintptr_t)|checksum (uint8_t)
- *
- * 2. Out-of-band free list entries. These utilitise the remaining padding in the 8mb region that
- * follows the blocks, metadata and region trailer. Out-of-band entries are used *iff* the
- * freed address lies on a page boundary and the freed region spans more than a page. If we were
- * to store the free list entry in-line in that memory, it would keep the entire page dirty,
- * so an out-of-band entry is used.
- *
- * An out-of-band free list entry is laid out as:
- * |prev (uintptr_t)|next (uintptr_t)|ptr (uint16_t)|
- *
- * The szone maintains an array of 256 freelists, each of which is used to hold free objects
- * of the corresponding quantum size.
- */
-
-#define MEDIUM_IS_FREE (1 << 15)
-#define MEDIUM_IS_ADVISED (1 << 15)
-#define FOLLOWING_MEDIUM_PTR(ptr, msize) (((unsigned char *)(ptr)) + ((msize) << SHIFT_MEDIUM_QUANTUM))
-#define MEDIUM_MAX_MSIZE ((uint16_t)(NUM_MEDIUM_BLOCKS >> SHIFT_MEDIUM_QUANTUM) \
- & ~(uint16_t)MEDIUM_IS_FREE)
-
-// Ensure that the we don't overflow the number of blocks that msize can
-// represent (without running into the free bit).
-MALLOC_STATIC_ASSERT(NUM_MEDIUM_BLOCKS <= (uint16_t)(~MEDIUM_IS_FREE),
- "NUM_MEDIUM_BLOCKS should fit into a msize_t");
-
-/*
- * MEDIUM_IS_OOB is used mark to the MSB of OOB free list entries to show that they are in use, and
- * distinguish them from their initial, empty, state.
- */
-#define MEDIUM_IS_OOB (1 << 15)
-
-#define MEDIUM_ENTROPY_BITS 13
-#define MEDIUM_ENTROPY_MASK ((1 << MEDIUM_ENTROPY_BITS) - 1)
-
-/*
- * Avoid having so much entropy that the end of a valid medium allocation
- * might overrun the end of the medium region.
- */
-#if MEDIUM_ENTROPY_MASK + NUM_MEDIUM_SLOTS > NUM_MEDIUM_BLOCKS
-#error Too many entropy bits for medium region requested
-#endif
-
-#define MEDIUM_METADATA_SIZE (sizeof(region_trailer_t) + \
- (NUM_MEDIUM_BLOCKS * sizeof(msize_t)) + \
- (NUM_MEDIUM_BLOCKS * sizeof(msize_t)))
-// Note: The other instances of x_REGION_SIZE use PAGE_MAX_SIZE as the rounding
-// and truncating constant but because medium's quanta size is larger than a
-// page, it's used instead.
-#define MEDIUM_REGION_SIZE ((NUM_MEDIUM_BLOCKS * MEDIUM_QUANTUM + \
- MEDIUM_METADATA_SIZE + MEDIUM_QUANTUM - 1) & ~(MEDIUM_QUANTUM - 1))
-
-#define MEDIUM_METADATA_START (NUM_MEDIUM_BLOCKS * MEDIUM_QUANTUM)
-
-/*
- * Beginning and end pointers for a region's heap.
- */
-#define MEDIUM_REGION_ADDRESS(region) ((unsigned char *)region)
-#define MEDIUM_REGION_END(region) (MEDIUM_REGION_ADDRESS(region) + (NUM_MEDIUM_BLOCKS * MEDIUM_QUANTUM))
-
-/*
- * Locate the heap base for a pointer known to be within a medium region.
- */
-#define MEDIUM_REGION_FOR_PTR(_p) ((void *)((uintptr_t)(_p) & ~((1ull << MEDIUM_BLOCKS_ALIGN) - 1)))
-#define MEDIUM_OFFSET_FOR_PTR(_p) ((uintptr_t)(_p) & ((1ull << MEDIUM_BLOCKS_ALIGN) - 1))
-
-/*
- * Convert between byte and msize units.
- */
-#define MEDIUM_BYTES_FOR_MSIZE(_m) ((uint32_t)(_m) << SHIFT_MEDIUM_QUANTUM)
-#define MEDIUM_MSIZE_FOR_BYTES(_b) ((_b) >> SHIFT_MEDIUM_QUANTUM)
-
-#define MEDIUM_PREVIOUS_MSIZE(ptr) (*MEDIUM_METADATA_FOR_PTR(ptr - 1) & ~MEDIUM_IS_FREE)
-
-/*
- * Convert from msize unit to free list slot.
- */
-#define MEDIUM_FREE_SLOT_COUNT(_r) (NUM_MEDIUM_SLOTS + 1)
-#define MEDIUM_FREE_SLOT_FOR_MSIZE(_r, _m) \
- (((_m) <= MEDIUM_FREE_SLOT_COUNT(_r)) ? ((_m) - 1) : (MEDIUM_FREE_SLOT_COUNT(_r) - 1))
-/* compare with MAGAZINE_FREELIST_BITMAP_WORDS */
-#define MEDIUM_FREELIST_BITMAP_WORDS(_r) ((MEDIUM_FREE_SLOT_COUNT(_r) + 31) >> 5)
-
-/*
- * Offset back to an szone_t given prior knowledge that this rack_t
- * is contained within an szone_t.
- *
- * Note: the only place this is used, the dtrace probes, only occurs
- * when the rack has been set up inside a scalable zone. Should
- * this ever be used somewhere that this does not hold true
- * (say, the test cases) then the pointer returned will be junk.
- */
-#define MEDIUM_SZONE_FROM_RACK(_r) \
- (szone_t *)((uintptr_t)(_r) - offsetof(struct szone_s, medium_rack))
-
-/*
- * Layout of a medium region
- */
-typedef uint32_t medium_block_t[MEDIUM_QUANTUM / sizeof(uint32_t)];
-#define MEDIUM_HEAP_SIZE (NUM_MEDIUM_BLOCKS * sizeof(medium_block_t))
-#define MEDIUM_OOB_COUNT ((MEDIUM_REGION_SIZE - MEDIUM_HEAP_SIZE - \
- MEDIUM_METADATA_SIZE) / sizeof(oob_free_entry_s))
-#define MEDIUM_OOB_SIZE (MEDIUM_OOB_COUNT * sizeof(oob_free_entry_s))
-#define MEDIUM_REGION_PAD (MEDIUM_REGION_SIZE - MEDIUM_HEAP_SIZE - \
- MEDIUM_METADATA_SIZE - MEDIUM_OOB_SIZE)
-
-typedef struct medium_region {
- medium_block_t blocks[NUM_MEDIUM_BLOCKS];
- region_trailer_t trailer;
- msize_t medium_meta_words[NUM_MEDIUM_BLOCKS];
- msize_t medium_madvise_words[NUM_MEDIUM_BLOCKS];
- oob_free_entry_s medium_oob_free_entries[MEDIUM_OOB_COUNT];
- uint8_t pad[MEDIUM_REGION_PAD];
-} * medium_region_t;
-
-// The layout described above should result in a medium_region_t being 512MB.
-MALLOC_STATIC_ASSERT(sizeof(struct medium_region) == 512 * 1024 * 1024,
- "incorrect medium_region_size");
-
-/*
- * Per-region meta data for medium allocator
- */
-#define REGION_TRAILER_FOR_MEDIUM_REGION(r) (&(((medium_region_t)(r))->trailer))
-#define MAGAZINE_INDEX_FOR_MEDIUM_REGION(r) (REGION_TRAILER_FOR_MEDIUM_REGION(r)->mag_index)
-#define BYTES_USED_FOR_MEDIUM_REGION(r) (REGION_TRAILER_FOR_MEDIUM_REGION(r)->bytes_used)
-
-/*
- * Locate the metadata base for a pointer known to be within a medium region.
- */
-#define MEDIUM_META_HEADER_FOR_PTR(_p) (((medium_region_t)MEDIUM_REGION_FOR_PTR(_p))->medium_meta_words)
-#define MEDIUM_MADVISE_HEADER_FOR_PTR(_p) (((medium_region_t)MEDIUM_REGION_FOR_PTR(_p))->medium_madvise_words)
-
-/*
- * Compute the metadata index for a pointer known to be within a medium region.
- */
-#define MEDIUM_META_INDEX_FOR_PTR(_p) (((uintptr_t)(_p) >> SHIFT_MEDIUM_QUANTUM) & (NUM_MEDIUM_CEIL_BLOCKS - 1))
-#define MEDIUM_PTR_FOR_META_INDEX(_region, _i) ((uintptr_t)(_region) + MEDIUM_BYTES_FOR_MSIZE(_i))
-
-/*
- * Find the metadata word for a pointer known to be within a medium region.
- */
-#define MEDIUM_METADATA_FOR_PTR(_p) (MEDIUM_META_HEADER_FOR_PTR(_p) + MEDIUM_META_INDEX_FOR_PTR(_p))
-
-/*
- * Determine whether a pointer known to be within a medium region points to memory which is free.
- */
-#define MEDIUM_PTR_IS_FREE(_p) (*MEDIUM_METADATA_FOR_PTR(_p) & MEDIUM_IS_FREE)
-
-/*
- * Extract the msize value for a pointer known to be within a medium region.
- */
-#define MEDIUM_PTR_SIZE(_p) (*MEDIUM_METADATA_FOR_PTR(_p) & ~MEDIUM_IS_FREE)
-
-#if !CONFIG_MEDIUM_CACHE
-#warning CONFIG_MEDIUM_CACHE turned off
-#endif
-
-#define MEDIUM_REGION_PAYLOAD_BYTES (NUM_MEDIUM_BLOCKS * MEDIUM_QUANTUM)
-
/************************* DEFINITIONS for large ****************************/
@@ -646,14 +449,6 @@
#if !CONFIG_LARGE_CACHE && DEBUG_MALLOC
#warning CONFIG_LARGE_CACHE turned off
#endif
-
-#if CONFIG_MEDIUM_ALLOCATOR
-#define LARGE_THRESHOLD(szone) ((szone)->is_medium_engaged ? \
- (MEDIUM_LIMIT_THRESHOLD) : (SMALL_LIMIT_THRESHOLD))
-#else // CONFIG_MEDIUM_ALLOCATOR
-#define LARGE_THRESHOLD(szone) (SMALL_LIMIT_THRESHOLD)
-#endif // CONFIG_MEDIUM_ALLOCATOR
-
/*******************************************************************************
* Per-processor magazine for tiny and small allocators
@@ -741,7 +536,6 @@
/* Allocation racks per allocator type. */
struct rack_s tiny_rack;
struct rack_s small_rack;
- struct rack_s medium_rack;
/* large objects: all the rest */
_malloc_lock_s large_szone_lock MALLOC_CACHE_ALIGN; // One customer at a time for large
@@ -762,7 +556,9 @@
/* flag and limits pertaining to altered malloc behavior for systems with
* large amounts of physical memory */
- bool is_medium_engaged;
+ unsigned is_largemem;
+ unsigned large_threshold;
+ unsigned vm_copy_threshold;
/* security cookie */
uintptr_t cookie;