Loading...
--- libmalloc/libmalloc-166.200.60/src/magazine_zone.h
+++ libmalloc/libmalloc-116/src/magazine_zone.h
@@ -242,19 +242,6 @@
*/
#define TINY_INDEX_FOR_PTR(_p) (((uintptr_t)(_p) >> SHIFT_TINY_QUANTUM) & (NUM_TINY_CEIL_BLOCKS - 1))
-/*
- * Offset back to an szone_t given prior knowledge that this rack_t
- * is contained within an szone_t.
- *
- * Note: the only place this is used, the dtrace probes, only occurs
- * when the rack has been set up inside a scalable zone. Should
- * this ever be used somewhere that this does not hold true
- * (say, the test cases) then the pointer returned will be junk.
- */
-#define TINY_SZONE_FROM_RACK(_r) \
- (szone_t *)((uintptr_t)(_r) - offsetof(struct szone_s, tiny_rack))
-
-
#if !CONFIG_TINY_CACHE
#warning CONFIG_TINY_CACHE turned off
#endif
@@ -351,33 +338,10 @@
/*
* Convert between byte and msize units.
*/
-#define SMALL_BYTES_FOR_MSIZE(_m) ((uint32_t)(_m) << SHIFT_SMALL_QUANTUM)
+#define SMALL_BYTES_FOR_MSIZE(_m) ((_m) << SHIFT_SMALL_QUANTUM)
#define SMALL_MSIZE_FOR_BYTES(_b) ((_b) >> SHIFT_SMALL_QUANTUM)
#define SMALL_PREVIOUS_MSIZE(ptr) (*SMALL_METADATA_FOR_PTR(ptr - 1) & ~SMALL_IS_FREE)
-
-/*
- * Convert from msize unit to free list slot.
- */
-#define SMALL_FREE_SLOT_COUNT(_r) \
- (((_r)->debug_flags & MALLOC_EXTENDED_SMALL_SLOTS) ? \
- NUM_SMALL_SLOTS_LARGEMEM + 1 : NUM_SMALL_SLOTS + 1)
-#define SMALL_FREE_SLOT_FOR_MSIZE(_r, _m) \
- (((_m) <= SMALL_FREE_SLOT_COUNT(_r)) ? ((_m) - 1) : (SMALL_FREE_SLOT_COUNT(_r) - 1))
-/* compare with MAGAZINE_FREELIST_BITMAP_WORDS */
-#define SMALL_FREELIST_BITMAP_WORDS(_r) ((SMALL_FREE_SLOT_COUNT(_r) + 31) >> 5)
-
-/*
- * Offset back to an szone_t given prior knowledge that this rack_t
- * is contained within an szone_t.
- *
- * Note: the only place this is used, the dtrace probes, only occurs
- * when the rack has been set up inside a scalable zone. Should
- * this ever be used somewhere that this does not hold true
- * (say, the test cases) then the pointer returned will be junk.
- */
-#define SMALL_SZONE_FROM_RACK(_r) \
- (szone_t *)((uintptr_t)(_r) - offsetof(struct szone_s, small_rack))
/*
* Layout of a small region
@@ -450,6 +414,27 @@
#warning CONFIG_LARGE_CACHE turned off
#endif
+
+/*******************************************************************************
+ * Definitions for region hash
+ ******************************************************************************/
+
+typedef void *region_t;
+typedef region_t *rgnhdl_t; /* A pointer into hashed_regions array. */
+
+#define INITIAL_NUM_REGIONS_SHIFT 6 // log2(INITIAL_NUM_REGIONS)
+#define INITIAL_NUM_REGIONS (1 << INITIAL_NUM_REGIONS_SHIFT) // Must be a power of 2!
+#define HASHRING_OPEN_ENTRY ((region_t)0) // Initial value and sentinel marking end of collision chain
+#define HASHRING_REGION_DEALLOCATED ((region_t)-1) // Region at this slot reclaimed by OS
+#define HASH_BLOCKS_ALIGN TINY_BLOCKS_ALIGN // MIN( TINY_BLOCKS_ALIGN, SMALL_BLOCKS_ALIGN, ... )
+
+typedef struct region_hash_generation {
+ size_t num_regions_allocated;
+ size_t num_regions_allocated_shift; // log2(num_regions_allocated)
+ region_t *hashed_regions; // hashed by location
+ struct region_hash_generation *nextgen;
+} region_hash_generation_t;
+
/*******************************************************************************
* Per-processor magazine for tiny and small allocators
******************************************************************************/
@@ -461,15 +446,11 @@
volatile boolean_t alloc_underway;
// One element deep "death row", optimizes malloc/free/malloc for identical size.
- void *mag_last_free;
- msize_t mag_last_free_msize; // msize for mag_last_free
-#if MALLOC_TARGET_64BIT
- uint32_t _pad;
-#endif
+ void *mag_last_free; // low SHIFT_{TINY,SMALL}_QUANTUM bits indicate the msize
region_t mag_last_free_rgn; // holds the region for mag_last_free
- free_list_t mag_free_list[MAGAZINE_FREELIST_SLOTS];
- uint32_t mag_bitmap[MAGAZINE_FREELIST_BITMAP_WORDS];
+ free_list_t mag_free_list[256]; // assert( 256 >= MAX( NUM_TINY_SLOTS, NUM_SMALL_SLOTS_LARGEMEM ))
+ unsigned mag_bitmap[8]; // assert( sizeof(mag_bitmap) << 3 >= sizeof(mag_free_list)/sizeof(free_list_t) )
// the first and last free region in the last block are treated as big blocks in use that are not accounted for
size_t mag_bytes_free_at_end;
@@ -477,9 +458,9 @@
region_t mag_last_region; // Valid iff mag_bytes_free_at_end || mag_bytes_free_at_start > 0
// bean counting ...
+ unsigned mag_num_objects;
size_t mag_num_bytes_in_objects;
size_t num_bytes_in_magazine;
- unsigned mag_num_objects;
// recirculation list -- invariant: all regions owned by this magazine that meet the emptiness criteria
// are located nearer to the head of the list than any region that doesn't satisfy that criteria.
@@ -488,14 +469,7 @@
region_trailer_t *firstNode;
region_trailer_t *lastNode;
-#if MALLOC_TARGET_64BIT
- uintptr_t pad[320 - 14 - MAGAZINE_FREELIST_SLOTS -
- (MAGAZINE_FREELIST_BITMAP_WORDS + 1) / 2];
-#else
- uintptr_t pad[320 - 16 - MAGAZINE_FREELIST_SLOTS -
- MAGAZINE_FREELIST_BITMAP_WORDS];
-#endif
-
+ uintptr_t pad[50 - MALLOC_CACHE_LINE / sizeof(uintptr_t)];
} magazine_t;
#if MALLOC_TARGET_64BIT
@@ -504,12 +478,12 @@
MALLOC_STATIC_ASSERT(sizeof(magazine_t) == 1280, "Incorrect padding in magazine_t");
#endif
-#define TINY_MAX_MAGAZINES 64 /* MUST BE A POWER OF 2! */
+#define TINY_MAX_MAGAZINES 32 /* MUST BE A POWER OF 2! */
#define TINY_MAGAZINE_PAGED_SIZE \
(((sizeof(magazine_t) * (TINY_MAX_MAGAZINES + 1)) + vm_page_quanta_size - 1) & \
~(vm_page_quanta_size - 1)) /* + 1 for the Depot */
-#define SMALL_MAX_MAGAZINES 64 /* MUST BE A POWER OF 2! */
+#define SMALL_MAX_MAGAZINES 32 /* MUST BE A POWER OF 2! */
#define SMALL_MAGAZINE_PAGED_SIZE \
(((sizeof(magazine_t) * (SMALL_MAX_MAGAZINES + 1)) + vm_page_quanta_size - 1) & \
~(vm_page_quanta_size - 1)) /* + 1 for the Depot */
@@ -533,9 +507,35 @@
unsigned debug_flags;
void *log_address;
- /* Allocation racks per allocator type. */
- struct rack_s tiny_rack;
- struct rack_s small_rack;
+ /* Regions for tiny objects */
+ _malloc_lock_s tiny_regions_lock MALLOC_CACHE_ALIGN;
+ size_t num_tiny_regions;
+ size_t num_tiny_regions_dealloc;
+ region_hash_generation_t *tiny_region_generation;
+ region_hash_generation_t trg[2];
+
+ int num_tiny_magazines;
+ unsigned num_tiny_magazines_mask;
+ int num_tiny_magazines_mask_shift;
+ magazine_t *tiny_magazines; // array of per-processor magazines
+
+ uintptr_t last_tiny_advise;
+
+ /* Regions for small objects */
+ _malloc_lock_s small_regions_lock MALLOC_CACHE_ALIGN;
+ size_t num_small_regions;
+ size_t num_small_regions_dealloc;
+ region_hash_generation_t *small_region_generation;
+ region_hash_generation_t srg[2];
+
+ unsigned num_small_slots; // determined by physmem size
+
+ int num_small_magazines;
+ unsigned num_small_magazines_mask;
+ int num_small_magazines_mask_shift;
+ magazine_t *small_magazines; // array of per-processor magazines
+
+ uintptr_t last_small_advise;
/* large objects: all the rest */
_malloc_lock_s large_szone_lock MALLOC_CACHE_ALIGN; // One customer at a time for large
@@ -563,6 +563,10 @@
/* security cookie */
uintptr_t cookie;
+ /* Initial region list */
+ region_t initial_tiny_regions[INITIAL_NUM_REGIONS];
+ region_t initial_small_regions[INITIAL_NUM_REGIONS];
+
/* The purgeable zone constructed by create_purgeable_zone() would like to hand off tiny and small
* allocations to the default scalable zone. Record the latter as the "helper" zone here. */
struct szone_s *helper_zone;