Loading...
--- libmalloc/libmalloc-792.1.1/src/magazine_medium.c
+++ libmalloc/libmalloc-317.40.8/src/magazine_medium.c
@@ -32,29 +32,15 @@
static MALLOC_INLINE uint64_t
medium_sliding_madvise_granularity(magazine_t *magazine)
{
- uint64_t granularity = MEDIUM_MADVISE_MIN;
// Use a sliding madvise granularity based on how many bytes the region
// currently has allocated. This way we will advise at a finer granularity
// as the region becomes more and more empty.
// region_trailer_t *t = REGION_TRAILER_FOR_MEDIUM_REGION(region);
- if (magazine->mag_num_bytes_in_objects > 0) {
- if (magazine_medium_madvise_window_scale_factor == 1) {
- // NB: This code has a bug causing undefined behavior whenever the result of clzl is > 32
- // because it's shifting a signed integer. It should be 1ULL << (64 - ...) but
- // fixing this bug will introduce memory regressions, so for now we've only fixed it on very
- // large memory configs that are scaling up their window anyways.
- // See rdar://82128925 for details
- granularity = MAX(granularity, 1 << (64 -
- __builtin_clzl(magazine->mag_num_bytes_in_objects >> MEDIUM_MADVISE_SHIFT)));
- } else {
- granularity = MAX(granularity, 1ULL << (64 -
- __builtin_clzl(magazine->mag_num_bytes_in_objects >> MEDIUM_MADVISE_SHIFT)));
- if (os_mul_overflow(granularity, magazine_medium_madvise_window_scale_factor, &granularity)) {
- return UINT64_MAX;
- }
- }
- }
- return granularity;
+ if (magazine->mag_num_bytes_in_objects == 0) {
+ return MEDIUM_MADVISE_MIN;
+ }
+ return MAX(MEDIUM_MADVISE_MIN, 1 << (64 -
+ __builtin_clzl(magazine->mag_num_bytes_in_objects >> MEDIUM_MADVISE_SHIFT)));
}
static MALLOC_INLINE void
@@ -130,6 +116,25 @@
medium_meta_header_set_middle(msize_t *meta_headers, msize_t index)
{
meta_headers[index] = 0;
+}
+
+static MALLOC_INLINE MALLOC_ALWAYS_INLINE
+mag_index_t
+medium_mag_get_thread_index(void)
+{
+#if CONFIG_MEDIUM_USES_HYPER_SHIFT
+ if (os_likely(_os_cpu_number_override == -1)) {
+ return _os_cpu_number() >> hyper_shift;
+ } else {
+ return _os_cpu_number_override >> hyper_shift;
+ }
+#else // CONFIG_MEDIUM_USES_HYPER_SHIFT
+ if (os_likely(_os_cpu_number_override == -1)) {
+ return _os_cpu_number();
+ } else {
+ return _os_cpu_number_override;
+ }
+#endif // CONFIG_MEDIUM_USES_HYPER_SHIFT
}
#pragma mark in-place free list
@@ -861,7 +866,6 @@
medium_advisory_t mat = (medium_advisory_t)pgLo;
mat->next = advisories;
mat->size = pgHi - pgLo;
- advisories = mat;
}
break;
}
@@ -889,7 +893,6 @@
medium_advisory_t mat = (medium_advisory_t)pgLo;
mat->next = advisories;
mat->size = pgHi - pgLo;
- advisories = mat;
}
memset(&madv_headers[index], 0, sizeof(uint16_t) * alloc_msize);
@@ -990,6 +993,11 @@
{
magazine_t *depot_ptr = &(rack->magazines[DEPOT_MAGAZINE_INDEX]);
+ /* FIXME: Would Uniprocessor benefit from recirc and MADV_FREE? */
+ if (rack->num_magazines == 1) { // Uniprocessor, single magazine, so no recirculation necessary
+ return 0;
+ }
+
#if DEBUG_MALLOC
if (DEPOT_MAGAZINE_INDEX == mag_index) {
malloc_zone_error(rack->debug_flags, true, "medium_get_region_from_depot called for magazine index -1\n", NULL, NULL);
@@ -1002,32 +1010,22 @@
// Appropriate a Depot'd region that can satisfy requested msize.
region_trailer_t *node;
region_t sparse_region;
- msize_t try_msize = msize;
while (1) {
- sparse_region = medium_find_msize_region(rack, depot_ptr, DEPOT_MAGAZINE_INDEX, try_msize);
+ sparse_region = medium_find_msize_region(rack, depot_ptr, DEPOT_MAGAZINE_INDEX, msize);
if (NULL == sparse_region) { // Depot empty?
SZONE_MAGAZINE_PTR_UNLOCK(depot_ptr);
return 0;
}
node = REGION_TRAILER_FOR_MEDIUM_REGION(sparse_region);
- if (0 == node->pinned_to_depot) {
- // Found one!
+ if (0 >= node->pinned_to_depot) {
break;
}
- // Try the next msize up - maybe the head of its free list will be in
- // a region we can use. Once we get the region we'll still allocate the
- // original msize.
- try_msize++;
-
- if (try_msize > NUM_MEDIUM_SLOTS) {
- // Tried all the msizes but couldn't get a usable region. Let's
- // give up for now and we'll allocate a new region from the kernel.
- SZONE_MAGAZINE_PTR_UNLOCK(depot_ptr);
- return 0;
- }
+ SZONE_MAGAZINE_PTR_UNLOCK(depot_ptr);
+ yield();
+ SZONE_MAGAZINE_PTR_LOCK(depot_ptr);
}
// disconnect node from Depot
@@ -1038,7 +1036,7 @@
// Transfer ownership of the region
MAGAZINE_INDEX_FOR_MEDIUM_REGION(sparse_region) = mag_index;
- MALLOC_ASSERT(node->pinned_to_depot == 0);
+ node->pinned_to_depot = 0;
// Iterate the region putting its free entries on its new (locked) magazine's free list
size_t bytes_inplay = medium_free_reattach_region(rack, medium_mag_ptr, sparse_region);
@@ -1072,31 +1070,19 @@
for (mag_index = 0; mag_index < rack->num_magazines; mag_index++) {
size_t index;
for (index = 0; index < rack->region_generation->num_regions_allocated; ++index) {
- rack_region_lock(rack);
+ SZONE_LOCK(MEDIUM_SZONE_FROM_RACK(rack));
region_t medium = rack->region_generation->hashed_regions[index];
if (!medium || medium == HASHRING_REGION_DEALLOCATED) {
- rack_region_unlock(rack);
+ SZONE_UNLOCK(MEDIUM_SZONE_FROM_RACK(rack));
continue;
}
- region_trailer_t *trailer =
- REGION_TRAILER_FOR_MEDIUM_REGION(medium);
- // Make sure that the owning magazine doesn't try and take this out
- // from under our feet.
- trailer->dispose_flags |= RACK_DISPOSE_DELAY;
- rack_region_unlock(rack);
-
magazine_t *mag_ptr = mag_lock_zine_for_region_trailer(rack->magazines,
- trailer, MAGAZINE_INDEX_FOR_MEDIUM_REGION(medium));
-
- // If acquiring the region lock was enough to prevent the owning
- // magazine from deallocating the region, free it now so we don't
- // do wasted work.
- if (rack_region_maybe_dispose(rack, medium, MEDIUM_REGION_SIZE, trailer)) {
- SZONE_MAGAZINE_PTR_UNLOCK(mag_ptr);
- continue;
- }
+ REGION_TRAILER_FOR_MEDIUM_REGION(medium),
+ MAGAZINE_INDEX_FOR_MEDIUM_REGION(medium));
+
+ SZONE_UNLOCK(MEDIUM_SZONE_FROM_RACK(rack));
/* Ordering is important here, the magazine of a region may potentially change
* during mag_lock_zine_for_region_trailer, so src_mag_index must be taken
@@ -1129,7 +1115,7 @@
SZONE_MAGAZINE_PTR_LOCK(medium_depot_ptr);
MAGAZINE_INDEX_FOR_MEDIUM_REGION(medium) = DEPOT_MAGAZINE_INDEX;
- MALLOC_ASSERT(REGION_TRAILER_FOR_MEDIUM_REGION(medium)->pinned_to_depot == 0);
+ REGION_TRAILER_FOR_MEDIUM_REGION(medium)->pinned_to_depot = 0;
size_t bytes_inplay = medium_free_reattach_region(rack, medium_depot_ptr, medium);
@@ -1170,6 +1156,7 @@
{
region_trailer_t *node = REGION_TRAILER_FOR_MEDIUM_REGION(region);
msize_t *madvh = MEDIUM_MADVISE_HEADER_FOR_PTR(ptr);
+
msize_t trigger_msize = trigger_level >> SHIFT_MEDIUM_QUANTUM;
size_t free_header_size = sizeof(medium_inplace_free_entry_s) + sizeof(msize_t);
@@ -1218,7 +1205,7 @@
}
msize_t right_dirty_msz = 0;
- if (right_end_idx > src_end_idx) {
+ if (right_end_idx < src_end_idx) {
// Same as above, if we had trailing data coalesced with this entry
// and that was not madvised, consider it, too.
right_dirty_msz = medium_madvise_header_dirty_len(madvh, right_start_idx);
@@ -1231,7 +1218,7 @@
medium_madvise_header_mark_middle(madvh, right_end_idx);
}
- // We absolutely can't madvise lower than the free-list entry pointer plus
+ // We absolutely can't madvise lower the the free-list entry pointer plus
// the header size. When the entry is OOB, there's no header or footer to
// store in memory.
uintptr_t safe_start_ptr = round_page_kernel(rangep + free_header_size);
@@ -1241,33 +1228,21 @@
// If the target region is madvisable, then madvise whatever we can but
// bound it by the safe_start/end pointers to make sure we don't clobber
// the free-list.
- bool should_madvise = (vote_force == 2) || (dirty_msz >= trigger_msize);
- if (magazine_medium_madvise_window_scale_factor > 1) {
- // trigger_msize is an unsigned short, but it's possible that trigger level was larger than UINT16_MAX
- // even after we shifted it. In this case the window rolls around to 0. This is even more likely
- // if we're scaling the window up. We should fix the truncation bug above, but doing so
- // will cause a memory regression. For now, we avoid using trigger_msize iff we're scaling
- // the window up since truncating to 0 would be self-defeating in this case.
- // See rdar://82128639 for details
- should_madvise = (vote_force == 2) || (MEDIUM_BYTES_FOR_MSIZE(dirty_msz) >= trigger_level);
- }
- if (should_madvise) {
+ if ((vote_force == 2) || (dirty_msz >= trigger_msize)) {
uintptr_t lo = MAX((uintptr_t)MEDIUM_PTR_FOR_META_INDEX(region, range_idx),
safe_start_ptr);
uintptr_t hi = MIN((uintptr_t)MEDIUM_PTR_FOR_META_INDEX(region, range_idx) +
MEDIUM_BYTES_FOR_MSIZE(range_msz), safe_end_ptr);
// The page that contains the freelist entry needs to be marked as not
- // having been madvised. Note that the quantum is larger than the kernel page size
- // so if safe_start_ptr and rangep are on different pages, we just mark
- // the whole block as clean.
+ // having been madvised.
if (range_idx < MEDIUM_META_INDEX_FOR_PTR(safe_start_ptr)) {
medium_madvise_header_mark_dirty(madvh, range_idx,
MEDIUM_META_INDEX_FOR_PTR(safe_start_ptr) - range_idx);
}
if (range_idx + range_msz > MEDIUM_META_INDEX_FOR_PTR(safe_end_ptr)) {
medium_madvise_header_mark_dirty(madvh,
- MEDIUM_META_INDEX_FOR_PTR(safe_end_ptr), range_idx +
+ MEDIUM_META_INDEX_FOR_PTR(safe_end_ptr) + 1, range_idx +
range_msz - MEDIUM_META_INDEX_FOR_PTR(safe_end_ptr));
}
@@ -1292,12 +1267,10 @@
// We chose not to madvise, we need to re-mark the region as dirty
// for when we come back to it later.
if (left_dirty_msz < left_msz) {
- /* The preceding block was clean. */
medium_madvise_header_mark_clean(madvh, range_idx,
left_msz - left_dirty_msz);
}
if (right_dirty_msz < right_msz) {
- /* The trailing block was clean. */
medium_madvise_header_mark_clean(madvh, right_start_idx +
right_dirty_msz, right_msz - right_dirty_msz);
}
@@ -1384,10 +1357,24 @@
int objects_in_use = medium_free_detach_region(rack, depot_ptr, sparse_region);
if (0 == objects_in_use) {
- if (!rack_region_remove(rack, sparse_region, node)) {
+ // Invalidate the hash table entry for this region with HASHRING_REGION_DEALLOCATED.
+ // Using HASHRING_REGION_DEALLOCATED preserves the collision chain, using HASHRING_OPEN_ENTRY (0) would not.
+ rgnhdl_t pSlot = hash_lookup_region_no_lock(rack->region_generation->hashed_regions,
+ rack->region_generation->num_regions_allocated,
+ rack->region_generation->num_regions_allocated_shift,
+ sparse_region);
+ if (NULL == pSlot) {
+ malloc_zone_error(rack->debug_flags, true, "medium_free_try_depot_unmap_no_lock hash lookup failed: %p\n", sparse_region);
return NULL;
}
+ *pSlot = HASHRING_REGION_DEALLOCATED;
depot_ptr->num_bytes_in_magazine -= MEDIUM_REGION_PAYLOAD_BYTES;
+ // Atomically increment num_regions_dealloc
+#ifdef __LP64___
+ OSAtomicIncrement64(&rack->num_regions_dealloc);
+#else
+ OSAtomicIncrement32((int32_t *)&rack->num_regions_dealloc);
+#endif
// Caller will transfer ownership of the region back to the OS with no locks held
MAGMALLOC_DEALLOCREGION(MEDIUM_SZONE_FROM_RACK(rack), (void *)sparse_region, (int)MEDIUM_REGION_SIZE); // DTrace USDT Probe
@@ -1490,7 +1477,11 @@
region_trailer_t *node = REGION_TRAILER_FOR_MEDIUM_REGION(region);
size_t bytes_used = node->bytes_used;
- if (DEPOT_MAGAZINE_INDEX != mag_index) {
+ /* FIXME: Would Uniprocessor benefit from recirc and MADV_FREE? */
+ if (rack->num_magazines == 1) { // Uniprocessor, single magazine, so no recirculation necessary
+ /* NOTHING */
+ return TRUE; // Caller must do SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr)
+ } else if (DEPOT_MAGAZINE_INDEX != mag_index) {
// Emptiness discriminant
if (bytes_used < DENSITY_THRESHOLD(MEDIUM_REGION_PAYLOAD_BYTES)) {
/* Region has crossed threshold from density to sparsity. Mark it "suitable" on the
@@ -1876,8 +1867,7 @@
// use is adjusted by the medium_meta_header_set_middle() call below.
medium_meta_header_set_in_use(meta_headers, index + new_msize, leftover_msize);
- /* Propagate the madvise information from the block we're using to the leftover block. */
- if (madv_headers[next_index] & MEDIUM_IS_ADVISED) {
+ if (madv_headers[index] & MEDIUM_IS_ADVISED) {
medium_madvise_header_mark_clean(madv_headers, index + new_msize, leftover_msize);
} else {
medium_madvise_header_mark_dirty(madv_headers, index + new_msize, leftover_msize);
@@ -1944,12 +1934,6 @@
/* there's some left, so put the remainder back */
leftover = (unsigned char *)ptr + MEDIUM_BYTES_FOR_MSIZE(new_msize);
medium_free_list_add_ptr(rack, medium_mag_ptr, leftover, leftover_msize);
- msize_t leftover_index = MEDIUM_META_INDEX_FOR_PTR(leftover);
- if (madv_headers[leftover_index] & MEDIUM_IS_ADVISED) {
- medium_madvise_header_mark_clean(madv_headers, leftover_index, leftover_msize);
- } else {
- medium_madvise_header_mark_dirty(madv_headers, leftover_index, leftover_msize);
- }
}
medium_meta_header_set_in_use(meta_headers, index, new_msize);
medium_madvise_header_mark_dirty(madv_headers, index, new_msize);
@@ -2278,7 +2262,7 @@
if ((ptr = medium_free_list_get_ptr(rack, *free_list))) {
this_msize = MEDIUM_PTR_SIZE(ptr);
was_madvised = (medium_madvise_header_dirty_len(
- MEDIUM_MADVISE_HEADER_FOR_PTR(ptr), MEDIUM_META_INDEX_FOR_PTR(ptr)) == 0);
+ MEDIUM_MADVISE_HEADER_FOR_PTR(ptr), this_msize) == 0);
medium_free_list_remove_ptr(rack, medium_mag_ptr, *free_list, this_msize);
goto add_leftover_and_proceed;
}
@@ -2384,7 +2368,7 @@
medium_malloc_should_clear(rack_t *rack, msize_t msize, boolean_t cleared_requested)
{
void *ptr;
- mag_index_t mag_index = rack_get_thread_index(rack) % rack->num_magazines;
+ mag_index_t mag_index = medium_mag_get_thread_index() % rack->num_magazines;
magazine_t *medium_mag_ptr = &(rack->magazines[mag_index]);
MALLOC_TRACE(TRACE_medium_malloc, (uintptr_t)rack, MEDIUM_BYTES_FOR_MSIZE(msize), (uintptr_t)medium_mag_ptr, cleared_requested);
@@ -2433,17 +2417,16 @@
// The magazine is exhausted. A new region (heap) must be allocated to satisfy this call to malloc().
// The allocation, an mmap() system call, will be performed outside the magazine spin locks by the first
- // thread that suffers the exhaustion. That thread accquires the magazine_alloc_lock, then drops the
- // magazine lock to allow freeing threads to proceed. Allocating thrads that arrive later are excluded
- // from the critial section by the alloc lock. When those are unblocked, they succeed in the code above.
- //
- // Note that we need to trylock the alloc lock to avoid a deadlock, since we can't block on the alloc
- // lock while holding the magazine lock
- if (os_likely(_malloc_lock_trylock(&medium_mag_ptr->magazine_alloc_lock))) {
- // We got the alloc lock, so we are the thread that should allocate a new region
+ // thread that suffers the exhaustion. That thread sets "alloc_underway" and enters a critical section.
+ // Threads arriving here later are excluded from the critical section, yield the CPU, and then retry the
+ // allocation. After some time the magazine is resupplied, the original thread leaves with its allocation,
+ // and retry-ing threads succeed in the code just above.
+ if (!medium_mag_ptr->alloc_underway) {
void *fresh_region;
// time to create a new region (do this outside the magazine lock)
+ medium_mag_ptr->alloc_underway = TRUE;
+ OSMemoryBarrier();
SZONE_MAGAZINE_PTR_UNLOCK(medium_mag_ptr);
fresh_region = mvm_allocate_pages(MEDIUM_REGION_SIZE,
MEDIUM_BLOCKS_ALIGN,
@@ -2456,8 +2439,9 @@
fresh_region, MEDIUM_REGION_SIZE);
if (!fresh_region) { // out of memory!
+ medium_mag_ptr->alloc_underway = FALSE;
+ OSMemoryBarrier();
SZONE_MAGAZINE_PTR_UNLOCK(medium_mag_ptr);
- _malloc_lock_unlock(&medium_mag_ptr->magazine_alloc_lock);
return NULL;
}
@@ -2466,20 +2450,14 @@
mag_index, msize, fresh_region);
// we don't clear because this freshly allocated space is pristine
+ medium_mag_ptr->alloc_underway = FALSE;
+ OSMemoryBarrier();
SZONE_MAGAZINE_PTR_UNLOCK(medium_mag_ptr);
- _malloc_lock_unlock(&medium_mag_ptr->magazine_alloc_lock);
CHECK(szone, __PRETTY_FUNCTION__);
return ptr;
} else {
- // We failed to get the alloc lock, so someone else is allocating.
- // Drop the magazine lock...
SZONE_MAGAZINE_PTR_UNLOCK(medium_mag_ptr);
-
- // Wait for the other thread on the alloc lock
- _malloc_lock_lock(&medium_mag_ptr->magazine_alloc_lock);
- _malloc_lock_unlock(&medium_mag_ptr->magazine_alloc_lock);
-
- // Reacquire the magazine lock to go around the loop again
+ yield();
SZONE_MAGAZINE_PTR_LOCK(medium_mag_ptr);
}
}