Loading...
--- libmalloc/libmalloc-409.40.6/src/magazine_medium.c
+++ libmalloc/libmalloc-317.140.5/src/magazine_medium.c
@@ -32,29 +32,15 @@
static MALLOC_INLINE uint64_t
medium_sliding_madvise_granularity(magazine_t *magazine)
{
- uint64_t granularity = MEDIUM_MADVISE_MIN;
// Use a sliding madvise granularity based on how many bytes the region
// currently has allocated. This way we will advise at a finer granularity
// as the region becomes more and more empty.
// region_trailer_t *t = REGION_TRAILER_FOR_MEDIUM_REGION(region);
- if (magazine->mag_num_bytes_in_objects > 0) {
- if (magazine_medium_madvise_window_scale_factor == 1) {
- // NB: This code has a bug causing undefined behavior whenever the result of clzl is > 32
- // because it's shifting a signed integer. It should be 1ULL << (64 - ...) but
- // fixing this bug will introduce memory regressions, so for now we've only fixed it on very
- // large memory configs that are scaling up their window anyways.
- // See rdar://82128925 for details
- granularity = MAX(granularity, 1 << (64 -
- __builtin_clzl(magazine->mag_num_bytes_in_objects >> MEDIUM_MADVISE_SHIFT)));
- } else {
- granularity = MAX(granularity, 1ULL << (64 -
- __builtin_clzl(magazine->mag_num_bytes_in_objects >> MEDIUM_MADVISE_SHIFT)));
- if (os_mul_overflow(granularity, magazine_medium_madvise_window_scale_factor, &granularity)) {
- return UINT64_MAX;
- }
- }
- }
- return granularity;
+ if (magazine->mag_num_bytes_in_objects == 0) {
+ return MEDIUM_MADVISE_MIN;
+ }
+ return MAX(MEDIUM_MADVISE_MIN, 1 << (64 -
+ __builtin_clzl(magazine->mag_num_bytes_in_objects >> MEDIUM_MADVISE_SHIFT)));
}
static MALLOC_INLINE void
@@ -1009,6 +995,11 @@
{
magazine_t *depot_ptr = &(rack->magazines[DEPOT_MAGAZINE_INDEX]);
+ /* FIXME: Would Uniprocessor benefit from recirc and MADV_FREE? */
+ if (rack->num_magazines == 1) { // Uniprocessor, single magazine, so no recirculation necessary
+ return 0;
+ }
+
#if DEBUG_MALLOC
if (DEPOT_MAGAZINE_INDEX == mag_index) {
malloc_zone_error(rack->debug_flags, true, "medium_get_region_from_depot called for magazine index -1\n", NULL, NULL);
@@ -1021,32 +1012,22 @@
// Appropriate a Depot'd region that can satisfy requested msize.
region_trailer_t *node;
region_t sparse_region;
- msize_t try_msize = msize;
while (1) {
- sparse_region = medium_find_msize_region(rack, depot_ptr, DEPOT_MAGAZINE_INDEX, try_msize);
+ sparse_region = medium_find_msize_region(rack, depot_ptr, DEPOT_MAGAZINE_INDEX, msize);
if (NULL == sparse_region) { // Depot empty?
SZONE_MAGAZINE_PTR_UNLOCK(depot_ptr);
return 0;
}
node = REGION_TRAILER_FOR_MEDIUM_REGION(sparse_region);
- if (0 == node->pinned_to_depot) {
- // Found one!
+ if (0 >= node->pinned_to_depot) {
break;
}
- // Try the next msize up - maybe the head of its free list will be in
- // a region we can use. Once we get the region we'll still allocate the
- // original msize.
- try_msize++;
-
- if (try_msize > NUM_MEDIUM_SLOTS) {
- // Tried all the msizes but couldn't get a usable region. Let's
- // give up for now and we'll allocate a new region from the kernel.
- SZONE_MAGAZINE_PTR_UNLOCK(depot_ptr);
- return 0;
- }
+ SZONE_MAGAZINE_PTR_UNLOCK(depot_ptr);
+ yield();
+ SZONE_MAGAZINE_PTR_LOCK(depot_ptr);
}
// disconnect node from Depot
@@ -1057,7 +1038,7 @@
// Transfer ownership of the region
MAGAZINE_INDEX_FOR_MEDIUM_REGION(sparse_region) = mag_index;
- MALLOC_ASSERT(node->pinned_to_depot == 0);
+ node->pinned_to_depot = 0;
// Iterate the region putting its free entries on its new (locked) magazine's free list
size_t bytes_inplay = medium_free_reattach_region(rack, medium_mag_ptr, sparse_region);
@@ -1148,7 +1129,7 @@
SZONE_MAGAZINE_PTR_LOCK(medium_depot_ptr);
MAGAZINE_INDEX_FOR_MEDIUM_REGION(medium) = DEPOT_MAGAZINE_INDEX;
- MALLOC_ASSERT(REGION_TRAILER_FOR_MEDIUM_REGION(medium)->pinned_to_depot == 0);
+ REGION_TRAILER_FOR_MEDIUM_REGION(medium)->pinned_to_depot = 0;
size_t bytes_inplay = medium_free_reattach_region(rack, medium_depot_ptr, medium);
@@ -1260,17 +1241,7 @@
// If the target region is madvisable, then madvise whatever we can but
// bound it by the safe_start/end pointers to make sure we don't clobber
// the free-list.
- bool should_madvise = (vote_force == 2) || (dirty_msz >= trigger_msize);
- if (magazine_medium_madvise_window_scale_factor > 1) {
- // trigger_msize is an unsigned short, but it's possible that trigger level was larger than UINT16_MAX
- // even after we shifted it. In this case the window rolls around to 0. This is even more likely
- // if we're scaling the window up. We should fix the truncation bug above, but doing so
- // will cause a memory regression. For now, we avoid using trigger_msize iff we're scaling
- // the window up since truncating to 0 would be self-defeating in this case.
- // See rdar://82128639 for details
- should_madvise = (vote_force == 2) || (MEDIUM_BYTES_FOR_MSIZE(dirty_msz) >= trigger_level);
- }
- if (should_madvise) {
+ if ((vote_force == 2) || (dirty_msz >= trigger_msize)) {
uintptr_t lo = MAX((uintptr_t)MEDIUM_PTR_FOR_META_INDEX(region, range_idx),
safe_start_ptr);
uintptr_t hi = MIN((uintptr_t)MEDIUM_PTR_FOR_META_INDEX(region, range_idx) +
@@ -1509,7 +1480,11 @@
region_trailer_t *node = REGION_TRAILER_FOR_MEDIUM_REGION(region);
size_t bytes_used = node->bytes_used;
- if (DEPOT_MAGAZINE_INDEX != mag_index) {
+ /* FIXME: Would Uniprocessor benefit from recirc and MADV_FREE? */
+ if (rack->num_magazines == 1) { // Uniprocessor, single magazine, so no recirculation necessary
+ /* NOTHING */
+ return TRUE; // Caller must do SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr)
+ } else if (DEPOT_MAGAZINE_INDEX != mag_index) {
// Emptiness discriminant
if (bytes_used < DENSITY_THRESHOLD(MEDIUM_REGION_PAYLOAD_BYTES)) {
/* Region has crossed threshold from density to sparsity. Mark it "suitable" on the
@@ -1895,8 +1870,7 @@
// use is adjusted by the medium_meta_header_set_middle() call below.
medium_meta_header_set_in_use(meta_headers, index + new_msize, leftover_msize);
- /* Propagate the madvise information from the block we're using to the leftover block. */
- if (madv_headers[next_index] & MEDIUM_IS_ADVISED) {
+ if (madv_headers[index] & MEDIUM_IS_ADVISED) {
medium_madvise_header_mark_clean(madv_headers, index + new_msize, leftover_msize);
} else {
medium_madvise_header_mark_dirty(madv_headers, index + new_msize, leftover_msize);
@@ -2297,7 +2271,7 @@
if ((ptr = medium_free_list_get_ptr(rack, *free_list))) {
this_msize = MEDIUM_PTR_SIZE(ptr);
was_madvised = (medium_madvise_header_dirty_len(
- MEDIUM_MADVISE_HEADER_FOR_PTR(ptr), MEDIUM_META_INDEX_FOR_PTR(ptr)) == 0);
+ MEDIUM_MADVISE_HEADER_FOR_PTR(ptr), this_msize) == 0);
medium_free_list_remove_ptr(rack, medium_mag_ptr, *free_list, this_msize);
goto add_leftover_and_proceed;
}