Loading...
--- libmalloc/libmalloc-374.60.3/src/magazine_large.c
+++ libmalloc/libmalloc-646.40.3/src/magazine_large.c
@@ -23,6 +23,8 @@
#include "internal.h"
+static large_entry_t *large_entries_grow_no_lock(szone_t *szone, vm_range_t *range_to_deallocate);
+
void
large_debug_print(task_t task, unsigned level, vm_address_t zone_address,
memory_reader_t reader, print_task_printer_t printer)
@@ -53,8 +55,15 @@
if (range->address) {
_simple_sprintf(b, " Slot %5d: %p, size %y", index,
(void *)range->address, range->size);
+#if CONFIG_DEFERRED_RECLAIM
_simple_sprintf(b, "%s\n",
- (range->did_madvise_reusable ? ", madvised" : ""));
+ ((range->size + 2 * large_vm_page_quanta_size <= UINT32_MAX &&
+ mvm_reclaim_is_available(range->reclaim_index))
+ ? "" : ", kernel reclaimed" ));
+#else
+ _simple_sprintf(b, "%s\n",
+ (range->did_madvise_reusable ? ", madvised" : ""));
+#endif // CONFIG_DEFERRED_RECLAIM
}
}
@@ -79,8 +88,15 @@
} else if (index == mapped_szone->large_entry_cache_oldest) {
age = "[oldest]";
}
- _simple_sprintf(b, " %s %s\n", age,
- (range->did_madvise_reusable ? " madvised" : ""));
+#if CONFIG_DEFERRED_RECLAIM
+ _simple_sprintf(b, "%s\n",
+ ((range->size + 2 * large_vm_page_quanta_size <= UINT32_MAX &&
+ mvm_reclaim_is_available(range->reclaim_index))
+ ? "" :", kernel reclaimed"));
+#else
+ _simple_sprintf(b, "%s\n",
+ (range->did_madvise_reusable ? ", madvised" : ""));
+#endif // CONFIG_DEFERRED_RECLAIM
}
_simple_sprintf(b, "\n");
}
@@ -158,14 +174,25 @@
hash_index = ((uintptr_t)ptr >> vm_page_quanta_shift) % num_large_entries;
index = hash_index;
+#if DEBUG_MALLOC
+ large_entry_t *found = NULL;
+#endif /* DEBUG_MALLOC */
do {
range = szone->large_entries + index;
if (range->address == (vm_address_t)ptr) {
+#if DEBUG_MALLOC
+ if (found != NULL) {
+ malloc_zone_error(szone->debug_flags, true,
+ "Duplicate entry in large table %p!\n", ptr);
+ }
+ found = range;
+#else
return range;
+#endif /* DEBUG_MALLOC */
}
if (0 == range->address) {
- return NULL; // end of chain
+ break;
}
index++;
if (index == num_large_entries) {
@@ -173,7 +200,11 @@
}
} while (index != hash_index);
+#if DEBUG_MALLOC
+ return found;
+#else
return NULL;
+#endif /* DEBUG_MALLOC */
}
static void
@@ -201,6 +232,40 @@
// assert(0); /* must not fallthrough! */
}
+/*
+ * Insert the entry into the hash-table
+ * growing it if needed. Caller should hold the szone lock.
+ * Returns false if unable to allocate memory to grow hash table. Otherwise returns true.
+ */
+static bool
+large_entry_grow_and_insert_no_lock(szone_t *szone, vm_address_t addr, vm_size_t size,
+ vm_range_t *range_to_deallocate)
+{
+ bool should_grow = (szone->num_large_objects_in_use + 1) * 4 > szone->num_large_entries;
+ if (should_grow) {
+ // density of hash table too high; grow table
+ // we do that under lock to avoid a race
+ large_entry_t *entries = large_entries_grow_no_lock(szone, range_to_deallocate);
+ if (entries == NULL) {
+ return false;
+ }
+ }
+
+ large_entry_t large_entry;
+ large_entry.address = addr;
+ large_entry.size = size;
+#if CONFIG_DEFERRED_RECLAIM
+ large_entry.reclaim_index = VM_RECLAIM_INDEX_NULL;
+#else
+ large_entry.did_madvise_reusable = FALSE;
+#endif // CONFIG_DEFERRED_RECLAIM
+ large_entry_insert_no_lock(szone, large_entry);
+
+ szone->num_large_objects_in_use++;
+ szone->num_bytes_in_large_objects += size;
+ return true;
+}
+
// FIXME: can't we simply swap the (now empty) entry with the last entry on the collision chain for this hash slot?
static MALLOC_INLINE void
large_entries_rehash_after_entry_no_lock(szone_t *szone, large_entry_t *entry)
@@ -223,7 +288,11 @@
}
szone->large_entries[index].address = (vm_address_t)0;
szone->large_entries[index].size = 0;
+#if CONFIG_DEFERRED_RECLAIM
+ szone->large_entries[index].reclaim_index = VM_RECLAIM_INDEX_NULL;
+#else
szone->large_entries[index].did_madvise_reusable = FALSE;
+#endif // CONFIG_DEFERRED_RECLAIM
large_entry_insert_no_lock(szone, range); // this will reinsert in the
// proper place
} while (index != hash_index);
@@ -313,7 +382,11 @@
entry->address = 0;
entry->size = 0;
+#if CONFIG_DEFERRED_RECLAIM
+ entry->reclaim_index = VM_RECLAIM_INDEX_NULL;
+#else
entry->did_madvise_reusable = FALSE;
+#endif // CONFIG_DEFERRED_RECLAIM
large_entries_rehash_after_entry_no_lock(szone, entry);
#if DEBUG_MALLOC
@@ -373,13 +446,255 @@
return 0;
}
+#if CONFIG_LARGE_CACHE
+/*
+ * Remove the entry at idx from the death row cache.
+ * If supplied, adjusts best to account for compaction.
+ * Does not operate on the entry itself.
+ * Caller must hold the szone lock.
+ *
+ * Returns the oldest entry idx after the removed entry (or -1 if the oldest entry was removed)
+ * so that the caller can iterate through the buffer while removing entries.
+ */
+static int
+remove_from_death_row_no_lock(szone_t *szone, int idx, int *best)
+{
+ int i, next_idx = -1;
+ // Compact live ring to fill entry now vacated at large_entry_cache[best]
+ // while preserving time-order
+ if (szone->large_entry_cache_oldest < szone->large_entry_cache_newest) {
+ // Ring hasn't wrapped. Fill in from right.
+ for (i = idx; i < szone->large_entry_cache_newest; ++i) {
+ szone->large_entry_cache[i] = szone->large_entry_cache[i + 1];
+ }
+
+ if (best && *best != -1) {
+ (*best)--;
+ }
+
+ if (idx == szone->large_entry_cache_oldest) {
+ next_idx = -1;
+ } else {
+ next_idx = idx - 1;
+ }
+ szone->large_entry_cache_newest--; // Pull in right endpoint.
+ } else if (szone->large_entry_cache_newest < szone->large_entry_cache_oldest) {
+ // Ring has wrapped. Arrange to fill in from the contiguous side.
+ if (idx <= szone->large_entry_cache_newest) {
+ // Fill from right.
+ for (i = idx; i < szone->large_entry_cache_newest; ++i) {
+ szone->large_entry_cache[i] = szone->large_entry_cache[i + 1];
+ }
+
+ if (best && *best != -1) {
+ (*best)--;
+ }
+
+ if (0 < szone->large_entry_cache_newest) {
+ szone->large_entry_cache_newest--;
+ } else {
+ szone->large_entry_cache_newest = szone->large_cache_depth - 1;
+ }
+ if (idx == 0) {
+ next_idx = szone->large_cache_depth - 1;
+ } else {
+ next_idx = idx - 1;
+ }
+ } else {
+ // Fill from left.
+ for (i = idx; i > szone->large_entry_cache_oldest; --i) {
+ szone->large_entry_cache[i] = szone->large_entry_cache[i - 1];
+ }
+
+ // best does not need adjustment
+
+ if (idx == szone->large_entry_cache_oldest) {
+ next_idx = -1;
+ } else {
+ next_idx = idx;
+ }
+ if (szone->large_entry_cache_oldest < szone->large_cache_depth - 1) {
+ szone->large_entry_cache_oldest++;
+ } else {
+ szone->large_entry_cache_oldest = 0;
+ }
+
+ }
+ } else {
+ // By trichotomy, large_entry_cache_newest == large_entry_cache_oldest.
+ // That implies best == large_entry_cache_newest == large_entry_cache_oldest
+ // and the ring is now empty.
+
+ // If we are removing the only entry, there must be no current best.
+ if (best && *best != -1) {
+ malloc_zone_error(szone->debug_flags, true, "Invalid best: %d\n", *best);
+ }
+
+ szone->large_entry_cache[idx].address = 0;
+ szone->large_entry_cache[idx].size = 0;
+#if CONFIG_DEFERRED_RECLAIM
+ szone->large_entry_cache[idx].reclaim_index = VM_RECLAIM_INDEX_NULL;
+#else
+ szone->large_entry_cache[idx].did_madvise_reusable = FALSE;
+#endif // CONFIG_DEFERRED_RECLAIM
+ next_idx = -1;
+ }
+
+ return next_idx;
+}
+
+/*
+ * Look for the best fit in the death row cache.
+ * Returns an entry with address NULL iff there is no suitable
+ * entry in the cache.
+ */
+static large_entry_t
+large_malloc_best_fit_in_cache(szone_t *szone, size_t size, unsigned char alignment)
+{
+ int best = -1, idx = szone->large_entry_cache_newest, stop_idx = szone->large_entry_cache_oldest;
+ size_t best_size = SIZE_T_MAX;
+ large_entry_t entry;
+ memset(&entry, 0, sizeof(entry));
+ // Scan large_entry_cache for best fit, starting with most recent entry
+ while (1) {
+ size_t this_size = szone->large_entry_cache[idx].size;
+ vm_address_t addr = szone->large_entry_cache[idx].address;
+
+ if (0 == alignment || 0 == (((uintptr_t)addr) & (((uintptr_t)1 << alignment) - 1))) {
+ if (size == this_size || (size < this_size && this_size < best_size)) {
+#if CONFIG_DEFERRED_RECLAIM
+ uint64_t reclaim_index = szone->large_entry_cache[idx].reclaim_index;
+ if (best_size + 2 * large_vm_page_quanta_size <= UINT32_MAX &&
+ !mvm_reclaim_is_available(reclaim_index)) {
+ // Kernel has already reclaimed this entry or
+ // is in the process of trying to reclaim it.
+ // Remove it from death row & keep looking
+ idx = remove_from_death_row_no_lock(szone, idx, &best);
+ stop_idx = szone->large_entry_cache_oldest;
+ if (idx == -1) {
+ // We've looked at all entries in the cache
+ break;
+ } else {
+ continue;
+ }
+ }
+#endif // CONFIG_DEFERRED_RECLAIM
+ best = idx;
+ best_size = this_size;
+ if (size == this_size) {
+ // Perfect fit. No need to keep looking.
+ break;
+ }
+ }
+ }
+
+ if (idx == stop_idx) { // exhausted live ring?
+ break;
+ }
+
+ if (idx) {
+ idx--; // bump idx down
+ } else {
+ idx = szone->large_cache_depth - 1; // wrap idx
+ }
+ }
+
+ if (best == -1 || (best_size - size) >= size) { // limit fragmentation to 50%
+ return entry;
+ }
+
+ entry = szone->large_entry_cache[best];
+
+ remove_from_death_row_no_lock(szone, best, NULL);
+
+ return entry;
+}
+
+/*
+ * Attempt to handle the allocation from the death-row cache
+ * Caller should not hold the szone lock & it will be unlocked on return.
+ * Returns NULL if unable to satisfy the allocation from the death-row cache.
+ */
+static void *
+large_malloc_from_cache(szone_t *szone, size_t size, unsigned char alignment, boolean_t cleared_requested)
+{
+ SZONE_LOCK(szone);
+
+ bool was_madvised_reusable;
+ large_entry_t entry;
+
+ while (true) {
+ entry = large_malloc_best_fit_in_cache(szone, size, alignment);
+ if (entry.address == (vm_address_t)NULL) {
+ // The cache does not contain an entry that we can use.
+ SZONE_UNLOCK(szone);
+ return NULL;
+ } else {
+#if CONFIG_DEFERRED_RECLAIM
+ if (entry.size + 2 * large_vm_page_quanta_size <= UINT32_MAX &&
+ !mvm_reclaim_mark_used(entry.reclaim_index, entry.address,
+ (uint32_t) entry.size, szone->debug_flags)) {
+ // Entry has been reclaimed by the kernel since we put it in the death row cache
+ // large_malloc_best_fit_in_cache already removed it from the cache.
+ // Let's search the cache again to see if there's another entry we can use.
+ // Note that we never dropped the SZONE lock so this search is bounded by the size
+ // of the cache and mvm_reclaim_mark_used synchronized with the kernel so
+ // subsequent calls to large_malloc_best_fit_in_cache
+ // will clear out any entries that were reclaimed before this one.
+ continue;
+ }
+#endif // CONFIG_DEFERRED_RECLAIM
+ /* Got an entry */
+ break;
+ }
+ }
+
+ vm_range_t range_to_deallocate;
+ range_to_deallocate.size = 0;
+ range_to_deallocate.address = 0;
+ bool success = large_entry_grow_and_insert_no_lock(szone, entry.address, entry.size,
+ &range_to_deallocate);
+
+ if (!success) {
+ SZONE_UNLOCK(szone);
+ return NULL;
+ }
+#if CONFIG_DEFERRED_RECLAIM
+ was_madvised_reusable = true;
+#else
+ was_madvised_reusable = entry.did_madvise_reusable;
+#endif // CONFIG_DEFERRED_RECLAIM
+ if (!was_madvised_reusable) {
+ szone->large_entry_cache_reserve_bytes -= entry.size;
+ }
+
+ szone->large_entry_cache_bytes -= entry.size;
+
+ if (szone->flotsam_enabled && szone->large_entry_cache_bytes < SZONE_FLOTSAM_THRESHOLD_LOW) {
+ szone->flotsam_enabled = FALSE;
+ }
+
+ SZONE_UNLOCK(szone);
+
+ if (range_to_deallocate.size) {
+ // we deallocate outside the lock
+ mvm_deallocate_pages((void *)range_to_deallocate.address, range_to_deallocate.size, 0);
+ }
+
+ if (cleared_requested) {
+ memset((void *) entry.address, 0, size);
+ }
+
+ return (void *)entry.address;
+}
+#endif /* CONFIG_LARGE_CACHE */
+
void *
large_malloc(szone_t *szone, size_t num_kernel_pages, unsigned char alignment, boolean_t cleared_requested)
{
void *addr;
vm_range_t range_to_deallocate;
size_t size;
- large_entry_t large_entry;
MALLOC_TRACE(TRACE_large_malloc, (uintptr_t)szone, num_kernel_pages, alignment, cleared_requested);
@@ -391,135 +706,13 @@
range_to_deallocate.address = 0;
#if CONFIG_LARGE_CACHE
- if (large_cache_enabled && size <= szone->large_cache_entry_limit) { // Look for a large_entry_t on the death-row cache?
- SZONE_LOCK(szone);
-
- int i, best = -1, idx = szone->large_entry_cache_newest, stop_idx = szone->large_entry_cache_oldest;
- size_t best_size = SIZE_T_MAX;
-
- while (1) { // Scan large_entry_cache for best fit, starting with most recent entry
- size_t this_size = szone->large_entry_cache[idx].size;
- addr = (void *)szone->large_entry_cache[idx].address;
-
- if (0 == alignment || 0 == (((uintptr_t)addr) & (((uintptr_t)1 << alignment) - 1))) {
- if (size == this_size) { // size match!
- best = idx;
- best_size = this_size;
- break;
- }
-
- if (size <= this_size && this_size < best_size) { // improved fit?
- best = idx;
- best_size = this_size;
- }
- }
-
- if (idx == stop_idx) { // exhausted live ring?
- break;
- }
-
- if (idx) {
- idx--; // bump idx down
- } else {
- idx = szone->large_cache_depth - 1; // wrap idx
- }
- }
-
- if (best > -1 && (best_size - size) < size) { // limit fragmentation to 50%
- addr = (void *)szone->large_entry_cache[best].address;
- boolean_t was_madvised_reusable = szone->large_entry_cache[best].did_madvise_reusable;
-
- // Compact live ring to fill entry now vacated at large_entry_cache[best]
- // while preserving time-order
- if (szone->large_entry_cache_oldest < szone->large_entry_cache_newest) {
- // Ring hasn't wrapped. Fill in from right.
- for (i = best; i < szone->large_entry_cache_newest; ++i) {
- szone->large_entry_cache[i] = szone->large_entry_cache[i + 1];
- }
-
- szone->large_entry_cache_newest--; // Pull in right endpoint.
-
- } else if (szone->large_entry_cache_newest < szone->large_entry_cache_oldest) {
- // Ring has wrapped. Arrange to fill in from the contiguous side.
- if (best <= szone->large_entry_cache_newest) {
- // Fill from right.
- for (i = best; i < szone->large_entry_cache_newest; ++i) {
- szone->large_entry_cache[i] = szone->large_entry_cache[i + 1];
- }
-
- if (0 < szone->large_entry_cache_newest) {
- szone->large_entry_cache_newest--;
- } else {
- szone->large_entry_cache_newest = szone->large_cache_depth - 1;
- }
- } else {
- // Fill from left.
- for (i = best; i > szone->large_entry_cache_oldest; --i) {
- szone->large_entry_cache[i] = szone->large_entry_cache[i - 1];
- }
-
- if (szone->large_entry_cache_oldest < szone->large_cache_depth - 1) {
- szone->large_entry_cache_oldest++;
- } else {
- szone->large_entry_cache_oldest = 0;
- }
- }
-
- } else {
- // By trichotomy, large_entry_cache_newest == large_entry_cache_oldest.
- // That implies best == large_entry_cache_newest == large_entry_cache_oldest
- // and the ring is now empty.
- szone->large_entry_cache[best].address = 0;
- szone->large_entry_cache[best].size = 0;
- szone->large_entry_cache[best].did_madvise_reusable = FALSE;
- }
-
- if ((szone->num_large_objects_in_use + 1) * 4 > szone->num_large_entries) {
- // density of hash table too high; grow table
- // we do that under lock to avoid a race
- large_entry_t *entries = large_entries_grow_no_lock(szone, &range_to_deallocate);
- if (entries == NULL) {
- SZONE_UNLOCK(szone);
- return NULL;
- }
- }
-
- large_entry.address = (vm_address_t)addr;
- large_entry.size = best_size;
- large_entry.did_madvise_reusable = FALSE;
- large_entry_insert_no_lock(szone, large_entry);
-
- szone->num_large_objects_in_use++;
- szone->num_bytes_in_large_objects += best_size;
- if (!was_madvised_reusable) {
- szone->large_entry_cache_reserve_bytes -= best_size;
- }
-
- szone->large_entry_cache_bytes -= best_size;
-
- if (szone->flotsam_enabled && szone->large_entry_cache_bytes < SZONE_FLOTSAM_THRESHOLD_LOW) {
- szone->flotsam_enabled = FALSE;
- }
-
- SZONE_UNLOCK(szone);
-
- if (range_to_deallocate.size) {
- // we deallocate outside the lock
- mvm_deallocate_pages((void *)range_to_deallocate.address, range_to_deallocate.size, 0);
- }
-
- if (cleared_requested) {
- memset(addr, 0, size);
- }
-
+ // Look for a large_entry_t on the death-row cache?
+ if (large_cache_enabled && size <= szone->large_cache_entry_limit) {
+ addr = large_malloc_from_cache(szone, size, alignment, cleared_requested);
+ if (addr != NULL) {
return addr;
- } else {
- SZONE_UNLOCK(szone);
- }
- }
-
- range_to_deallocate.size = 0;
- range_to_deallocate.address = 0;
+ }
+ }
#endif /* CONFIG_LARGE_CACHE */
// NOTE: we do not use MALLOC_FIX_GUARD_PAGE_FLAGS(szone->debug_flags) here
@@ -530,24 +723,12 @@
}
SZONE_LOCK(szone);
- if ((szone->num_large_objects_in_use + 1) * 4 > szone->num_large_entries) {
- // density of hash table too high; grow table
- // we do that under lock to avoid a race
- large_entry_t *entries = large_entries_grow_no_lock(szone, &range_to_deallocate);
- if (entries == NULL) {
- SZONE_UNLOCK(szone);
- return NULL;
- }
- }
-
- large_entry.address = (vm_address_t)addr;
- large_entry.size = size;
- large_entry.did_madvise_reusable = FALSE;
- large_entry_insert_no_lock(szone, large_entry);
-
- szone->num_large_objects_in_use++;
- szone->num_bytes_in_large_objects += size;
+ bool success = large_entry_grow_and_insert_no_lock(szone, (vm_address_t) addr, (vm_size_t) size,
+ &range_to_deallocate);
SZONE_UNLOCK(szone);
+ if (!success) {
+ return NULL;
+ }
if (range_to_deallocate.size) {
// we deallocate outside the lock
@@ -556,36 +737,80 @@
return addr;
}
-void
-free_large(szone_t *szone, void *ptr)
+bool
+free_large(szone_t *szone, void *ptr, bool try)
{
// We have established ptr is page-aligned and neither tiny nor small
large_entry_t *entry;
vm_range_t vm_range_to_deallocate;
+ vm_range_to_deallocate.size = 0;
+ vm_range_to_deallocate.address = 0;
SZONE_LOCK(szone);
entry = large_entry_for_pointer_no_lock(szone, ptr);
if (entry) {
#if CONFIG_LARGE_CACHE
if (large_cache_enabled &&
- entry->size <= szone->large_cache_entry_limit &&
- -1 != madvise((void *)(entry->address), entry->size,
- MADV_CAN_REUSE)) { // Put the large_entry_t on the death-row cache?
+ entry->size <= szone->large_cache_entry_limit
+#if !CONFIG_DEFERRED_RECLAIM
+ && -1 != madvise((void *)(entry->address), entry->size, MADV_CAN_REUSE)
+#endif // CONFIG_DEFERRED_RECLAIM
+ ) { // Put the large_entry_t on the death-row cache?
int idx = szone->large_entry_cache_newest, stop_idx = szone->large_entry_cache_oldest;
- large_entry_t this_entry = *entry; // Make a local copy, "entry" is volatile when lock is let go.
+ // Make a local copy, we'll free the entry from the lookup table
+ // before dropping the lock.
+ large_entry_t this_entry = *entry;
+#if !CONFIG_DEFERRED_RECLAIM
+ boolean_t should_madvise = szone->large_entry_cache_reserve_bytes +
+ this_entry.size > szone->large_entry_cache_reserve_limit;
+#endif // !CONFIG_DEFERRED_RECLAIM
boolean_t reusable = TRUE;
- boolean_t should_madvise =
- szone->large_entry_cache_reserve_bytes + this_entry.size > szone->large_entry_cache_reserve_limit;
// Already freed?
// [Note that repeated entries in death-row risk vending the same entry subsequently
// to two different malloc() calls. By checking here the (illegal) double free
// is accommodated, matching the behavior of the previous implementation.]
while (1) { // Scan large_entry_cache starting with most recent entry
- if (szone->large_entry_cache[idx].address == entry->address) {
+ vm_address_t addr = szone->large_entry_cache[idx].address;
+#if CONFIG_DEFERRED_RECLAIM
+ vm_size_t curr_size = szone->large_entry_cache[idx].size;
+ uint64_t reclaim_index = szone->large_entry_cache[idx].reclaim_index;
+ if (curr_size + 2 * large_vm_page_quanta_size <= UINT32_MAX &&
+ !mvm_reclaim_is_available(reclaim_index)) {
+ // Entry has been reclaimed
+ // Remove it from the cache
+
+ idx = remove_from_death_row_no_lock(szone, idx, NULL);
+ stop_idx = szone->large_entry_cache_oldest;
+
+ if (idx == -1) {
+ // Ring buffer is now empty
+ break;
+ }
+
+ continue;
+ }
+#endif // CONFIG_DEFERRED_RECLAIM
+ if (addr == entry->address) {
+#if CONFIG_DEFERRED_RECLAIM
+ if (curr_size + 2 * large_vm_page_quanta_size <= UINT32_MAX) {
+ // mvm_reclaim_is_available doesn't actually synchronize with the kernel,
+ // so in order to confidently say this was a double free
+ // we need to make sure the entry was not reclaimed.
+ if (!mvm_reclaim_mark_used(reclaim_index, addr, (uint32_t) curr_size, szone->debug_flags)) {
+ // This entry has been reclaimed, so it's not a double-free. continue
+ break;
+ }
+ // This is a double free, but we just took the entry
+ // out of the reclaim buffer. Put it back.
+ szone->large_entry_cache[idx].reclaim_index =
+ mvm_reclaim_mark_free(addr, (uint32_t) curr_size, szone->debug_flags);
+ reclaim_index = szone->large_entry_cache[idx].reclaim_index;
+ }
+#endif // CONFIG_DEFERRED_RECLAIM
malloc_zone_error(szone->debug_flags, true, "pointer %p being freed already on death-row\n", ptr);
SZONE_UNLOCK(szone);
- return;
+ return true;
}
if (idx == stop_idx) { // exhausted live ring?
@@ -598,6 +823,9 @@
idx = szone->large_cache_depth - 1; // wrap idx
}
}
+
+ vm_range_to_deallocate = large_entry_free_no_lock(szone, entry);
+ entry = NULL;
SZONE_UNLOCK(szone);
@@ -621,9 +849,26 @@
}
// madvise(..., MADV_REUSABLE) death-row arrivals if hoarding would exceed large_entry_cache_reserve_limit
+
+#if CONFIG_DEFERRED_RECLAIM
+ if (reusable) {
+ if ((szone->debug_flags & MALLOC_DO_SCRIBBLE)) {
+ memset((void *)(this_entry.address), SCRUBBLE_BYTE, this_entry.size);
+ }
+ // Only put this in the reclaim buffer if its size (plus any guard pages)
+ // can fit in a uint32_t.
+ if (this_entry.size + 2 * large_vm_page_quanta_size > UINT32_MAX) {
+ reusable = false;
+ }
+ this_entry.reclaim_index = mvm_reclaim_mark_free(this_entry.address,
+ (uint32_t) this_entry.size, szone->debug_flags);
+ // NB: At this point this_entry.address could be reclaimed
+ }
+#else
if (should_madvise) {
// Issue madvise to avoid paging out the dirtied free()'d pages in "entry"
- MAGMALLOC_MADVFREEREGION((void *)szone, (void *)0, (void *)(this_entry.address), (int)this_entry.size); // DTrace USDT Probe
+ MAGMALLOC_MADVFREEREGION((void *)szone, (void *)0,
+ (void *)(this_entry.address), (int)this_entry.size); // DTrace USDT Probe
// Ok to do this madvise on embedded because we won't call MADV_FREE_REUSABLE on a large
// cache block twice without MADV_FREE_REUSE in between.
@@ -638,22 +883,21 @@
reusable = FALSE;
}
}
+#endif // CONFIG_DEFERRED_RECLAIM
SZONE_LOCK(szone);
- // Re-acquire "entry" after interval just above where we let go the lock.
- entry = large_entry_for_pointer_no_lock(szone, ptr);
- if (NULL == entry) {
- malloc_zone_error(szone->debug_flags, true, "entry for pointer %p being freed from death-row vanished\n", ptr);
- SZONE_UNLOCK(szone);
- return;
- }
-
- // Add "entry" to death-row ring
+ szone->num_large_objects_in_use--;
+ szone->num_bytes_in_large_objects -= this_entry.size;
+
+ // Add "this_entry" to death-row ring
if (reusable) {
int idx = szone->large_entry_cache_newest; // Most recently occupied
vm_address_t addr;
size_t adjsize;
+#if CONFIG_DEFERRED_RECLAIM
+ uint64_t old_reclaim_index;
+#endif // CONFIG_DEFERRED_RECLAIM
if (szone->large_entry_cache_newest == szone->large_entry_cache_oldest &&
0 == szone->large_entry_cache[idx].address) {
@@ -672,9 +916,13 @@
addr = szone->large_entry_cache[idx].address;
adjsize = szone->large_entry_cache[idx].size;
szone->large_entry_cache_bytes -= adjsize;
+#if CONFIG_DEFERRED_RECLAIM
+ old_reclaim_index = szone->large_entry_cache[idx].reclaim_index;
+#else
if (!szone->large_entry_cache[idx].did_madvise_reusable) {
szone->large_entry_cache_reserve_bytes -= adjsize;
}
+#endif // CONFIG_DEFERRED_RECLAIM
} else {
// Using an unoccupied cache slot
addr = 0;
@@ -682,32 +930,30 @@
}
}
+#if !CONFIG_DEFERRED_RECLAIM
if ((szone->debug_flags & MALLOC_DO_SCRIBBLE)) {
- memset((void *)(entry->address), should_madvise ? SCRUBBLE_BYTE : SCRABBLE_BYTE, entry->size);
- }
-
- entry->did_madvise_reusable = should_madvise; // Was madvise()'d above?
- if (!should_madvise) { // Entered on death-row without madvise() => up the hoard total
- szone->large_entry_cache_reserve_bytes += entry->size;
- }
-
- szone->large_entry_cache_bytes += entry->size;
+ memset((void *)(this_entry.address), should_madvise ?
+ SCRUBBLE_BYTE : SCRABBLE_BYTE, this_entry.size);
+ }
+ this_entry.did_madvise_reusable = should_madvise; // Was madvise()'d above?
+ if (!should_madvise) {
+ // Entered on death-row without madvise() => up the hoard total
+ szone->large_entry_cache_reserve_bytes += this_entry.size;
+ }
+#endif // !CONFIG_DEFERRED_RECLAIM
+
+ szone->large_entry_cache_bytes += this_entry.size;
if (!szone->flotsam_enabled && szone->large_entry_cache_bytes > SZONE_FLOTSAM_THRESHOLD_HIGH) {
szone->flotsam_enabled = TRUE;
}
- szone->large_entry_cache[idx] = *entry;
+ szone->large_entry_cache[idx] = this_entry;
szone->large_entry_cache_newest = idx;
-
- szone->num_large_objects_in_use--;
- szone->num_bytes_in_large_objects -= entry->size;
-
- (void)large_entry_free_no_lock(szone, entry);
if (0 == addr) {
SZONE_UNLOCK(szone);
- return;
+ return true;
}
// Fall through to drop large_entry_cache_oldest from the cache,
@@ -722,25 +968,39 @@
// we deallocate_pages, including guard pages, outside the lock
SZONE_UNLOCK(szone);
+
+#if CONFIG_DEFERRED_RECLAIM
+ // Need to take ownership of the allocation before trying to deallocate it.
+ if (adjsize + 2 * large_vm_page_quanta_size <= UINT32_MAX &&
+ mvm_reclaim_mark_used(old_reclaim_index, addr,
+ (uint32_t) adjsize, szone->debug_flags)) {
+ mvm_deallocate_pages((void *)addr, (size_t)adjsize, szone->debug_flags);
+ }
+#else
mvm_deallocate_pages((void *)addr, (size_t)adjsize, 0);
- return;
+#endif // CONFIG_DEFERRED_RECLAIM
+ return true;
} else {
- /* fall through to discard an allocation that is not reusable */
+ // fall through to deallocate vm_range_to_deallocate
}
}
#endif /* CONFIG_LARGE_CACHE */
- szone->num_large_objects_in_use--;
- szone->num_bytes_in_large_objects -= entry->size;
-
- vm_range_to_deallocate = large_entry_free_no_lock(szone, entry);
+ if (!vm_range_to_deallocate.address) {
+ szone->num_large_objects_in_use--;
+ szone->num_bytes_in_large_objects -= entry->size;
+
+ vm_range_to_deallocate = large_entry_free_no_lock(szone, entry);
+ }
} else {
+ if (!try) {
#if DEBUG_MALLOC
- large_debug_print_self(szone, 1);
+ large_debug_print_self(szone, 1);
#endif
- malloc_zone_error(szone->debug_flags, true, "pointer %p being freed was not allocated\n", ptr);
+ malloc_zone_error(szone->debug_flags, true, "pointer %p being freed was not allocated\n", ptr);
+ }
SZONE_UNLOCK(szone);
- return;
+ return false;
}
SZONE_UNLOCK(szone); // we release the lock asap
CHECK(szone, __PRETTY_FUNCTION__);
@@ -757,6 +1017,8 @@
#endif
mvm_deallocate_pages((void *)vm_range_to_deallocate.address, (size_t)vm_range_to_deallocate.size, 0);
}
+
+ return true;
}
void *
@@ -849,3 +1111,64 @@
SZONE_UNLOCK(szone);
return result;
}
+
+#if CONFIG_LARGE_CACHE
+static void
+large_clear_cache_locked(szone_t *szone)
+{
+ szone->large_entry_cache_oldest = szone->large_entry_cache_newest = 0;
+ szone->large_entry_cache[0].address = 0x0;
+ szone->large_entry_cache[0].size = 0;
+ szone->large_entry_cache_bytes = 0;
+ szone->large_entry_cache_reserve_bytes = 0;
+}
+
+static void
+large_deallocate_cache_entry(szone_t *szone, large_entry_t *entry)
+{
+#if CONFIG_DEFERRED_RECLAIM
+ // If we're using deferred reclaim, we have to first take ownership of the entry back
+ // out of the reclaim buffer. If we fail to get the entry, then it's already been
+ // reclaimed.
+ if (entry->size > UINT32_MAX ||
+ mvm_reclaim_mark_used(entry->reclaim_index, entry->address,
+ (uint32_t) entry->size, szone->debug_flags)) {
+ mvm_deallocate_pages((void *)entry->address, entry->size, szone->debug_flags);
+ }
+#else // CONFIG_DEFERRED_RECLAIM
+ mvm_deallocate_pages((void *)entry->address, entry->size, szone->debug_flags);
+#endif // CONFIG_DEFERRED_RECLAIM
+}
+
+void
+large_destroy_cache(szone_t *szone)
+{
+ SZONE_LOCK(szone);
+
+ // disable any memory pressure responder
+ szone->flotsam_enabled = FALSE;
+ // stack allocated copy of the death-row cache
+ int idx = szone->large_entry_cache_oldest, idx_max = szone->large_entry_cache_newest;
+ large_entry_t local_entry_cache[LARGE_ENTRY_CACHE_SIZE_HIGH];
+
+ memcpy((void *)local_entry_cache, (void *)szone->large_entry_cache, sizeof(local_entry_cache));
+
+ large_clear_cache_locked(szone);
+ SZONE_UNLOCK(szone);
+
+ // deallocate the death-row cache entries outside the zone lock
+ while (idx != idx_max) {
+ large_entry_t *entry = &local_entry_cache[idx];
+
+ large_deallocate_cache_entry(szone, entry);
+ if (++idx == szone->large_cache_depth) {
+ idx = 0;
+ }
+ }
+
+ if (0 != local_entry_cache[idx].address && 0 != local_entry_cache[idx].size) {
+ large_deallocate_cache_entry(szone, &local_entry_cache[idx]);
+ }
+}
+
+#endif // CONFIG_LARGE_CACHE