Loading...
--- libmalloc/libmalloc-283.100.6/src/magazine_medium.c
+++ libmalloc/libmalloc-317.121.1/src/magazine_medium.c
@@ -124,13 +124,13 @@
{
#if CONFIG_MEDIUM_USES_HYPER_SHIFT
if (os_likely(_os_cpu_number_override == -1)) {
- return _os_cpu_number() >> hyper_shift;
+ return _malloc_cpu_number() >> hyper_shift;
} else {
return _os_cpu_number_override >> hyper_shift;
}
#else // CONFIG_MEDIUM_USES_HYPER_SHIFT
if (os_likely(_os_cpu_number_override == -1)) {
- return _os_cpu_number();
+ return _malloc_cpu_number();
} else {
return _os_cpu_number_override;
}
@@ -1070,19 +1070,31 @@
for (mag_index = 0; mag_index < rack->num_magazines; mag_index++) {
size_t index;
for (index = 0; index < rack->region_generation->num_regions_allocated; ++index) {
- SZONE_LOCK(MEDIUM_SZONE_FROM_RACK(rack));
+ rack_region_lock(rack);
region_t medium = rack->region_generation->hashed_regions[index];
if (!medium || medium == HASHRING_REGION_DEALLOCATED) {
- SZONE_UNLOCK(MEDIUM_SZONE_FROM_RACK(rack));
+ rack_region_unlock(rack);
continue;
}
+ region_trailer_t *trailer =
+ REGION_TRAILER_FOR_MEDIUM_REGION(medium);
+ // Make sure that the owning magazine doesn't try and take this out
+ // from under our feet.
+ trailer->dispose_flags |= RACK_DISPOSE_DELAY;
+ rack_region_unlock(rack);
+
magazine_t *mag_ptr = mag_lock_zine_for_region_trailer(rack->magazines,
- REGION_TRAILER_FOR_MEDIUM_REGION(medium),
- MAGAZINE_INDEX_FOR_MEDIUM_REGION(medium));
-
- SZONE_UNLOCK(MEDIUM_SZONE_FROM_RACK(rack));
+ trailer, MAGAZINE_INDEX_FOR_MEDIUM_REGION(medium));
+
+ // If acquiring the region lock was enough to prevent the owning
+ // magazine from deallocating the region, free it now so we don't
+ // do wasted work.
+ if (rack_region_maybe_dispose(rack, medium, MEDIUM_REGION_SIZE, trailer)) {
+ SZONE_MAGAZINE_PTR_UNLOCK(mag_ptr);
+ continue;
+ }
/* Ordering is important here, the magazine of a region may potentially change
* during mag_lock_zine_for_region_trailer, so src_mag_index must be taken
@@ -1357,24 +1369,10 @@
int objects_in_use = medium_free_detach_region(rack, depot_ptr, sparse_region);
if (0 == objects_in_use) {
- // Invalidate the hash table entry for this region with HASHRING_REGION_DEALLOCATED.
- // Using HASHRING_REGION_DEALLOCATED preserves the collision chain, using HASHRING_OPEN_ENTRY (0) would not.
- rgnhdl_t pSlot = hash_lookup_region_no_lock(rack->region_generation->hashed_regions,
- rack->region_generation->num_regions_allocated,
- rack->region_generation->num_regions_allocated_shift,
- sparse_region);
- if (NULL == pSlot) {
- malloc_zone_error(rack->debug_flags, true, "medium_free_try_depot_unmap_no_lock hash lookup failed: %p\n", sparse_region);
+ if (!rack_region_remove(rack, sparse_region, node)) {
return NULL;
}
- *pSlot = HASHRING_REGION_DEALLOCATED;
depot_ptr->num_bytes_in_magazine -= MEDIUM_REGION_PAYLOAD_BYTES;
- // Atomically increment num_regions_dealloc
-#ifdef __LP64___
- OSAtomicIncrement64(&rack->num_regions_dealloc);
-#else
- OSAtomicIncrement32((int32_t *)&rack->num_regions_dealloc);
-#endif
// Caller will transfer ownership of the region back to the OS with no locks held
MAGMALLOC_DEALLOCREGION(MEDIUM_SZONE_FROM_RACK(rack), (void *)sparse_region, (int)MEDIUM_REGION_SIZE); // DTrace USDT Probe
@@ -1446,10 +1444,13 @@
MAGMALLOC_RECIRCREGION(MEDIUM_SZONE_FROM_RACK(rack), (int)mag_index, (void *)sparse_region, MEDIUM_REGION_SIZE,
(int)BYTES_USED_FOR_MEDIUM_REGION(sparse_region)); // DTrace USDT Probe
-#if !CONFIG_AGGRESSIVE_MADVISE
- // Mark free'd dirty pages with MADV_FREE to reduce memory pressure
- medium_free_scan_madvise_free(rack, depot_ptr, sparse_region);
+#if CONFIG_AGGRESSIVE_MADVISE
+ if (!aggressive_madvise_enabled)
#endif
+ {
+ // Mark free'd dirty pages with MADV_FREE to reduce memory pressure
+ medium_free_scan_madvise_free(rack, depot_ptr, sparse_region);
+ }
// If the region is entirely empty vm_deallocate() it outside the depot lock
region_t r_dealloc = medium_free_try_depot_unmap_no_lock(rack, depot_ptr, node);
@@ -1501,12 +1502,15 @@
}
} else {
-#if !CONFIG_AGGRESSIVE_MADVISE
- // We are free'ing into the depot, so madvise as we do so unless we were madvising every incoming
- // allocation anyway.
- medium_madvise_free_range_no_lock(rack, medium_mag_ptr,
- vm_kernel_page_size, region, freee, msize, headptr, headsize);
+#if CONFIG_AGGRESSIVE_MADVISE
+ if (!aggressive_madvise_enabled)
#endif
+ {
+ // We are free'ing into the depot, so madvise as we do so unless we were madvising every incoming
+ // allocation anyway.
+ medium_madvise_free_range_no_lock(rack, medium_mag_ptr,
+ vm_kernel_page_size, region, freee, msize, headptr, headsize);
+ }
if (0 < bytes_used || 0 < node->pinned_to_depot) {
/* Depot'd region is still live. Leave it in place on the Depot's recirculation list
@@ -1537,9 +1541,7 @@
MALLOC_TRACE(TRACE_medium_free, (uintptr_t)rack, (uintptr_t)medium_mag_ptr, (uintptr_t)ptr, MEDIUM_BYTES_FOR_MSIZE(msize));
-#if CONFIG_AGGRESSIVE_MADVISE || CONFIG_RECIRC_DEPOT
void *original_ptr = ptr;
-#endif
#if DEBUG_MALLOC
if (!msize) {
@@ -1604,11 +1606,20 @@
size_t bytes_used = trailer->bytes_used - original_size;
trailer->bytes_used = (unsigned int)bytes_used;
- // Always attempt to madvise free regions that exceed the conditional
- // madvise limit size.
- medium_madvise_free_range_conditional_no_lock(rack, medium_mag_ptr,
- medium_sliding_madvise_granularity(medium_mag_ptr), region, &freee,
- msize, original_ptr, original_size);
+#if CONFIG_AGGRESSIVE_MADVISE
+ if (aggressive_madvise_enabled) {
+ medium_madvise_free_range_no_lock(rack, medium_mag_ptr,
+ vm_kernel_page_size, region, freee, msize, original_ptr, original_size);
+ }
+ else
+#endif
+ {
+ // Always attempt to madvise free regions that exceed the conditional
+ // madvise limit size.
+ medium_madvise_free_range_conditional_no_lock(rack, medium_mag_ptr,
+ medium_sliding_madvise_granularity(medium_mag_ptr), region, &freee,
+ msize, original_ptr, original_size);
+ }
// Caller must do SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr) if this function
// returns TRUE.
@@ -2491,9 +2502,6 @@
static MALLOC_NOINLINE void
free_medium_botch(rack_t *rack, void *ptr)
{
- mag_index_t mag_index = MAGAZINE_INDEX_FOR_MEDIUM_REGION(MEDIUM_REGION_FOR_PTR(ptr));
- magazine_t *medium_mag_ptr = &(rack->magazines[mag_index]);
- SZONE_MAGAZINE_PTR_UNLOCK(medium_mag_ptr);
malloc_zone_error(rack->debug_flags, true, "double free for ptr %p\n", ptr);
}
@@ -2526,6 +2534,7 @@
/* check that we don't already have this pointer in the cache */
if (ptr == ptr2) {
+ SZONE_MAGAZINE_PTR_UNLOCK(medium_mag_ptr);
free_medium_botch(rack, ptr);
return;
}