Loading...
--- libmalloc/libmalloc-715.140.5/src/magazine_tiny.c
+++ libmalloc/libmalloc-409.40.6/src/magazine_tiny.c
@@ -38,6 +38,25 @@
// reduce the msize part below zero (not checked).
#define TINY_MAG_LAST_FREE_PTR_DEC_MSIZE(x, msize_delta) (x) = ((void *)(x) - (msize_delta))
+static MALLOC_INLINE MALLOC_ALWAYS_INLINE
+mag_index_t
+tiny_mag_get_thread_index(void)
+{
+#if CONFIG_TINY_USES_HYPER_SHIFT
+ if (os_likely(_os_cpu_number_override == -1)) {
+ return _malloc_cpu_number() >> hyper_shift;
+ } else {
+ return _os_cpu_number_override >> hyper_shift;
+ }
+#else // CONFIG_SMALL_USES_HYPER_SHIFT
+ if (os_likely(_os_cpu_number_override == -1)) {
+ return _malloc_cpu_number();
+ } else {
+ return _os_cpu_number_override;
+ }
+#endif // CONFIG_SMALL_USES_HYPER_SHIFT
+}
+
static inline grain_t
tiny_slot_from_msize(msize_t msize)
{
@@ -186,7 +205,7 @@
static MALLOC_INLINE void
zero_tiny_free_inline_meta(void *ptr, msize_t msize)
{
- if (malloc_zero_policy == MALLOC_ZERO_ON_FREE) {
+ if (malloc_zero_on_free) {
*((tiny_free_list_t *)ptr) = (tiny_free_list_t){ 0 };
if (msize > 1) {
TINY_FREE_SIZE(ptr) = 0;
@@ -201,7 +220,7 @@
static MALLOC_INLINE void
zero_tiny_free_inline_meta_following(void *ptr, msize_t msize)
{
- if (malloc_zero_policy == MALLOC_ZERO_ON_FREE) {
+ if (malloc_zero_on_free) {
if (msize > 1) {
void *follower = FOLLOWING_TINY_PTR(ptr, msize);
TINY_PREVIOUS_MSIZE(follower) = 0;
@@ -264,23 +283,17 @@
}
static MALLOC_INLINE void
-tiny_check_zero_or_clear(void *ptr, msize_t msize, boolean_t clear)
-{
- switch (malloc_zero_policy) {
- case MALLOC_ZERO_ON_FREE:
+tiny_check_zero_and_clear(void *ptr, msize_t msize, boolean_t clear)
+{
+ if (malloc_zero_on_free) {
if (zero_on_free_should_sample() &&
_malloc_memcmp_zero_aligned8(ptr, TINY_BYTES_FOR_MSIZE(msize))) {
tiny_zero_corruption_abort(ptr, msize);
}
- break;
- case MALLOC_ZERO_NONE:
- if (!clear) {
- break;
- }
- MALLOC_FALLTHROUGH;
- case MALLOC_ZERO_ON_ALLOC:
- memset(ptr, '\0', TINY_BYTES_FOR_MSIZE(msize));
- break;
+ } else {
+ if (clear) {
+ memset(ptr, '\0', TINY_BYTES_FOR_MSIZE(msize));
+ }
}
}
@@ -329,7 +342,7 @@
tiny_check_and_zero_inline_meta_from_freelist(rack_t *rack, void *ptr,
msize_t msize)
{
- if (malloc_zero_policy != MALLOC_ZERO_ON_FREE) {
+ if (!malloc_zero_on_free) {
return;
}
@@ -1493,7 +1506,7 @@
msize += next_msize;
}
- if (malloc_zero_policy != MALLOC_ZERO_ON_FREE) {
+ if (!malloc_zero_on_free) {
// The tiny cache already scribbles free blocks as they go through the
// cache, so we do not need to do it here.
//
@@ -1768,7 +1781,7 @@
trailer->objects_in_use--;
}
set_tiny_meta_header_middle(next_block);
- tiny_check_zero_or_clear(last_free_ptr, coalesced_msize, false);
+ tiny_check_zero_and_clear(last_free_ptr, coalesced_msize, false);
coalesced_msize = 0; // No net change in memory use
} else {
#endif // CONFIG_TINY_CACHE
@@ -1793,7 +1806,7 @@
// Mark the first block of the remaining free area as a header and in-use.
set_tiny_meta_header_in_use_1(ptr + TINY_BYTES_FOR_MSIZE(new_msize));
}
- tiny_check_zero_or_clear(unused_start, coalesced_msize, false);
+ tiny_check_zero_and_clear(unused_start, coalesced_msize, false);
} else {
/*
* Look for a free block immediately afterwards. If it's large
@@ -1817,7 +1830,7 @@
set_tiny_meta_header_middle(next_block); // clear the meta_header to enable coalescing backwards
tiny_check_and_zero_inline_meta_from_freelist(rack, next_block, next_msize);
- tiny_check_zero_or_clear(next_block, coalesced_msize, false);
+ tiny_check_zero_and_clear(next_block, coalesced_msize, false);
leftover_msize = next_msize - coalesced_msize;
if (leftover_msize) {
@@ -2250,12 +2263,6 @@
BITMAPV_CLR(tiny_mag_ptr->mag_bitmap, slot);
}
this_msize = get_tiny_free_size(ptr);
- if (os_unlikely(this_msize < msize)) {
- malloc_zone_error(MALLOC_ABORT_ON_CORRUPTION, true,
- "Corruption of tiny freelist %p: size too small (%u/%u)\n",
- ptr, this_msize, msize);
-
- }
tiny_update_region_free_list_for_remove(slot, ptr, next);
tiny_check_and_zero_inline_meta_from_freelist(rack, ptr, this_msize);
goto add_leftover_and_proceed;
@@ -2272,11 +2279,6 @@
ptr = limit->p;
if (ptr) {
this_msize = get_tiny_free_size(ptr);
- if (os_unlikely(this_msize < msize)) {
- malloc_zone_error(MALLOC_ABORT_ON_CORRUPTION, true,
- "Corruption of tiny freelist %p: size too small (%u/%u)\n",
- ptr, this_msize, msize);
- }
next = free_list_unchecksum_ptr(rack, &ptr->next);
if (this_msize - msize > NUM_TINY_SLOTS) {
// the leftover will go back to the free list, so we optimize by
@@ -2407,7 +2409,7 @@
tiny_malloc_should_clear(rack_t *rack, msize_t msize, boolean_t cleared_requested)
{
void *ptr;
- mag_index_t mag_index = rack_get_thread_index(rack) % rack->num_magazines;
+ mag_index_t mag_index = tiny_mag_get_thread_index() % rack->num_magazines;
magazine_t *tiny_mag_ptr = &(rack->magazines[mag_index]);
MALLOC_TRACE(TRACE_tiny_malloc, (uintptr_t)rack, TINY_BYTES_FOR_MSIZE(msize), (uintptr_t)tiny_mag_ptr, cleared_requested);
@@ -2437,7 +2439,7 @@
SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
CHECK(szone, __PRETTY_FUNCTION__);
- tiny_check_zero_or_clear(ptr, msize, cleared_requested);
+ tiny_check_zero_and_clear(ptr, msize, cleared_requested);
#if DEBUG_MALLOC
if (LOG(szone, ptr)) {
@@ -2453,7 +2455,7 @@
if (ptr) {
SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
CHECK(szone, __PRETTY_FUNCTION__);
- tiny_check_zero_or_clear(ptr, msize, cleared_requested);
+ tiny_check_zero_and_clear(ptr, msize, cleared_requested);
return ptr;
}
@@ -2463,7 +2465,7 @@
if (ptr) {
SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
CHECK(szone, __PRETTY_FUNCTION__);
- tiny_check_zero_or_clear(ptr, msize, cleared_requested);
+ tiny_check_zero_and_clear(ptr, msize, cleared_requested);
return ptr;
}
}
@@ -2471,17 +2473,16 @@
// The magazine is exhausted. A new region (heap) must be allocated to satisfy this call to malloc().
// The allocation, an mmap() system call, will be performed outside the magazine spin locks by the first
- // thread that suffers the exhaustion. That thread accquires the magazine_alloc_lock, then drops the
- // magazine lock to allow freeing threads to proceed. Allocating thrads that arrive later are excluded
- // from the critial section by the alloc lock. When those are unblocked, they succeed in the code above.
- //
- // Note that we need to trylock the alloc lock to avoid a deadlock, since we can't block on the alloc
- // lock while holding the magazine lock
- if (os_likely(_malloc_lock_trylock(&tiny_mag_ptr->magazine_alloc_lock))) {
- // We got the alloc lock, so we are the thread that should allocate a new region
+ // thread that suffers the exhaustion. That thread sets "alloc_underway" and enters a critical section.
+ // Threads arriving here later are excluded from the critical section, yield the CPU, and then retry the
+ // allocation. After some time the magazine is resupplied, the original thread leaves with its allocation,
+ // and retry-ing threads succeed in the code just above.
+ if (!tiny_mag_ptr->alloc_underway) {
void *fresh_region;
// time to create a new region (do this outside the magazine lock)
+ tiny_mag_ptr->alloc_underway = TRUE;
+ OSMemoryBarrier();
SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
fresh_region = mvm_allocate_pages(TINY_REGION_SIZE,
TINY_BLOCKS_ALIGN,
@@ -2493,8 +2494,9 @@
MAGMALLOC_ALLOCREGION(TINY_SZONE_FROM_RACK(rack), (int)mag_index, fresh_region, TINY_REGION_SIZE);
if (!fresh_region) { // out of memory!
+ tiny_mag_ptr->alloc_underway = FALSE;
+ OSMemoryBarrier();
SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
- _malloc_lock_unlock(&tiny_mag_ptr->magazine_alloc_lock);
return NULL;
}
@@ -2503,20 +2505,14 @@
// we don't clear or zero-check because this freshly allocated space
// is pristine
+ tiny_mag_ptr->alloc_underway = FALSE;
+ OSMemoryBarrier();
SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
- _malloc_lock_unlock(&tiny_mag_ptr->magazine_alloc_lock);
CHECK(szone, __PRETTY_FUNCTION__);
return ptr;
} else {
- // We failed to get the alloc lock, so someone else is allocating.
- // Drop the magazine lock...
SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
-
- // Wait for the other thread on the alloc lock
- _malloc_lock_lock(&tiny_mag_ptr->magazine_alloc_lock);
- _malloc_lock_unlock(&tiny_mag_ptr->magazine_alloc_lock);
-
- // Reacquire the magazine lock to go around the loop again
+ yield();
SZONE_MAGAZINE_PTR_LOCK(tiny_mag_ptr);
}
}
@@ -2599,7 +2595,7 @@
}
#endif
- if (malloc_zero_policy == MALLOC_ZERO_ON_FREE) {
+ if (malloc_zero_on_free) {
memset(ptr, '\0', TINY_BYTES_FOR_MSIZE(msize));
}
@@ -2620,7 +2616,7 @@
return;
}
- if (malloc_zero_policy != MALLOC_ZERO_ON_FREE) {
+ if (!malloc_zero_on_free) {
if ((rack->debug_flags & MALLOC_DO_SCRIBBLE) && msize) {
memset(ptr, SCRABBLE_BYTE, TINY_BYTES_FOR_MSIZE(msize));
}
@@ -2675,8 +2671,7 @@
{
msize_t msize = TINY_MSIZE_FOR_BYTES(size + TINY_QUANTUM - 1);
unsigned found = 0;
-
- mag_index_t mag_index = rack_get_thread_index(&szone->tiny_rack) % szone->tiny_rack.num_magazines;
+ mag_index_t mag_index = tiny_mag_get_thread_index() % szone->tiny_rack.num_magazines;
magazine_t *tiny_mag_ptr = &(szone->tiny_rack.magazines[mag_index]);
// make sure to return objects at least one quantum in size
@@ -2751,7 +2746,7 @@
if (is_free) {
break; // a double free; let the standard free deal with it
}
- if (malloc_zero_policy == MALLOC_ZERO_ON_FREE) {
+ if (malloc_zero_on_free) {
memset(ptr, '\0', TINY_BYTES_FOR_MSIZE(msize));
}
if (!tiny_free_no_lock(&szone->tiny_rack, tiny_mag_ptr, mag_index, tiny_region, ptr, msize, 0)) {