Loading...
--- libmalloc/libmalloc-409.60.6/src/magazine_tiny.c
+++ libmalloc/libmalloc-792.60.6/src/magazine_tiny.c
@@ -37,25 +37,6 @@
 // Decrements the size part of mag_last_free by a given msize value. Must not
 // reduce the msize part below zero (not checked).
 #define TINY_MAG_LAST_FREE_PTR_DEC_MSIZE(x, msize_delta) (x) = ((void *)(x) - (msize_delta))
-
-static MALLOC_INLINE MALLOC_ALWAYS_INLINE
-mag_index_t
-tiny_mag_get_thread_index(void)
-{
-#if CONFIG_TINY_USES_HYPER_SHIFT
-	if (os_likely(_os_cpu_number_override == -1)) {
-		return _malloc_cpu_number() >> hyper_shift;
-	} else {
-		return _os_cpu_number_override >> hyper_shift;
-	}
-#else // CONFIG_SMALL_USES_HYPER_SHIFT
-	if (os_likely(_os_cpu_number_override == -1)) {
-		return _malloc_cpu_number();
-	} else {
-		return _os_cpu_number_override;
-	}
-#endif // CONFIG_SMALL_USES_HYPER_SHIFT
-}
 
 static inline grain_t
 tiny_slot_from_msize(msize_t msize)
@@ -296,7 +277,7 @@
 		if (!clear) {
 			break;
 		}
-		// fall through
+		MALLOC_FALLTHROUGH;
 	case MALLOC_ZERO_ON_ALLOC:
 		memset(ptr, '\0', TINY_BYTES_FOR_MSIZE(msize));
 		break;
@@ -2269,6 +2250,12 @@
 				BITMAPV_CLR(tiny_mag_ptr->mag_bitmap, slot);
 			}
 			this_msize = get_tiny_free_size(ptr);
+			if (os_unlikely(this_msize < msize)) {
+				malloc_zone_error(MALLOC_ABORT_ON_CORRUPTION, true,
+						"Corruption of tiny freelist %p: size too small (%u/%u)\n",
+						ptr, this_msize, msize);
+
+			}
 			tiny_update_region_free_list_for_remove(slot, ptr, next);
 			tiny_check_and_zero_inline_meta_from_freelist(rack, ptr, this_msize);
 			goto add_leftover_and_proceed;
@@ -2285,6 +2272,11 @@
 	ptr = limit->p;
 	if (ptr) {
 		this_msize = get_tiny_free_size(ptr);
+        if (os_unlikely(this_msize < msize)) {
+            malloc_zone_error(MALLOC_ABORT_ON_CORRUPTION, true,
+                    "Corruption of tiny freelist %p: size too small (%u/%u)\n",
+                    ptr, this_msize, msize);
+        }
 		next = free_list_unchecksum_ptr(rack, &ptr->next);
 		if (this_msize - msize > NUM_TINY_SLOTS) {
 			// the leftover will go back to the free list, so we optimize by
@@ -2415,7 +2407,7 @@
 tiny_malloc_should_clear(rack_t *rack, msize_t msize, boolean_t cleared_requested)
 {
 	void *ptr;
-	mag_index_t mag_index = tiny_mag_get_thread_index() % rack->num_magazines;
+	mag_index_t mag_index = rack_get_thread_index(rack) % rack->num_magazines;
 	magazine_t *tiny_mag_ptr = &(rack->magazines[mag_index]);
 
 	MALLOC_TRACE(TRACE_tiny_malloc, (uintptr_t)rack, TINY_BYTES_FOR_MSIZE(msize), (uintptr_t)tiny_mag_ptr, cleared_requested);
@@ -2479,16 +2471,17 @@
 
 		// The magazine is exhausted. A new region (heap) must be allocated to satisfy this call to malloc().
 		// The allocation, an mmap() system call, will be performed outside the magazine spin locks by the first
-		// thread that suffers the exhaustion. That thread sets "alloc_underway" and enters a critical section.
-		// Threads arriving here later are excluded from the critical section, yield the CPU, and then retry the
-		// allocation. After some time the magazine is resupplied, the original thread leaves with its allocation,
-		// and retry-ing threads succeed in the code just above.
-		if (!tiny_mag_ptr->alloc_underway) {
+		// thread that suffers the exhaustion. That thread accquires the magazine_alloc_lock, then drops the
+		// magazine lock to allow freeing threads to proceed. Allocating thrads that arrive later  are excluded
+		// from the critial section by the alloc lock. When those are unblocked, they succeed in the code above.
+		//
+		// Note that we need to trylock the alloc lock to avoid a deadlock, since we can't block on the alloc
+		// lock while holding the magazine lock
+		if (os_likely(_malloc_lock_trylock(&tiny_mag_ptr->magazine_alloc_lock))) {
+			// We got the alloc lock, so we are the thread that should allocate a new region
 			void *fresh_region;
 
 			// time to create a new region (do this outside the magazine lock)
-			tiny_mag_ptr->alloc_underway = TRUE;
-			OSMemoryBarrier();
 			SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
 			fresh_region = mvm_allocate_pages(TINY_REGION_SIZE,
 					TINY_BLOCKS_ALIGN,
@@ -2500,9 +2493,8 @@
 			MAGMALLOC_ALLOCREGION(TINY_SZONE_FROM_RACK(rack), (int)mag_index, fresh_region, TINY_REGION_SIZE);
 
 			if (!fresh_region) { // out of memory!
-				tiny_mag_ptr->alloc_underway = FALSE;
-				OSMemoryBarrier();
 				SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
+				_malloc_lock_unlock(&tiny_mag_ptr->magazine_alloc_lock);
 				return NULL;
 			}
 
@@ -2511,14 +2503,20 @@
 
 			// we don't clear or zero-check because this freshly allocated space
 			// is pristine
-			tiny_mag_ptr->alloc_underway = FALSE;
-			OSMemoryBarrier();
 			SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
+			_malloc_lock_unlock(&tiny_mag_ptr->magazine_alloc_lock);
 			CHECK(szone, __PRETTY_FUNCTION__);
 			return ptr;
 		} else {
+			// We failed to get the alloc lock, so someone else is allocating.
+			// Drop the magazine lock...
 			SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr);
-			yield();
+
+			// Wait for the other thread on the alloc lock
+			_malloc_lock_lock(&tiny_mag_ptr->magazine_alloc_lock);
+			_malloc_lock_unlock(&tiny_mag_ptr->magazine_alloc_lock);
+
+			// Reacquire the magazine lock to go around the loop again
 			SZONE_MAGAZINE_PTR_LOCK(tiny_mag_ptr);
 		}
 	}
@@ -2677,7 +2675,8 @@
 {
 	msize_t msize = TINY_MSIZE_FOR_BYTES(size + TINY_QUANTUM - 1);
 	unsigned found = 0;
-	mag_index_t mag_index = tiny_mag_get_thread_index() % szone->tiny_rack.num_magazines;
+
+	mag_index_t mag_index = rack_get_thread_index(&szone->tiny_rack) % szone->tiny_rack.num_magazines;
 	magazine_t *tiny_mag_ptr = &(szone->tiny_rack.magazines[mag_index]);
 
 	// make sure to return objects at least one quantum in size