Loading...
--- libmalloc/libmalloc-792.80.2/src/magazine_small.c
+++ libmalloc/libmalloc-374.60.3/src/magazine_small.c
@@ -66,6 +66,25 @@
 	meta_headers[index] = 0;
 }
 
+static MALLOC_INLINE MALLOC_ALWAYS_INLINE
+mag_index_t
+small_mag_get_thread_index(void)
+{
+#if CONFIG_SMALL_USES_HYPER_SHIFT
+	if (os_likely(_os_cpu_number_override == -1)) {
+		return _malloc_cpu_number() >> hyper_shift;
+	} else {
+		return _os_cpu_number_override >> hyper_shift;
+	}
+#else // CONFIG_SMALL_USES_HYPER_SHIFT
+	if (os_likely(_os_cpu_number_override == -1)) {
+		return _malloc_cpu_number();
+	} else {
+		return _os_cpu_number_override;
+	}
+#endif // CONFIG_SMALL_USES_HYPER_SHIFT
+}
+
 #pragma mark in-place free list
 
 static MALLOC_INLINE void
@@ -814,8 +833,8 @@
 		OSAtomicIncrement32Barrier(&(REGION_TRAILER_FOR_SMALL_REGION(r)->pinned_to_depot));
 		SZONE_MAGAZINE_PTR_UNLOCK(depot_ptr);
 		for (i = 0; i < advisories; ++i) {
-			uintptr_t addr = (advisory[i].pnum << vm_kernel_page_shift) + (uintptr_t)r;
-			size_t size = advisory[i].size << vm_kernel_page_shift;
+			uintptr_t addr = (advisory[i].pnum << vm_page_quanta_shift) + (uintptr_t)r;
+			size_t size = advisory[i].size << vm_page_quanta_shift;
 
 			mvm_madvise_free(rack, r, addr, addr + size, NULL, rack->debug_flags & MALLOC_DO_SCRIBBLE);
 		}
@@ -896,6 +915,11 @@
 small_get_region_from_depot(rack_t *rack, magazine_t *small_mag_ptr, mag_index_t mag_index, msize_t msize)
 {
 	magazine_t *depot_ptr = &(rack->magazines[DEPOT_MAGAZINE_INDEX]);
+
+	/* FIXME: Would Uniprocessor benefit from recirc and MADV_FREE? */
+	if (rack->num_magazines == 1) { // Uniprocessor, single magazine, so no recirculation necessary
+		return 0;
+	}
 
 #if DEBUG_MALLOC
 	if (DEPOT_MAGAZINE_INDEX == mag_index) {
@@ -1253,7 +1277,11 @@
 	region_trailer_t *node = REGION_TRAILER_FOR_SMALL_REGION(region);
 	size_t bytes_used = node->bytes_used;
 
-	if (DEPOT_MAGAZINE_INDEX != mag_index) {
+	/* FIXME: Would Uniprocessor benefit from recirc and MADV_FREE? */
+	if (rack->num_magazines == 1) { // Uniprocessor, single magazine, so no recirculation necessary
+		/* NOTHING */
+		return TRUE; // Caller must do SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr)
+	} else if (DEPOT_MAGAZINE_INDEX != mag_index) {
 		// Emptiness discriminant
 		if (small_region_below_recirc_threshold(region)) {
 			/* Region has crossed threshold from density to sparsity. Mark it "suitable" on the
@@ -2062,7 +2090,7 @@
 small_malloc_should_clear(rack_t *rack, msize_t msize, boolean_t cleared_requested)
 {
 	void *ptr;
-	mag_index_t mag_index = rack_get_thread_index(rack) % rack->num_magazines;
+	mag_index_t mag_index = small_mag_get_thread_index() % rack->num_magazines;
 	magazine_t *small_mag_ptr = &(rack->magazines[mag_index]);
 
 	MALLOC_TRACE(TRACE_small_malloc, (uintptr_t)rack, SMALL_BYTES_FOR_MSIZE(msize), (uintptr_t)small_mag_ptr, cleared_requested);
@@ -2113,17 +2141,16 @@
 
 		// The magazine is exhausted. A new region (heap) must be allocated to satisfy this call to malloc().
 		// The allocation, an mmap() system call, will be performed outside the magazine spin locks by the first
-		// thread that suffers the exhaustion. That thread accquires the magazine_alloc_lock, then drops the
-		// magazine lock to allow freeing threads to proceed. Allocating thrads that arrive later  are excluded
-		// from the critial section by the alloc lock. When those are unblocked, they succeed in the code above.
-		//
-		// Note that we need to trylock the alloc lock to avoid a deadlock, since we can't block on the alloc
-		// lock while holding the magazine lock
-		if (os_likely(_malloc_lock_trylock(&small_mag_ptr->magazine_alloc_lock))) {
-			// We got the alloc lock, so we are the thread that should allocate a new region
+		// thread that suffers the exhaustion. That thread sets "alloc_underway" and enters a critical section.
+		// Threads arriving here later are excluded from the critical section, yield the CPU, and then retry the
+		// allocation. After some time the magazine is resupplied, the original thread leaves with its allocation,
+		// and retry-ing threads succeed in the code just above.
+		if (!small_mag_ptr->alloc_underway) {
 			void *fresh_region;
 
 			// time to create a new region (do this outside the magazine lock)
+			small_mag_ptr->alloc_underway = TRUE;
+			OSMemoryBarrier();
 			SZONE_MAGAZINE_PTR_UNLOCK(small_mag_ptr);
 			fresh_region = mvm_allocate_pages(SMALL_REGION_SIZE,
 					SMALL_BLOCKS_ALIGN,
@@ -2135,8 +2162,9 @@
 			MAGMALLOC_ALLOCREGION(SMALL_SZONE_FROM_RACK(rack), (int)mag_index, fresh_region, SMALL_REGION_SIZE);
 
 			if (!fresh_region) { // out of memory!
+				small_mag_ptr->alloc_underway = FALSE;
+				OSMemoryBarrier();
 				SZONE_MAGAZINE_PTR_UNLOCK(small_mag_ptr);
-				_malloc_lock_unlock(&small_mag_ptr->magazine_alloc_lock);
 				return NULL;
 			}
 
@@ -2144,20 +2172,14 @@
 			ptr = small_malloc_from_region_no_lock(rack, small_mag_ptr, mag_index, msize, fresh_region);
 
 			// we don't clear because this freshly allocated space is pristine
+			small_mag_ptr->alloc_underway = FALSE;
+			OSMemoryBarrier();
 			SZONE_MAGAZINE_PTR_UNLOCK(small_mag_ptr);
-			_malloc_lock_unlock(&small_mag_ptr->magazine_alloc_lock);
 			CHECK(szone, __PRETTY_FUNCTION__);
 			return ptr;
 		} else {
-			// We failed to get the alloc lock, so someone else is allocating.
-			// Drop the magazine lock...
 			SZONE_MAGAZINE_PTR_UNLOCK(small_mag_ptr);
-
-			// Wait for the other thread on the alloc lock
-			_malloc_lock_lock(&small_mag_ptr->magazine_alloc_lock);
-			_malloc_lock_unlock(&small_mag_ptr->magazine_alloc_lock);
-
-			// Reacquire the magazine lock to go around the loop again
+			yield();
 			SZONE_MAGAZINE_PTR_LOCK(small_mag_ptr);
 		}
 	}