Loading...
--- libmalloc/libmalloc-317.100.9/src/magazine_small.c
+++ libmalloc/libmalloc-646.0.13/src/magazine_small.c
@@ -66,25 +66,6 @@
 	meta_headers[index] = 0;
 }
 
-static MALLOC_INLINE MALLOC_ALWAYS_INLINE
-mag_index_t
-small_mag_get_thread_index(void)
-{
-#if CONFIG_SMALL_USES_HYPER_SHIFT
-	if (os_likely(_os_cpu_number_override == -1)) {
-		return _malloc_cpu_number() >> hyper_shift;
-	} else {
-		return _os_cpu_number_override >> hyper_shift;
-	}
-#else // CONFIG_SMALL_USES_HYPER_SHIFT
-	if (os_likely(_os_cpu_number_override == -1)) {
-		return _malloc_cpu_number();
-	} else {
-		return _os_cpu_number_override;
-	}
-#endif // CONFIG_SMALL_USES_HYPER_SHIFT
-}
-
 #pragma mark in-place free list
 
 static MALLOC_INLINE void
@@ -833,8 +814,8 @@
 		OSAtomicIncrement32Barrier(&(REGION_TRAILER_FOR_SMALL_REGION(r)->pinned_to_depot));
 		SZONE_MAGAZINE_PTR_UNLOCK(depot_ptr);
 		for (i = 0; i < advisories; ++i) {
-			uintptr_t addr = (advisory[i].pnum << vm_page_quanta_shift) + (uintptr_t)r;
-			size_t size = advisory[i].size << vm_page_quanta_shift;
+			uintptr_t addr = (advisory[i].pnum << vm_kernel_page_shift) + (uintptr_t)r;
+			size_t size = advisory[i].size << vm_kernel_page_shift;
 
 			mvm_madvise_free(rack, r, addr, addr + size, NULL, rack->debug_flags & MALLOC_DO_SCRIBBLE);
 		}
@@ -916,11 +897,6 @@
 {
 	magazine_t *depot_ptr = &(rack->magazines[DEPOT_MAGAZINE_INDEX]);
 
-	/* FIXME: Would Uniprocessor benefit from recirc and MADV_FREE? */
-	if (rack->num_magazines == 1) { // Uniprocessor, single magazine, so no recirculation necessary
-		return 0;
-	}
-
 #if DEBUG_MALLOC
 	if (DEPOT_MAGAZINE_INDEX == mag_index) {
 		malloc_zone_error(rack->debug_flags, true, "small_get_region_from_depot called for magazine index -1\n", NULL, NULL);
@@ -933,22 +909,32 @@
 	// Appropriate a Depot'd region that can satisfy requested msize.
 	region_trailer_t *node;
 	region_t sparse_region;
+	msize_t try_msize = msize;
 
 	while (1) {
-		sparse_region = small_find_msize_region(rack, depot_ptr, DEPOT_MAGAZINE_INDEX, msize);
+		sparse_region = small_find_msize_region(rack, depot_ptr, DEPOT_MAGAZINE_INDEX, try_msize);
 		if (NULL == sparse_region) { // Depot empty?
 			SZONE_MAGAZINE_PTR_UNLOCK(depot_ptr);
 			return 0;
 		}
 
 		node = REGION_TRAILER_FOR_SMALL_REGION(sparse_region);
-		if (0 >= node->pinned_to_depot) {
+		if (0 == node->pinned_to_depot) {
+			// Found one!
 			break;
 		}
 
-		SZONE_MAGAZINE_PTR_UNLOCK(depot_ptr);
-		yield();
-		SZONE_MAGAZINE_PTR_LOCK(depot_ptr);
+		// Try the next msize up - maybe the head of its free list will be in
+		// a region we can use. Once we get the region we'll still allocate the
+		// original msize.
+		try_msize++;
+
+		if (try_msize > NUM_SMALL_SLOTS) {
+			// Tried all the msizes but couldn't get a usable region. Let's
+			// give up for now and we'll allocate a new region from the kernel.
+			SZONE_MAGAZINE_PTR_UNLOCK(depot_ptr);
+			return 0;
+		}
 	}
 
 	// disconnect node from Depot
@@ -959,7 +945,7 @@
 
 	// Transfer ownership of the region
 	MAGAZINE_INDEX_FOR_SMALL_REGION(sparse_region) = mag_index;
-	node->pinned_to_depot = 0;
+	MALLOC_ASSERT(node->pinned_to_depot == 0);
 
 	// Iterate the region putting its free entries on its new (locked) magazine's free list
 	size_t bytes_inplay = small_free_reattach_region(rack, small_mag_ptr, sparse_region);
@@ -1031,6 +1017,11 @@
 				continue;
 			}
 
+			if (REGION_TRAILER_FOR_SMALL_REGION(small)->pinned_to_depot > 0) {
+				SZONE_MAGAZINE_PTR_UNLOCK(mag_ptr);
+				continue;
+			}
+
 			if (small == mag_ptr->mag_last_region && (mag_ptr->mag_bytes_free_at_end || mag_ptr->mag_bytes_free_at_start)) {
 				small_finalize_region(rack, mag_ptr);
 			}
@@ -1045,7 +1036,7 @@
 
 			SZONE_MAGAZINE_PTR_LOCK(small_depot_ptr);
 			MAGAZINE_INDEX_FOR_SMALL_REGION(small) = DEPOT_MAGAZINE_INDEX;
-			REGION_TRAILER_FOR_SMALL_REGION(small)->pinned_to_depot = 0;
+			MALLOC_ASSERT(REGION_TRAILER_FOR_SMALL_REGION(small)->pinned_to_depot == 0);
 
 			size_t bytes_inplay = small_free_reattach_region(rack, small_depot_ptr, small);
 
@@ -1262,11 +1253,7 @@
 	region_trailer_t *node = REGION_TRAILER_FOR_SMALL_REGION(region);
 	size_t bytes_used = node->bytes_used;
 
-	/* FIXME: Would Uniprocessor benefit from recirc and MADV_FREE? */
-	if (rack->num_magazines == 1) { // Uniprocessor, single magazine, so no recirculation necessary
-		/* NOTHING */
-		return TRUE; // Caller must do SZONE_MAGAZINE_PTR_UNLOCK(tiny_mag_ptr)
-	} else if (DEPOT_MAGAZINE_INDEX != mag_index) {
+	if (DEPOT_MAGAZINE_INDEX != mag_index) {
 		// Emptiness discriminant
 		if (small_region_below_recirc_threshold(region)) {
 			/* Region has crossed threshold from density to sparsity. Mark it "suitable" on the
@@ -2075,7 +2062,7 @@
 small_malloc_should_clear(rack_t *rack, msize_t msize, boolean_t cleared_requested)
 {
 	void *ptr;
-	mag_index_t mag_index = small_mag_get_thread_index() % rack->num_magazines;
+	mag_index_t mag_index = rack_get_thread_index(rack) % rack->num_magazines;
 	magazine_t *small_mag_ptr = &(rack->magazines[mag_index]);
 
 	MALLOC_TRACE(TRACE_small_malloc, (uintptr_t)rack, SMALL_BYTES_FOR_MSIZE(msize), (uintptr_t)small_mag_ptr, cleared_requested);