Loading...
--- libmalloc/libmalloc-283.60.1/src/magazine_inline.h
+++ libmalloc/libmalloc-409.60.6/src/magazine_inline.h
@@ -34,10 +34,19 @@
* and immediately return an error if the requested size exceeds this maximum.
* Of course, values less than this absolute max can fail later if the value
* is still too large for the available memory. The largest value added
- * seems to be PAGE_SIZE (in the macro round_page()), so to be safe, we set
+ * seems to be large_vm_page_quanta_size (in the macro round_large_page_quanta()), so to be safe, we set
* the maximum to be 2 * PAGE_SIZE less than SIZE_T_MAX.
- */
-#define MALLOC_ABSOLUTE_MAX_SIZE (SIZE_T_MAX - (2 * PAGE_SIZE))
+ *
+ * This value needs to be calculated at runtime, so we'll cache it rather than
+ * recalculate on each use.
+ */
+#define _MALLOC_ABSOLUTE_MAX_SIZE (SIZE_T_MAX - (2 * large_vm_page_quanta_size))
+
+#if defined(MALLOC_BUILDING_XCTESTS)
+#define malloc_absolute_max_size _MALLOC_ABSOLUTE_MAX_SIZE
+#else
+extern size_t malloc_absolute_max_size; // caches the definition above
+#endif
// Gets the allocation size for a calloc(). Multiples size by num_items and adds
// extra_size, storing the result in *total_size. Returns 0 on success, -1 (with
@@ -47,13 +56,13 @@
{
size_t alloc_size = size;
if (num_items != 1 && (os_mul_overflow(num_items, size, &alloc_size)
- || alloc_size > MALLOC_ABSOLUTE_MAX_SIZE)) {
- errno = ENOMEM;
+ || alloc_size > malloc_absolute_max_size)) {
+ malloc_set_errno_fast(MZ_POSIX, ENOMEM);
return -1;
}
if (extra_size && (os_add_overflow(alloc_size, extra_size, &alloc_size)
- || alloc_size > MALLOC_ABSOLUTE_MAX_SIZE)) {
- errno = ENOMEM;
+ || alloc_size > malloc_absolute_max_size)) {
+ malloc_set_errno_fast(MZ_POSIX, ENOMEM);
return -1;
}
*total_size = alloc_size;
@@ -197,23 +206,18 @@
"Corrupt value: %p\n", ptr, value);
}
+// TODO: replace uses in small and medium with data PAC when possible
static MALLOC_INLINE uintptr_t
free_list_gen_checksum(uintptr_t ptr)
{
- uint8_t chk;
-
- chk = (unsigned char)(ptr >> 0);
- chk += (unsigned char)(ptr >> 8);
- chk += (unsigned char)(ptr >> 16);
- chk += (unsigned char)(ptr >> 24);
#if __LP64__
- chk += (unsigned char)(ptr >> 32);
- chk += (unsigned char)(ptr >> 40);
- chk += (unsigned char)(ptr >> 48);
- chk += (unsigned char)(ptr >> 56);
+ uint32_t level1 = (uint32_t)ptr + ((uint32_t)(ptr >> 32));
+#else
+ uint32_t level1 = (uint32_t)ptr;
#endif
-
- return chk;
+ uint16_t level2 = (uint16_t)level1 + ((uint16_t)(level1 >> 16));
+ uint8_t level3 = (uint8_t)level2 + ((uint8_t)(level2 >> 8));
+ return level3;
}
static unsigned
@@ -236,6 +240,41 @@
return count;
}
+#if __has_feature(ptrauth_calls) && defined(__arm64e__) && !TARGET_OS_SIMULATOR
+
+// We can use data PAC to protect the free list pointers
+static MALLOC_INLINE uintptr_t
+free_list_checksum_ptr(rack_t *rack, void *ptr)
+{
+ uintptr_t signed_ptr = (uintptr_t)ptrauth_sign_unauthenticated(ptr,
+ ptrauth_key_process_dependent_data,
+ ptrauth_blend_discriminator(rack,
+ ptrauth_string_discriminator("malloc freelist")));
+ return signed_ptr;
+}
+
+static MALLOC_INLINE void *
+free_list_unchecksum_ptr(rack_t *rack, inplace_union *ptr)
+{
+ void *stored_ptr = ptr->p;
+ // N.B. we don't use ptrauth_auth_data() because we want to be able to call
+ // free_list_checksum_botch() on failure, which prints a diagnostic first,
+ // rather than trapping directly
+ void *stripped_ptr = ptrauth_strip(stored_ptr, ptrauth_key_process_dependent_data);
+ uintptr_t resigned_ptr = free_list_checksum_ptr(rack, stripped_ptr);
+ if ((uintptr_t)stored_ptr != resigned_ptr) {
+ free_list_checksum_botch(rack, ptr, (void *)ptr->u);
+ __builtin_trap();
+ }
+ return stripped_ptr;
+}
+
+#else // __has_feature(ptrauth_calls) && defined(__arm64e__) && !TARGET_OS_SIMULATOR
+
+// We can't use data PAC so we manually calculate and store a checksum instead
+// TODO: use the high bits on LP64
+// TODO: this can likely still be faster
+
#define NYBBLE 4
#if __LP64__
#define ANTI_NYBBLE (64 - NYBBLE)
@@ -268,6 +307,8 @@
#undef ANTI_NYBBLE
#undef NYBBLE
+
+#endif // __has_feature(ptrauth_calls) && defined(__arm64e__) && !TARGET_OS_SIMULATOR
#pragma mark recirc helpers
@@ -410,7 +451,7 @@
hash_regions_alloc_no_lock(size_t num_entries)
{
size_t size = num_entries * sizeof(region_t);
- return mvm_allocate_pages(round_page_quanta(size), 0, 0, VM_MEMORY_MALLOC);
+ return mvm_allocate_pages(round_page_quanta(size), 0, DISABLE_ASLR, VM_MEMORY_MALLOC);
}
/*
@@ -492,28 +533,28 @@
extern uint64_t malloc_entropy[2];
-static uint32_t
+static region_cookie_t
region_cookie(void)
{
- return (uint32_t)(malloc_entropy[0] >> 8) & 0xffff;
+ return (region_cookie_t)(malloc_entropy[0] >> 8) & 0xffff;
}
static MALLOC_INLINE void
-region_check_cookie(region_t region, region_trailer_t *trailer)
-{
- if (trailer->region_cookie != region_cookie())
+region_check_cookie(region_t region, region_cookie_t *cookiep)
+{
+ if (*cookiep != region_cookie())
{
malloc_zone_error(MALLOC_ABORT_ON_ERROR, true,
- "Region cookie corrupted for region %p (value is %x)\n",
- region, trailer->region_cookie);
+ "Region cookie corrupted for region %p (value is %x)[%p]\n",
+ region, *cookiep, cookiep);
__builtin_unreachable();
}
}
static MALLOC_INLINE void
-region_set_cookie(region_trailer_t *trailer)
-{
- trailer->region_cookie = region_cookie();
+region_set_cookie(region_cookie_t *cookiep)
+{
+ *cookiep = region_cookie();
}
#pragma mark tiny allocator
@@ -540,7 +581,7 @@
get_tiny_free_size_offset(const void *ptr, off_t mapped_offset)
{
void *next_block = (void *)((uintptr_t)ptr + TINY_QUANTUM);
- void *region_end = TINY_REGION_END(TINY_REGION_FOR_PTR(ptr));
+ void *region_end = TINY_REGION_HEAP_END(TINY_REGION_FOR_PTR(ptr));
// check whether the next block is outside the tiny region or a block header
// if so, then the size of this block is one, and there is no stored size.
@@ -626,7 +667,7 @@
tiny_region_below_recirc_threshold(region_t region)
{
region_trailer_t *trailer = REGION_TRAILER_FOR_TINY_REGION(region);
- return trailer->bytes_used < DENSITY_THRESHOLD(TINY_REGION_PAYLOAD_BYTES);
+ return trailer->bytes_used < DENSITY_THRESHOLD(TINY_HEAP_SIZE);
}
/**
@@ -639,7 +680,7 @@
size_t a = mag_ptr->num_bytes_in_magazine; // Total bytes allocated to this magazine
size_t u = mag_ptr->mag_num_bytes_in_objects; // In use (malloc'd) from this magaqzine
- return a - u > ((3 * TINY_REGION_PAYLOAD_BYTES) / 2)
+ return a - u > ((3 * TINY_HEAP_SIZE) / 2)
&& u < DENSITY_THRESHOLD(a);
}
#endif // CONFIG_RECIRC_DEPOT
@@ -668,7 +709,7 @@
small_region_below_recirc_threshold(region_t region)
{
region_trailer_t *trailer = REGION_TRAILER_FOR_SMALL_REGION(region);
- return trailer->bytes_used < DENSITY_THRESHOLD(SMALL_REGION_PAYLOAD_BYTES);
+ return trailer->bytes_used < DENSITY_THRESHOLD(SMALL_HEAP_SIZE);
}
/**
@@ -681,8 +722,7 @@
size_t a = mag_ptr->num_bytes_in_magazine; // Total bytes allocated to this magazine
size_t u = mag_ptr->mag_num_bytes_in_objects; // In use (malloc'd) from this magaqzine
- return a - u > ((3 * SMALL_REGION_PAYLOAD_BYTES) / 2)
- && u < DENSITY_THRESHOLD(a);
+ return a - u > ((3 * SMALL_HEAP_SIZE) / 2) && u < DENSITY_THRESHOLD(a);
}
#endif // CONFIG_RECIRC_DEPOT
@@ -711,4 +751,28 @@
return r ? *r : r;
}
+#pragma mark zero on free
+
+MALLOC_NOEXPORT
+extern unsigned malloc_zero_on_free_sample_period;
+
+static MALLOC_INLINE bool
+zero_on_free_should_sample(void)
+{
+ bool sample = false;
+ if (malloc_zero_on_free_sample_period != 0) {
+ uintptr_t value = (uintptr_t)_pthread_getspecific_direct(
+ __TSD_MALLOC_ZERO_CORRUPTION_COUNTER);
+ value++;
+ if (value == malloc_zero_on_free_sample_period) {
+ sample = true;
+ value = 0;
+ }
+ _pthread_setspecific_direct(__TSD_MALLOC_ZERO_CORRUPTION_COUNTER,
+ (void *)value);
+ }
+
+ return sample;
+}
+
#endif // __MAGAZINE_INLINE_H