Loading...
--- libmalloc/libmalloc-792.1.1/src/magazine_inline.h
+++ libmalloc/libmalloc-317.40.8/src/magazine_inline.h
@@ -24,10 +24,41 @@
#ifndef __MAGAZINE_INLINE_H
#define __MAGAZINE_INLINE_H
-#include <malloc/_ptrcheck.h>
-__ptrcheck_abi_assume_single()
-
extern unsigned int _os_cpu_number_override;
+
+/*
+ * MALLOC_ABSOLUTE_MAX_SIZE - There are many instances of addition to a
+ * user-specified size_t, which can cause overflow (and subsequent crashes)
+ * for values near SIZE_T_MAX. Rather than add extra "if" checks everywhere
+ * this occurs, it is easier to just set an absolute maximum request size,
+ * and immediately return an error if the requested size exceeds this maximum.
+ * Of course, values less than this absolute max can fail later if the value
+ * is still too large for the available memory. The largest value added
+ * seems to be large_vm_page_quanta_size (in the macro round_large_page_quanta()), so to be safe, we set
+ * the maximum to be 2 * PAGE_SIZE less than SIZE_T_MAX.
+ */
+#define MALLOC_ABSOLUTE_MAX_SIZE (SIZE_T_MAX - (2 * large_vm_page_quanta_size))
+
+// Gets the allocation size for a calloc(). Multiples size by num_items and adds
+// extra_size, storing the result in *total_size. Returns 0 on success, -1 (with
+// errno set to ENOMEM) on overflow.
+static int MALLOC_INLINE MALLOC_ALWAYS_INLINE
+calloc_get_size(size_t num_items, size_t size, size_t extra_size, size_t *total_size)
+{
+ size_t alloc_size = size;
+ if (num_items != 1 && (os_mul_overflow(num_items, size, &alloc_size)
+ || alloc_size > MALLOC_ABSOLUTE_MAX_SIZE)) {
+ errno = ENOMEM;
+ return -1;
+ }
+ if (extra_size && (os_add_overflow(alloc_size, extra_size, &alloc_size)
+ || alloc_size > MALLOC_ABSOLUTE_MAX_SIZE)) {
+ errno = ENOMEM;
+ return -1;
+ }
+ *total_size = alloc_size;
+ return 0;
+}
/********************* FREE LIST UTILITIES ************************/
@@ -64,6 +95,19 @@
static MALLOC_INLINE void recirc_list_splice_last(rack_t *rack, magazine_t *mag_ptr, region_trailer_t *node) MALLOC_ALWAYS_INLINE;
static MALLOC_INLINE void recirc_list_splice_first(rack_t *rack, magazine_t *mag_ptr, region_trailer_t *node) MALLOC_ALWAYS_INLINE;
+static MALLOC_INLINE void
+yield(void)
+{
+ thread_switch(MACH_PORT_NULL, SWITCH_OPTION_DEPRESS, 1);
+}
+
+static MALLOC_INLINE kern_return_t
+_malloc_default_reader(task_t task, vm_address_t address, vm_size_t size, void **ptr)
+{
+ *ptr = (void *)address;
+ return 0;
+}
+
#pragma mark helpers
static MALLOC_INLINE MALLOC_ALWAYS_INLINE
@@ -153,18 +197,23 @@
"Corrupt value: %p\n", ptr, value);
}
-// TODO: replace uses in small and medium with data PAC when possible
static MALLOC_INLINE uintptr_t
free_list_gen_checksum(uintptr_t ptr)
{
-#if defined(__LP64__)
- uint32_t level1 = (uint32_t)ptr + ((uint32_t)(ptr >> 32));
-#else
- uint32_t level1 = (uint32_t)ptr;
+ uint8_t chk;
+
+ chk = (unsigned char)(ptr >> 0);
+ chk += (unsigned char)(ptr >> 8);
+ chk += (unsigned char)(ptr >> 16);
+ chk += (unsigned char)(ptr >> 24);
+#if __LP64__
+ chk += (unsigned char)(ptr >> 32);
+ chk += (unsigned char)(ptr >> 40);
+ chk += (unsigned char)(ptr >> 48);
+ chk += (unsigned char)(ptr >> 56);
#endif
- uint16_t level2 = (uint16_t)level1 + ((uint16_t)(level1 >> 16));
- uint8_t level3 = (uint8_t)level2 + ((uint8_t)(level2 >> 8));
- return level3;
+
+ return chk;
}
static unsigned
@@ -178,7 +227,7 @@
while (ptr.p) {
count++;
if (reader(task, (vm_address_t)ptr.inplace, sizeof(*ptr.inplace),
- __unsafe_forge_single(void **, &mapped_inplace_free_entry))) {
+ (void **)&mapped_inplace_free_entry)) {
printer("** invalid pointer in free list: %p\n", ptr.inplace);
break;
}
@@ -187,43 +236,8 @@
return count;
}
-#if __has_feature(ptrauth_calls) && defined(__arm64e__) && !TARGET_OS_SIMULATOR
-
-// We can use data PAC to protect the free list pointers
-static MALLOC_INLINE uintptr_t
-free_list_checksum_ptr(rack_t *rack, void *ptr)
-{
- uintptr_t signed_ptr = (uintptr_t)ptrauth_sign_unauthenticated(ptr,
- ptrauth_key_process_dependent_data,
- ptrauth_blend_discriminator(rack,
- ptrauth_string_discriminator("malloc freelist")));
- return signed_ptr;
-}
-
-static MALLOC_INLINE void *
-free_list_unchecksum_ptr(rack_t *rack, inplace_union *ptr)
-{
- void * __single stored_ptr = ptr->p;
- // N.B. we don't use ptrauth_auth_data() because we want to be able to call
- // free_list_checksum_botch() on failure, which prints a diagnostic first,
- // rather than trapping directly
- void * __single stripped_ptr = ptrauth_strip(stored_ptr, ptrauth_key_process_dependent_data);
- uintptr_t resigned_ptr = free_list_checksum_ptr(rack, stripped_ptr);
- if ((uintptr_t)stored_ptr != resigned_ptr) {
- free_list_checksum_botch(rack, ptr, __unsafe_forge_single(void *, ptr->u));
- __builtin_trap();
- }
- return stripped_ptr;
-}
-
-#else // __has_feature(ptrauth_calls) && defined(__arm64e__) && !TARGET_OS_SIMULATOR
-
-// We can't use data PAC so we manually calculate and store a checksum instead
-// TODO: use the high bits on LP64
-// TODO: this can likely still be faster
-
#define NYBBLE 4
-#if defined(__LP64__)
+#if __LP64__
#define ANTI_NYBBLE (64 - NYBBLE)
#else
#define ANTI_NYBBLE (32 - NYBBLE)
@@ -246,7 +260,7 @@
p.u = t & ~(uintptr_t)0xF;
if ((t ^ free_list_gen_checksum(p.u ^ rack->cookie)) & (uintptr_t)0xF) {
- free_list_checksum_botch(rack, ptr, __unsafe_forge_single(void *, ptr->u));
+ free_list_checksum_botch(rack, ptr, (void *)ptr->u);
__builtin_trap();
}
return p.p;
@@ -254,8 +268,6 @@
#undef ANTI_NYBBLE
#undef NYBBLE
-
-#endif // __has_feature(ptrauth_calls) && defined(__arm64e__) && !TARGET_OS_SIMULATOR
#pragma mark recirc helpers
@@ -327,7 +339,7 @@
* cache would likely be a significant performance win here.
*/
static MALLOC_INLINE rgnhdl_t
-hash_lookup_region_no_lock(region_t * __counted_by(num_entries) regions, size_t num_entries, size_t shift, region_t r)
+hash_lookup_region_no_lock(region_t *regions, size_t num_entries, size_t shift, region_t r)
{
size_t index, hash_index;
rgnhdl_t entry;
@@ -339,7 +351,7 @@
// Multiplicative hash where the multiplier is a prime near (ULONG_MAX / phi). [phi = 1.618033...]
// Since the values of (((uintptr_t)r >> HASH_BLOCKS_ALIGN) are (roughly) an ascending sequence of integers,
// this hash works really well. See Knuth TAOCP, Vol. 3.
-#if defined(__LP64__)
+#if __LP64__
index = hash_index = (((uintptr_t)r >> HASH_BLOCKS_ALIGN) * 11400714819323198549ULL) >> (64 - shift);
#else
index = hash_index = (((uintptr_t)r >> HASH_BLOCKS_ALIGN) * 2654435761UL) >> (32 - shift);
@@ -363,7 +375,7 @@
* hash_region_insert_no_lock - Insert a region into the hash ring.
*/
static void
-hash_region_insert_no_lock(region_t * __counted_by(num_entries) regions, size_t num_entries, size_t shift, region_t r)
+hash_region_insert_no_lock(region_t *regions, size_t num_entries, size_t shift, region_t r)
{
size_t index, hash_index;
rgnhdl_t entry;
@@ -371,7 +383,7 @@
// Multiplicative hash where the multiplier is a prime near (ULONG_MAX / phi). [phi = 1.618033...]
// Since the values of (((uintptr_t)r >> HASH_BLOCKS_ALIGN) are (roughly) an ascending sequence of integers,
// this hash works really well. See Knuth TAOCP, Vol. 3.
-#if defined(__LP64__)
+#if __LP64__
index = hash_index = (((uintptr_t)r >> HASH_BLOCKS_ALIGN) * 11400714819323198549ULL) >> (64 - shift);
#else
index = hash_index = (((uintptr_t)r >> HASH_BLOCKS_ALIGN) * 2654435761UL) >> (32 - shift);
@@ -394,7 +406,7 @@
* region, and asking the small region to allocate space for the new list of
* regions.
*/
-static region_t * __counted_by(num_entries)
+static region_t *
hash_regions_alloc_no_lock(size_t num_entries)
{
size_t size = num_entries * sizeof(region_t);
@@ -407,25 +419,34 @@
* the old entries since someone may still be allocating them.
*/
static MALLOC_INLINE region_t *
-hash_regions_grow_no_lock(region_t * __counted_by(old_count) regions, size_t old_count, size_t *mutable_shift, size_t *new_count)
+hash_regions_grow_no_lock(region_t *regions, size_t old_size, size_t *mutable_shift, size_t *new_size)
{
// double in size and allocate memory for the regions
- *new_count = old_count + old_count;
+ *new_size = old_size + old_size;
*mutable_shift = *mutable_shift + 1;
- region_t *new_regions = hash_regions_alloc_no_lock(*new_count);
+ region_t *new_regions = hash_regions_alloc_no_lock(*new_size);
// rehash the entries into the new list
size_t index;
- for (index = 0; index < old_count; ++index) {
+ for (index = 0; index < old_size; ++index) {
region_t r = regions[index];
if (r != HASHRING_OPEN_ENTRY && r != HASHRING_REGION_DEALLOCATED) {
- hash_region_insert_no_lock(new_regions, *new_count, *mutable_shift, r);
+ hash_region_insert_no_lock(new_regions, *new_size, *mutable_shift, r);
}
}
return new_regions;
}
#pragma mark mag index
+
+/*
+ * These commpage routines provide fast access to the logical cpu number
+ * of the calling processor assuming no pre-emption occurs.
+ */
+
+extern unsigned int hyper_shift;
+extern unsigned int phys_ncpus;
+extern unsigned int logical_ncpus;
static MALLOC_INLINE MALLOC_ALWAYS_INLINE
unsigned int
@@ -444,10 +465,10 @@
#pragma mark mag lock
static MALLOC_INLINE magazine_t *
-mag_lock_zine_for_region_trailer(magazine_t * __unsafe_indexable magazines, region_trailer_t *trailer, mag_index_t mag_index)
+mag_lock_zine_for_region_trailer(magazine_t *magazines, region_trailer_t *trailer, mag_index_t mag_index)
{
mag_index_t refreshed_index;
- magazine_t *mag_ptr = __unsafe_forge_single(magazine_t *, &(magazines[mag_index]));
+ magazine_t *mag_ptr = &(magazines[mag_index]);
// Take the lock on entry.
SZONE_MAGAZINE_PTR_LOCK(mag_ptr);
@@ -460,7 +481,7 @@
SZONE_MAGAZINE_PTR_UNLOCK(mag_ptr);
mag_index = refreshed_index;
- mag_ptr = __unsafe_forge_single(magazine_t *, &(magazines[mag_index]));
+ mag_ptr = &(magazines[mag_index]);
SZONE_MAGAZINE_PTR_LOCK(mag_ptr);
}
@@ -468,6 +489,8 @@
}
#pragma mark Region Cookie
+
+extern uint64_t malloc_entropy[2];
static region_cookie_t
region_cookie(void)
@@ -516,8 +539,8 @@
static MALLOC_INLINE msize_t
get_tiny_free_size_offset(const void *ptr, off_t mapped_offset)
{
- void * __single next_block = __unsafe_forge_single(void *, (uintptr_t)ptr + TINY_QUANTUM);
- void * __single region_end = TINY_REGION_HEAP_END(TINY_REGION_FOR_PTR(ptr));
+ void *next_block = (void *)((uintptr_t)ptr + TINY_QUANTUM);
+ void *region_end = TINY_REGION_HEAP_END(TINY_REGION_FOR_PTR(ptr));
// check whether the next block is outside the tiny region or a block header
// if so, then the size of this block is one, and there is no stored size.
@@ -687,28 +710,4 @@
return r ? *r : r;
}
-#pragma mark zero on free
-
-MALLOC_NOEXPORT
-extern unsigned malloc_zero_on_free_sample_period;
-
-static MALLOC_INLINE bool
-zero_on_free_should_sample(void)
-{
- bool sample = false;
- if (malloc_zero_on_free_sample_period != 0) {
- uintptr_t value = (uintptr_t)_pthread_getspecific_direct(
- __TSD_MALLOC_ZERO_CORRUPTION_COUNTER);
- value++;
- if (value == malloc_zero_on_free_sample_period) {
- sample = true;
- value = 0;
- }
- _pthread_setspecific_direct(__TSD_MALLOC_ZERO_CORRUPTION_COUNTER,
- (void *)value);
- }
-
- return sample;
-}
-
#endif // __MAGAZINE_INLINE_H