Loading...
--- libmalloc/libmalloc-792.80.2/src/early_malloc.c
+++ libmalloc/libmalloc-474.0.13/src/early_malloc.c
@@ -25,9 +25,11 @@
#if CONFIG_EARLY_MALLOC
-#if MALLOC_TARGET_EXCLAVES_INTROSPECTOR
-#pragma GCC diagnostic ignored "-Wunused-function"
-#endif // MALLOC_TARGET_EXCLAVES_INTROSPECTOR
+#include <assert.h>
+#include <os/atomic_private.h>
+#include <os/crashlog_private.h>
+#include <os/lock_private.h>
+#include <sys/param.h>
/*
* An arena looks like this:
@@ -44,13 +46,7 @@
*/
#define MFM_TRACE 0
-#if MALLOC_TARGET_EXCLAVES || MALLOC_TARGET_EXCLAVES_INTROSPECTOR
-#define MFM_ARENA_SIZE (1ul << 20)
-#else
-// Originally this was set to 8M: we shrank it down to 4M to accomodate for
-// the guarded range reservation (as the arena was heavily underused anyway).
-#define MFM_ARENA_SIZE (4ul << 20)
-#endif /* MALLOC_TARGET_EXCLAVES || MALLOC_TARGET_EXCLAVES_INTROSPECTOR */
+#define MFM_ARENA_SIZE (8ul << 20)
#define MFM_QUANTUM 16ul
#define MFM_SIZE_CLASSES (__builtin_ctz(MFM_ALLOC_SIZE_MAX / MFM_QUANTUM) + 1)
#define MFM_USABLE_SIZE (MFM_ARENA_SIZE - sizeof(struct mfm_header) - 4 * sizeof(uint64_t))
@@ -71,9 +67,8 @@
static_assert(CHAR_BIT == 8, "CHAR_BIT is 8");
static_assert(powerof2(MFM_ALLOC_SIZE_MAX), "MFM_ALLOC_SIZE_MAX is a power of 2");
-static_assert(powerof2(MFM_BLOCK_SIZE_MAX), "MFM_BLOCK_SIZE_MAX is a power of 2");
-
-#if !MALLOC_TARGET_EXCLAVES
+static_assert(powerof2(MFM_BLOCK_SIZE_MAX), "MFM_ALLOC_SIZE_MAX is a power of 2");
+
#define MFM_INTERNAL_CRASH(code, msg) ({ \
_os_set_crash_log_cause_and_message(code, "BUG IN LIBMALLOC: " msg); \
__builtin_trap(); \
@@ -84,20 +79,13 @@
"BUG IN CLIENT OF LIBMALLOC: " msg); \
__builtin_trap(); \
})
-#else
-#define MFM_INTERNAL_CRASH(code, msg) \
- __liblibc_fatal_error("BUG IN LIBMALLOC (%llu): " msg, (uint64_t)code)
-
-#define MFM_CLIENT_CRASH(code, msg) \
- __liblibc_fatal_error("BUG IN CLIENT OF LIBMALLOC (%llu): " msg, (uint64_t)code)
-#endif // !MALLOC_TARGET_EXCLAVES
struct mfm_block {
#if __has_feature(ptrauth_calls)
void *__ptrauth(ptrauth_key_process_dependent_data, true,
ptrauth_string_discriminator("mfmb_next"),
"authenticates-null-values")
- mfmb_next;
+ mfmb_next;
#else
uint64_t mfmb_next;
#endif
@@ -111,9 +99,6 @@
size_t mfm_bump_hwm;
size_t mfm_alloc_count;
struct mfm_block mfm_freelist[MFM_SIZE_CLASSES];
-#if MALLOC_TARGET_EXCLAVES || MALLOC_TARGET_EXCLAVES_INTROSPECTOR
- plat_map_t mfm_map;
-#endif // MALLOC_TARGET_EXCLAVES || MALLOC_TARGET_EXCLAVES_INTROSPECTOR
};
struct mfm_arena {
@@ -159,9 +144,7 @@
#define mfmh_freelist mfm_header.mfm_freelist
static struct mfm_arena *mfm_arena;
-#if CONFIG_MTE
-static bool mfm_memtag_enabled = false;
-#endif // CONFIG_MTE
+
#pragma mark validation and helper functions
@@ -535,24 +518,6 @@
#endif
}
-#if CONFIG_MTE
-/*!
- * @function __mfm_block_fixup_ptr()
- *
- * @brief
- * Loads the tag for a given block pointer, and returns the
- * tagged pointer.
- *
- * @discussion
- * This must be called with mfm_memtag_enabled set to true.
- */
-static struct mfm_block *
-__mfm_block_fixup_ptr(struct mfm_block *blk)
-{
- return (struct mfm_block *)memtag_fixup_ptr((uint8_t *)blk);
-}
-#endif // CONFIG_MTE
-
/*!
* @function __mfm_block_insert_head()
*
@@ -566,24 +531,15 @@
struct mfm_block *blk)
{
uint64_t head, offs, next;
- struct mfm_block *next_blk;
head = __mfm_block_offset(arena, hblk);
next = __mfm_block_next(hblk);
offs = __mfm_block_offset(arena, blk);
- next_blk = &arena->mfm_base[next];
-
-#if CONFIG_MTE
- if (mfm_memtag_enabled) {
- blk = __mfm_block_fixup_ptr(blk);
- next_blk = __mfm_block_fixup_ptr(next_blk);
- }
-#endif // CONFIG_MTE
blk->mfmb_prev = head;
__mfm_block_set_next(blk, next);
__mfm_block_set_next(hblk, offs);
- next_blk->mfmb_prev = offs;
+ arena->mfm_base[next].mfmb_prev = offs;
}
/*!
@@ -596,26 +552,11 @@
__mfm_block_remove(struct mfm_arena *arena, struct mfm_block *blk)
{
uint64_t next, prev;
- struct mfm_block *next_blk, *prev_blk;
-
-#if CONFIG_MTE
- if (mfm_memtag_enabled) {
- blk = __mfm_block_fixup_ptr(blk);
- }
-#endif // CONFIG_MTE
next = __mfm_block_next(blk);
prev = blk->mfmb_prev;
- next_blk = &arena->mfm_base[next];
- prev_blk = &arena->mfm_base[prev];
-#if CONFIG_MTE
- if (mfm_memtag_enabled) {
- next_blk = __mfm_block_fixup_ptr(next_blk);
- prev_blk = __mfm_block_fixup_ptr(prev_blk);
- }
-#endif // CONFIG_MTE
- next_blk->mfmb_prev = prev;
- __mfm_block_set_next(prev_blk, next);
+ arena->mfm_base[next].mfmb_prev = prev;
+ __mfm_block_set_next(&arena->mfm_base[prev], next);
__builtin_bzero(blk, sizeof(struct mfm_block));
}
@@ -667,7 +608,6 @@
#pragma mark external interface
-#if !MALLOC_TARGET_EXCLAVES_INTROSPECTOR
static void
__mfm_free_block(struct mfm_arena *arena, size_t index, size_t size)
{
@@ -681,66 +621,29 @@
mfm_initialize(void)
{
struct mfm_arena *arena;
- int debug_flags;
-#if MALLOC_TARGET_EXCLAVES
- plat_map_t map = {0};
- debug_flags = MALLOC_NO_POPULATE;
-#else
- int alloc_flags = 0;
- debug_flags = DISABLE_ASLR | MALLOC_ADD_GUARD_PAGE_FLAGS;
-#endif // MALLOC_TARGET_EXCLAVES
-
-#if CONFIG_MTE
- // Tie the enablement of memory tagging support in MFM to
- // malloc_has_sec_transition: we rely on mfm_initialize
- // being called *after* this has been setup by malloc proper.
- mfm_memtag_enabled = malloc_has_sec_transition;
-#if MALLOC_TARGET_EXCLAVES
- if (mfm_memtag_enabled) {
- debug_flags |= MALLOC_MTE_TAGGABLE;
- }
-#else
- mfm_memtag_enabled =
- mfm_memtag_enabled && malloc_sec_transition_early_malloc_support;
-
- if (mfm_memtag_enabled) {
- alloc_flags |= VM_FLAGS_MTE;
- }
-#endif // !MALLOC_TARGET_EXCLAVES
-#endif // CONFIG_MTE
-
- /* this is called early, which means the address space _does_ have 4M */
- arena = mvm_allocate_pages_plat(MFM_ARENA_SIZE, 0, debug_flags,
- VM_MEMORY_MALLOC, mvm_plat_map(map));
+
+ /* this is called early, which means the address space _does_ have 8M */
+ arena = mvm_allocate_pages(MFM_ARENA_SIZE, 0,
+ DISABLE_ASLR | MALLOC_ADD_GUARD_PAGE_FLAGS, VM_MEMORY_MALLOC);
if (arena == NULL) {
- MFM_INTERNAL_CRASH(arena, "failed to allocate memory");
- }
-
-#if MALLOC_TARGET_EXCLAVES
- /* populate the header up to the block storage */
- mvm_madvise_plat(arena,
- roundup(offsetof(struct mfm_arena, mfm_blocks), PAGE_SIZE),
- MADV_FAULTABLE, MALLOC_ABORT_ON_ERROR, mvm_plat_map(map));
-
- arena->mfm_header.mfm_map = map;
-#else
+ MFM_INTERNAL_CRASH(0, "failed to allocate memory");
+ }
+
/* to make clear that this region is not purely metadata, we'll now
* overwrite the allocation we received with another at the same location
* and size using VM_MEMORY_MALLOC_TINY - we couldn't use that tag
* originally because the kernel would have placed it in the heap range */
mach_vm_address_t vm_addr = (mach_vm_address_t)arena;
mach_vm_size_t vm_size = (mach_vm_size_t)MFM_ARENA_SIZE;
- alloc_flags |= VM_FLAGS_OVERWRITE | VM_MAKE_TAG(VM_MEMORY_MALLOC_TINY);
-
+ int alloc_flags = VM_FLAGS_OVERWRITE | VM_MAKE_TAG(VM_MEMORY_MALLOC_TINY);
kern_return_t kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
/* mask */ 0, alloc_flags, MEMORY_OBJECT_NULL, /* offset */ 0,
- /* copy */ false, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
+ /* copy */ FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
if (kr != KERN_SUCCESS) {
MFM_INTERNAL_CRASH(kr, "failed to overwrite mfm arena");
}
-#endif // MALLOC_TARGET_EXCLAVES
-
- arena->mfmh_lock = _MALLOC_LOCK_INIT;
+
+ arena->mfmh_lock = OS_UNFAIR_LOCK_INIT;
/*
* mfm_before pretend the bitmap is larger
@@ -770,10 +673,6 @@
struct mfm_arena *arena = os_atomic_load(&mfm_arena, dependency);
size_t index;
-#if CONFIG_MTE
- ptr = memtag_strip_address((void *)ptr);
-#endif // CONFIG_MTE
-
if (!__mfm_address_owned(arena, ptr)) {
return 0ul;
}
@@ -823,7 +722,7 @@
if (blk_size > size) {
__mfm_block_mark_start(arena, blk_index + size);
__mfm_free_block(arena, blk_index + size,
- blk_size - size);
+ blk_size - size);
}
__mfm_block_mark_allocated(arena, blk_index, size);
@@ -840,27 +739,12 @@
__mfm_block_mark_start(arena, index + size);
__mfm_block_mark_allocated(arena, index, size);
mfm_arena->mfmh_bump += size;
-
+ if (mfm_arena->mfmh_bump_hwm < mfm_arena->mfmh_bump) {
+ mfm_arena->mfmh_bump_hwm = mfm_arena->mfmh_bump;
+ }
arena->mfmh_size += size;
arena->mfmh_alloc_count += 1;
ptr = arena->mfm_blocks + index;
-
- if (mfm_arena->mfmh_bump_hwm < mfm_arena->mfmh_bump) {
-#if MALLOC_TARGET_EXCLAVES
- const uintptr_t begin = roundup(
- (uintptr_t)(arena->mfm_blocks + mfm_arena->mfmh_bump_hwm),
- PAGE_SIZE);
- const uintptr_t end = roundup((uintptr_t)ptr + alloc_size, PAGE_SIZE);
- const size_t bytes = end - begin;
- if (bytes) {
- mvm_madvise_plat((void*)begin, bytes, MADV_FAULTABLE,
- MALLOC_ABORT_ON_ERROR,
- mvm_plat_map(arena->mfm_header.mfm_map));
- }
-#endif
-
- mfm_arena->mfmh_bump_hwm = mfm_arena->mfmh_bump;
- }
}
out:
@@ -871,15 +755,6 @@
#endif
__mfm_unlock(arena);
-#if CONFIG_MTE
- // Set the tag for the block we are returning to the caller.
- if (mfm_memtag_enabled) {
- ptr = memtag_assign_tag(ptr, size * MFM_QUANTUM);
- // Blocks from the early arena may have arbitrary 16B-alignment
- memtag_set_tag_unaligned(ptr, size * MFM_QUANTUM);
- }
-#endif // CONFIG_MTE
-
return ptr;
}
@@ -888,34 +763,20 @@
{
struct mfm_arena *arena = os_atomic_load(&mfm_arena, dependency);
size_t index, size;
- void *addr = ptr;
#if MFM_TRACE
dprintf(STDERR_FILENO, "{ -1, %p },\n", ptr);
#endif
-#if CONFIG_MTE
- addr = memtag_strip_address((void *)ptr);
-#endif // CONFIG_MTE
-
- if (!__mfm_address_owned(arena, addr)) {
+ if (!__mfm_address_owned(arena, ptr)) {
MFM_INTERNAL_CRASH(ptr, "not MFM owned");
}
- index = __mfm_block_index(arena, addr);
+ index = __mfm_block_index(arena, ptr);
if (!__mfm_block_is_allocated(arena, index)) {
MFM_CLIENT_CRASH(ptr, "not an allocated block");
}
size = __mfm_block_size(arena, index);
-
-#if CONFIG_MTE
- // Retag the block we are freeing.
- if (mfm_memtag_enabled) {
- ptr = memtag_assign_tag(ptr, MFM_QUANTUM * size);
- // Blocks from the early arena may have arbitrary 16B-alignment
- memtag_set_tag_unaligned(ptr, MFM_QUANTUM * size);
- }
-#endif // CONFIG_MTE
bzero(ptr, MFM_QUANTUM * size);
@@ -942,7 +803,7 @@
}
if (index + size < arena->mfmh_bump &&
- !__mfm_block_is_allocated(arena, index + size)) {
+ !__mfm_block_is_allocated(arena, index + size)) {
size_t next = index + size;
size_t nsize = __mfm_block_size(arena, next);
@@ -967,31 +828,7 @@
{
struct mfm_arena *arena = os_atomic_load(&mfm_arena, dependency);
-#if CONFIG_MTE
- // After checking that this pointer belongs to us, verify that the logical
- // tag of the pointer matches the physical tag stored in memory. If that is
- // not the case, report this pointer as not claimed.
- //
- // The layers above us (xzone and malloc proper) rely on this behaviour.
- // This is required to properly handle (i.e. fail) the lookup of a pointer
- // with a mismatching tag: in the free() and realloc() paths, the dispatch
- // layer (malloc proper) will try to validate the tag of the pointer, and
- // raise a fatal exception if it doesn't match the tag stored in memory.
- //
- // Note: we need to validate the tag after ensuring that we own the
- // pointer, otherwise we might incur in passing complete garbage to ldg,
- // which would lead to a crash even in paths that need to possibly operate
- // on completely invalid pointers (e.g. malloc_size()).
- bool claimed = __mfm_address_owned(arena, memtag_strip_address(ptr));
- if (claimed && mfm_memtag_enabled) {
- if (!memtag_tags_match(ptr, memtag_fixup_ptr(ptr))) {
- return false;
- }
- }
- return claimed;
-#else
return __mfm_address_owned(arena, ptr);
-#endif
}
void *
@@ -999,7 +836,6 @@
{
return os_atomic_load(&mfm_arena, relaxed);
}
-#endif // !MALLOC_TARGET_EXCLAVES_INTROSPECTOR
#pragma mark introspection
@@ -1027,7 +863,7 @@
size_t size = __mfm_block_size(arena, index);
P(" [%p, %p) size=%zd\n",
- blk, blk + size, size * MFM_QUANTUM);
+ blk, blk + size, size * MFM_QUANTUM);
}
}
P("\n");
@@ -1065,7 +901,9 @@
memory_reader_t reader,
struct mfm_arena **arena_out)
{
- reader = reader_or_in_memory_fallback(reader, task);
+ if (!reader && task == mach_task_self()) {
+ reader = _malloc_default_reader;
+ }
return reader(task, zone_address, MFM_ARENA_SIZE, (void **)arena_out);
}
@@ -1237,7 +1075,7 @@
static boolean_t
mfmi_check(malloc_zone_t *zone __unused)
{
- return true;
+ return TRUE;
}
/* locking */
@@ -1266,9 +1104,9 @@
if (arena && _malloc_lock_trylock(&arena->mfmh_lock)) {
_malloc_lock_unlock(&arena->mfmh_lock);
- return true;
- }
- return false;
+ return TRUE;
+ }
+ return FALSE;
}
const struct malloc_introspection_t mfm_introspect = {