Loading...
--- libmalloc/libmalloc-474.0.13/src/early_malloc.c
+++ libmalloc/libmalloc-657.80.3/src/early_malloc.c
@@ -25,11 +25,9 @@
#if CONFIG_EARLY_MALLOC
-#include <assert.h>
-#include <os/atomic_private.h>
-#include <os/crashlog_private.h>
-#include <os/lock_private.h>
-#include <sys/param.h>
+#if MALLOC_TARGET_EXCLAVES_INTROSPECTOR
+#pragma GCC diagnostic ignored "-Wunused-function"
+#endif // MALLOC_TARGET_EXCLAVES_INTROSPECTOR
/*
* An arena looks like this:
@@ -46,7 +44,11 @@
*/
#define MFM_TRACE 0
+#if MALLOC_TARGET_EXCLAVES || MALLOC_TARGET_EXCLAVES_INTROSPECTOR
+#define MFM_ARENA_SIZE (1ul << 20)
+#else
#define MFM_ARENA_SIZE (8ul << 20)
+#endif /* MALLOC_TARGET_EXCLAVES || MALLOC_TARGET_EXCLAVES_INTROSPECTOR */
#define MFM_QUANTUM 16ul
#define MFM_SIZE_CLASSES (__builtin_ctz(MFM_ALLOC_SIZE_MAX / MFM_QUANTUM) + 1)
#define MFM_USABLE_SIZE (MFM_ARENA_SIZE - sizeof(struct mfm_header) - 4 * sizeof(uint64_t))
@@ -67,8 +69,9 @@
static_assert(CHAR_BIT == 8, "CHAR_BIT is 8");
static_assert(powerof2(MFM_ALLOC_SIZE_MAX), "MFM_ALLOC_SIZE_MAX is a power of 2");
-static_assert(powerof2(MFM_BLOCK_SIZE_MAX), "MFM_ALLOC_SIZE_MAX is a power of 2");
-
+static_assert(powerof2(MFM_BLOCK_SIZE_MAX), "MFM_BLOCK_SIZE_MAX is a power of 2");
+
+#if !MALLOC_TARGET_EXCLAVES
#define MFM_INTERNAL_CRASH(code, msg) ({ \
_os_set_crash_log_cause_and_message(code, "BUG IN LIBMALLOC: " msg); \
__builtin_trap(); \
@@ -79,6 +82,13 @@
"BUG IN CLIENT OF LIBMALLOC: " msg); \
__builtin_trap(); \
})
+#else
+#define MFM_INTERNAL_CRASH(code, msg) \
+ __liblibc_fatal_error("BUG IN LIBMALLOC (%llu): " msg, (uint64_t)code)
+
+#define MFM_CLIENT_CRASH(code, msg) \
+ __liblibc_fatal_error("BUG IN CLIENT OF LIBMALLOC (%llu): " msg, (uint64_t)code)
+#endif // !MALLOC_TARGET_EXCLAVES
struct mfm_block {
#if __has_feature(ptrauth_calls)
@@ -99,6 +109,9 @@
size_t mfm_bump_hwm;
size_t mfm_alloc_count;
struct mfm_block mfm_freelist[MFM_SIZE_CLASSES];
+#if MALLOC_TARGET_EXCLAVES || MALLOC_TARGET_EXCLAVES_INTROSPECTOR
+ plat_map_t mfm_map;
+#endif // MALLOC_TARGET_EXCLAVES || MALLOC_TARGET_EXCLAVES_INTROSPECTOR
};
struct mfm_arena {
@@ -145,7 +158,6 @@
static struct mfm_arena *mfm_arena;
-
#pragma mark validation and helper functions
/*!
@@ -518,6 +530,7 @@
#endif
}
+
/*!
* @function __mfm_block_insert_head()
*
@@ -531,15 +544,18 @@
struct mfm_block *blk)
{
uint64_t head, offs, next;
+ struct mfm_block *next_blk;
head = __mfm_block_offset(arena, hblk);
next = __mfm_block_next(hblk);
offs = __mfm_block_offset(arena, blk);
+ next_blk = &arena->mfm_base[next];
+
blk->mfmb_prev = head;
__mfm_block_set_next(blk, next);
__mfm_block_set_next(hblk, offs);
- arena->mfm_base[next].mfmb_prev = offs;
+ next_blk->mfmb_prev = offs;
}
/*!
@@ -552,11 +568,15 @@
__mfm_block_remove(struct mfm_arena *arena, struct mfm_block *blk)
{
uint64_t next, prev;
+ struct mfm_block *next_blk, *prev_blk;
+
next = __mfm_block_next(blk);
prev = blk->mfmb_prev;
- arena->mfm_base[next].mfmb_prev = prev;
- __mfm_block_set_next(&arena->mfm_base[prev], next);
+ next_blk = &arena->mfm_base[next];
+ prev_blk = &arena->mfm_base[prev];
+ next_blk->mfmb_prev = prev;
+ __mfm_block_set_next(prev_blk, next);
__builtin_bzero(blk, sizeof(struct mfm_block));
}
@@ -608,6 +628,7 @@
#pragma mark external interface
+#if !MALLOC_TARGET_EXCLAVES_INTROSPECTOR
static void
__mfm_free_block(struct mfm_arena *arena, size_t index, size_t size)
{
@@ -621,14 +642,36 @@
mfm_initialize(void)
{
struct mfm_arena *arena;
-
+#if MALLOC_TARGET_EXCLAVES
+ plat_map_t map = {0};
+#endif // MALLOC_TARGET_EXCLAVES
+
+
+#if MALLOC_TARGET_EXCLAVES
+ arena = mvm_allocate_pages_plat(MFM_ARENA_SIZE, 0, MALLOC_NO_POPULATE,
+ VM_MEMORY_MALLOC, mvm_plat_map(map));
+#else
/* this is called early, which means the address space _does_ have 8M */
- arena = mvm_allocate_pages(MFM_ARENA_SIZE, 0,
- DISABLE_ASLR | MALLOC_ADD_GUARD_PAGE_FLAGS, VM_MEMORY_MALLOC);
+ arena = mvm_allocate_pages_plat(MFM_ARENA_SIZE, 0,
+ DISABLE_ASLR | MALLOC_ADD_GUARD_PAGE_FLAGS, VM_MEMORY_MALLOC,
+ NULL);
+#endif // MALLOC_TARGET_EXCLAVES
+
if (arena == NULL) {
- MFM_INTERNAL_CRASH(0, "failed to allocate memory");
- }
-
+ MFM_INTERNAL_CRASH(arena, "failed to allocate memory");
+ }
+
+#if MALLOC_TARGET_EXCLAVES
+ /* populate the header up to the block storage */
+ const uintptr_t addr = (uintptr_t)mvm_allocate_plat((uintptr_t)arena,
+ roundup(offsetof(struct mfm_arena, mfm_blocks), PAGE_SIZE), 0,
+ VM_FLAGS_FIXED, 0, 0, mvm_plat_map(map));
+ if (addr != (uintptr_t)arena) {
+ MFM_INTERNAL_CRASH(addr, "populate of header failed");
+ }
+
+ arena->mfm_header.mfm_map = map;
+#else
/* to make clear that this region is not purely metadata, we'll now
* overwrite the allocation we received with another at the same location
* and size using VM_MEMORY_MALLOC_TINY - we couldn't use that tag
@@ -636,14 +679,17 @@
mach_vm_address_t vm_addr = (mach_vm_address_t)arena;
mach_vm_size_t vm_size = (mach_vm_size_t)MFM_ARENA_SIZE;
int alloc_flags = VM_FLAGS_OVERWRITE | VM_MAKE_TAG(VM_MEMORY_MALLOC_TINY);
+
+
kern_return_t kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
/* mask */ 0, alloc_flags, MEMORY_OBJECT_NULL, /* offset */ 0,
- /* copy */ FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
+ /* copy */ false, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
if (kr != KERN_SUCCESS) {
MFM_INTERNAL_CRASH(kr, "failed to overwrite mfm arena");
}
-
- arena->mfmh_lock = OS_UNFAIR_LOCK_INIT;
+#endif // MALLOC_TARGET_EXCLAVES
+
+ arena->mfmh_lock = _MALLOC_LOCK_INIT;
/*
* mfm_before pretend the bitmap is larger
@@ -673,6 +719,7 @@
struct mfm_arena *arena = os_atomic_load(&mfm_arena, dependency);
size_t index;
+
if (!__mfm_address_owned(arena, ptr)) {
return 0ul;
}
@@ -739,12 +786,30 @@
__mfm_block_mark_start(arena, index + size);
__mfm_block_mark_allocated(arena, index, size);
mfm_arena->mfmh_bump += size;
- if (mfm_arena->mfmh_bump_hwm < mfm_arena->mfmh_bump) {
- mfm_arena->mfmh_bump_hwm = mfm_arena->mfmh_bump;
- }
+
arena->mfmh_size += size;
arena->mfmh_alloc_count += 1;
ptr = arena->mfm_blocks + index;
+
+ if (mfm_arena->mfmh_bump_hwm < mfm_arena->mfmh_bump) {
+#if MALLOC_TARGET_EXCLAVES
+ const uintptr_t begin = roundup(
+ (uintptr_t)(arena->mfm_blocks + mfm_arena->mfmh_bump_hwm),
+ PAGE_SIZE);
+ const uintptr_t end = roundup((uintptr_t)ptr + alloc_size, PAGE_SIZE);
+ const size_t bytes = end - begin;
+ if (bytes) {
+ const uintptr_t addr = (uintptr_t)mvm_allocate_plat(begin,
+ bytes, 0, VM_FLAGS_FIXED, 0, 0,
+ mvm_plat_map(arena->mfm_header.mfm_map));
+ if (addr != begin) {
+ MFM_INTERNAL_CRASH(ptr, "populate of pages failed");
+ }
+ }
+#endif
+
+ mfm_arena->mfmh_bump_hwm = mfm_arena->mfmh_bump;
+ }
}
out:
@@ -755,6 +820,7 @@
#endif
__mfm_unlock(arena);
+
return ptr;
}
@@ -763,20 +829,23 @@
{
struct mfm_arena *arena = os_atomic_load(&mfm_arena, dependency);
size_t index, size;
+ void *addr = ptr;
#if MFM_TRACE
dprintf(STDERR_FILENO, "{ -1, %p },\n", ptr);
#endif
- if (!__mfm_address_owned(arena, ptr)) {
+
+ if (!__mfm_address_owned(arena, addr)) {
MFM_INTERNAL_CRASH(ptr, "not MFM owned");
}
- index = __mfm_block_index(arena, ptr);
+ index = __mfm_block_index(arena, addr);
if (!__mfm_block_is_allocated(arena, index)) {
MFM_CLIENT_CRASH(ptr, "not an allocated block");
}
size = __mfm_block_size(arena, index);
+
bzero(ptr, MFM_QUANTUM * size);
@@ -828,6 +897,7 @@
{
struct mfm_arena *arena = os_atomic_load(&mfm_arena, dependency);
+
return __mfm_address_owned(arena, ptr);
}
@@ -836,6 +906,7 @@
{
return os_atomic_load(&mfm_arena, relaxed);
}
+#endif // !MALLOC_TARGET_EXCLAVES_INTROSPECTOR
#pragma mark introspection
@@ -901,9 +972,7 @@
memory_reader_t reader,
struct mfm_arena **arena_out)
{
- if (!reader && task == mach_task_self()) {
- reader = _malloc_default_reader;
- }
+ reader = reader_or_in_memory_fallback(reader, task);
return reader(task, zone_address, MFM_ARENA_SIZE, (void **)arena_out);
}
@@ -1075,7 +1144,7 @@
static boolean_t
mfmi_check(malloc_zone_t *zone __unused)
{
- return TRUE;
+ return true;
}
/* locking */
@@ -1104,9 +1173,9 @@
if (arena && _malloc_lock_trylock(&arena->mfmh_lock)) {
_malloc_lock_unlock(&arena->mfmh_lock);
- return TRUE;
- }
- return FALSE;
+ return true;
+ }
+ return false;
}
const struct malloc_introspection_t mfm_introspect = {