Loading...
--- libmalloc/libmalloc-646.40.3/src/vm.c
+++ libmalloc/libmalloc-792.80.2/src/vm.c
@@ -24,9 +24,12 @@
#include "internal.h"
#if !MALLOC_TARGET_EXCLAVES
+volatile uintptr_t entropic_base = 0;
static volatile uintptr_t entropic_address = 0;
-static volatile uintptr_t entropic_base = 0;
static volatile uintptr_t entropic_limit = 0;
+
+MALLOC_NOEXPORT
+struct mvm_guarded_range_config_s malloc_guarded_range_config = {0};
#endif // !MALLOC_TARGET_EXCLAVES
MALLOC_NOEXPORT
@@ -110,9 +113,80 @@
#endif // TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
}
+#if !MALLOC_TARGET_EXCLAVES
+static size_t
+mvm_random_page_aligned(uint32_t upper_bound)
+{
+ uint32_t max_pages = upper_bound / PAGE_SIZE;
+ uint32_t pages = arc4random_uniform(max_pages + 1);
+ return pages * PAGE_SIZE;
+}
+
+// The "guarded range" is a synthetic VA range that we create in order to
+// randomize the distance of malloc metadata from the fixed mappings that
+// are laid out at process launch.
+//
+// We implement this by mapping a PROT_NONE region: the size of this region is
+// randomly chosen between 4M and 5M. Within that region, we then punch a 1M
+// carveout, by deallocating a random piece of the guarded region VA.
+// The address of this carveout is then passed as a hint in mvm_allocate_plat
+// (whenever the provided address is zero), to nudge the VM into serving the
+// requested mappings through the carveout.
+//
+// Specifically, this is aimed at randomizing the distance between the binary's
+// __DATA segment and the malloc metadata allocated at process launch, namely
+// the initial zone allocation. Moreover, by having a variable-length tail at
+// the end of the region, we also randomize the distance between __DATA and
+// the regions mapped after malloc's metadata.
+//
+// rdar://146578480
+void
+mvm_guarded_range_init(void)
+{
+ size_t tail_size = mvm_random_page_aligned(MiB(1));
+ size_t range_size = MiB(4) + tail_size;
+ size_t carveout_size = MiB(1);
+ size_t carveout_max_offset = range_size - carveout_size - 2 * PAGE_SIZE;
+ size_t carveout_offset = mvm_random_page_aligned((uint32_t)carveout_max_offset);
+ mach_vm_address_t range_addr = 0;
+ mach_vm_address_t carveout_addr = 0;
+ kern_return_t kr = KERN_FAILURE;
+
+ // Reserve an inaccessible VA range.
+ kr = mach_vm_map(mach_task_self(), &range_addr, range_size, 0,
+ VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_MEMORY_MALLOC),
+ MEMORY_OBJECT_NULL, 0, FALSE, VM_PROT_NONE, VM_PROT_NONE,
+ VM_INHERIT_DEFAULT);
+ if (kr != KERN_SUCCESS) {
+ malloc_zone_error(MALLOC_ABORT_ON_ERROR, false,
+ "Failed to map guarded range: %d\n", kr);
+ }
+
+ // Punch a hole through the inaccessible VA range, creating a sub-region
+ // that can be used by the VM to satisfy mapping requests.
+ carveout_addr = range_addr + PAGE_SIZE + carveout_offset;
+ kr = mach_vm_deallocate(mach_task_self(), carveout_addr, carveout_size);
+ if (kr != KERN_SUCCESS) {
+ malloc_zone_error(MALLOC_ABORT_ON_ERROR, false,
+ "Failed to create carveout at 0x%lx"
+ " in malloc guarded range 0x%lx: %d\n",
+ (unsigned long) carveout_addr, (unsigned long) range_addr, kr);
+ }
+
+ malloc_guarded_range_config = (struct mvm_guarded_range_config_s){
+ .base_address = range_addr,
+ .size = range_size,
+ .carveout_address = carveout_addr
+ };
+}
+#endif // !MALLOC_TARGET_EXCLAVES
+
void * __sized_by_or_null(size)
mvm_allocate_plat(uintptr_t addr, size_t size, uint8_t align, int flags, int debug_flags, int vm_page_label, plat_map_t *map_out)
{
+ void * __unsafe_indexable mapped;
+ kern_return_t kr;
+
if (addr && (flags & VM_FLAGS_ANYWHERE)) {
// Pass MALLOC_ABORT_ON_ERROR to make this call abort
malloc_zone_error(MALLOC_ABORT_ON_ERROR | debug_flags, false,
@@ -120,46 +194,63 @@
(unsigned long) addr, (unsigned long) size, flags);
}
#if MALLOC_TARGET_EXCLAVES
- // This call can have different behavior depending on `flags` and `map_out`:
- // 1. If the input handle is invalid and MALLOC_NO_POPULATE is not present,
- // the handle is initialized and memory is both reserved and populated
- // 2. If the input handle is invalid and MALLOC_NO_POPULATE is present,
- // the handle is initialized and memory is only reserved
- // 3. If the input handle is valid and MALLOC_NO_POPULATE is not present,
- // memory is populated
+ // Memory will be reserved and/or populated, and the handle initialized
const _liblibc_map_type_t type = LIBLIBC_MAP_TYPE_PRIVATE |
- ((flags & VM_FLAGS_ANYWHERE) ? LIBLIBC_MAP_TYPE_NONE : LIBLIBC_MAP_TYPE_FIXED) |
- ((debug_flags & MALLOC_NO_POPULATE) ? LIBLIBC_MAP_TYPE_NOCOMMIT : LIBLIBC_MAP_TYPE_NONE) |
- ((debug_flags & DISABLE_ASLR) ? LIBLIBC_MAP_TYPE_NORAND : LIBLIBC_MAP_TYPE_NONE);
- const _liblibc_map_perm_t perm = LIBLIBC_MAP_PERM_READ | LIBLIBC_MAP_PERM_WRITE;
- void * __unsafe_indexable map = mmap_plat(map_out, addr, size, perm,
- type, align, (unsigned)vm_page_label);
- if (!map) {
+ ((flags & VM_FLAGS_ANYWHERE) ? LIBLIBC_MAP_TYPE_NONE : LIBLIBC_MAP_TYPE_FIXED) |
+ ((debug_flags & MALLOC_CAN_FAULT) ? LIBLIBC_MAP_TYPE_FAULTABLE : LIBLIBC_MAP_TYPE_NONE) |
+#if CONFIG_MTE
+ ((debug_flags & MALLOC_MTE_TAGGABLE) ? LIBLIBC_MAP_TYPE_MEMTAG : LIBLIBC_MAP_TYPE_NONE) |
+#endif // CONFIG_MTE
+ ((debug_flags & MALLOC_NO_POPULATE) ? LIBLIBC_MAP_TYPE_NOCOMMIT : LIBLIBC_MAP_TYPE_NONE) |
+ ((debug_flags & DISABLE_ASLR) ? LIBLIBC_MAP_TYPE_NORAND : LIBLIBC_MAP_TYPE_NONE);
+ const _liblibc_map_perm_t perm = LIBLIBC_MAP_PERM_READ |
+ LIBLIBC_MAP_PERM_WRITE;
+ mapped = mmap_plat(map_out, addr, size, perm, type, align,
+ (unsigned)vm_page_label);
+ kr = errno;
+ // This message is not printed on non-exclaves targets. Certain code paths,
+ // like xzm_segment_group_try_realloc_huge_chunk, may fail under normal
+ // conditions, and would print a spurious message, but are disabled on
+ // exclaves.
+ if (!mapped) {
malloc_zone_error(debug_flags, false,
- "Failed to allocate memory at address 0x%lx of size 0x%lx with flags %d\n", addr, size, flags);
- }
- return __unsafe_forge_bidi_indexable(void *, map, size);
+ "Failed to allocate memory at address 0x%lx of size 0x%lx with flags %d: %d\n", addr, size, flags, kr);
+ }
#else
(void)map_out;
- if (debug_flags & MALLOC_NO_POPULATE) {
+ if (debug_flags & (MALLOC_CAN_FAULT | MALLOC_NO_POPULATE)) {
// Pass MALLOC_ABORT_ON_ERROR to make this call abort
malloc_zone_error(MALLOC_ABORT_ON_ERROR | debug_flags, false,
"Unsupported unpopulated allocation at address 0x%lx of size 0x%lx with flags %d\n",
(unsigned long) addr, (unsigned long) size, flags);
}
+ if (debug_flags & MALLOC_GUARDED_METADATA) {
+ if (addr || vm_page_label != VM_MEMORY_MALLOC) {
+ malloc_zone_error(MALLOC_ABORT_ON_ERROR | debug_flags, false,
+ "Unsupported guarded metadata allocation at address 0x%lx of size 0x%lx with flags %d and label %d\n",
+ (unsigned long) addr, (unsigned long) size, flags, vm_page_label);
+ }
+ // Pass the address of the carveout as a hint.
+ addr = (uintptr_t)malloc_guarded_range_config.base_address;
+ }
+
+#if CONFIG_MTE
+ if (debug_flags & MALLOC_MTE_TAGGABLE) {
+ flags |= VM_FLAGS_MTE;
+ }
+#endif
mach_vm_address_t vm_addr = addr;
mach_vm_offset_t allocation_mask = ((mach_vm_offset_t)1 << align) - 1;
- kern_return_t kr = mach_vm_map(mach_task_self(), &vm_addr,
- (mach_vm_size_t)size, allocation_mask,
- flags | VM_MAKE_TAG(vm_page_label), MEMORY_OBJECT_NULL, 0, FALSE,
- VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
- if (kr) {
- return NULL;
- }
- return __unsafe_forge_bidi_indexable(void *, vm_addr, size);
+ kr = mach_vm_map(mach_task_self(), &vm_addr, (mach_vm_size_t)size,
+ allocation_mask, flags | VM_MAKE_TAG(vm_page_label),
+ MEMORY_OBJECT_NULL, 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL,
+ VM_INHERIT_DEFAULT);
+ mapped = (kr == KERN_SUCCESS) ? (void *)vm_addr : NULL;
#endif // MALLOC_TARGET_EXCLAVES
+
+ return __unsafe_forge_bidi_indexable(void *, mapped, size);
}
void * __sized_by_or_null(size)
@@ -205,6 +296,11 @@
alloc_flags |= VM_FLAGS_PURGABLE;
}
+#if CONFIG_MTE
+ if (debug_flags & MALLOC_MTE_TAGGABLE) {
+ alloc_flags |= VM_FLAGS_MTE;
+ }
+#endif
if (allocation_size < size) { // size_t arithmetic wrapped!
return NULL;
@@ -303,20 +399,20 @@
void
mvm_deallocate_plat(void * __sized_by(size) addr, size_t size, int debug_flags, plat_map_t *map)
{
-#if MALLOC_TARGET_EXCLAVES
- if (!munmap_plat(map, addr, size)) {
+ kern_return_t kr;
+
+#if MALLOC_TARGET_EXCLAVES
+ kr = munmap_plat(map, addr, size) ? KERN_SUCCESS : errno;
+#else
+ (void)map;
+ kr = mach_vm_deallocate(mach_task_self(), (mach_vm_address_t)addr,
+ (mach_vm_size_t)size);
+#endif // MALLOC_TARGET_EXCLAVES
+
+ if (kr != KERN_SUCCESS) {
malloc_zone_error(debug_flags, false,
- "Failed to deallocate at address %p of size 0x%lx\n", addr, size);
- }
-#else
- (void)map;
- kern_return_t kr = mach_vm_deallocate(mach_task_self(),
- (mach_vm_address_t)addr, (mach_vm_size_t)size);
- if (kr) {
- malloc_zone_error(debug_flags, false,
- "Failed to deallocate at address %p of size 0x%lx\n", addr, size);
- }
-#endif // MALLOC_TARGET_EXCLAVES
+ "Failed to deallocate at address %p of size 0x%lx: %d\n", addr, size, kr);
+ }
}
void
@@ -377,7 +473,7 @@
}
if (!mprotect_plat(map, address, size, perm)) {
malloc_zone_error(MALLOC_ABORT_ON_ERROR | debug_flags, true,
- "Unsupported deallocation address %p or size %lu\n", address, size);
+ "Unsupported deallocation address %p or size %lu: %d\n", address, size, errno);
}
#else
(void)map;
@@ -409,24 +505,29 @@
int
mvm_madvise_plat(void * __sized_by(sz) addr, size_t sz, int advice, unsigned debug_flags, plat_map_t *map)
{
-#if MALLOC_TARGET_EXCLAVES
- if (!(advice == MADV_FREE || advice == MADV_FREE_REUSABLE) ||
- (debug_flags & (MALLOC_ADD_GUARD_PAGE_FLAGS | MALLOC_PURGEABLE))) {
+ kern_return_t kr;
+
+#if MALLOC_TARGET_EXCLAVES
+ if ((debug_flags & (MALLOC_ADD_GUARD_PAGE_FLAGS | MALLOC_PURGEABLE))) {
malloc_zone_error(MALLOC_ABORT_ON_ERROR | debug_flags, true,
- "Unsupported allocation advice %d or debug flags %u\n", advice, debug_flags);
- }
-
- if (!madvise_plat(map, addr, sz, LIBLIBC_MAP_HINT_UNUSED)) {
- return 1;
+ "Unsupported debug flags %u\n", debug_flags);
+ }
+
+ kr = !madvise_plat(map, addr, sz, advice) ? KERN_SUCCESS : errno;
+ if (kr != KERN_SUCCESS) {
+ malloc_zone_error(debug_flags, false,
+ "Failed to madvise %d at address %p of size 0x%lx: %d\n", advice,
+ addr, sz, kr);
}
#else
(void)map;
- if (madvise(addr, sz, advice) == -1) {
- return 1;
- }
+ kr = !madvise(addr, sz, advice) ? KERN_SUCCESS : errno;
#endif // MALLOC_TARGET_EXCLAVES
- return 0;
-}
+
+ return !(kr == KERN_SUCCESS);
+}
+
+#if !defined(TESTING_XZONE_MALLOC)
int
mvm_madvise_free(void *rack, void *r, uintptr_t pgLo, uintptr_t pgHi, uintptr_t *last, boolean_t scribble)
@@ -477,21 +578,36 @@
return 0;
}
-#if CONFIG_DEFERRED_RECLAIM
-static struct mach_vm_reclaim_ringbuffer_v1_s reclaim_buffer;
+#if CONFIG_MAGAZINE_DEFERRED_RECLAIM
+static mach_vm_reclaim_ring_t reclaim_buffer;
static _malloc_lock_s reclaim_buffer_lock = _MALLOC_LOCK_INIT;
-kern_return_t
+mach_vm_reclaim_error_t
mvm_deferred_reclaim_init(void)
{
- return mach_vm_reclaim_ringbuffer_init(&reclaim_buffer);
+ // Pick a sane minimum number of entries and let vm_reclaim round up
+ // to a page boundary. The intention is for the initial size to be
+ // one page. We don't support ringbuffer growth on the legacy DRC, so
+ // the maximum size will be unmodified.
+ mach_vm_reclaim_count_t capacity = mach_vm_reclaim_round_capacity(512);
+ return mach_vm_reclaim_ring_allocate(&reclaim_buffer, capacity, capacity);
}
bool
-mvm_reclaim_mark_used(uint64_t id, mach_vm_address_t ptr, uint32_t size, unsigned int debug_flags)
-{
- bool used;
+mvm_reclaim_mark_used(mach_vm_reclaim_id_t id, mach_vm_address_t ptr, mach_vm_size_t size, unsigned int debug_flags)
+{
+ mach_vm_reclaim_error_t kr;
+ mach_vm_reclaim_state_t state;
+ bool update_accounting;
+
+ if (id == VM_RECLAIM_ID_NULL) {
+ // Region was never entered into ring
+ // FIXME: Understand why the all cache entries aren't being
+ // assigned reclaim IDs (rdar://137709029)
+ return true;
+ }
+
if (debug_flags & MALLOC_ADD_GUARD_PAGE_FLAGS) {
if (os_add_overflow(size, 2 * large_vm_page_quanta_size, &size)) {
return false;
@@ -499,35 +615,70 @@
ptr -= large_vm_page_quanta_size;
}
_malloc_lock_lock(&reclaim_buffer_lock);
- used = mach_vm_reclaim_mark_used(&reclaim_buffer, id, ptr, size);
+ kr = mach_vm_reclaim_try_cancel(reclaim_buffer, id, ptr, size,
+ VM_RECLAIM_DEALLOCATE, &state, &update_accounting);
+ MALLOC_ASSERT(kr == VM_RECLAIM_SUCCESS);
_malloc_lock_unlock(&reclaim_buffer_lock);
- return used;
-}
-
-uint64_t
-mvm_reclaim_mark_free(vm_address_t ptr, uint32_t size, unsigned int debug_flags)
-{
- uint64_t id;
+ if (update_accounting) {
+ mach_vm_reclaim_update_kernel_accounting(reclaim_buffer);
+ }
+ return mach_vm_reclaim_is_reusable(state);
+}
+
+mach_vm_reclaim_id_t
+mvm_reclaim_mark_free(mach_vm_address_t ptr, mach_vm_size_t size, unsigned int debug_flags)
+{
+ mach_vm_reclaim_error_t kr;
+ mach_vm_reclaim_id_t id;
bool should_update_kernel_accounting = false;
if (debug_flags & MALLOC_ADD_GUARD_PAGE_FLAGS) {
if (os_add_overflow(size, 2 * large_vm_page_quanta_size, &size)) {
- return VM_RECLAIM_INDEX_NULL;
+ return VM_RECLAIM_ID_NULL;
}
ptr -= large_vm_page_quanta_size;
}
+
_malloc_lock_lock(&reclaim_buffer_lock);
- id = mach_vm_reclaim_mark_free(&reclaim_buffer, ptr, size,
- MACH_VM_RECLAIM_DEALLOCATE, &should_update_kernel_accounting);
+
+ do {
+ id = VM_RECLAIM_ID_NULL;
+ kr = mach_vm_reclaim_try_enter(reclaim_buffer, ptr, size,
+ VM_RECLAIM_DEALLOCATE, &id, &should_update_kernel_accounting);
+ MALLOC_ASSERT(kr == VM_RECLAIM_SUCCESS);
+ if (id == VM_RECLAIM_ID_NULL) {
+ mach_vm_reclaim_count_t capacity;
+ kr = mach_vm_reclaim_ring_capacity(reclaim_buffer, &capacity);
+ MALLOC_ASSERT(kr == VM_RECLAIM_SUCCESS);
+ kr = mach_vm_reclaim_ring_flush(reclaim_buffer, capacity);
+ MALLOC_ASSERT(kr == VM_RECLAIM_SUCCESS);
+ }
+ } while (id == VM_RECLAIM_ID_NULL);
+
_malloc_lock_unlock(&reclaim_buffer_lock);
+
if (should_update_kernel_accounting) {
- mach_vm_reclaim_update_kernel_accounting(&reclaim_buffer);
+ mach_vm_reclaim_update_kernel_accounting(reclaim_buffer);
}
return id;
}
bool
-mvm_reclaim_is_available(uint64_t id)
-{
- return mach_vm_reclaim_is_available(&reclaim_buffer, id);
-}
-#endif // CONFIG_DEFERRED_RECLAIM
+mvm_reclaim_is_available(mach_vm_reclaim_id_t id)
+{
+ mach_vm_reclaim_error_t err;
+ mach_vm_reclaim_state_t state;
+
+ if (id == VM_RECLAIM_ID_NULL) {
+ // Region was never entered into ring
+ // FIXME: Understand why the all cache entries aren't being
+ // assigned reclaim IDs (rdar://137709029)
+ return true;
+ }
+
+ err = mach_vm_reclaim_query_state(reclaim_buffer, id, VM_RECLAIM_DEALLOCATE, &state);
+ MALLOC_ASSERT(err == VM_RECLAIM_SUCCESS);
+ return mach_vm_reclaim_is_reusable(state);
+}
+#endif // CONFIG_MAGAZINE_DEFERRED_RECLAIM
+
+#endif // !defined(TESTING_XZONE_MALLOC)