Loading...
--- /dev/null
+++ libmalloc/libmalloc-792.41.1/src/xzone_malloc/xzone_malloc.c
@@ -0,0 +1,8025 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2022, Microsoft Research, Daan Leijen
+Copyright © 2025 Apple Inc.
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" in the same directory as this file.
+-----------------------------------------------------------------------------*/
+
+#include "../internal.h"
+
+#if CONFIG_XZONE_MALLOC
+
+#if defined(DEBUG) && !MALLOC_TARGET_EXCLAVES
+#define xzm_trace(name, ...) MALLOC_TRACE(TRACE_xzone_##name, __VA_ARGS__)
+#else
+#define xzm_trace(...)
+#endif
+
+#pragma mark xzone lookup
+
+// mimalloc: mi_bin
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static uint8_t
+_xzm_bin(size_t size)
+{
+ if (size == 0) {
+ // TODO: fancy unmapped page
+ return 0;
+ } else if (size <= 128) {
+ return howmany(size, XZM_GRANULE) - 1;
+ } else {
+ xzm_debug_assert(size <= XZM_SMALL_BLOCK_SIZE_MAX);
+
+ // The bin calculation rounds up, so subtract to handle perfect fits
+ size--;
+
+ int msb = 63 - __builtin_clzl(size);
+ return ((msb << 2) + ((size >> (msb - 2)) & 0x3)) - 20;
+ }
+}
+
+MALLOC_STATIC_ASSERT(
+ XZM_POINTER_BUCKETS_MAX <= 4,
+ "The bucketing function supports up to 4 pointer buckets");
+
+MALLOC_STATIC_ASSERT(
+ sizeof(xzm_bucketing_keys_t) == 2 * sizeof(uint64_t),
+ "Invalid size for struct xzm_bucketing_keys_t");
+
+// This function assigns a type descriptor to a pointer bucket.
+// It takes as input the type descriptor itself and a set of keys, which are
+// obtained from the executable_boothash provided to the process by the kernel.
+// It is both security and performance sensitive.
+//
+// The function is designed to fulfill three requirements:
+// 1. It is stable with respect to identical type hashes, for executions of
+// the same process in the same boot session: this is required to prevent
+// an attacker from brute-forcing the bucketing assignment by repeatedly
+// crashing a given process.
+// 2. It distributes the same type hash pseudo-uniformly across buckets, when
+// the execution hash changes: this is required to prevent an attacker from
+// predicting the bucket assignment across processes, or to statically
+// determine the assignment for a given binary across boot sessions.
+// 3. It distributes different type hashes pseudo-uniformly across buckets,
+// given a fixed execution hash: it should not be possible for any
+// random pair of type hashes to always be assigned to the same bucket,
+// independently of the execution hash.
+//
+// The first requirement only holds when the key material is derived from the
+// executable_boothash. It does not hold otherwise.
+MALLOC_ALWAYS_INLINE MALLOC_INLINE MALLOC_USED
+static uint8_t
+_xzm_type_choose_ptr_bucket(const xzm_bucketing_keys_t *const keys,
+ uint8_t ptr_bucket_count, malloc_type_descriptor_t type_desc)
+{
+ xzm_debug_assert(ptr_bucket_count <= XZM_POINTER_BUCKETS_MAX);
+ uint8_t bucket = 0;
+
+ if (ptr_bucket_count > 1) {
+ const uint32_t type_hash = type_desc.hash;
+ const uint64_t key_a = keys->xbk_key_data[0];
+ const uint64_t key_b = keys->xbk_key_data[1];
+ const uint32_t hash = ((key_a * (uint64_t)type_hash) + key_b) >> 32;
+
+ switch (ptr_bucket_count) {
+ case 2:
+ bucket = hash & 0x1;
+ break;
+ case 3:
+ bucket = hash % 0x3;
+ break;
+ case 4:
+ bucket = hash & 0x3;
+ break;
+ default:
+ __builtin_unreachable();
+ }
+ }
+
+ return bucket;
+}
+
+// mimalloc: mi_page_queue
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static xzm_xzone_index_t
+_xzm_xzone_lookup(xzm_malloc_zone_t zone, size_t size,
+ malloc_type_descriptor_t type_desc)
+{
+ xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
+ uint8_t bin = _xzm_bin(size);
+ uint8_t bin_bucket_count = main->xzmz_xzone_bin_bucket_counts[bin];
+ uint8_t bin_offset = main->xzmz_xzone_bin_offsets[bin];
+
+ uint8_t bucket;
+#if XZM_BUCKET_DATA_ONLY || XZM_BUCKET_POINTER_ONLY
+ (void)bin_bucket_count;
+ bucket = 0;
+#else // XZM_BUCKET_DATA_ONLY || XZM_BUCKET_POINTER_ONLY
+
+#if XZM_NARROW_BUCKETING
+ if (main->xzmz_narrow_bucketing) {
+ if (type_desc.summary.type_kind == MALLOC_TYPE_KIND_OBJC) {
+ // - There are pure-data callsites that provide a 0 type hash
+ // because they assume that nothing is done with it.
+ // - The ObjC runtime callsite also provides a 0 type hash, for the
+ // same reason.
+ //
+ // To prevent these callsites from deterministically bucketing
+ // together, pick a different arbitrary type hash for ObjC.
+ //
+ // TODO: update the callsite in the ObjC runtime directly so that
+ // this is no longer necessary
+ type_desc.hash = 1;
+ }
+ bucket = _xzm_type_choose_ptr_bucket(&main->xzmz_bucketing_keys,
+ bin_bucket_count, type_desc);
+ goto bucket_computed;
+ }
+#endif // XZM_NARROW_BUCKETING
+
+ bool pure_data = malloc_type_descriptor_is_pure_data(type_desc);
+ if (pure_data) {
+ bucket = XZM_XZONE_BUCKET_DATA;
+ } else if (type_desc.summary.type_kind == MALLOC_TYPE_KIND_OBJC) {
+ bucket = XZM_XZONE_BUCKET_OBJC;
+#if XZM_BUCKET_VISIBILITY
+ } else if (type_desc.type_id == MALLOC_TYPE_ID_NONE) {
+ bucket = XZM_XZONE_BUCKET_PLAIN;
+ } else if (malloc_type_descriptor_is_uninferred(type_desc)) {
+ bucket = XZM_XZONE_BUCKET_UNINFERRED;
+#endif // XZM_BUCKET_VISIBILITY
+ } else {
+ bool fallback_hash = (type_desc.type_id == MALLOC_TYPE_ID_NONE);
+ if (fallback_hash) {
+ type_desc.hash = malloc_entropy[0] >> 32;
+ }
+ uint8_t ptr_bucket = _xzm_type_choose_ptr_bucket(
+ &main->xzmz_bucketing_keys,
+ (bin_bucket_count - XZM_XZONE_BUCKET_POINTER_BASE), type_desc);
+ bucket = XZM_XZONE_BUCKET_POINTER_BASE + ptr_bucket;
+ }
+
+#if XZM_NARROW_BUCKETING
+bucket_computed:
+#endif // XZM_NARROW_BUCKETING
+
+#endif // XZM_BUCKET_DATA_ONLY || XZM_BUCKET_POINTER_ONLY
+ xzm_debug_assert(bucket < bin_bucket_count);
+
+ return (xzm_xzone_index_t)(bin_offset + bucket);
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static xzm_allocation_index_t
+_xzm_get_allocation_index(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_slot_config_t *cur_slot_config, bool is_xas)
+{
+ xzm_slot_config_t *slot = is_xas ? &xz->xz_slot_config :
+ &xz->xz_list_config;
+ xzm_slot_config_t slot_config = os_atomic_load(slot, relaxed);
+ xzm_debug_assert(slot_config <= zone->xzz_max_slot_config);
+
+ if (cur_slot_config) {
+ *cur_slot_config = slot_config;
+ }
+
+ switch (slot_config) {
+ case XZM_SLOT_CPU:
+ return _malloc_cpu_number();
+ case XZM_SLOT_CLUSTER:
+#if CONFIG_XZM_CLUSTER_AWARE
+ return _malloc_cpu_cluster_number();
+#else // CONFIG_XZM_CLUSTER_AWARE
+ return _malloc_cpu_number() % 2;
+#endif // CONFIG_XZM_CLUSTER_AWARE
+ case XZM_SLOT_SINGLE:
+ default:
+ return 0;
+ }
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static xzm_allocation_index_t
+_xzm_get_limit_allocation_index(xzm_slot_config_t slot_config)
+{
+ switch (slot_config) {
+ case XZM_SLOT_CPU:
+ return logical_ncpus;
+ case XZM_SLOT_CLUSTER:
+#if CONFIG_XZM_CLUSTER_AWARE
+ return ncpuclusters;
+#else // CONFIG_XZM_CLUSTER_AWARE
+ return MIN(2, logical_ncpus);
+#endif // CONFIG_XZM_CLUSTER_AWARE
+ case XZM_SLOT_SINGLE:
+ default:
+ return 1;
+ }
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static xzm_chunk_list_t
+_xzm_xzone_chunk_list_for_index(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_chunk_list_t lists, xzm_allocation_index_t alloc_idx)
+{
+ xzm_xzone_index_t xz_idx = xz->xz_idx;
+ size_t alloc_base_idx = alloc_idx * zone->xzz_xzone_count;
+ xzm_debug_assert(alloc_base_idx + xz_idx <
+ zone->xzz_slot_count * zone->xzz_xzone_count);
+ return &lists[alloc_base_idx + xz_idx];
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static xzm_segment_group_t
+_xzm_segment_group_for_id_and_front(xzm_malloc_zone_t zone,
+ xzm_segment_group_id_t sgid, xzm_front_index_t front, bool huge)
+{
+ xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
+#if CONFIG_XZM_CLUSTER_AWARE
+ const uint32_t clusterid = _malloc_cpu_cluster_number();
+ xzm_debug_assert(clusterid < ncpuclusters);
+#else
+ const uint32_t clusterid = _malloc_cpu_number() % 2;
+ xzm_debug_assert(clusterid < logical_ncpus);
+#endif // CONFIG_XZM_CLUSTER_AWARE
+
+ xzm_debug_assert(sgid < main->xzmz_segment_group_ids_count);
+ if (sgid < XZM_SEGMENT_GROUP_POINTER_XZONES) {
+ xzm_debug_assert(front == XZM_FRONT_INDEX_DEFAULT);
+ } else {
+ xzm_debug_assert(front < main->xzmz_allocation_front_count);
+ }
+
+ uint8_t sg_front_index = sgid + front;
+
+ if (main->xzmz_segment_group_front_count < main->xzmz_segment_group_count) {
+ uint8_t sg_index;
+ bool use_data_large = false;
+#if CONFIG_XZM_CLUSTER_AWARE
+ // Route all huge allocations to the same global segment group that
+ // has the huge cache enabled
+ xzm_debug_assert(main->xzmz_defer_large);
+ use_data_large = huge;
+#endif // CONFIG_XZM_CLUSTER_AWARE
+ if (use_data_large) {
+ sg_index = XZM_SEGMENT_GROUP_DATA_LARGE;
+ } else {
+ sg_index = main->xzmz_segment_group_front_count * clusterid +
+ sg_front_index;
+ }
+ xzm_debug_assert(sg_index < main->xzmz_segment_group_count);
+ return &main->xzmz_segment_groups[sg_index];
+ } else {
+ return &main->xzmz_segment_groups[sg_front_index];
+ }
+}
+
+MALLOC_NOINLINE
+static void
+_xzm_fork_lock_wait(xzm_malloc_zone_t zone)
+{
+ // This lock is taken first during fork, so that anything that needs to be
+ // locked during fork that otherwise doesn't have one can be sent here.
+ _malloc_lock_lock(&zone->xzz_fork_lock);
+ _malloc_lock_unlock(&zone->xzz_fork_lock);
+}
+
+MALLOC_NOINLINE MALLOC_PRESERVE_MOST
+static void
+_xzm_walk_lock_wait(xzm_malloc_zone_t zone)
+{
+ // We take this lock prior to walking any chunk freelist. A chunk that is
+ // marked as being walked will send any allocating and deallocating threads
+ // here.
+ _malloc_lock_lock(&zone->xzz_lock);
+ _malloc_lock_unlock(&zone->xzz_lock);
+}
+
+#pragma mark Large allocation and deallocation
+
+// mimalloc: mi_large_huge_page_alloc
+static void * __alloc_size(2)
+_xzm_malloc_large_huge(xzm_malloc_zone_t zone, size_t size, size_t alignment,
+ malloc_type_descriptor_t type_desc, xzm_malloc_options_t opt)
+{
+ bool clear = (opt & XZM_MALLOC_CLEAR);
+ void *ptr = NULL;
+
+ xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
+
+ // Consider: _mi_os_good_alloc_size()?
+ size_t rounded_size = roundup(size, XZM_SEGMENT_SLICE_SIZE);
+ xzm_slice_kind_t kind;
+ if (rounded_size > XZM_LARGE_BLOCK_SIZE_MAX ||
+ alignment > XZM_ALIGNMENT_MAX) {
+ kind = XZM_SLICE_KIND_HUGE_CHUNK;
+ } else {
+ kind = XZM_SLICE_KIND_LARGE_CHUNK;
+ }
+ xzm_slice_count_t slice_count;
+ if (os_convert_overflow(rounded_size / XZM_SEGMENT_SLICE_SIZE,
+ &slice_count)) {
+ goto out;
+ }
+
+ bool use_data_for_large = true;
+#if XZM_NARROW_BUCKETING
+ if (main->xzmz_narrow_bucketing && !main->xzmz_use_ranges) {
+ use_data_for_large = false;
+ }
+#endif
+
+ xzm_segment_group_id_t sg_id;
+ if ((use_data_for_large && malloc_type_descriptor_is_pure_data(type_desc)) ||
+ kind == XZM_SLICE_KIND_HUGE_CHUNK ||
+ main->xzmz_segment_group_ids_count == XZM_SEGMENT_GROUP_IDS_COUNT_DATA_ONLY) {
+ // Use a separate segment group for large data only when using deferred
+ // reclamation
+ bool use_data_large = main->xzmz_defer_large;
+
+#if CONFIG_MTE
+ struct xzm_memtag_config_s *memtag_config = &zone->xzz_memtag_config;
+ if (memtag_config->tag_data &&
+ memtag_config->max_block_size <= XZM_SMALL_BLOCK_SIZE_MAX) {
+ use_data_large = true;
+ }
+#endif
+
+ sg_id = use_data_large ? XZM_SEGMENT_GROUP_DATA_LARGE :
+ XZM_SEGMENT_GROUP_DATA;
+ } else {
+ sg_id = XZM_SEGMENT_GROUP_POINTER_LARGE;
+ }
+ xzm_debug_assert(sg_id < main->xzmz_segment_group_ids_count);
+ xzm_segment_group_t sg = _xzm_segment_group_for_id_and_front(zone, sg_id,
+ XZM_FRONT_INDEX_DEFAULT, kind == XZM_SLICE_KIND_HUGE_CHUNK);
+
+ bool purgeable = (zone->xzz_flags & MALLOC_PURGEABLE);
+ xzm_chunk_t chunk = xzm_segment_group_alloc_chunk(sg, kind, NULL,
+ slice_count, NULL, alignment, clear, purgeable);
+ if (os_unlikely(!chunk)) {
+ goto out;
+ }
+
+ xzm_debug_assert(!clear || chunk->xzc_bits.xzcb_is_pristine);
+
+ // Set the mzone_idx last to publish this chunk for the enumerator protocol
+ chunk->xzc_mzone_idx = zone->xzz_mzone_idx;
+
+ _malloc_lock_lock(&zone->xzz_lock);
+ LIST_INSERT_HEAD(&zone->xzz_chunkq_large, chunk, xzc_entry);
+ _malloc_lock_unlock(&zone->xzz_lock);
+
+ size_t chunk_size = 0;
+ ptr = (uint8_t *)_xzm_chunk_start(zone, chunk, &chunk_size);
+
+#if CONFIG_MTE
+ bool memtag_enabled = _xzm_segment_group_memtag_block(sg, chunk_size);
+ bool canonical_tag = (opt & XZM_MALLOC_CANONICAL_TAG);
+ if (memtag_enabled) {
+ if (canonical_tag) {
+ ptr = memtag_tag_canonical(ptr, chunk_size);
+ } else {
+ ptr = memtag_retag(ptr, chunk_size);
+ }
+ }
+#endif
+
+
+out:
+ if (os_unlikely(!ptr)) {
+ malloc_set_errno_fast(MZ_POSIX, ENOMEM);
+ }
+ return ptr;
+}
+
+// mimalloc: _mi_segment_huge_page_free
+MALLOC_NOINLINE
+static void
+_xzm_free_large_huge(xzm_malloc_zone_t zone, xzm_chunk_t chunk)
+{
+ _malloc_lock_lock(&zone->xzz_lock);
+
+ // Unset mzone_idx first to unpublish it for the enumerator protocol
+ chunk->xzc_mzone_idx = XZM_MZONE_INDEX_INVALID;
+ LIST_REMOVE(chunk, xzc_entry);
+
+ _malloc_lock_unlock(&zone->xzz_lock);
+
+ xzm_segment_group_t sg = _xzm_segment_group_for_slice(zone, chunk);
+ xzm_segment_group_free_chunk(sg, chunk, zone->xzz_flags & MALLOC_PURGEABLE,
+ /* small_madvise_needed */ false);
+}
+
+#if CONFIG_MTE
+
+#pragma mark MTE
+
+// FIXME: `zone` parameter of the following functions is unused.
+
+// Initialize the tags for the given chunk.
+static void *
+_xzm_xzone_chunk_memtag_init(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_chunk_t chunk)
+{
+ size_t chunk_size = 0;
+ uint8_t *chunk_start = _xzm_chunk_start_ptr(zone, chunk, &chunk_size);
+ return memtag_init_chunk(chunk_start, chunk_size, xz->xz_block_size);
+}
+
+// Retag a single block.
+static xzm_block_t
+_xzm_xzone_block_memtag_retag(xzm_malloc_zone_t zone, xzm_block_t block,
+ size_t block_size)
+{
+ return (xzm_block_t)memtag_retag((uint8_t *)block, block_size);
+}
+#endif // CONFIG_MTE
+
+static inline xzm_malloc_options_t
+_xzm_xzone_get_malloc_thread_options(void)
+{
+#if CONFIG_MTE
+ // rdar://140822174
+ // Check the flag in the TSD to see if canonical tagging was requested.
+ malloc_thread_options_t options = malloc_get_thread_options();
+ if (options.ReservedFlag) {
+ return XZM_MALLOC_CANONICAL_TAG;
+ }
+#endif // CONFIG_MTE
+
+ return 0;
+}
+
+#pragma mark chunk list
+
+static void
+_xzm_chunk_list_fork_lock(xzm_chunk_list_head_t list)
+{
+ xzm_chunk_list_head_u locked = {
+ .xzch_fork_locked = true,
+ };
+
+ __assert_only xzm_chunk_list_head_u old_head = {
+ .xzch_value = os_atomic_or_orig(&list->xzch_value, locked.xzch_value,
+ relaxed),
+ };
+ xzm_debug_assert(!old_head.xzch_fork_locked);
+}
+
+static void
+_xzm_chunk_list_fork_unlock(xzm_chunk_list_head_t list)
+{
+ xzm_chunk_list_head_u locked = {
+ .xzch_fork_locked = true,
+ };
+ uint64_t unlocked = ~locked.xzch_value;
+
+ __assert_only xzm_chunk_list_head_u old_head = {
+ .xzch_value = os_atomic_and_orig(&list->xzch_value, unlocked, relaxed),
+ };
+ xzm_debug_assert(old_head.xzch_fork_locked);
+}
+
+static xzm_chunk_t
+_xzm_chunk_list_pop(xzm_malloc_zone_t zone, xzm_chunk_list_head_t list,
+ xzm_chunk_linkage_t linkage, bool *contended_out)
+{
+ xzm_chunk_t chunk = NULL;
+ const bool is_batch = (linkage == XZM_CHUNK_LINKAGE_BATCH);
+ xzm_chunk_list_head_u head = {
+ .xzch_value = os_atomic_load(&list->xzch_value, dependency),
+ };
+
+ while (true) {
+ if (os_unlikely(is_batch ?
+ head.xzch_batch_fork_locked : head.xzch_fork_locked)) {
+ _xzm_fork_lock_wait(zone);
+ head.xzch_value = os_atomic_load(&list->xzch_value, dependency);
+ continue;
+ }
+
+ chunk = (xzm_chunk_t)head.xzch_ptr;
+ if (!chunk) {
+ break;
+ }
+
+ xzm_chunk_list_head_u new_head;
+ xzm_chunk_t next;
+ if (!is_batch) {
+ next = chunk->xzc_linkages[linkage];
+ new_head = (xzm_chunk_list_head_u){
+ .xzch_ptr = (uint64_t)next,
+ .xzch_gen = head.xzch_gen + 1,
+ };
+ } else {
+ xzm_debug_assert(head.xzch_batch_count);
+ next = *_xzm_segment_slice_meta_batch_next(zone, chunk);
+ new_head = (xzm_chunk_list_head_u){
+ .xzch_batch_ptr = (uint64_t)next,
+ .xzch_batch_gen = head.xzch_batch_gen + 1,
+ .xzch_batch_count = head.xzch_batch_count - 1,
+ };
+ xzm_debug_assert(new_head.xzch_batch_count < head.xzch_batch_count);
+ }
+ xzm_debug_assert(chunk != next);
+ bool success = os_atomic_cmpxchgv(&list->xzch_value, head.xzch_value,
+ new_head.xzch_value, &head.xzch_value, dependency);
+ if (success) {
+ if (is_batch) {
+ xzm_debug_assert(_xzm_slice_meta_is_batch_pointer(zone,
+ (xzm_chunk_t)new_head.xzch_batch_ptr));
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ *_xzm_slice_meta_reclaim_id(zone, chunk) = VM_RECLAIM_ID_NULL;
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+ }
+ break;
+ } else if (contended_out) {
+ *contended_out = true;
+ }
+ }
+
+ return chunk;
+}
+
+static void
+_xzm_chunk_list_push(xzm_malloc_zone_t zone, xzm_chunk_list_head_t list,
+ xzm_chunk_t chunk, xzm_chunk_linkage_t linkage, bool *contended_out)
+{
+ xzm_debug_assert(linkage != XZM_CHUNK_LINKAGE_BATCH);
+
+ xzm_chunk_list_head_u head = {
+ .xzch_value = os_atomic_load(&list->xzch_value, dependency),
+ };
+
+ while (true) {
+ if (os_unlikely(head.xzch_fork_locked)) {
+ _xzm_fork_lock_wait(zone);
+ head.xzch_value = os_atomic_load(&list->xzch_value, dependency);
+ continue;
+ }
+
+ xzm_chunk_t head_chunk = (xzm_chunk_t)head.xzch_ptr;
+ xzm_debug_assert(head_chunk != chunk);
+ xzm_chunk_list_head_u new_head = {
+ .xzch_ptr = (uint64_t)chunk,
+ .xzch_gen = head.xzch_gen + 1,
+ };
+ chunk->xzc_linkages[linkage] = head_chunk;
+
+ bool success = os_atomic_cmpxchgv(&list->xzch_value, head.xzch_value,
+ new_head.xzch_value, &head.xzch_value, release);
+ if (success) {
+ break;
+ } else if (contended_out) {
+ *contended_out = true;
+ }
+ }
+}
+
+static xzm_chunk_list_t
+_xzm_chunk_list_get(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_slot_config_t *slot_config, xzm_chunk_list_t lists)
+{
+ xzm_allocation_index_t alloc_idx = _xzm_get_allocation_index(zone, xz,
+ slot_config, false);
+ return _xzm_xzone_chunk_list_for_index(zone, xz, lists, alloc_idx);
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static void
+_xzm_xzone_slot_record_contention(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_xzone_slot_counters_u *xsc, xzm_slot_config_t slot_config,
+ bool is_xas, bool contended);
+
+static xzm_chunk_t
+_xzm_chunk_list_slot_pop(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_chunk_list_t lists)
+{
+ xzm_slot_config_t slot_config;
+ xzm_chunk_list_t xcl = _xzm_chunk_list_get(zone, xz, &slot_config, lists);
+
+ bool contended = false;
+ xzm_chunk_t chunk = _xzm_chunk_list_pop(zone, &xcl->xcl_list,
+ XZM_CHUNK_LINKAGE_MAIN, &contended);
+ if (chunk) {
+ _xzm_xzone_slot_record_contention(zone, xz, &xcl->xcl_counters,
+ slot_config, false, contended);
+ }
+
+ return chunk;
+}
+
+static void
+_xzm_chunk_list_slot_push(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_chunk_list_t lists, xzm_chunk_t chunk)
+{
+ xzm_slot_config_t slot_config;
+ xzm_chunk_list_t xcl = _xzm_chunk_list_get(zone, xz, &slot_config, lists);
+ bool contended = false;
+ _xzm_chunk_list_push(zone, &xcl->xcl_list, chunk, XZM_CHUNK_LINKAGE_MAIN,
+ &contended);
+ _xzm_xzone_slot_record_contention(zone, xz, &xcl->xcl_counters, slot_config,
+ false, contended);
+}
+
+static void
+_xzm_chunk_batch_list_fork_lock(xzm_chunk_list_head_t list)
+{
+ xzm_chunk_list_head_u locked = {
+ .xzch_batch_fork_locked = true,
+ };
+
+ __assert_only xzm_chunk_list_head_u old_head = {
+ .xzch_value = os_atomic_or_orig(&list->xzch_value, locked.xzch_value,
+ relaxed),
+ };
+ xzm_debug_assert(!old_head.xzch_batch_fork_locked);
+}
+
+static void
+_xzm_chunk_batch_list_fork_unlock(xzm_chunk_list_head_t list)
+{
+ xzm_chunk_list_head_u locked = {
+ .xzch_batch_fork_locked = true,
+ };
+ uint64_t unlocked = ~locked.xzch_value;
+
+ __assert_only xzm_chunk_list_head_u old_head = {
+ .xzch_value = os_atomic_and_orig(&list->xzch_value, unlocked, relaxed),
+ };
+ xzm_debug_assert(old_head.xzch_batch_fork_locked);
+}
+
+static void
+_xzm_xzone_madvise_batch(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_chunk_t chunk);
+
+static void
+_xzm_chunk_batch_list_push(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_chunk_t chunk, size_t max)
+{
+ xzm_chunk_list_head_t list = &xz->xz_batch_list;
+ xzm_chunk_list_head_u head = {
+ .xzch_value = os_atomic_load(&list->xzch_value, dependency),
+ };
+
+ bool was_full;
+ while (true) {
+ if (os_unlikely(head.xzch_batch_fork_locked)) {
+ _xzm_fork_lock_wait(zone);
+ head.xzch_value = os_atomic_load(&list->xzch_value, dependency);
+ continue;
+ }
+
+ xzm_debug_assert(head.xzch_batch_count <= max);
+ was_full = (head.xzch_batch_count >= max);
+
+ xzm_chunk_list_head_u new_head = {
+ .xzch_batch_ptr = (uint64_t)chunk,
+ .xzch_batch_gen = head.xzch_batch_gen + 1,
+ .xzch_batch_count = os_likely(!was_full) ?
+ head.xzch_batch_count + 1 : 1,
+ };
+ xzm_debug_assert(was_full ||
+ new_head.xzch_batch_count > head.xzch_batch_count);
+ xzm_chunk_t next = os_likely(!was_full) ?
+ (xzm_chunk_t)head.xzch_ptr : NULL;
+ xzm_debug_assert(chunk != next);
+ *_xzm_segment_slice_meta_batch_next(zone, chunk) = next;
+
+ bool success = os_atomic_cmpxchgv(&list->xzch_value, head.xzch_value,
+ new_head.xzch_value, &head.xzch_value, release);
+ if (success) {
+ break;
+ }
+ }
+
+ // Perform the batch madvise on the old head
+ if (os_unlikely(was_full)) {
+ _xzm_xzone_madvise_batch(zone, xz, (xzm_chunk_t)head.xzch_batch_ptr);
+ }
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static bool
+_xzm_chunk_list_empty(xzm_chunk_list_head_t list)
+{
+ xzm_chunk_list_head_u head = {
+ .xzch_value = os_atomic_load(&list->xzch_value, relaxed),
+ };
+
+ return !head.xzch_ptr;
+}
+
+#pragma mark Tiny allocation
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static void *
+_xzm_xzone_malloc_from_freelist_chunk_inline(xzm_malloc_zone_t zone,
+ xzm_xzone_t xz, xzm_allocation_index_t alloc_idx,
+ xzm_xzone_thread_cache_t cache, xzm_chunk_t chunk, bool small,
+ bool walk_wait, bool *corrupt_out, bool *contended_out,
+ bool *install_empty_out)
+{
+#if CONFIG_MTE
+ bool memtag_enabled = chunk->xzc_tagged;
+#endif
+
+ bool install = !!install_empty_out;
+ size_t granule = small ? XZM_SMALL_GRANULE : XZM_GRANULE;
+
+ uint8_t *start = (uint8_t *)_xzm_chunk_start(zone, chunk, NULL);
+ if (small) {
+ xzm_debug_assert(chunk->xzc_bits.xzcb_kind ==
+ XZM_SLICE_KIND_SMALL_FREELIST_CHUNK);
+ } else {
+ xzm_debug_assert(chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_TINY_CHUNK);
+ }
+
+ // If we're trying to cache this chunk, it's important for the enumerator
+ // that we initialize the cache to point to it before we mark it as
+ // installed to a thread
+ if (cache) {
+ if (install) {
+ cache->xztc_chunk = chunk;
+ cache->xztc_chunk_start = start;
+ } else {
+ xzm_debug_assert(cache->xztc_chunk == chunk);
+ xzm_debug_assert(cache->xztc_chunk_start == start);
+ }
+ }
+
+ void *ptr = NULL;
+ bool from_free_list = false;
+ struct xzm_block_inline_meta_s block_meta;
+
+ uint32_t contentions = 0;
+
+ // Dependency order to observe initialization and zeroing of the block we
+ // allocated
+ xzm_chunk_atomic_meta_u old_meta = {
+ .xca_value = os_atomic_load_wide(
+ &chunk->xzc_atomic_meta.xca_value, dependency),
+ };
+ while (true) {
+ if (os_unlikely(old_meta.xca_walk_locked)) {
+ if (walk_wait) {
+ _xzm_walk_lock_wait(zone);
+ old_meta.xca_value = os_atomic_load_wide(
+ &chunk->xzc_atomic_meta.xca_value, dependency);
+ continue;
+ } else {
+ return NULL;
+ }
+ }
+
+ xzm_chunk_atomic_meta_u new_meta = old_meta;
+
+ if (install) {
+ // We're considering this chunk as a candidate to install. The
+ // chunk we're looking at should not be installed in any slot, and
+ // should be marked as being on the partial list (though in fact
+ // we'll have just popped it).
+ xzm_debug_assert(old_meta.xca_alloc_idx == XZM_SLOT_INDEX_EMPTY);
+ xzm_debug_assert(old_meta.xca_on_partial_list);
+ xzm_debug_assert(!old_meta.xca_on_empty_list);
+
+ new_meta.xca_on_partial_list = false;
+
+ // Does the chunk have any blocks?
+ if (old_meta.xca_free_count == 0) {
+ // No, no blocks to allocate.
+ if (old_meta.xca_alloc_head == XZM_FREE_MADVISED) {
+ // If the chunk is fully madvised and it was on the partial
+ // list, it's the caller's responsibility to move it to the
+ // empty list.
+ new_meta.xca_on_empty_list = true;
+ *install_empty_out = true;
+ } else {
+ // It shouldn't be possible for us to have gotten a full
+ // chunk from the partial list, so the only remaining
+ // possibility is an in-progress madvise.
+ xzm_debug_assert(
+ old_meta.xca_alloc_head == XZM_FREE_MADVISING);
+ }
+
+ // We may have initialized our thread cache entry to this chunk
+ // on a previous iteration of the loop. It's important that we
+ // invalidate it now, before it can potentially be reused by
+ // another thread cache, so that there's no ambiguity to the
+ // enumerator about which cache owns the chunk.
+ if (cache) {
+ cache->xztc_state = XZM_XZONE_CACHE_EMPTY;
+ }
+
+ bool success = os_atomic_cmpxchgv(
+ &chunk->xzc_atomic_meta.xca_value, old_meta.xca_value,
+ new_meta.xca_value, &old_meta.xca_value, dependency);
+ if (!success) {
+ // The only way this can happen is MADVISING -> MADVISED
+ xzm_debug_assert(!(*install_empty_out));
+ continue;
+ }
+
+ xzm_trace(malloc_install_skip, (uint64_t)chunk,
+ old_meta.xca_value_lo, new_meta.xca_value_lo,
+ cache ? cache->xztc_freelist_state : 0);
+ return NULL;
+ } else {
+ // Yes, we want to proceed with the allocation attempt and mark
+ // the chunk as installed to our slot if successful.
+ new_meta.xca_alloc_idx = alloc_idx + 1;
+
+#if CONFIG_TINY_ALLOCATION_SLOT_LOCK
+ // In this configuration, we actually can't proceed with the
+ // attempt until after we've marked the chunk as installed to
+ // our slot, as that's what allows a thread depopulating the
+ // chunk to be able to wait until we've exited the critical
+ // section. So, we're going to mark the chunk as installed
+ // directly, and then transition to the non-install logic to try
+ // to actually allocate a block.
+ bool success = os_atomic_cmpxchgv(
+ &chunk->xzc_atomic_meta.xca_value, old_meta.xca_value,
+ new_meta.xca_value, &old_meta.xca_value, dependency);
+ if (!success) {
+ continue;
+ }
+
+ // Update the chunk state for the next loop iteration
+ old_meta = new_meta;
+
+ install = false;
+ continue;
+#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
+ }
+ } else {
+ // We found this chunk as the one currently installed in our slot,
+ // and only want to allocate from it if it still is.
+ if (old_meta.xca_alloc_idx != alloc_idx + 1) {
+ // Shouldn't be possible when the chunk is installed to the
+ // thread cache
+ xzm_debug_assert(!cache);
+
+ // No. Go back to try to get a different one.
+ return NULL;
+ }
+
+ // Does the chunk have any blocks?
+ if (old_meta.xca_free_count == 0) {
+ // We should try to uninstall this chunk.
+ new_meta.xca_alloc_idx = XZM_SLOT_INDEX_EMPTY;
+
+ // It's marked as installed, so it should be full.
+ xzm_debug_assert(old_meta.xca_alloc_head == XZM_FREE_NULL);
+
+ // It shouldn't be on any chunk lists either, and doesn't need
+ // to be. If it eventually becomes unfull, it's the
+ // responsibility of that thread to move it back to the partial
+ // list.
+ xzm_debug_assert(!old_meta.xca_on_partial_list);
+ xzm_debug_assert(!old_meta.xca_on_empty_list);
+
+ if (cache) {
+ // We need to mark our thread cache entry as invalid before
+ // we mark the chunk as uninstalled, so that there's no
+ // ambiguity about which cache owns the chunk if we get
+ // pre-empted and that chunk gets installed to another cache
+ cache->xztc_state = XZM_XZONE_CACHE_EMPTY;
+ }
+
+ bool success = os_atomic_cmpxchgv(
+ &chunk->xzc_atomic_meta.xca_value, old_meta.xca_value,
+ new_meta.xca_value, &old_meta.xca_value, dependency);
+ if (!success) {
+ continue;
+ }
+
+ xzm_trace(malloc_full_skip, (uint64_t)chunk,
+ old_meta.xca_value_lo, new_meta.xca_value_lo,
+ cache ? cache->xztc_freelist_state : 0);
+ return NULL;
+ }
+ }
+
+ // It looks like we should be able to allocate.
+ from_free_list = false;
+
+ if (cache) {
+ // Allocating all blocks for this cache
+
+ // We initialize the cache entry to the freelist _before_ we mark
+ // the chunk as installed so that the enumerator can reliably walk
+ // the local freelist in the cache if it sees the chunk marked as
+ // installed.
+
+ new_meta.xca_alloc_head = XZM_FREE_NULL;
+ new_meta.xca_free_count = 0;
+
+ cache->xztc_free_count = old_meta.xca_free_count - 1;
+
+ if (old_meta.xca_alloc_head < XZM_FREE_LIMIT) {
+ // Pop from the alloc head
+ ptr = start + (old_meta.xca_alloc_head * granule);
+#if CONFIG_MTE
+ if (memtag_enabled) {
+ memtag_disable_checking();
+ block_meta = *(struct xzm_block_inline_meta_s *)ptr;
+ memtag_enable_checking();
+ } else {
+#endif
+ block_meta = *(struct xzm_block_inline_meta_s *)ptr;
+#if CONFIG_MTE
+ }
+#endif
+
+ cache->xztc_head = block_meta.xzb_linkage.xzbl_next_offset;
+ cache->xztc_head_seqno = block_meta.xzb_linkage.xzbl_next_seqno;
+
+ from_free_list = true;
+ } else {
+ xzm_debug_assert(old_meta.xca_alloc_head == XZM_FREE_NULL);
+ xzm_debug_assert(old_meta.xca_free_count);
+
+ // Note: it's important that we compute ptr in the loop, before
+ // we pop, for the benefit of the memory tools: if we don't,
+ // then the block will appear allocated but the tools won't be
+ // able to find any references to it. Getting it into a
+ // register first will ensure that there's always a reference.
+ size_t capacity = xz->xz_chunk_capacity;
+ size_t n_allocated = capacity - old_meta.xca_free_count;
+ ptr = start + (n_allocated * xz->xz_block_size);
+
+ cache->xztc_head = XZM_FREE_NULL;
+ }
+ } else {
+ // Allocating one block
+ new_meta.xca_free_count--;
+ if (old_meta.xca_alloc_head < XZM_FREE_LIMIT) {
+ // Pop from the alloc head
+ ptr = start + (old_meta.xca_alloc_head * granule);
+
+#if CONFIG_MTE
+ if (memtag_enabled) {
+ memtag_disable_checking();
+ block_meta = *(struct xzm_block_inline_meta_s *)ptr;
+ memtag_enable_checking();
+ } else {
+#endif
+ block_meta = *(struct xzm_block_inline_meta_s *)ptr;
+#if CONFIG_MTE
+ }
+#endif
+
+ new_meta.xca_alloc_head = block_meta.xzb_linkage.xzbl_next_offset;
+ new_meta.xca_head_seqno = block_meta.xzb_linkage.xzbl_next_seqno;
+ from_free_list = true;
+ } else {
+ xzm_debug_assert(old_meta.xca_free_count);
+ xzm_debug_assert(old_meta.xca_alloc_head == XZM_FREE_NULL);
+
+ // Note: it's important that we compute ptr in the loop, before we
+ // pop, for the benefit of the memory tools: if we don't, then the
+ // block will appear allocated but the tools won't be able to find
+ // any references to it. Getting it into a register first will
+ // ensure that there's always a reference.
+ size_t capacity = xz->xz_chunk_capacity;
+ ptr = start + ((capacity - old_meta.xca_free_count) *
+ xz->xz_block_size);
+ }
+ }
+
+ bool success = os_atomic_cmpxchgv(&chunk->xzc_atomic_meta.xca_value,
+ old_meta.xca_value, new_meta.xca_value, &old_meta.xca_value,
+ dependency);
+ if (!success) {
+ // XXX: possible madvise race - record? Retry?
+
+ // Record the contention
+ *contended_out = true;
+
+ contentions++;
+
+ // Try again.
+ continue;
+ }
+
+ // We got it.
+ xzm_trace(malloc_success, (uint64_t)chunk, old_meta.xca_value_lo,
+ new_meta.xca_value_lo,
+ cache ? cache->xztc_freelist_state : 0);
+ break;
+ }
+
+#if CONFIG_MTE
+ if (memtag_enabled) {
+ if (small) {
+ // Small blocks are tagged only on alloc
+ ptr = memtag_retag(ptr, xz->xz_block_size);
+ } else {
+ if (from_free_list) {
+ uint8_t tag = memtag_extract_tag((uint8_t *)block_meta.xzb_cookie);
+ xzm_debug_assert(tag != 0);
+ ptr = (void *)memtag_mix_tag(ptr, tag);
+ } else {
+ ptr = memtag_fixup_ptr(ptr);
+ }
+ }
+ }
+#endif
+
+ if (from_free_list) {
+ uint64_t expected_cookie = (uint64_t)ptr ^ zone->xzz_freelist_cookie;
+#if CONFIG_MTE
+ if (small) {
+ // Small chunks do not encode the tag in the freelist cookie
+ expected_cookie = (uint64_t)memtag_strip_address(
+ (uint8_t *)expected_cookie);
+ }
+#endif
+ if (expected_cookie != block_meta.xzb_cookie) {
+ *corrupt_out = true;
+ goto freelist_done;
+ }
+
+ // Check the PAC by adding the seqno back
+ union xzm_block_linkage_u linkage = {
+ .xzbl_next_offset = block_meta.xzb_linkage.xzbl_next_offset,
+ .xzbl_next_seqno = block_meta.xzb_linkage.xzbl_next_seqno,
+ .xzbl_seqno = old_meta.xca_head_seqno,
+ };
+
+ linkage.xzbl_next_value = (uintptr_t)ptrauth_sign_unauthenticated(
+ linkage.xzbl_next, ptrauth_key_process_dependent_data,
+ ptrauth_blend_discriminator(ptr,
+ ptrauth_string_discriminator("xzb_linkage")));
+
+ // Take the seqno back out of the signed value for the comparison with
+ // the stored one
+ linkage.xzbl_seqno = 0;
+
+ if (block_meta.xzb_linkage.xzbl_next_value != linkage.xzbl_next_value) {
+ *corrupt_out = true;
+ goto freelist_done;
+ }
+
+freelist_done:
+ ;
+ }
+
+ return ptr;
+}
+
+static void *
+_xzm_xzone_malloc_from_freelist_chunk(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_allocation_index_t alloc_idx, xzm_xzone_thread_cache_t cache,
+ xzm_chunk_t chunk, bool small, bool *contended_out,
+ bool *install_empty_out)
+{
+ const bool walk_wait = true;
+ bool corrupt = false;
+ void *ptr = _xzm_xzone_malloc_from_freelist_chunk_inline(zone, xz,
+ alloc_idx, cache, chunk, small, walk_wait, &corrupt, contended_out,
+ install_empty_out);
+ if (corrupt) {
+ _xzm_corruption_detected(ptr);
+ }
+
+ return ptr;
+}
+
+static void *
+_xzm_xzone_malloc_from_empty_freelist_chunk(xzm_malloc_zone_t zone,
+ xzm_xzone_t xz, xzm_allocation_index_t alloc_idx,
+ xzm_xzone_thread_cache_t cache, xzm_chunk_t chunk,
+ bool is_fresh)
+{
+ xzm_debug_assert(chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_TINY_CHUNK ||
+ chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK);
+#if CONFIG_MTE
+ bool small = chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK;
+#endif
+
+ // acquire to make sure the reclaim index is visible
+ // TODO make this load relaxed. the address of the reclaim index is derived
+ // from the chunk pointer that we popped from the list, and stores to the
+ // list are done with a release barrier
+ xzm_chunk_atomic_meta_u old_meta = {
+ .xca_value = os_atomic_load_wide(
+ &chunk->xzc_atomic_meta.xca_value, acquire),
+ };
+
+ // We found this on the batch list or the empty list, so it is either
+ // enqueued for madvising or has been madvised
+ xzm_debug_assert(old_meta.xca_alloc_head == XZM_FREE_MADVISED ||
+ old_meta.xca_alloc_head == XZM_FREE_MADVISING);
+
+ // A chunk is unusable if it is both on the partial list and the batch list;
+ // e.g., if its insertion raced with our emptying of the partial list
+ if (os_unlikely(old_meta.xca_on_partial_list)) {
+ xzm_debug_assert(old_meta.xca_alloc_head == XZM_FREE_MADVISING);
+ return NULL;
+ }
+
+
+ // We found this chunk on the batch or isolation list. It is not installed
+ // anywhere.
+ xzm_debug_assert(!old_meta.xca_on_partial_list);
+ xzm_debug_assert(old_meta.xca_alloc_idx == XZM_SLOT_INDEX_EMPTY);
+
+ uint8_t *start = (uint8_t *)_xzm_chunk_start(zone, chunk, NULL);
+ void *ptr = (void *)start;
+#if CONFIG_MTE
+ const bool memtag_enabled = xz->xz_tagged;
+ if (memtag_enabled) {
+ if (small) {
+ // For small, retag blocks lazily instead of for the entire chunk
+ // up front
+ ptr = memtag_retag(start, xz->xz_block_size);
+ } else {
+ if (is_fresh) {
+ ptr = _xzm_xzone_chunk_memtag_init(zone, xz, chunk);
+ } else {
+ ptr = memtag_fixup_ptr(ptr);
+ }
+ }
+ }
+#else
+ (void)is_fresh;
+#endif
+
+ // Preserve the seqno across reuses - this is important for anti-replay
+ xzm_chunk_atomic_meta_u new_meta = old_meta;
+
+ // We've taken it off the isolation list and won't be putting it back.
+ new_meta.xca_on_empty_list = false;
+
+ // In the absence of deferred reclaim, we're guaranteed to be able to bring
+ // this chunk back into use.
+ new_meta.xca_alloc_idx = alloc_idx + 1;
+ if (cache) {
+ new_meta.xca_alloc_head = XZM_FREE_NULL;
+ new_meta.xca_free_count = 0;
+
+ cache->xztc_chunk = chunk;
+ cache->xztc_chunk_start = start;
+ cache->xztc_head = XZM_FREE_NULL;
+ cache->xztc_free_count = xz->xz_chunk_capacity - 1;
+ // preserve xztc_seqno
+ } else {
+ new_meta.xca_alloc_head = XZM_FREE_NULL;
+ new_meta.xca_free_count = xz->xz_chunk_capacity - 1;
+ }
+
+ // This needs an acquire barrier because the pointer we're returning is not
+ // derived from the metadata, so the dependency ordering we usually rely on
+ // doesn't work.
+ bool success = os_atomic_cmpxchgv(&chunk->xzc_atomic_meta.xca_value,
+ old_meta.xca_value, new_meta.xca_value, &old_meta.xca_value,
+ acquire);
+ xzm_assert(success);
+
+ xzm_trace(malloc_from_empty, (uint64_t)chunk, old_meta.xca_value_lo,
+ new_meta.xca_value_lo, old_meta.xca_seqno);
+
+ return ptr;
+}
+
+// Called whenever a tiny or small chunk is received from the span queues. This
+// function needs to perform all the initialization needed to allocate a block
+// from this chunk, or to free the chunk during zone destruction.
+static void
+_xzm_xzone_fresh_chunk_init(xzm_xzone_t xz, xzm_chunk_t chunk,
+ xzm_slice_kind_t kind)
+{
+ xzm_debug_assert(chunk->xzc_xzone_idx == xz->xz_idx);
+
+ chunk->xzc_bits.xzcb_preallocated = false;
+
+ switch (kind) {
+ case XZM_SLICE_KIND_SMALL_CHUNK:
+ _xzm_chunk_reset_free(xz, chunk, true);
+ break;
+ case XZM_SLICE_KIND_TINY_CHUNK:
+ case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
+ xzm_debug_assert(xz->xz_block_size <= UINT16_MAX);
+ chunk->xzc_freelist_block_size = (uint16_t)xz->xz_block_size;
+
+ xzm_debug_assert(xz->xz_chunk_capacity <= UINT16_MAX);
+ chunk->xzc_freelist_chunk_capacity = (uint16_t)xz->xz_chunk_capacity;
+#if CONFIG_MTE
+ chunk->xzc_tagged = xz->xz_tagged;
+#endif
+ break;
+ default:
+ xzm_abort("Unexpected chunk kind");
+ break;
+ }
+}
+
+static void *
+_xzm_xzone_malloc_from_fresh_freelist_chunk(xzm_malloc_zone_t zone,
+ xzm_xzone_t xz, xzm_allocation_index_t alloc_idx,
+ xzm_xzone_thread_cache_t cache, xzm_chunk_t chunk, bool small)
+{
+ xzm_debug_assert(chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_TINY_CHUNK ||
+ chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK);
+
+ _xzm_xzone_fresh_chunk_init(xz, chunk, chunk->xzc_bits.xzcb_kind);
+
+ xzm_chunk_atomic_meta_u new_meta;
+
+ uint8_t *start = (uint8_t *)_xzm_chunk_start(zone, chunk, NULL);
+
+ if (cache) {
+ new_meta = (xzm_chunk_atomic_meta_u){
+ .xca_alloc_head = XZM_FREE_NULL,
+ .xca_free_count = 0,
+ .xca_alloc_idx = alloc_idx + 1,
+ };
+
+ cache->xztc_chunk = chunk;
+ cache->xztc_chunk_start = start;
+ cache->xztc_head = XZM_FREE_NULL;
+ cache->xztc_free_count = xz->xz_chunk_capacity - 1;
+ // preserve xztc_seqno
+ } else {
+ new_meta = (xzm_chunk_atomic_meta_u){
+ .xca_alloc_head = XZM_FREE_NULL,
+ .xca_free_count = xz->xz_chunk_capacity - 1,
+ .xca_alloc_idx = alloc_idx + 1,
+ };
+ }
+
+ xzm_trace(malloc_from_fresh, (uint64_t)chunk, xz->xz_idx,
+ new_meta.xca_value_lo, 0);
+ os_atomic_store_wide(&chunk->xzc_atomic_meta.xca_value, new_meta.xca_value,
+ relaxed);
+
+ // Enumerator protocol: mzone_idx is initialized last to "publish" the
+ // chunk. Because there are no references to this chunk yet, this store is
+ // guaranteed by dependency ordering to be visible to all future threads
+ // that observe it when the chunk is store-released to its initial slot.
+ chunk->xzc_mzone_idx = xz->xz_mzone_idx;
+
+#if CONFIG_MTE
+ if (chunk->xzc_tagged) {
+ void *ptr;
+ if (!small) {
+ ptr = _xzm_xzone_chunk_memtag_init(zone, xz, chunk);
+ } else {
+ // For small, retag blocks lazily instead of for the entire chunk
+ // up front
+ ptr = (void *)_xzm_xzone_block_memtag_retag(zone, (xzm_block_t)start,
+ xz->xz_block_size);
+ }
+ return ptr;
+ }
+#endif
+ return (void *)start;
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static xzm_slot_config_t
+_xzm_next_slot_config(xzm_slot_config_t slot_config)
+{
+ switch (slot_config) {
+ case XZM_SLOT_SINGLE:
+#if CONFIG_XZM_CLUSTER_AWARE
+ if (ncpuclusters > 1) {
+ return XZM_SLOT_CLUSTER;
+ } else {
+ return XZM_SLOT_CPU;
+ }
+#else // CONFIG_XZM_CLUSTER_AWARE
+ return XZM_SLOT_CLUSTER;
+#endif // CONFIG_XZM_CLUSTER_AWARE
+ case XZM_SLOT_CLUSTER:
+ return XZM_SLOT_CPU;
+ case XZM_SLOT_CPU:
+ xzm_debug_abort("Can't upgrade from XZM_SLOT_CPU");
+ return XZM_SLOT_CPU;
+ default:
+ xzm_debug_abort("Invalid xzone slot config");
+ return XZM_SLOT_CPU;
+ }
+}
+
+#if CONFIG_XZM_THREAD_CACHE
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static void
+_xzm_xzone_thread_cache_record_contention(xzm_malloc_zone_t zone,
+ xzm_xzone_t xz, xzm_xzone_thread_cache_t cache);
+
+#endif // CONFIG_XZM_THREAD_CACHE
+
+static void
+_xzm_xzone_upgrade_freelist_slot_config(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_xzone_slot_counters_u *xsc, xzm_xzone_slot_counters_u counter,
+ xzm_slot_config_t next_slot_config, bool is_xas)
+{
+ if (is_xas) {
+ xzm_trace(slot_upgrade, xz->xz_idx, (uint64_t)xsc,
+ next_slot_config, counter.xsc_value);
+ } else {
+ xzm_trace(list_upgrade, xz->xz_idx, (uint64_t)xsc,
+ next_slot_config, counter.xsc_value);
+ }
+
+ xzm_slot_config_t *slot_config_p = is_xas ? &xz->xz_slot_config :
+ &xz->xz_list_config;
+ os_atomic_store(slot_config_p, next_slot_config, relaxed);
+
+ // Try to upgrade/reset the counters in all the slots,
+ // regardless of whether or not we actually upgraded the xzone
+ counter = (xzm_xzone_slot_counters_u){
+ .xsc_slot_config = next_slot_config,
+ };
+ xzm_allocation_index_t limit_idx =
+ _xzm_get_limit_allocation_index(next_slot_config);
+ for (xzm_allocation_index_t alloc_idx = 0; alloc_idx < limit_idx;
+ alloc_idx++) {
+ xsc = is_xas ?
+ &_xzm_xzone_allocation_slot_for_index(zone, xz,
+ alloc_idx)->xas_counters :
+ &_xzm_xzone_chunk_list_for_index(zone, xz,
+ zone->xzz_partial_lists, alloc_idx)->xcl_counters;
+ os_atomic_store(&xsc->xsc_value, counter.xsc_value, relaxed);
+ }
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static void
+_xzm_xzone_slot_record_contention(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_xzone_slot_counters_u *xsc, xzm_slot_config_t slot_config,
+ bool is_xas, bool contended)
+{
+#if CONFIG_XZM_THREAD_CACHE
+ // XXX Somewhat wasteful to re-check this condition here - consider plumbing
+ // thread cache loaded earlier through to here
+ if (is_xas &&
+ zone->xzz_thread_cache_enabled &&
+ xz->xz_block_size <= XZM_THREAD_CACHE_THRESHOLD) {
+ if (contended) {
+ xzm_thread_cache_t tc = _xzm_get_thread_cache();
+ if (os_likely(tc)) {
+ xzm_xzone_index_t xz_idx = xz->xz_idx;
+ xzm_xzone_thread_cache_t cache = &tc->xtc_xz_caches[xz_idx];
+ _xzm_xzone_thread_cache_record_contention(zone, xz, cache);
+ }
+ }
+ return;
+ }
+#endif // CONFIG_XZM_THREAD_CACHE
+
+ xzm_xzone_slot_counters_u old_counters = {
+ .xsc_value = os_atomic_load(&xsc->xsc_value,
+ relaxed),
+ };
+
+ xzm_slot_config_t max_config = is_xas ? zone->xzz_max_slot_config :
+ zone->xzz_max_list_config;
+ if (old_counters.xsc_slot_config == max_config) {
+ return;
+ }
+
+ // Only need to record if:
+ // - We detected a contention, or
+ // - Some contention has already been detected, so we need to record the
+ // operation to assess the rate
+ if (contended || old_counters.xsc_contentions) {
+ uint64_t inc = 1; // inc xsc_ops
+ if (contended) {
+ inc |= (1ull << 32); // inc xsc_contentions
+ }
+
+ xzm_xzone_slot_counters_u new_counters = {
+ .xsc_value = os_atomic_add(&xsc->xsc_value, inc,
+ relaxed),
+ };
+
+ // The "current" slot config that we observe in the slot
+ // after incrementing the counters may be different than the one we
+ // observed originally on the xzone:
+ // - It could have a lower slot config if it was just upgraded
+ //
+ // Regardless, we'll use the slot config that we saw in the counters as
+ // the one that dictates what we should do next from here on.
+ xzm_slot_config_t current_slot_config = new_counters.xsc_slot_config;
+
+ if (current_slot_config == max_config) {
+ return;
+ }
+
+ uint32_t period = is_xas ? zone->xzz_slot_upgrade_period :
+ zone->xzz_list_upgrade_period;
+ uint32_t *thresholds = is_xas ? zone->xzz_slot_upgrade_threshold :
+ zone->xzz_list_upgrade_threshold;
+ if (os_unlikely(new_counters.xsc_contentions >=
+ thresholds[current_slot_config])) {
+ if (new_counters.xsc_contentions >
+ thresholds[current_slot_config]) {
+ // Someone else is upgrading all the slots, do nothing
+ return;
+ }
+
+ // If we've detected contention above the threshold, try to
+ // upgrade the xzone
+ xzm_slot_config_t next_slot_config = _xzm_next_slot_config(
+ new_counters.xsc_slot_config);
+ xzm_debug_assert(new_counters.xsc_slot_config < next_slot_config);
+
+ _xzm_xzone_upgrade_freelist_slot_config(zone, xz, xsc,
+ new_counters, next_slot_config, is_xas);
+ } else if (os_unlikely(new_counters.xsc_ops >= period)) {
+ if (new_counters.xsc_ops > period) {
+ // Someone else is resetting the counter, do nothing
+ return;
+ }
+ // If the detection period has elapsed without meeting the
+ // threshold, reset the counters for the next period.
+ xzm_xzone_slot_counters_u orig_counters = new_counters;
+
+ new_counters = (xzm_xzone_slot_counters_u){
+ .xsc_slot_config = current_slot_config,
+ };
+
+ os_atomic_rmw_loop(&xsc->xsc_value,
+ old_counters.xsc_value, new_counters.xsc_value, relaxed, {
+ if (old_counters.xsc_value < orig_counters.xsc_value ||
+ old_counters.xsc_slot_config >
+ orig_counters.xsc_slot_config) {
+ os_atomic_rmw_loop_give_up(break);
+ }
+ });
+ }
+ }
+}
+
+#if !CONFIG_TINY_ALLOCATION_SLOT_LOCK
+// This inlines badly: the compiler unconditionally hoists the mrs out of the
+// loop, which is exactly what we don't want since we normally won't need it
+MALLOC_NOINLINE
+static uint32_t
+_malloc_ulock_self_owner_value(void)
+{
+ mach_port_t self_port = _pthread_mach_thread_self_direct();
+ return (uint32_t)self_port >> 2;
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static xzm_allocation_slot_atomic_meta_u
+_xzm_allocation_slot_gate_wait(xzm_xzone_allocation_slot_t xas,
+ xzm_allocation_slot_atomic_meta_u slot_meta)
+{
+ xzm_allocation_slot_atomic_meta_u *xasa = &xas->xas_atomic;
+
+ xzm_debug_assert(slot_meta.xasa_gate.xsg_locked);
+
+ if (!slot_meta.xasa_gate.xsg_waiters) {
+ // We're the first waiter, so we first need to mark ourselves as
+ // waiting.
+ xzm_allocation_slot_atomic_meta_u new_slot_meta = slot_meta;
+ new_slot_meta.xasa_gate.xsg_waiters = true;
+ bool success = os_atomic_cmpxchgv(&xasa->xasa_value,
+ slot_meta.xasa_value, new_slot_meta.xasa_value,
+ &slot_meta.xasa_value, dependency);
+ if (!success) {
+ return slot_meta;
+ }
+ slot_meta.xasa_gate.xsg_waiters = true;
+ }
+
+ uint32_t wait_op = UL_UNFAIR_LOCK | ULF_NO_ERRNO |
+ ULF_WAIT_ADAPTIVE_SPIN | ULF_WAIT_WORKQ_DATA_CONTENTION;
+ int rc = __ulock_wait(wait_op, &xasa->xasa_ulock,
+ slot_meta.xasa_ulock, 0);
+ if (os_unlikely(rc < 0)) {
+ switch (-rc) {
+ case EINTR:
+ case EFAULT:
+ break;
+ default:
+ xzm_abort_with_reason("ulock_wait failure", -rc);
+ }
+ } else {
+ // We should only wake as part of a wake-all broadcast, but
+ // there's no way to assert that. rc indicates the number of
+ // waiters still in the ulock, but we can't assert anything
+ // useful about that.
+ }
+
+ slot_meta = (xzm_allocation_slot_atomic_meta_u){
+ .xasa_value = os_atomic_load(&xasa->xasa_value, dependency),
+ };
+ return slot_meta;
+}
+
+static void
+_xzm_xzone_allocation_slot_fork_lock(xzm_xzone_allocation_slot_t xas)
+{
+ xzm_allocation_slot_atomic_meta_u *xasa = &xas->xas_atomic;
+
+ xzm_allocation_slot_atomic_meta_u slot_meta = {
+ .xasa_value = os_atomic_load(&xasa->xasa_value, dependency),
+ };
+ while (true) {
+ if (slot_meta.xasa_gate.xsg_locked) {
+ slot_meta = _xzm_allocation_slot_gate_wait(xas, slot_meta);
+ continue;
+ }
+
+ xzm_allocation_slot_atomic_meta_u new_slot_meta = slot_meta;
+ new_slot_meta.xasa_chunk.xsc_fork_locked = true;
+
+ bool success = os_atomic_cmpxchgv(&xasa->xasa_value,
+ slot_meta.xasa_value, new_slot_meta.xasa_value,
+ &slot_meta.xasa_value, relaxed);
+ if (!success) {
+ continue;
+ }
+
+ break;
+ }
+}
+
+static void
+_xzm_xzone_allocation_slot_fork_unlock(xzm_xzone_allocation_slot_t xas)
+{
+ xzm_allocation_slot_atomic_meta_u *xasa = &xas->xas_atomic;
+
+ xzm_allocation_slot_atomic_meta_u slot_meta = {
+ .xasa_value = os_atomic_load(&xasa->xasa_value, dependency),
+ };
+
+ xzm_debug_assert(slot_meta.xasa_chunk.xsc_fork_locked);
+
+ xzm_allocation_slot_atomic_meta_u new_slot_meta = slot_meta;
+ new_slot_meta.xasa_chunk.xsc_fork_locked = false;
+
+ uint64_t prev_slot_value = os_atomic_xchg(&xasa->xasa_value,
+ new_slot_meta.xasa_value, release);
+ xzm_assert(prev_slot_value == slot_meta.xasa_value);
+}
+#endif // !CONFIG_TINY_ALLOCATION_SLOT_LOCK
+
+static xzm_chunk_t
+_xzm_xzone_allocate_chunk_from_isolation(xzm_main_malloc_zone_t main,
+ xzm_xzone_t xz)
+{
+ xzm_debug_assert(xz->xz_sequestered);
+ xzm_isolation_zone_t iz = &main->xzmz_isolation_zones[xz->xz_idx];
+
+ xzm_chunk_t chunk = NULL;
+
+ // Don't bother trying to take the iz lock if it doesn't look like
+ // there's anything in there
+ //
+ // XXX TODO: this should be a proper atomic relaxed load
+ if (LIST_FIRST(&iz->xziz_chunkq)) {
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ LIST_HEAD(, xzm_slice_s) busy_list = LIST_HEAD_INITIALIZER(busy_list);
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+ _malloc_lock_lock(&iz->xziz_lock);
+ chunk = LIST_FIRST(&iz->xziz_chunkq);
+ while (chunk) {
+ xzm_debug_assert(_xzm_chunk_is_empty(&main->xzmz_base, xz, chunk));
+ LIST_REMOVE(chunk, xzc_entry);
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ // Chunks stay in the reclaim buffer when removed from the
+ // isolation zone, and must be removed from the buffer
+ if ((xz->xz_block_size > XZM_TINY_BLOCK_SIZE_MAX &&
+ main->xzmz_defer_small) ||
+ ((xz->xz_block_size <= XZM_TINY_BLOCK_SIZE_MAX &&
+ main->xzmz_defer_tiny))) {
+ if (!xzm_chunk_mark_used(&main->xzmz_base, chunk, NULL)) {
+ // this chunk is busy being reclaimed by the kernel
+ LIST_INSERT_HEAD(&busy_list, chunk, xzc_entry);
+ chunk = LIST_FIRST(&iz->xziz_chunkq);
+ continue;
+ }
+ }
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+ // chunk is already initialized, other than the mzone idx
+ chunk->xzc_mzone_idx = xz->xz_mzone_idx;
+ break;
+ }
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ if (!LIST_EMPTY(&busy_list)) {
+ // place any busy chunks back on the sequester list
+ xzm_chunk_t busy_chunk, tmp_chunk;
+ LIST_FOREACH_SAFE(busy_chunk, &busy_list, xzc_entry,
+ tmp_chunk) {
+ LIST_REMOVE(busy_chunk, xzc_entry);
+ LIST_INSERT_HEAD(&iz->xziz_chunkq, busy_chunk, xzc_entry);
+ }
+ }
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+ _malloc_lock_unlock(&iz->xziz_lock);
+ }
+
+
+ return chunk;
+}
+
+static void *
+_xzm_xzone_find_and_malloc_from_freelist_chunk(xzm_malloc_zone_t zone,
+ xzm_xzone_t xz, xzm_allocation_index_t alloc_idx,
+ xzm_xzone_thread_cache_t cache, xzm_chunk_t *chunk_out,
+ bool *contended_out)
+{
+ void *ptr = NULL;
+ xzm_chunk_t chunk = NULL;
+ bool small = (xz->xz_block_size > XZM_TINY_BLOCK_SIZE_MAX);
+
+ // The first place to try to get a new chunk from is the partial list.
+ while ((chunk = _xzm_chunk_list_slot_pop(zone, xz,
+ zone->xzz_partial_lists))) {
+ xzm_debug_assert(chunk->xzc_atomic_meta.xca_on_partial_list &&
+ !chunk->xzc_atomic_meta.xca_on_empty_list);
+ bool empty = false;
+ ptr = _xzm_xzone_malloc_from_freelist_chunk(zone, xz, alloc_idx, cache,
+ chunk, small, contended_out, &empty);
+ if (ptr) {
+ // Great, we allocated from this chunk and can install it.
+ goto done;
+ } else if (empty) {
+ xzm_debug_assert(!chunk->xzc_atomic_meta.xca_on_partial_list &&
+ chunk->xzc_atomic_meta.xca_on_empty_list);
+ // It's our responsibility to move this chunk to the empty list.
+ _xzm_chunk_list_push(zone, &xz->xz_empty_list, chunk,
+ XZM_CHUNK_LINKAGE_MAIN, NULL);
+ }
+ }
+
+ xzm_debug_assert(!chunk);
+
+ // Try the batch list. Some chunks may not be usable, so we need to build a
+ // list of the busy ones that can be reinserted. Since some of these chunks
+ // may still be on the partial list, we must reuse the batch linkage instead
+ // of the inline linkage (c.f. empty list busy chunks below).
+ xzm_chunk_t busy_chunk = NULL;
+ while ((chunk = _xzm_chunk_list_pop(zone, &xz->xz_batch_list,
+ XZM_CHUNK_LINKAGE_BATCH, NULL))) {
+ xzm_debug_assert(!chunk->xzc_atomic_meta.xca_on_empty_list);
+ ptr = _xzm_xzone_malloc_from_empty_freelist_chunk(zone, xz, alloc_idx,
+ cache, chunk, false);
+ if (ptr) {
+ break;
+ }
+
+ *_xzm_segment_slice_meta_batch_next(zone, chunk) = busy_chunk;
+ busy_chunk = chunk;
+ }
+
+ // Push unusable chunks back onto the batch list
+ xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
+ while (busy_chunk) {
+ xzm_debug_assert(!busy_chunk->xzc_atomic_meta.xca_on_empty_list);
+ xzm_chunk_t next = *_xzm_segment_slice_meta_batch_next(zone, busy_chunk);
+ _xzm_chunk_batch_list_push(zone, xz, busy_chunk, main->xzmz_batch_size);
+ busy_chunk = next;
+ }
+
+ if (ptr) {
+ // Great, we allocated from this chunk and can install it.
+ goto done;
+ }
+
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ SLIST_HEAD(, xzm_slice_s) busy_list = SLIST_HEAD_INITIALIZER(busy_list);
+#endif
+ // Next, try the empty list. Some chunks may not be usable, so we need to
+ // build a list of the busy ones that can be reinserted. Note that this list
+ // is intrusive, and reuses the same linkage as that used by the partial and
+ // empty lists.
+ while ((chunk = _xzm_chunk_list_pop(zone, &xz->xz_empty_list,
+ XZM_CHUNK_LINKAGE_MAIN, NULL))) {
+ xzm_debug_assert(chunk->xzc_atomic_meta.xca_on_empty_list);
+
+ bool was_reclaimed = true;
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ // This relies on dependency ordering from the pop of the chunk
+ // list to obtain the correct reclaim buffer index
+ if (((main->xzmz_defer_tiny && !small) ||
+ (main->xzmz_defer_small && small)) &&
+ !xzm_chunk_mark_used(zone, chunk, &was_reclaimed)) {
+ // this chunk is busy so we can't use it
+ goto empty_busy;
+ }
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+
+ ptr = _xzm_xzone_malloc_from_empty_freelist_chunk(zone, xz, alloc_idx,
+ cache, chunk, was_reclaimed);
+ if (ptr) {
+ break;
+ }
+
+#if CONFIG_XZM_DEFERRED_RECLAIM
+empty_busy:
+ SLIST_INSERT_HEAD(&busy_list, chunk, xzc_slist_entry);
+#else // CONFIG_XZM_DEFERRED_RECLAIM
+ xzm_debug_abort("_xzm_xzone_malloc_from_empty_freelist_chunk failed");
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+ }
+
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ // Push unusable chunks back onto the empty list
+ while ((busy_chunk = SLIST_FIRST(&busy_list))) {
+ xzm_debug_assert(busy_chunk->xzc_atomic_meta.xca_on_empty_list);
+ SLIST_REMOVE_HEAD(&busy_list, xzc_slist_entry);
+ _xzm_chunk_list_push(zone, &xz->xz_empty_list, busy_chunk,
+ XZM_CHUNK_LINKAGE_MAIN, NULL);
+ }
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+
+ if (ptr) {
+ // Great, we allocated from this chunk and can install it.
+ goto done;
+ }
+
+ // try the global isolation list
+ if (xz->xz_sequestered) {
+ xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
+ chunk = _xzm_xzone_allocate_chunk_from_isolation(main, xz);
+ if (chunk) {
+ xzm_debug_assert(_xzm_chunk_is_empty(zone, xz, chunk));
+ ptr = _xzm_xzone_malloc_from_empty_freelist_chunk(zone, xz, alloc_idx,
+ cache, chunk, true);
+ xzm_debug_assert(ptr);
+ goto push_to_all_list_and_unlock;
+ }
+ }
+
+ bool zero_on_free = (xz->xz_block_size <= XZM_ZERO_ON_FREE_THRESHOLD);
+ if (!(chunk = _xzm_chunk_list_pop(zone, &xz->xz_preallocated_list,
+ XZM_CHUNK_LINKAGE_MAIN, NULL))) {
+ // Failed to get a chunk from the preallocated list, need to go to the
+ // segment group to get a fresh chunk.
+ bool clear_chunk = zero_on_free;
+ bool purgeable = false; // TINY chunks can't be purgeable
+ xzm_preallocate_list_s preallocated;
+ SLIST_INIT(&preallocated);
+ xzm_preallocate_list_s *list = &preallocated;
+ xzm_segment_group_t sg = _xzm_segment_group_for_id_and_front(zone,
+ xz->xz_segment_group_id, xz->xz_front, false);
+
+ xzm_slice_kind_t kind = XZM_SLICE_KIND_TINY_CHUNK;
+ xzm_slice_count_t slice_count = 1;
+ if (small) {
+ kind = XZM_SLICE_KIND_SMALL_FREELIST_CHUNK;
+ slice_count =
+ XZM_SMALL_FREELIST_CHUNK_SIZE / XZM_SEGMENT_SLICE_SIZE;
+ }
+
+ chunk = xzm_segment_group_alloc_chunk(sg, kind,
+ &xz->xz_guard_config, slice_count, list, 0, clear_chunk,
+ purgeable);
+ if (!chunk) {
+ // Not able to get anything from the segment group, so give up
+ // entirely.
+ xzm_debug_assert(!list || !SLIST_FIRST(list));
+ goto done;
+ } else {
+ chunk->xzc_xzone_idx = xz->xz_idx;
+ xzm_chunk_t c, t;
+ SLIST_FOREACH_SAFE(c, list, xzc_slist_entry, t) {
+ SLIST_REMOVE_HEAD(list, xzc_slist_entry);
+ c->xzc_xzone_idx = xz->xz_idx;
+ c->xzc_bits.xzcb_preallocated = true;
+ _xzm_chunk_list_push(zone, &xz->xz_preallocated_list, c,
+ XZM_CHUNK_LINKAGE_MAIN, NULL);
+ }
+
+ if (xz->xz_slot_config < zone->xzz_initial_slot_config &&
+ xz->xz_block_size < zone->xzz_slot_initial_threshold) {
+ xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
+ uint64_t count = os_atomic_inc(&xz->xz_chunk_count, relaxed);
+ if (count > main->xzmz_xzone_chunk_threshold) {
+ // Upgrade/reset the counters in all the slots
+ xzm_xzone_slot_counters_u counter = {
+ .xsc_slot_config = xz->xz_slot_config,
+ };
+ _xzm_xzone_upgrade_freelist_slot_config(zone, xz, NULL,
+ counter, zone->xzz_initial_slot_config, true);
+ }
+ }
+ }
+ } else {
+ // We got a chunk from the preallocated list, which is not guaranteed to
+ // be zero'd
+ if (zero_on_free && !chunk->xzc_bits.xzcb_is_pristine) {
+ size_t chunk_size = 0;
+ void *ptr = _xzm_chunk_start_ptr(zone, chunk, &chunk_size);
+ xzm_debug_assert(ptr);
+ xzm_debug_assert(chunk_size == XZM_SEGMENT_SLICE_SIZE);
+ bzero(ptr, chunk_size);
+ }
+ }
+
+ ptr = _xzm_xzone_malloc_from_fresh_freelist_chunk(zone, xz, alloc_idx,
+ cache, chunk, small);
+ xzm_debug_assert(ptr);
+
+push_to_all_list_and_unlock:
+ // Put the chunk on the "all" list for the xzone so that it can be found
+ // during fork
+ _xzm_chunk_list_push(zone, &xz->xz_all_list, chunk, XZM_CHUNK_LINKAGE_ALL,
+ NULL);
+
+done:
+ xzm_debug_assert((ptr && chunk) || (!ptr && !chunk));
+
+ *chunk_out = chunk;
+ return ptr;
+}
+
+MALLOC_NOINLINE
+static void *
+_xzm_xzone_malloc_freelist_outlined(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_allocation_index_t alloc_idx, xzm_xzone_allocation_slot_t xas,
+ void *corrupt_block, xzm_malloc_options_t opt)
+{
+ if (os_unlikely(corrupt_block)) {
+ _xzm_corruption_detected(corrupt_block);
+ }
+
+ bool clear = (opt & XZM_MALLOC_CLEAR);
+ bool zero_on_free = (xz->xz_block_size <= XZM_ZERO_ON_FREE_THRESHOLD);
+#if !CONFIG_TINY_ALLOCATION_SLOT_LOCK
+ bool small = (xz->xz_block_size > XZM_TINY_BLOCK_SIZE_MAX);
+#endif
+#if CONFIG_MTE
+ const bool memtag_enabled = xz->xz_tagged;
+ bool needs_canonical_tagging = (opt & XZM_MALLOC_CANONICAL_TAG);
+#endif
+
+#if CONFIG_TINY_ALLOCATION_SLOT_LOCK
+#ifdef DEBUG
+ _malloc_lock_assert_owner(&xas->xas_lock);
+#endif // DEBUG
+#else // CONFIG_TINY_ALLOCATION_SLOT_LOCK
+ uint32_t self_owner_value = 0;
+#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
+
+ xzm_allocation_slot_atomic_meta_u *xasa = &xas->xas_atomic;
+
+ xzm_allocation_slot_atomic_meta_u slot_meta = {
+ .xasa_value = os_atomic_load(&xasa->xasa_value, dependency),
+ };
+
+ void *ptr = NULL;
+ xzm_chunk_t chunk = NULL;
+ bool contended = false;
+
+#if !CONFIG_TINY_ALLOCATION_SLOT_LOCK
+ while (true) {
+ if (slot_meta.xasa_gate.xsg_locked) {
+ // No chunk is currently installed, and another thread is in the
+ // middle of fetching and installing one. Wait for them and then
+ // try again.
+ slot_meta = _xzm_allocation_slot_gate_wait(xas, slot_meta);
+ continue;
+ } else if (os_unlikely(slot_meta.xasa_chunk.xsc_fork_locked)) {
+ // We can't acquire the gate because a fork is in progress that we
+ // need to wait out.
+ _xzm_fork_lock_wait(zone);
+
+ slot_meta = (xzm_allocation_slot_atomic_meta_u){
+ .xasa_value = os_atomic_load(&xasa->xasa_value, dependency),
+ };
+ continue;
+ }
+
+ // The gate isn't locked, so the next question is whether a chunk is
+ // installed.
+ chunk = (xzm_chunk_t)slot_meta.xasa_chunk.xsc_ptr;
+ if (chunk) {
+ // A chunk is installed, so let's try to allocate from it.
+ ptr = _xzm_xzone_malloc_from_freelist_chunk(zone, xz, alloc_idx,
+ NULL, chunk, small, &contended,
+ /* install_empty_out */ NULL);
+ if (ptr) {
+ // Success!
+ goto done;
+ }
+ }
+
+ if (!self_owner_value) {
+ self_owner_value = _malloc_ulock_self_owner_value();
+ }
+
+ // Either we had no chunk reference, or the one we had couldn't be
+ // allocated from. Let's try to become an allocator for our slot to go
+ // get a new one.
+ xzm_allocation_slot_atomic_meta_u new_slot_meta = {
+ .xasa_gate = {
+ .xsg_locked = true,
+ .xsg_owner = self_owner_value,
+ .xsg_gen = slot_meta.xasa_gate.xsg_gen,
+ },
+ };
+ bool success = os_atomic_cmpxchgv(&xasa->xasa_value,
+ slot_meta.xasa_value, new_slot_meta.xasa_value,
+ &slot_meta.xasa_value, dependency);
+ if (!success) {
+ continue;
+ }
+
+ break;
+ }
+
+ xzm_debug_assert(self_owner_value);
+#else // CONFIG_TINY_ALLOCATION_SLOT_LOCK
+ xzm_debug_assert(!slot_meta.xasa_gate.xsg_locked);
+#if CONFIG_MTE
+ chunk = (xzm_chunk_t)slot_meta.xasa_chunk.xsc_ptr;
+ if (chunk && (xz->xz_tagged && (opt & XZM_MALLOC_CANONICAL_TAG))) {
+ // If we're serving a canonically-tagged request, we won't have actually
+ // attempted to allocate from the chunk yet, so we need to do that first
+ ptr = _xzm_xzone_malloc_from_freelist_chunk(zone, xz, alloc_idx,
+ NULL, chunk, /* small */ false, &contended,
+ /* install_empty_out */ NULL);
+ if (ptr) {
+ // Success!
+ goto unlock;
+ }
+ }
+#endif
+#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
+
+ // It's now up to us to get a usable chunk and install it.
+
+ xzm_debug_assert(!ptr);
+
+ ptr = _xzm_xzone_find_and_malloc_from_freelist_chunk(zone, xz, alloc_idx,
+ NULL, &chunk, &contended);
+
+ // Unlock the gate and publish our new chunk (or lack thereof)
+ xzm_allocation_slot_atomic_meta_u new_slot_meta = {
+ .xasa_chunk = {
+ .xsc_ptr = (uint64_t)chunk,
+ .xsc_gen = slot_meta.xasa_gate.xsg_gen + 1,
+ },
+ };
+
+ uint64_t prev_slot_value = os_atomic_xchg(&xasa->xasa_value,
+ new_slot_meta.xasa_value, release);
+ xzm_allocation_slot_atomic_meta_u prev_slot_meta = {
+ .xasa_value = prev_slot_value,
+ };
+#if CONFIG_TINY_ALLOCATION_SLOT_LOCK
+ (void)prev_slot_meta;
+ xzm_debug_assert(prev_slot_meta.xasa_value == slot_meta.xasa_value);
+#else // CONFIG_TINY_ALLOCATION_SLOT_LOCK
+ xzm_debug_assert(prev_slot_meta.xasa_gate.xsg_locked);
+ xzm_debug_assert(prev_slot_meta.xasa_gate.xsg_owner == self_owner_value);
+ xzm_debug_assert(prev_slot_meta.xasa_gate.xsg_unused == 0);
+ xzm_debug_assert(prev_slot_meta.xasa_gate.xsg_gen ==
+ slot_meta.xasa_gate.xsg_gen);
+
+ if (prev_slot_meta.xasa_gate.xsg_waiters) {
+ uint32_t wake_op = UL_UNFAIR_LOCK | ULF_WAKE_ALL | ULF_NO_ERRNO;
+ int rc = __ulock_wake(wake_op, &xasa->xasa_ulock, 0);
+ if (rc && rc != -ENOENT) {
+ xzm_abort_with_reason("ulock_wake failure", -rc);
+ }
+ }
+#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
+
+#if CONFIG_TINY_ALLOCATION_SLOT_LOCK
+#if CONFIG_MTE
+unlock:
+#endif
+ _malloc_lock_unlock(&xas->xas_lock);
+#else // CONFIG_TINY_ALLOCATION_SLOT_LOCK
+done:
+#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
+
+ // Now that we've unlocked the gate, we can safely touch the chunk to
+ // perform any necessary zeroing to the block we allocated. It's very
+ // important that we not do this inside the gate because taking a fault in
+ // there makes it much more likely for other threads to contend and get
+ // stuck waiting for us.
+ if (ptr) {
+ // The true contention information we'd want to record in the locking
+ // case would be from the trylock rather than the list pop, but we
+ // didn't forward that information so there's nothing to do here
+#if !CONFIG_TINY_ALLOCATION_SLOT_LOCK
+ xzm_slot_config_t slot_config = os_atomic_load(&xz->xz_slot_config,
+ relaxed);
+ _xzm_xzone_slot_record_contention(zone, xz, &xas->xas_counters,
+ slot_config, true, contended);
+#endif // !CONFIG_TINY_ALLOCATION_SLOT_LOCK
+
+#if CONFIG_MTE
+ if (memtag_enabled) {
+ if (needs_canonical_tagging) {
+ ptr = memtag_tag_canonical(ptr, xz->xz_block_size);
+ } else {
+ xzm_debug_assert(ptr == memtag_fixup_ptr(ptr));
+ }
+ }
+#endif
+
+ xzm_block_t block = ptr;
+ *block = (struct xzm_block_inline_meta_s){ 0 };
+
+ if (clear && !zero_on_free) {
+ bzero(ptr, xz->xz_block_size);
+ }
+ } else {
+ malloc_set_errno_fast(MZ_POSIX, ENOMEM);
+ }
+ return ptr;
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static void *
+_xzm_xzone_malloc_freelist(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_malloc_options_t opt, const bool small)
+{
+ bool clear = (opt & XZM_MALLOC_CLEAR);
+
+ xzm_slot_config_t slot_config;
+ xzm_allocation_index_t alloc_idx = _xzm_get_allocation_index(zone, xz,
+ &slot_config, true);
+ xzm_xzone_allocation_slot_t xas =
+ _xzm_xzone_allocation_slot_for_index(zone, xz, alloc_idx);
+
+ bool contended = false;
+#if CONFIG_TINY_ALLOCATION_SLOT_LOCK
+ bool contended_dummy = false;
+ if (!_malloc_lock_trylock(&xas->xas_lock)) {
+ contended = true;
+ _malloc_lock_lock(&xas->xas_lock);
+ }
+#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
+
+ xzm_allocation_slot_atomic_meta_u *xasa = &xas->xas_atomic;
+
+ xzm_allocation_slot_atomic_meta_u slot_meta = {
+ .xasa_value = os_atomic_load(&xasa->xasa_value, dependency),
+ };
+
+ void *ptr = NULL;
+#if CONFIG_MTE
+ const bool fast_path =
+ !slot_meta.xasa_gate.xsg_locked && slot_meta.xasa_chunk.xsc_ptr &&
+ !(xz->xz_tagged && (opt & XZM_MALLOC_CANONICAL_TAG));
+#else
+ const bool fast_path =
+ !slot_meta.xasa_gate.xsg_locked && slot_meta.xasa_chunk.xsc_ptr;
+#endif
+ // Fast path: the slot is not locked and a chunk is installed
+ if (fast_path) {
+ xzm_chunk_t chunk = (xzm_chunk_t)slot_meta.xasa_chunk.xsc_ptr;
+
+ // Try to allocate from the installed chunk
+ const bool walk_wait = false;
+ bool corrupt = false;
+ ptr = _xzm_xzone_malloc_from_freelist_chunk_inline(zone, xz, alloc_idx,
+ NULL, chunk, small, walk_wait, &corrupt,
+#if CONFIG_TINY_ALLOCATION_SLOT_LOCK
+ &contended_dummy,
+#else // CONFIG_TINY_ALLOCATION_SLOT_LOCK
+ &contended,
+#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
+ /* install_empty_out */ NULL);
+ if (ptr && !corrupt) {
+ // Success!
+
+#if CONFIG_TINY_ALLOCATION_SLOT_LOCK
+ _malloc_lock_unlock(&xas->xas_lock);
+#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
+
+ _xzm_xzone_slot_record_contention(zone, xz, &xas->xas_counters,
+ slot_config, true, contended);
+
+ xzm_block_t block = ptr;
+ *block = (struct xzm_block_inline_meta_s){ 0 };
+
+ if (clear && xz->xz_block_size > XZM_ZERO_ON_FREE_THRESHOLD) {
+ return memset(ptr, 0, xz->xz_block_size);
+ }
+
+ return ptr;
+ }
+ }
+
+ return _xzm_xzone_malloc_freelist_outlined(zone, xz, alloc_idx, xas, ptr,
+ opt);
+}
+
+MALLOC_NOINLINE
+static void *
+_xzm_xzone_malloc_tiny(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_malloc_options_t opt)
+{
+ return _xzm_xzone_malloc_freelist(zone, xz, opt, false);
+}
+
+MALLOC_NOINLINE
+static void *
+_xzm_xzone_malloc_small_freelist(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_malloc_options_t opt)
+{
+ return _xzm_xzone_malloc_freelist(zone, xz, opt, true);
+}
+
+#pragma mark Tiny deallocation
+
+static void
+_xzm_xzone_freelist_chunks_mark_empty(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_chunk_t *chunks, size_t chunk_count)
+{
+ for (size_t i = 0; i < chunk_count; ++i) {
+ xzm_chunk_t chunk = chunks[i];
+ xzm_chunk_atomic_meta_u old_meta = {
+ .xca_value = os_atomic_load_wide(
+ &chunk->xzc_atomic_meta.xca_value, relaxed),
+ };
+ while (true) {
+ xzm_chunk_atomic_meta_u new_meta = old_meta;
+
+ xzm_debug_assert(old_meta.xca_alloc_head == XZM_FREE_MADVISING);
+ new_meta.xca_alloc_head = XZM_FREE_MADVISED;
+ if (!old_meta.xca_on_partial_list) {
+ new_meta.xca_on_empty_list = true;
+ }
+ // release to publish the reclaim index if we stored one
+ // TODO make this store relaxed. the reclaim index is no longer
+ // set in this function
+ bool success = os_atomic_cmpxchgv(&chunk->xzc_atomic_meta.xca_value,
+ old_meta.xca_value, new_meta.xca_value, &old_meta.xca_value,
+ release);
+ if (!success) {
+ continue;
+ }
+
+ xzm_trace(free_madvise, (uint64_t)chunk, old_meta.xca_value_lo,
+ new_meta.xca_value_lo, old_meta.xca_seqno);
+ break;
+ }
+
+ // because the partial list is atomically singly-linked, an empty chunk
+ // could remain there until it is popped off the head, so we cannot put
+ // it onto the empty list until after that happens
+ if (!old_meta.xca_on_partial_list) {
+ _xzm_chunk_list_push(zone, &xz->xz_empty_list, chunk,
+ XZM_CHUNK_LINKAGE_MAIN, NULL);
+ }
+ }
+}
+
+static void
+_xzm_xzone_madvise_freelist_chunk(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_chunk_t chunk)
+{
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
+ if ((chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_TINY_CHUNK &&
+ main->xzmz_defer_tiny) ||
+ (chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK &&
+ main->xzmz_defer_small)) {
+ xzm_chunk_mark_free(zone, chunk);
+ } else
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+ {
+ xzm_segment_group_t sg = _xzm_segment_group_for_slice(zone, chunk);
+ xzm_segment_group_segment_madvise_chunk(sg, chunk);
+ }
+
+ _xzm_xzone_freelist_chunks_mark_empty(zone, xz, &chunk, 1);
+}
+
+static void
+_xzm_xzone_small_chunks_mark_empty(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_chunk_t *chunk, size_t chunk_count);
+
+static void
+_xzm_xzone_madvise_batch(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_chunk_t chunk)
+{
+ const xzm_slice_kind_t kind = chunk->xzc_bits.xzcb_kind;
+ // We need to store the local list of entries in the batch. Otherwise,
+ // the list pointers could be replaced by the deferred reclaim id, and
+ // attempting to update metadata while under the reclaim buffer lock could
+ // deadlock against a concurrent allocator that holds the isolation zone
+ // lock and wants the reclaim buffer lock.
+ xzm_chunk_t batch_list[1u << XZM_BATCH_SIZE_BITS];
+ unsigned batch_list_size = 0;
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
+ xzm_reclaim_buffer_t buffer = main->xzmz_reclaim_buffer;
+ const bool deferred_reclaim = (kind == XZM_SLICE_KIND_TINY_CHUNK) ?
+ main->xzmz_defer_tiny : main->xzmz_defer_small;
+ bool should_update_kernel_accounting = false;
+
+ if (deferred_reclaim) {
+ _malloc_lock_lock(&buffer->xrb_lock);
+ }
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+
+ // Walk the batch, make a local copy, and perform the reclaim
+ while (chunk) {
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ xzm_debug_assert(batch_list_size < main->xzmz_batch_size &&
+ batch_list_size < (1u << XZM_BATCH_SIZE_BITS));
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+ batch_list[batch_list_size++] = chunk;
+
+ xzm_debug_assert(kind == chunk->xzc_bits.xzcb_kind);
+ // Fetch the pointer for the next chunk first, because madvising it
+ // may replace the linkage with the deferred reclaim id
+ xzm_chunk_t next = *_xzm_segment_slice_meta_batch_next(zone, chunk);
+ xzm_debug_assert(_xzm_slice_meta_is_batch_pointer(zone, next));
+
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ if (deferred_reclaim) {
+ size_t size;
+ uint8_t *addr = _xzm_chunk_start_ptr(zone, chunk, &size);
+ uint64_t *reclaim_id = _xzm_slice_meta_reclaim_id(zone, chunk);
+
+ bool should_update_kernel_chunk = false;
+ // This relies on release ordering from the push of the batch
+ // list to obtain the correct reclaim buffer index
+ *reclaim_id = xzm_reclaim_mark_free_locked(buffer, addr, size, true,
+ &should_update_kernel_chunk);
+ should_update_kernel_accounting |= should_update_kernel_chunk;
+ } else
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+ {
+ xzm_segment_group_t sg = _xzm_segment_group_for_slice(zone,
+ chunk);
+ xzm_segment_group_segment_madvise_chunk(sg, chunk);
+ }
+
+ chunk = next;
+ }
+
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ if (deferred_reclaim) {
+ _malloc_lock_unlock(&buffer->xrb_lock);
+
+ if (should_update_kernel_accounting) {
+ mach_vm_reclaim_update_kernel_accounting(buffer->xrb_ringbuffer);
+ }
+ }
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+
+ if (kind == XZM_SLICE_KIND_TINY_CHUNK ||
+ kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK) {
+ _xzm_xzone_freelist_chunks_mark_empty(zone, xz, batch_list,
+ batch_list_size);
+ } else if (kind == XZM_SLICE_KIND_SMALL_CHUNK) {
+ _xzm_xzone_small_chunks_mark_empty(zone, xz, batch_list, batch_list_size);
+ } else {
+ xzm_abort_with_reason("Unexpected chunk kind", kind);
+ }
+}
+
+MALLOC_NOINLINE
+static void
+_xzm_free_abort(void *ptr)
+{
+ xzm_client_abort_with_reason(
+ "free to empty or invalid chunk detected (likely double-free)",
+ ptr);
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static void
+_xzm_xzone_free_freelist_inline(xzm_malloc_zone_t zone,
+ xzm_xzone_index_t xz_idx, xzm_chunk_t chunk, xzm_block_t block,
+ uint16_t block_ref, size_t block_size, size_t chunk_capacity)
+{
+ xzm_xzone_t xz = NULL;
+ union xzm_block_linkage_u linkage = { 0 };
+
+ uint64_t now = 0;
+ uint64_t last_empty_ts = 0;
+ bool should_madvise = true;
+ bool madvise_considered = false;
+
+ uint32_t contentions = 0;
+
+ xzm_xzone_allocation_slot_t xas = NULL;
+
+ xzm_chunk_atomic_meta_u old_meta = {
+ .xca_value = os_atomic_load_wide(
+ &chunk->xzc_atomic_meta.xca_value, dependency),
+ };
+
+ xzm_chunk_atomic_meta_u new_meta;
+
+ bool push_to_partial = false;
+ while (true) {
+ if (os_unlikely(old_meta.xca_walk_locked)) {
+ _xzm_walk_lock_wait(zone);
+ old_meta.xca_value = os_atomic_load_wide(
+ &chunk->xzc_atomic_meta.xca_value, dependency);
+ continue;
+ }
+
+ new_meta = old_meta;
+ new_meta.xca_seqno++;
+
+ xzm_debug_assert(!push_to_partial);
+
+ if (old_meta.xca_free_count + 1 == chunk_capacity) {
+ // We're freeing the last block.
+
+ // Evaluate the madvise heuristic the first time through.
+ if (!madvise_considered) {
+ xz = &zone->xzz_xzones[xz_idx];
+ if (old_meta.xca_alloc_idx &&
+ old_meta.xca_alloc_idx != XZM_SLOT_INDEX_THREAD_INSTALLED) {
+ xzm_chunk_list_t partial_list = _xzm_chunk_list_get(zone,
+ xz, NULL, zone->xzz_partial_lists);
+ bool last_chunk = _xzm_chunk_list_empty(&partial_list->xcl_list);
+ if (last_chunk) {
+ now = mach_absolute_time();
+ xas = _xzm_xzone_allocation_slot_for_index(zone, xz,
+ old_meta.xca_alloc_idx - 1);
+ last_empty_ts = os_atomic_load(
+ &xas->xas_last_chunk_empty_ts, relaxed);
+ if (now - last_empty_ts < zone->xzz_tiny_thrash_threshold) {
+ should_madvise = false;
+ }
+ }
+ }
+ madvise_considered = true;
+ }
+
+ if (should_madvise &&
+ old_meta.xca_alloc_idx != XZM_SLOT_INDEX_THREAD_INSTALLED) {
+ new_meta.xca_alloc_head = XZM_FREE_MADVISING;
+ new_meta.xca_free_count = 0;
+ new_meta.xca_alloc_idx = XZM_SLOT_INDEX_EMPTY;
+ goto do_cmpxchg;
+ }
+ } else if (old_meta.xca_free_count == 0) {
+ if (os_unlikely(old_meta.xca_alloc_head != XZM_FREE_NULL)) {
+ return _xzm_free_abort((void *)block);
+ }
+
+ if (old_meta.xca_alloc_idx == XZM_SLOT_INDEX_EMPTY) {
+ // This chunk is not currently installed anywhere. We
+ // should move it to the partial list so it can be
+ // considered for future allocations.
+ xzm_debug_assert(!old_meta.xca_on_partial_list &&
+ !old_meta.xca_on_empty_list);
+ new_meta.xca_on_partial_list = true;
+ push_to_partial = true;
+
+ xz = &zone->xzz_xzones[xz_idx];
+ } else {
+ xzm_debug_assert(!old_meta.xca_on_partial_list);
+ }
+ }
+
+ uint32_t seqno = (uint32_t)old_meta.xca_seqno & XZM_SEQNO_COUNTER_MASK;
+
+ // Link the new block to the current head of the free list.
+ linkage = (union xzm_block_linkage_u){
+ .xzbl_next_offset = old_meta.xca_alloc_head,
+ .xzbl_next_seqno = old_meta.xca_head_seqno,
+ .xzbl_seqno = seqno,
+ };
+ linkage.xzbl_next_value = (uintptr_t)ptrauth_sign_unauthenticated(
+ linkage.xzbl_next, ptrauth_key_process_dependent_data,
+ ptrauth_blend_discriminator(block,
+ ptrauth_string_discriminator("xzb_linkage")));
+ linkage.xzbl_seqno = 0;
+
+ os_atomic_store(&block->xzb_linkage.xzbl_next_value,
+ linkage.xzbl_next_value, relaxed);
+
+ // Install the new block as the head.
+ new_meta.xca_alloc_head = block_ref;
+ new_meta.xca_free_count++;
+ new_meta.xca_head_seqno = seqno;
+
+do_cmpxchg:;
+ // Release ordering to publish our zeroing and initialization of the
+ // block - pairs with the dependency order loads/cmpxchgs on allocation
+ bool success = os_atomic_cmpxchgv(&chunk->xzc_atomic_meta.xca_value,
+ old_meta.xca_value, new_meta.xca_value, &old_meta.xca_value,
+ release);
+ if (!success) {
+ // Try again.
+ push_to_partial = false;
+ contentions++;
+ continue;
+ }
+
+ xzm_trace(free, (uint64_t)chunk, old_meta.xca_value_lo,
+ new_meta.xca_value_lo,
+ (uint32_t)old_meta.xca_seqno | ((uint64_t)contentions) << 32);
+ break;
+ }
+
+ // If we checked the time while evaluating the madvise heuristic, update it
+ // in the allocation slot.
+ if (now) {
+ xzm_debug_assert(xas);
+ os_atomic_store(&xas->xas_last_chunk_empty_ts, now, relaxed);
+ }
+
+ if (new_meta.xca_alloc_head == XZM_FREE_MADVISING) {
+ xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
+ xzm_debug_assert(xz);
+
+#if CONFIG_TINY_ALLOCATION_SLOT_LOCK
+ // We need to ensure that there are no threads in the critical section
+ // of the allocation slot that we uninstalled the chunk from
+ if (old_meta.xca_alloc_idx != XZM_SLOT_INDEX_EMPTY) {
+ xas = _xzm_xzone_allocation_slot_for_index(zone, xz,
+ old_meta.xca_alloc_idx - 1);
+ _malloc_lock_lock(&xas->xas_lock);
+ _malloc_lock_unlock(&xas->xas_lock);
+ }
+#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
+
+ if (!main->xzmz_batch_size) {
+ // Batching is disabled, do the madvise we committed to
+ _xzm_xzone_madvise_freelist_chunk(zone, xz, chunk);
+ } else {
+ // Batching is enabled, enqueue the chunk. Note that the batch list
+ // is not exclusive, the chunk could also still be on the partial
+ // list as well
+ _xzm_chunk_batch_list_push(zone, xz, chunk,
+ main->xzmz_batch_size);
+ }
+ } else if (push_to_partial) {
+ xzm_debug_assert(xz);
+ _xzm_chunk_list_slot_push(zone, xz, zone->xzz_partial_lists, chunk);
+ }
+}
+
+MALLOC_NOINLINE
+static void
+_xzm_xzone_free_freelist(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_chunk_t chunk, xzm_block_t block)
+{
+ size_t block_size = xz->xz_block_size;
+ if (block_size <= XZM_ZERO_ON_FREE_THRESHOLD) {
+ bzero(block, block_size);
+ }
+
+ void *ptr = block;
+#if CONFIG_MTE
+ ptr = memtag_strip_address((uint8_t *)ptr);
+#endif
+
+#if CONFIG_MTE
+ if (xz->xz_tagged && block_size <= XZM_TINY_BLOCK_SIZE_MAX) {
+ // Only tiny chunks are tagged on free
+ block = _xzm_xzone_block_memtag_retag(zone, block, block_size);
+ }
+#endif
+
+ uint64_t free_cookie = zone->xzz_freelist_cookie ^ (uint64_t)block;
+#if CONFIG_MTE
+ if (block_size > XZM_TINY_BLOCK_SIZE_MAX) {
+ // Small blocks do not encode the tag in the freelist cookie
+ free_cookie = (uint64_t)memtag_strip_address((uint8_t *)free_cookie);
+ }
+#endif
+ os_atomic_store(&block->xzb_cookie, free_cookie, relaxed);
+
+ uint16_t block_ref;
+ if (xz->xz_block_size > XZM_TINY_BLOCK_SIZE_MAX) {
+ size_t block_offset = _xzm_chunk_block_offset(zone, chunk, block);
+ block_ref = block_offset / XZM_SMALL_GRANULE;
+ } else {
+ size_t block_offset = (uintptr_t)ptr & XZM_SEGMENT_SLICE_MASK;
+ block_ref = block_offset / XZM_GRANULE;
+ }
+
+ _xzm_xzone_free_freelist_inline(zone, xz->xz_idx, chunk, block, block_ref,
+ block_size, xz->xz_chunk_capacity);
+}
+
+#if CONFIG_XZM_THREAD_CACHE
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static void
+_xzm_xzone_thread_cache_free_tiny(xzm_malloc_zone_t zone,
+ xzm_xzone_thread_cache_t cache, xzm_block_t block, uint16_t block_ref)
+{
+ uint16_t seqno = cache->xztc_seqno & XZM_SEQNO_COUNTER_MASK;
+ seqno |= XZM_SEQNO_THREAD_LOCAL;
+
+ union xzm_block_linkage_u linkage = {
+ .xzbl_next_offset = cache->xztc_head,
+ .xzbl_next_seqno = cache->xztc_head_seqno,
+ .xzbl_seqno = seqno,
+ };
+ linkage.xzbl_next_value = (uintptr_t)ptrauth_sign_unauthenticated(
+ linkage.xzbl_next, ptrauth_key_process_dependent_data,
+ ptrauth_blend_discriminator(block,
+ ptrauth_string_discriminator("xzb_linkage")));
+ linkage.xzbl_seqno = 0;
+
+ os_atomic_store(&block->xzb_linkage.xzbl_next_value,
+ linkage.xzbl_next_value, relaxed);
+
+#ifdef DEBUG
+ uint64_t orig_state = cache->xztc_freelist_state;
+#endif
+
+ xzm_xzone_thread_cache_atomic_meta_u tc_meta = {
+ .xztcam_head = block_ref,
+ .xztcam_free_count = cache->xztc_free_count + 1,
+ };
+ os_atomic_store(&cache->xztc_atomic_meta.xztcam_value,
+ tc_meta.xztcam_value, relaxed);
+ cache->xztc_head_seqno = seqno;
+ cache->xztc_seqno++;
+
+#ifdef DEBUG
+ uint64_t new_state = cache->xztc_freelist_state;
+#endif
+
+ xzm_trace(thread_cache_free, (uint64_t)cache->xztc_chunk, orig_state,
+ new_state, 0);
+}
+
+static void
+_xzm_xzone_thread_cache_detach_link_to_remote(xzm_malloc_zone_t zone,
+ xzm_xzone_thread_cache_t cache, xzm_chunk_atomic_meta_u old_meta)
+{
+ xzm_chunk_t chunk = cache->xztc_chunk;
+
+ uint8_t *start = cache->xztc_chunk_start;
+ uint64_t block_granules = chunk->xzc_freelist_block_size / XZM_GRANULE;
+ uint64_t max_offset = (chunk->xzc_freelist_chunk_capacity - 1) * block_granules;
+
+ uint64_t zone_freelist_cookie = zone->xzz_freelist_cookie;
+
+ xzm_debug_assert(old_meta.xca_alloc_head < XZM_FREE_LIMIT);
+ xzm_debug_assert(old_meta.xca_free_count);
+
+ xzm_block_t cur_block = NULL;
+ uint64_t cur_block_offset = old_meta.xca_alloc_head;
+ uint64_t cur_seqno = old_meta.xca_head_seqno;
+ size_t freelist_count = 1;
+ while (true) {
+ cur_block = (xzm_block_t)(
+ start + (cur_block_offset * XZM_GRANULE));
+
+#if CONFIG_MTE
+ if (chunk->xzc_tagged) {
+ cur_block = (xzm_block_t)memtag_fixup_ptr((void *)cur_block);
+ }
+#endif
+ uint64_t expected_cookie = zone_freelist_cookie ^ (uintptr_t)cur_block;
+
+ struct xzm_block_inline_meta_s block_meta = *cur_block;
+ if (os_unlikely(block_meta.xzb_cookie != expected_cookie)) {
+ xzm_client_abort_with_reason(
+ "corrupt tiny freelist - cookie, client likely has a buffer"
+ " overflow or use-after-free bug", block_meta.xzb_cookie);
+ }
+
+ union xzm_block_linkage_u linkage = {
+ .xzbl_next_offset = block_meta.xzb_linkage.xzbl_next_offset,
+ .xzbl_next_seqno = block_meta.xzb_linkage.xzbl_next_seqno,
+ .xzbl_seqno = cur_seqno,
+ };
+
+ linkage.xzbl_next_value = (uintptr_t)ptrauth_sign_unauthenticated(
+ linkage.xzbl_next, ptrauth_key_process_dependent_data,
+ ptrauth_blend_discriminator(cur_block,
+ ptrauth_string_discriminator("xzb_linkage")));
+
+ linkage.xzbl_seqno = 0;
+
+ if (block_meta.xzb_linkage.xzbl_next_value != linkage.xzbl_next_value) {
+ xzm_client_abort_with_reason(
+ "corrupt tiny freelist - linkage, client likely has a "
+ "buffer overflow or use-after-free bug",
+ block_meta.xzb_linkage.xzbl_next_value);
+ }
+
+ uint64_t next_seqno = (uint16_t)block_meta.xzb_linkage.xzbl_next_seqno;
+ uint64_t next_block_offset = block_meta.xzb_linkage.xzbl_next_offset;
+
+ if (next_block_offset == XZM_FREE_NULL) {
+ // freelist walk should be complete
+ if (os_unlikely(freelist_count != old_meta.xca_free_count)) {
+ xzm_client_abort_with_reason(
+ "corrupt tiny freelist - free count, client likely has a"
+ " buffer overflow or use-after-free bug",
+ freelist_count);
+ }
+
+ break;
+ }
+
+ if (os_unlikely(next_block_offset % block_granules != 0 ||
+ next_block_offset > max_offset ||
+ freelist_count >= old_meta.xca_free_count)) {
+ xzm_client_abort_with_reason(
+ "corrupt tiny freelist - inconsistent walk, client likely"
+ " has a buffer overflow or use-after-free bug",
+ freelist_count);
+ }
+
+ cur_block_offset = next_block_offset;
+ cur_seqno = next_seqno;
+ freelist_count++;
+ }
+
+ xzm_block_t tail_block = cur_block;
+ uint64_t tail_seqno = cur_seqno;
+
+ union xzm_block_linkage_u linkage = {
+ .xzbl_next_offset = cache->xztc_head,
+ .xzbl_next_seqno = cache->xztc_head_seqno,
+ .xzbl_seqno = tail_seqno,
+ };
+ linkage.xzbl_next_value = (uintptr_t)ptrauth_sign_unauthenticated(
+ linkage.xzbl_next, ptrauth_key_process_dependent_data,
+ ptrauth_blend_discriminator(tail_block,
+ ptrauth_string_discriminator("xzb_linkage")));
+ linkage.xzbl_seqno = 0;
+
+ os_atomic_store(&tail_block->xzb_linkage.xzbl_next_value,
+ linkage.xzbl_next_value, relaxed);
+}
+
+static void
+_xzm_xzone_thread_cache_detach(xzm_main_malloc_zone_t main, xzm_xzone_t xz,
+ xzm_xzone_thread_cache_t cache)
+{
+ xzm_malloc_zone_t zone = &main->xzmz_base;
+ xzm_chunk_t chunk = cache->xztc_chunk;
+
+ size_t chunk_capacity = xz->xz_chunk_capacity;
+ uint16_t local_free_count = cache->xztc_free_count;
+
+ xzm_chunk_atomic_meta_u old_meta = {
+ .xca_value = os_atomic_load_wide(
+ &chunk->xzc_atomic_meta.xca_value, dependency),
+ };
+
+ xzm_chunk_atomic_meta_u new_meta;
+
+ bool needs_link = (cache->xztc_head < XZM_FREE_LIMIT);
+ bool push_to_partial = false;
+ while (true) {
+ if (os_unlikely(old_meta.xca_walk_locked)) {
+ _xzm_walk_lock_wait(zone);
+ old_meta.xca_value = os_atomic_load_wide(
+ &chunk->xzc_atomic_meta.xca_value, dependency);
+ continue;
+ }
+
+ new_meta = old_meta;
+ new_meta.xca_alloc_idx = XZM_SLOT_INDEX_EMPTY;
+
+ xzm_debug_assert(!push_to_partial);
+
+ uint16_t new_free_count = old_meta.xca_free_count + local_free_count;
+ if (new_free_count == chunk_capacity) {
+ // The entire chunk will now be free
+
+ // TODO madvise heuristic? Doesn't seem like it would really make
+ // sense here
+ new_meta.xca_free_count = 0;
+ new_meta.xca_alloc_head = XZM_FREE_MADVISING;
+ } else if (new_free_count) {
+ xzm_debug_assert(!old_meta.xca_on_partial_list &&
+ !old_meta.xca_on_empty_list);
+ new_meta.xca_free_count = new_free_count;
+ new_meta.xca_on_partial_list = true;
+ push_to_partial = true;
+
+ if (old_meta.xca_alloc_head == XZM_FREE_NULL) {
+ new_meta.xca_alloc_head = cache->xztc_head;
+ new_meta.xca_head_seqno = cache->xztc_head_seqno;
+ } else {
+ xzm_debug_assert(old_meta.xca_alloc_head < XZM_FREE_LIMIT);
+ if (needs_link) {
+ _xzm_xzone_thread_cache_detach_link_to_remote(zone, cache,
+ old_meta);
+ needs_link = false;
+ }
+ }
+ }
+
+ // Release ordering pairs with the dependency order loads/cmpxchgs on
+ // allocation
+ bool success = os_atomic_cmpxchgv(&chunk->xzc_atomic_meta.xca_value,
+ old_meta.xca_value, new_meta.xca_value, &old_meta.xca_value,
+ release);
+ if (!success) {
+ // Try again.
+ push_to_partial = false;
+ continue;
+ }
+
+ xzm_trace(thread_detach, (uint64_t)chunk, old_meta.xca_value_lo,
+ new_meta.xca_value_lo, cache->xztc_freelist_state);
+ break;
+ }
+
+ if (new_meta.xca_alloc_head == XZM_FREE_MADVISING) {
+#if CONFIG_TINY_ALLOCATION_SLOT_LOCK
+#error "not compatible with thread caching"
+#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
+
+ // Do the madvise we committed to.
+ _xzm_xzone_madvise_freelist_chunk(zone, xz, chunk);
+ } else if (push_to_partial) {
+ _xzm_chunk_list_slot_push(zone, xz, zone->xzz_partial_lists, chunk);
+ }
+}
+
+static void
+_xzm_xzone_thread_cache_destructor(void *arg)
+{
+ xzm_thread_cache_t tc = arg;
+ xzm_main_malloc_zone_t main = tc->xtc_main;
+ xzm_malloc_zone_t zone = &main->xzmz_base;
+
+ tc->xtc_teardown_gen = os_atomic_inc(
+ &main->xzmz_thread_cache_teardown_gen, relaxed);
+
+ for (size_t i = XZM_XZONE_INDEX_FIRST;
+ i < zone->xzz_thread_cache_xzone_count; i++) {
+ xzm_xzone_thread_cache_t cache = &tc->xtc_xz_caches[i];
+ if (cache->xztc_head <= XZM_FREE_LIMIT) {
+ xzm_xzone_t xz = &zone->xzz_xzones[i];
+ _xzm_xzone_thread_cache_detach(main, xz, cache);
+ }
+ }
+
+ _malloc_lock_lock(&main->xzmz_thread_cache_list_lock);
+ LIST_REMOVE(tc, xtc_linkage);
+ _malloc_lock_unlock(&main->xzmz_thread_cache_list_lock);
+
+ xzm_metapool_t mp = &main->xzmz_metapools[XZM_METAPOOL_THREAD_CACHE];
+ xzm_metapool_free(mp, tc);
+}
+
+#endif // CONFIG_XZM_THREAD_CACHE
+
+#pragma mark Small allocation
+
+// mimalloc: mi_page_init
+static void
+_xzm_xzone_small_chunk_init(xzm_xzone_t xz, xzm_chunk_t chunk)
+{
+ // Precondition: xzcb_kind, xzcs_slice_count and xzsl_slice_offset are
+ // already initialized
+ xzm_debug_assert(chunk->xzc_free == 0);
+ xzm_debug_assert(chunk->xzc_used == 0);
+ xzm_debug_assert(chunk->xzc_mzone_idx == XZM_MZONE_INDEX_INVALID);
+
+ _xzm_xzone_fresh_chunk_init(xz, chunk, XZM_SLICE_KIND_SMALL_CHUNK);
+
+ // Enumerator protocol: mzone_idx is initialized last to "publish" the chunk
+ chunk->xzc_mzone_idx = xz->xz_mzone_idx;
+}
+
+// mimalloc: mi_page_fresh_alloc
+static xzm_chunk_t
+_xzm_xzone_small_chunk_alloc(xzm_malloc_zone_t zone, xzm_xzone_t xz)
+{
+ // First, check the preallocated list
+ xzm_chunk_t chunk = NULL;
+ if (LIST_FIRST(&xz->xz_chunkq_preallocated)) {
+ _malloc_lock_lock(&xz->xz_lock);
+ if ((chunk = LIST_FIRST(&xz->xz_chunkq_preallocated))) {
+ LIST_REMOVE(chunk, xzc_entry);
+ }
+ _malloc_lock_unlock(&xz->xz_lock);
+ }
+
+ // If none are present on the preallocated list, request a new run from the
+ // segment group, and enqueue the spares into the preallocated list
+ if (!chunk) {
+ xzm_slice_kind_t kind = XZM_SLICE_KIND_SMALL_CHUNK;
+ size_t span_size = XZM_SMALL_CHUNK_SIZE;
+ xzm_slice_count_t slice_count =
+ (xzm_slice_count_t)(span_size / XZM_SEGMENT_SLICE_SIZE);
+ bool clear = false;
+ bool purgeable = false; // Purgeable only applies to LARGE/HUGE chunks
+ xzm_preallocate_list_s preallocated;
+ SLIST_INIT(&preallocated);
+ xzm_preallocate_list_s *list = &preallocated;
+ xzm_segment_group_t sg = _xzm_segment_group_for_id_and_front(zone,
+ xz->xz_segment_group_id, xz->xz_front, false);
+ chunk = xzm_segment_group_alloc_chunk(sg, kind, &xz->xz_guard_config,
+ slice_count, list, 0, clear, purgeable);
+ if (!chunk) {
+ xzm_debug_assert(!list || !SLIST_FIRST(list));
+ return NULL;
+ }
+ chunk->xzc_xzone_idx = xz->xz_idx;
+ // Most processes don't get guard pages, so will only allocate one chunk
+ // from the segment group. Avoid taking the lock in this common case
+ if (SLIST_FIRST(list)) {
+ xzm_chunk_t c, t;
+ _malloc_lock_lock(&xz->xz_lock);
+ SLIST_FOREACH_SAFE(c, list, xzc_slist_entry, t) {
+ SLIST_REMOVE_HEAD(list, xzc_slist_entry);
+ c->xzc_xzone_idx = xz->xz_idx;
+ c->xzc_bits.xzcb_preallocated = true;
+ LIST_INSERT_HEAD(&xz->xz_chunkq_preallocated, c, xzc_entry);
+ }
+ _malloc_lock_unlock(&xz->xz_lock);
+ }
+ }
+
+ xzm_debug_assert(chunk);
+ _xzm_xzone_small_chunk_init(xz, chunk);
+
+ return chunk;
+}
+
+static xzm_slice_t
+_xzm_chunk_find_dirtiest_slice(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
+ uint32_t *slice_free_bitmap)
+{
+ // Attempt to find the dirtiest slice with at least one free block
+ xzm_slice_count_t slice_count = _xzm_chunk_slice_count(chunk);
+ struct xzm_slice_s *slices = _xzm_chunk_slices_of(chunk, slice_count);
+ uint32_t lowest_free = UINT32_MAX;
+ xzm_slice_t best_slice = NULL;
+ for (xzm_slice_count_t i = 0; i < slice_count; ++i) {
+ xzm_slice_t slice = &slices[i];
+ uint32_t slice_mask = _xzm_xzone_slice_free_mask(zone, slice);
+ uint32_t slice_free = chunk->xzc_free & slice_mask;
+ if (slice_free && slice_free != slice_mask) {
+ // This slice contains at least one used and one free block
+ // TODO: consider replacing popcount with table (max blocks per
+ // slice is bounded)
+ uint32_t free_count = __builtin_popcount(slice_free);
+ if (free_count < lowest_free) {
+ lowest_free = free_count;
+ best_slice = slice;
+ *slice_free_bitmap = slice_free;
+ }
+ }
+ }
+ return best_slice;
+}
+
+// mimalloc: _mi_page_malloc
+static void *
+_xzm_xzone_alloc_from_chunk(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_chunk_t chunk, bool *is_zero_out)
+{
+ xzm_debug_assert(!_xzm_chunk_is_full(zone, xz, chunk));
+
+ *is_zero_out = false;
+
+ uintptr_t block;
+ xzm_block_index_t block_index = UINT32_MAX;
+ uintptr_t start = _xzm_chunk_start(zone, chunk, NULL);
+ const xzm_segment_t segment = _xzm_segment_for_slice(zone, chunk);
+ const size_t block_size = _xzm_chunk_block_size(zone, chunk);
+
+ switch (chunk->xzc_bits.xzcb_kind) {
+ case XZM_SLICE_KIND_SMALL_CHUNK:
+ // TODO: Find a way to determine if this block is zeroed to
+ // avoid excessive zeroing
+ if (_xzm_chunk_is_empty(zone, xz, chunk) ||
+ (block_size % XZM_SEGMENT_SLICE_SIZE) == 0) {
+ // It is impossible for a free block to exist on a dirty slice for
+ // this chunk -- don't bother searching for one
+ block_index = __builtin_ffs(chunk->xzc_free) - 1;
+ } else {
+ uint32_t slice_free;
+ xzm_slice_t best_slice = _xzm_chunk_find_dirtiest_slice(zone, chunk,
+ &slice_free);
+
+ if (best_slice != NULL) {
+ block_index = __builtin_ffs(slice_free) - 1;
+
+ if (slice_free && !powerof2(slice_free)) {
+ // This slice has more than one free block; check
+ // if the one we selected would dirty multiple slices
+ uintptr_t block_start = start +
+ (block_index + xz->xz_block_size);
+ xzm_slice_t start_slice = _xzm_segment_slice_of(segment,
+ block_start);
+ if (start_slice < best_slice) {
+ // This block would dirty multiple slices; choose the
+ // next free block instead. Note that the next free
+ // block may also dirty multiple slices; this implies
+ // that there are no free blocks in between that might
+ // dirty fewer slices.
+ block_index = __builtin_ffs(slice_free &
+ ~(1u << block_index)) - 1;
+ }
+ }
+ } else {
+ // Unable to find a dirty slice, fall back to the first
+ // available free block
+ block_index = __builtin_ffs(chunk->xzc_free) - 1;
+ }
+ }
+
+ xzm_debug_assert(block_index <= xz->xz_chunk_capacity);
+ block = start + (block_index * xz->xz_block_size);
+
+
+ // Mark the current block as not free
+ chunk->xzc_free &= ~(1u << block_index);
+
+ break;
+ default:
+ xzm_abort_with_reason("attempting to allocate from chunk of bad kind",
+ (unsigned int)chunk->xzc_bits.xzcb_kind);
+ }
+
+ chunk->xzc_used++;
+
+ return (void *)block;
+}
+
+static void
+_xzm_xzone_upgrade_small_slot_config(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_xzone_allocation_slot_t xas, xzm_slot_config_t slot_config)
+{
+ // This allocation slot is hot, upgrade this xzone's slot config
+ // for subsequent allocations and reset its contention counters
+ xas->xas_contentions = 0;
+
+ // Walk all the other allocation slots and reset their contention counters
+ xzm_allocation_index_t limit_idx = _xzm_get_limit_allocation_index(slot_config);
+ if (limit_idx > 1) {
+ _malloc_lock_unlock(&xas->xas_lock);
+ for (xzm_allocation_index_t alloc_idx = 0;
+ alloc_idx < limit_idx;
+ alloc_idx++) {
+ xzm_xzone_allocation_slot_t xas_other =
+ _xzm_xzone_allocation_slot_for_index(zone, xz,
+ alloc_idx);
+ if (xas_other != xas) {
+ _malloc_lock_lock(&xas_other->xas_lock);
+ xas_other->xas_contentions = 0;
+ _malloc_lock_unlock(&xas_other->xas_lock);
+ }
+ }
+ _malloc_lock_lock(&xas->xas_lock);
+ }
+
+ _malloc_lock_lock(&xz->xz_lock);
+ // Another thread may have already upgraded the slot config
+ if (xz->xz_slot_config == slot_config) {
+ switch (xz->xz_slot_config) {
+ case XZM_SLOT_SINGLE:
+#if CONFIG_XZM_CLUSTER_AWARE
+ if (ncpuclusters > 1) {
+ xz->xz_slot_config = XZM_SLOT_CLUSTER;
+ } else {
+ xz->xz_slot_config = XZM_SLOT_CPU;
+ }
+#else // CONFIG_XZM_CLUSTER_AWARE
+ xz->xz_slot_config = XZM_SLOT_CLUSTER;
+#endif // CONFIG_XZM_CLUSTER_AWARE
+ break;
+ case XZM_SLOT_CLUSTER:
+ xz->xz_slot_config = XZM_SLOT_CPU;
+ break;
+ case XZM_SLOT_CPU:
+ xzm_abort("Can't upgrade from XZM_SLOT_CPU");
+ break;
+ default:
+ xzm_abort("Invalid xzone slot config");
+ break;
+ }
+ xzm_debug_assert(xz->xz_slot_config <= zone->xzz_max_slot_config);
+ }
+ _malloc_lock_unlock(&xz->xz_lock);
+}
+
+MALLOC_NOINLINE
+static void *
+_xzm_xzone_malloc_small(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_malloc_options_t opt)
+{
+ bool clear = (opt & XZM_MALLOC_CLEAR);
+ void *ptr = NULL;
+ bool is_zero = false;
+ xzm_slot_config_t slot_config;
+
+ xzm_allocation_index_t alloc_idx = _xzm_get_allocation_index(zone, xz,
+ &slot_config, true);
+ xzm_xzone_allocation_slot_t xas =
+ _xzm_xzone_allocation_slot_for_index(zone, xz, alloc_idx);
+
+ if (slot_config < zone->xzz_max_slot_config) {
+ // This xzone's slot config is upgradable -- evaluate for contention
+ if (!_malloc_lock_trylock(&xas->xas_lock)) {
+ // Contended for this allocation slot
+ _malloc_lock_lock(&xas->xas_lock);
+ xas->xas_contentions++;
+ if (xas->xas_contentions > zone->xzz_slot_upgrade_threshold[slot_config]) {
+ _xzm_xzone_upgrade_small_slot_config(zone, xz, xas, slot_config);
+ }
+ }
+ xas->xas_allocs++;
+ // Reset the contention counter every T allocations -- this prevents
+ // low-traffic xzones in long-lived processes from being upgraded over
+ // time
+ if ((xas->xas_allocs % zone->xzz_slot_upgrade_period) == 0) {
+ xas->xas_contentions = 0;
+ }
+ } else {
+ _malloc_lock_lock(&xas->xas_lock);
+#ifdef DEBUG
+ xas->xas_allocs++;
+#endif
+ }
+
+ xzm_chunk_t chunk = xas->xas_chunk;
+ if (!chunk || _xzm_chunk_is_full(zone, xz, chunk)) {
+ // We need to go to the xzone to get a new chunk
+ _malloc_lock_lock(&xz->xz_lock);
+
+ if (chunk) {
+ // If we have an existing chunk that's full, we need to uninstall it and
+ // move it to the full list
+ LIST_INSERT_HEAD(&xz->xz_chunkq_full, chunk, xzc_entry);
+
+ // This is the final step of uninstallation - as soon as the chunk
+ // transitions to the uninstalled state, other threads freeing to it
+ // will go to its lock instead of the lock for this slot. Store
+ // release to publish all of the changes made to the chunk under the
+ // slot lock, which will pair with the acquire when they take the
+ // chunk lock
+ os_atomic_store(&chunk->xzc_alloc_idx, XZM_SLOT_INDEX_EMPTY,
+ release);
+ }
+
+ while ((chunk = LIST_FIRST(&xz->xz_chunkq_partial))) {
+ _malloc_lock_lock(&chunk->xzc_lock);
+ xzm_debug_assert(chunk->xzc_bits.xzcb_on_partial_list);
+ LIST_REMOVE(chunk, xzc_entry);
+ chunk->xzc_bits.xzcb_on_partial_list = false;
+ if (chunk->xzc_used) {
+ xzm_debug_assert(!_xzm_chunk_is_full(zone, xz, chunk));
+ // We can use this chunk! Install it. We hold both the slot
+ // and chunk locks now so threads that are freeing to this chunk
+ // are guaranteed visibility of our modifications regardless of
+ // what they observe for the slot index.
+ xas->xas_chunk = chunk;
+ os_atomic_store(&chunk->xzc_alloc_idx, alloc_idx + 1, relaxed);
+ _malloc_lock_unlock(&chunk->xzc_lock);
+ break;
+ } else {
+ // This chunk is fully free, and the thread that freed it is on
+ // its way to remove it, so we don't want to use it. We've
+ // pulled it off the partial list so nobody else spends time on
+ // it and marked it accordingly so that the freeing thread
+ // knows.
+ _malloc_lock_unlock(&chunk->xzc_lock);
+ }
+ }
+
+ // Attempt to reuse a chunk from the batch list
+ if (!chunk && xz->xz_chunkq_batch_count) {
+ chunk = xz->xz_chunkq_batch;
+ xzm_debug_assert(chunk);
+ xz->xz_chunkq_batch = *_xzm_segment_slice_meta_batch_next(zone,
+ chunk);
+ xzm_debug_assert(_xzm_slice_meta_is_batch_pointer(zone,
+ xz->xz_chunkq_batch));
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ *_xzm_slice_meta_reclaim_id(zone, chunk) = VM_RECLAIM_ID_NULL;
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+ --xz->xz_chunkq_batch_count;
+ xas->xas_chunk = chunk;
+ os_atomic_store(&chunk->xzc_alloc_idx, alloc_idx + 1, relaxed);
+ }
+
+ // We're done with the xzone now - either we got a partial chunk from
+ // it, or we didn't and we need to get a sequestered or fresh one
+ _malloc_lock_unlock(&xz->xz_lock);
+
+ if (!chunk && xz->xz_sequestered) {
+ xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
+ chunk = _xzm_xzone_allocate_chunk_from_isolation(main, xz);
+
+ if (chunk) {
+ // Now install the chunk to the slot (TODO: does this need to be
+ // done under the iz lock? TODO: does this need to be done
+ // before populating in exclaves?)
+ xas->xas_chunk = chunk;
+ os_atomic_store(&chunk->xzc_alloc_idx, alloc_idx + 1, relaxed);
+ }
+ }
+
+ if (!chunk) {
+ chunk = _xzm_xzone_small_chunk_alloc(zone, xz);
+ if (chunk) {
+ xzm_debug_assert(!_xzm_chunk_is_full(zone, xz, chunk));
+ // Succeeded at allocating a chunk. Install it to the slot.
+ xas->xas_chunk = chunk;
+ os_atomic_store(&chunk->xzc_alloc_idx, alloc_idx + 1, relaxed);
+ } else {
+ // There's no other way for us to get a chunk, so we need to
+ // give up.
+ xas->xas_chunk = NULL;
+ goto out;
+ }
+ }
+ }
+
+ xzm_debug_assert(xas->xas_chunk == chunk);
+ xzm_debug_assert(!_xzm_chunk_is_full(zone, xz, chunk));
+
+ ptr = _xzm_xzone_alloc_from_chunk(zone, xz, chunk, &is_zero);
+ xzm_debug_assert(ptr);
+
+out:
+ _malloc_lock_unlock(&xas->xas_lock);
+
+ if (os_likely(ptr)) {
+#if CONFIG_MTE
+ bool memtag_enabled = xz->xz_tagged;
+ bool needs_canonical_tagging = (opt & XZM_MALLOC_CANONICAL_TAG);
+ if (memtag_enabled) {
+ if (needs_canonical_tagging) {
+ // We need this specific block to have canonical tagging (i.e. tag = 0)
+ ptr = memtag_tag_canonical(ptr, xz->xz_block_size);
+ } else {
+ ptr = _xzm_xzone_block_memtag_retag(zone, ptr,
+ xz->xz_block_size);
+ }
+ }
+#endif
+
+ // TODO: MallocCheckZeroOnFreeCorruption support
+ if (!is_zero && clear &&
+ xz->xz_block_size > XZM_ZERO_ON_FREE_THRESHOLD) {
+ bzero(ptr, xz->xz_block_size);
+ }
+ } else {
+ malloc_set_errno_fast(MZ_POSIX, ENOMEM);
+ }
+ return ptr;
+}
+
+#pragma mark General allocation
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static bool
+_xzm_xzone_try_reserve_early_budget(xzm_malloc_zone_t zone, xzm_xzone_t xz)
+{
+ uint16_t early_budget = os_atomic_load(&xz->xz_early_budget, relaxed);
+ uint16_t updated_budget;
+ do {
+ if (os_likely(!early_budget)) {
+ return false;
+ }
+
+ updated_budget = early_budget - 1;
+ } while (!os_atomic_cmpxchgv(&xz->xz_early_budget, early_budget,
+ updated_budget, &early_budget, relaxed));
+
+ return true;
+}
+
+#ifdef DEBUG
+// mimalloc: mi_mem_is_zero
+static bool
+_xzm_mem_is_zero(uint8_t *ptr, size_t size)
+{
+ return !_malloc_memcmp_zero_aligned8(ptr, size);
+}
+#endif // DEBUG
+
+#if CONFIG_XZM_THREAD_CACHE
+
+static void *
+_xzm_xzone_malloc_tiny_or_early(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_malloc_options_t opt)
+{
+ if (!(opt & XZM_MALLOC_NO_MFM) &&
+ _xzm_xzone_try_reserve_early_budget(zone, xz)) {
+ void *ptr = mfm_alloc(xz->xz_block_size);
+ xzm_debug_assert(ptr);
+#ifdef DEBUG
+ if (opt & XZM_MALLOC_CLEAR) {
+ xzm_debug_assert(_xzm_mem_is_zero(ptr, xz->xz_block_size));
+ }
+#endif
+ return ptr;
+ }
+
+ void *ptr = _xzm_xzone_malloc_tiny(zone, xz, 0);
+#ifdef DEBUG
+ if (opt & XZM_MALLOC_CLEAR) {
+ xzm_debug_assert(_xzm_mem_is_zero(ptr, xz->xz_block_size));
+ }
+#endif
+ return ptr;
+}
+
+MALLOC_NOINLINE
+static void *
+_xzm_xzone_thread_cache_malloc_corrupt(void *corrupt_block)
+{
+ // This looks kind of weird, but I couldn't convince the optimizer not to
+ // push a frame in _xzm_xzone_malloc() if I put the abort here directly, so
+ // we need to launder it apparently
+ return _xzm_xzone_malloc_freelist_outlined(NULL, NULL, 0, NULL, corrupt_block,
+ 0);
+}
+
+MALLOC_NOINLINE
+static void *
+_xzm_xzone_thread_cache_fill_and_malloc(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_xzone_thread_cache_t cache)
+{
+ void *ptr = NULL;
+ bool contended = false;
+ if (cache->xztc_state != XZM_XZONE_CACHE_EMPTY) {
+ xzm_debug_assert(cache->xztc_head == XZM_FREE_NULL);
+
+ ptr = _xzm_xzone_malloc_from_freelist_chunk(zone, xz,
+ XZM_SLOT_INDEX_THREAD, cache, cache->xztc_chunk, false,
+ &contended, NULL);
+ if (ptr) {
+ goto done;
+ }
+ }
+
+ xzm_debug_assert(cache->xztc_state == XZM_XZONE_CACHE_EMPTY);
+
+ xzm_chunk_t chunk = NULL;
+ ptr = _xzm_xzone_find_and_malloc_from_freelist_chunk(zone, xz,
+ XZM_SLOT_INDEX_THREAD, cache, &chunk, &contended);
+
+done:
+ if (ptr) {
+ xzm_block_t block = ptr;
+ *block = (struct xzm_block_inline_meta_s){ 0 };
+ } else {
+ xzm_debug_assert(cache->xztc_state == XZM_XZONE_CACHE_EMPTY);
+ malloc_set_errno_fast(MZ_POSIX, ENOMEM);
+ }
+
+ return ptr;
+}
+
+static void
+_xzm_thread_cache_create(xzm_malloc_zone_t zone)
+{
+ xzm_main_malloc_zone_t main = (xzm_main_malloc_zone_t)zone;
+ xzm_metapool_t mp = &main->xzmz_metapools[XZM_METAPOOL_THREAD_CACHE];
+ xzm_thread_cache_t tc = xzm_metapool_alloc(mp);
+
+ *tc = (struct xzm_thread_cache_s){
+ .xtc_main = main,
+ .xtc_thread = pthread_self(),
+ };
+
+ uint64_t now = mach_absolute_time();
+ for (size_t i = 0; i < zone->xzz_thread_cache_xzone_count; i++) {
+ tc->xtc_xz_caches[i] = (xzm_xzone_thread_cache_u){
+ .xztc_state = XZM_XZONE_NOT_CACHED,
+ .xztc_timestamp = now,
+ };
+ }
+
+ _malloc_lock_lock(&main->xzmz_thread_cache_list_lock);
+ LIST_INSERT_HEAD(&main->xzmz_thread_cache_list, tc, xtc_linkage);
+ _malloc_lock_unlock(&main->xzmz_thread_cache_list_lock);
+
+ _pthread_setspecific_direct(__TSD_MALLOC_XZONE_THREAD_CACHE, tc);
+}
+
+MALLOC_NOINLINE
+static void *
+_xzm_thread_cache_create_and_malloc(xzm_malloc_zone_t zone,
+ xzm_xzone_index_t xz_idx, xzm_xzone_t xz, xzm_malloc_options_t opt)
+{
+ _xzm_thread_cache_create(zone);
+
+ return _xzm_xzone_malloc_tiny_or_early(zone, xz, opt);
+}
+
+MALLOC_NOINLINE
+static void *
+_xzm_xzone_thread_cache_record_and_malloc_outlined(xzm_malloc_zone_t zone,
+ xzm_xzone_t xz, xzm_malloc_options_t opt,
+ xzm_xzone_thread_cache_t cache)
+{
+ xzm_debug_assert(cache->xztc_head == XZM_XZONE_NOT_CACHED);
+
+ uint64_t now = mach_absolute_time();
+ uint64_t activation_time = zone->xzz_thread_cache_xzone_activation_time;
+ if (now - cache->xztc_timestamp < activation_time) {
+ // We reached the threshold number of allocations within the
+ // activation threshold time, so activate caching for this xzone
+ cache->xztc_head = XZM_XZONE_CACHE_EMPTY;
+
+ // Initialize the seqno for this cache with the low bits of the
+ // timestamp to add some non-determinism to its base
+ cache->xztc_seqno = cache->xztc_timestamp & XZM_SEQNO_COUNTER_MASK;
+
+ return _xzm_xzone_thread_cache_fill_and_malloc(zone, xz, cache);
+ }
+
+ // reset for the next period
+ cache->xztc_timestamp = now;
+ cache->xztc_allocs = 0;
+ cache->xztc_contentions = 0;
+
+ return _xzm_xzone_malloc_tiny_or_early(zone, xz, opt);
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static void *
+_xzm_xzone_thread_cache_record_and_malloc(xzm_malloc_zone_t zone,
+ xzm_xzone_t xz, xzm_malloc_options_t opt,
+ xzm_xzone_thread_cache_t cache)
+{
+ xzm_debug_assert(cache->xztc_head == XZM_XZONE_NOT_CACHED);
+
+ cache->xztc_allocs++;
+
+ uint32_t activation_period = zone->xzz_thread_cache_xzone_activation_period;
+ if (os_unlikely(cache->xztc_allocs == activation_period)) {
+ return _xzm_xzone_thread_cache_record_and_malloc_outlined(zone, xz, opt,
+ cache);
+ }
+
+ return _xzm_xzone_malloc_tiny_or_early(zone, xz, opt);
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static void
+_xzm_xzone_thread_cache_record_contention(xzm_malloc_zone_t zone,
+ xzm_xzone_t xz, xzm_xzone_thread_cache_t cache)
+{
+ // We can end up here after caching is already engaged if serving an
+ // allocation that can't be handled from the cache
+ if (cache->xztc_head != XZM_XZONE_NOT_CACHED) {
+ return;
+ }
+
+ cache->xztc_contentions++;
+
+ uint32_t activation_contentions =
+ zone->xzz_thread_cache_xzone_activation_contentions;
+ if (os_unlikely(cache->xztc_contentions == activation_contentions)) {
+ // We reached the threshold number of contentions within the threshold
+ // period, so activate caching for this xzone
+ cache->xztc_head = XZM_XZONE_CACHE_EMPTY;
+
+ // Initialize the seqno for this cache with the low bits of the
+ // timestamp to add some non-determinism to its base
+ cache->xztc_seqno = cache->xztc_timestamp & XZM_SEQNO_COUNTER_MASK;
+ }
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static void *
+_xzm_xzone_thread_cache_malloc(xzm_malloc_zone_t zone, xzm_xzone_index_t xz_idx,
+ xzm_xzone_t xz, xzm_malloc_options_t opt)
+{
+ xzm_thread_cache_t tc = _xzm_get_thread_cache();
+ if (os_unlikely(!tc)) {
+ return _xzm_thread_cache_create_and_malloc(zone, xz_idx, xz, opt);
+ }
+
+ xzm_debug_assert(xz_idx < zone->xzz_thread_cache_xzone_count);
+ xzm_xzone_thread_cache_t cache = &tc->xtc_xz_caches[xz_idx];
+ if (cache->xztc_head < XZM_FREE_LIMIT) {
+ uint8_t *start = cache->xztc_chunk_start;
+ void *ptr = start + (cache->xztc_head * XZM_GRANULE);
+
+#if CONFIG_MTE
+ if (_xzm_zone_memtag_enabled(zone)) {
+ ptr = memtag_fixup_ptr(ptr);
+ }
+#endif
+ struct xzm_block_inline_meta_s *block_meta_p =
+ (struct xzm_block_inline_meta_s *)ptr;
+ struct xzm_block_inline_meta_s block_meta = *block_meta_p;
+
+ // Check the PAC by adding the seqno back
+ union xzm_block_linkage_u linkage = {
+ .xzbl_next_offset = block_meta.xzb_linkage.xzbl_next_offset,
+ .xzbl_next_seqno = block_meta.xzb_linkage.xzbl_next_seqno,
+ .xzbl_seqno = cache->xztc_head_seqno,
+ };
+
+ linkage.xzbl_next_value = (uintptr_t)ptrauth_sign_unauthenticated(
+ linkage.xzbl_next, ptrauth_key_process_dependent_data,
+ ptrauth_blend_discriminator(ptr,
+ ptrauth_string_discriminator("xzb_linkage")));
+
+ // Take the seqno back out of the signed value for the comparison with
+ // the stored one
+ linkage.xzbl_seqno = 0;
+
+ if (os_unlikely(block_meta.xzb_linkage.xzbl_next_value !=
+ linkage.xzbl_next_value)) {
+ return _xzm_xzone_thread_cache_malloc_corrupt(ptr);
+ }
+
+ *block_meta_p = (struct xzm_block_inline_meta_s){ 0 };
+
+#ifdef DEBUG
+ uint64_t orig_state = cache->xztc_freelist_state;
+#endif
+
+ // We update the local freelist head and count together with an atomic
+ // store so that the enumerator gets a consistent picture of the
+ // freelist. Having the head and the count agree is required in order
+ // to handle the bump correctly:
+ // - If the head were updated before the count, we'd think the bump
+ // offset was higher than its true value, so we'd falsely report the
+ // block at the bump offset as allocated when it isn't
+ // - If the head were updated after the count, we'd think the bump
+ // offset was lower than its true value, so we'd falsely report the
+ // last block before the bump as free when it may not be
+ xzm_xzone_thread_cache_atomic_meta_u tc_meta = {
+ .xztcam_head = block_meta.xzb_linkage.xzbl_next_offset,
+ .xztcam_free_count = cache->xztc_free_count - 1,
+ };
+ os_atomic_store(&cache->xztc_atomic_meta.xztcam_value,
+ tc_meta.xztcam_value, relaxed);
+ cache->xztc_head_seqno = block_meta.xzb_linkage.xzbl_next_seqno;
+
+#ifdef DEBUG
+ uint64_t new_state = cache->xztc_freelist_state;
+#endif
+
+ xzm_trace(thread_cache_malloc, (uint64_t)cache->xztc_chunk, orig_state,
+ new_state, 0);
+
+ return ptr;
+ } else if (cache->xztc_head == XZM_FREE_NULL && cache->xztc_free_count) {
+ uint8_t *start = cache->xztc_chunk_start;
+ size_t capacity = xz->xz_chunk_capacity;
+ void *ptr = start + ((capacity - cache->xztc_free_count) *
+ xz->xz_block_size);
+#if CONFIG_MTE
+ if (_xzm_zone_memtag_enabled(zone)) {
+ ptr = memtag_fixup_ptr(ptr);
+ }
+#endif
+
+ struct xzm_block_inline_meta_s *block_meta_p =
+ (struct xzm_block_inline_meta_s *)ptr;
+ *block_meta_p = (struct xzm_block_inline_meta_s){ 0 };
+
+#ifdef DEBUG
+ uint64_t orig_state = cache->xztc_freelist_state;
+#endif
+
+ cache->xztc_free_count--;
+
+#ifdef DEBUG
+ uint64_t new_state = cache->xztc_freelist_state;
+#endif
+
+ xzm_trace(thread_cache_malloc, (uint64_t)cache->xztc_chunk, orig_state,
+ new_state, 0);
+
+ return ptr;
+ } else if (cache->xztc_head == XZM_XZONE_NOT_CACHED) {
+ return _xzm_xzone_thread_cache_record_and_malloc(zone, xz, opt, cache);
+ } else {
+ xzm_debug_assert(cache->xztc_head == XZM_FREE_NULL ||
+ cache->xztc_head == XZM_XZONE_CACHE_EMPTY);
+ return _xzm_xzone_thread_cache_fill_and_malloc(zone, xz, cache);
+ }
+}
+
+#endif // CONFIG_XZM_THREAD_CACHE
+
+static void *
+_xzm_xzone_malloc(xzm_malloc_zone_t zone, size_t size,
+ xzm_xzone_index_t xz_idx, xzm_malloc_options_t opt)
+{
+ void *ptr = NULL;
+
+ xzm_xzone_t xz = &zone->xzz_xzones[xz_idx];
+
+#if CONFIG_XZM_THREAD_CACHE
+ // First, try thread cache
+ if (zone->xzz_thread_cache_enabled &&
+ size <= XZM_THREAD_CACHE_THRESHOLD
+#if CONFIG_MTE
+ && !(opt & XZM_MALLOC_CANONICAL_TAG)
+#endif
+ ) {
+ return _xzm_xzone_thread_cache_malloc(zone, xz_idx, xz, opt);
+ }
+#endif // CONFIG_XZM_THREAD_CACHE
+
+ if (!(opt & XZM_MALLOC_NO_MFM) && _xzm_malloc_zone_is_main(zone) &&
+#if CONFIG_MTE
+ // With memory tagging enabled, we shouldn't call into the early
+ // allocator if we're being asked for a canonically tagged
+ // allocation, as we don't support that in MFM.
+ !((opt & XZM_MALLOC_CANONICAL_TAG) && zone->xzz_memtag_config.enabled) &&
+#endif // CONFIG_MTE
+ _xzm_xzone_try_reserve_early_budget(zone, xz)) {
+ // To maintain compatibility with code expecting
+ // malloc_size(malloc(size)) >= malloc_good_size(size), just always use
+ // the block size
+ ptr = mfm_alloc(xz->xz_block_size);
+ // The way that we use the early allocator, it can't ever legitimately
+ // fail, and we're depending on that here (to tail-call)
+ xzm_debug_assert(ptr);
+ return ptr;
+ }
+
+ if (size <= XZM_TINY_BLOCK_SIZE_MAX) {
+ return _xzm_xzone_malloc_tiny(zone, xz, opt);
+ } else if (zone->xzz_small_freelist_enabled) {
+ return _xzm_xzone_malloc_small_freelist(zone, xz, opt);
+ }
+
+ xzm_debug_assert(size <= XZM_SMALL_BLOCK_SIZE_MAX);
+ return _xzm_xzone_malloc_small(zone, xz, opt);
+}
+
+static size_t
+xzm_malloc_zone_size(xzm_malloc_zone_t zone, const void *ptr);
+
+void *
+xzm_malloc_inline(xzm_malloc_zone_t zone, size_t size,
+ malloc_type_descriptor_t type_desc, xzm_malloc_options_t opt)
+{
+ void *ptr;
+ if (size > XZM_SMALL_BLOCK_SIZE_MAX) {
+ ptr = _xzm_malloc_large_huge(zone, size, 0, type_desc, opt);
+ } else {
+ xzm_debug_assert((zone->xzz_flags & MALLOC_PURGEABLE) == 0);
+ xzm_xzone_index_t xz_idx = _xzm_xzone_lookup(zone, size, type_desc);
+ ptr = _xzm_xzone_malloc(zone, size, xz_idx, opt);
+ }
+
+#ifdef DEBUG
+ xzm_debug_assert(ptr);
+ if (opt & XZM_MALLOC_CLEAR) {
+ xzm_debug_assert(_xzm_mem_is_zero(ptr, size));
+ }
+#if CONFIG_MTE
+ // The `size` parameter means "requested size", but we make tagging decisions
+ // based on block size. Note that in the special case of allocations with
+ // large alignments we may get an incorrect answer from
+ // _xzm_zone_memtag_block(), since it can't account for the segment group
+ // min block size, but that case doesn't take this path so the assert here
+ // is still safe.
+ size_t block_size = xzm_malloc_zone_size(zone, ptr);
+ bool data = malloc_type_descriptor_is_pure_data(type_desc);
+ bool memtag = _xzm_zone_memtag_block(zone, block_size, data);
+ bool canonical_tag = (opt & XZM_MALLOC_CANONICAL_TAG);
+ if (!mfm_claimed_address(ptr)) {
+ if (memtag && !canonical_tag) {
+ xzm_debug_assert(memtag_strip_address(ptr) != ptr);
+ } else {
+ xzm_debug_assert(memtag_strip_address(ptr) == ptr);
+ }
+ }
+#endif // CONFIG_MTE
+#endif // DEBUG
+ return ptr;
+}
+
+MALLOC_NOINLINE
+void *
+xzm_malloc(xzm_malloc_zone_t zone, size_t size,
+ malloc_type_descriptor_t type_desc, xzm_malloc_options_t opt)
+{
+ return xzm_malloc_inline(zone, size, type_desc, opt);
+}
+
+// mimalloc: mi_heap_malloc_aligned
+static void * __alloc_align(2) __alloc_size(3)
+_xzm_memalign(xzm_malloc_zone_t zone, size_t alignment, size_t size,
+ malloc_type_descriptor_t type_desc, xzm_malloc_options_t opt)
+{
+ // - Assumption: alignment is a power of 2
+ // - Assumption: all xzm size classes are aligned to their MSb/4
+ // (e.g. the 3548-byte size class is aligned to 2048/4=512)
+ // - Assumption: all power-of-2 size classes less than or equal to page size
+ // are naturally aligned
+ //
+ // With those, the following definition of a memalign function will always
+ // return properly aligned memory (for values of size/align <= page size)
+ // memalign(align, size) =
+ // { alloc(align) , size <= align
+ // { alloc(2 * align) , align < size <= 2 * align
+ // { alloc(roundup(size, 4 * align)) , 2 * align < size < 4 * align
+ // { alloc(size) , 4 * align <= size
+ //
+ // The mimalloc memalign implementation (roundup(alloc(size + align - 1)))
+ // has a fixed internal fragmentation of align - 1. For the definition
+ // above, the internal fragmentation is:
+ // { align - size , size <= align
+ // { 2 * align - size , align < size <= 2 * align
+ // { roundup(size, 4 * align) <= 2 * align , 2 * align < size < 4 * align
+ // { 0 , 4 * align <= size
+ //
+ // In all but the third case, the internal fragmentation should be strictly
+ // better than the original implementation. An example of the third case
+ // being worse is memalign(2k, 5k), which will require an 8k allocation
+ // under the new scheme, but a 7k allocation under the original.
+ // Empirically this case seems rare enough that wins from the other three
+ // cases make up for it.
+
+ xzm_debug_assert(powerof2(alignment)); // should be guaranteed
+
+ // The early allocator doesn't support alignment
+ opt |= XZM_MALLOC_NO_MFM;
+
+ void *ptr;
+ if (size > XZM_SMALL_BLOCK_SIZE_MAX ||
+ alignment > XZM_SEGMENT_SLICE_SIZE) {
+ ptr = _xzm_malloc_large_huge(zone, size, alignment, type_desc, opt);
+ } else if (size <= alignment) {
+ xzm_debug_assert(alignment <= XZM_SMALL_BLOCK_SIZE_MAX);
+ ptr = xzm_malloc(zone, alignment, type_desc, opt);
+ } else if (size <= 2 * alignment) {
+ xzm_debug_assert(2 * alignment <= XZM_SMALL_BLOCK_SIZE_MAX);
+ ptr = xzm_malloc(zone, 2 * alignment, type_desc, opt);
+ } else if (size < 4 * alignment) {
+ xzm_debug_assert(roundup(size, 4 * alignment) <=
+ XZM_SMALL_BLOCK_SIZE_MAX);
+ ptr = xzm_malloc(zone, roundup(size, 4 * alignment), type_desc, opt);
+ } else {
+ xzm_debug_assert(size <= XZM_SMALL_BLOCK_SIZE_MAX);
+ ptr = xzm_malloc(zone, size, type_desc, opt);
+ }
+
+ xzm_debug_assert(ptr);
+ xzm_debug_assert((uintptr_t)ptr % alignment == 0);
+ return ptr;
+}
+
+void *
+xzm_memalign(xzm_malloc_zone_t zone, size_t alignment, size_t size,
+ malloc_type_descriptor_t type_desc, xzm_malloc_options_t opt)
+{
+ return _xzm_memalign(zone, alignment, size, type_desc, opt);
+}
+
+#pragma mark Small deallocation
+
+static void
+_xzm_xzone_chunk_free(xzm_malloc_zone_t zone, xzm_xzone_t xz, xzm_chunk_t chunk,
+ bool small_madvise_needed);
+
+static void
+_xzm_xzone_small_chunks_mark_empty(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_chunk_t *chunks, size_t chunk_count)
+{
+ xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
+ xzm_isolation_zone_t iz = &main->xzmz_isolation_zones[xz->xz_idx];
+ const bool sequestered = xz->xz_sequestered;
+
+ if (sequestered) {
+ _malloc_lock_lock(&iz->xziz_lock);
+ }
+
+ for (size_t i = 0; i < chunk_count; ++i) {
+ xzm_chunk_t chunk = chunks[i];
+
+ // See _xzm_xzone_chunk_free
+ chunk->xzc_mzone_idx = XZM_MZONE_INDEX_INVALID;
+
+ if (sequestered) {
+ // Reset such that the chunk is ready for next use immediately
+ _xzm_chunk_reset_free(xz, chunk, true);
+
+ // This chunk is no longer pristine
+ chunk->xzc_bits.xzcb_is_pristine = false;
+
+ LIST_INSERT_HEAD(&iz->xziz_chunkq, chunk, xzc_entry);
+ } else {
+ // Full reset - the chunk's span may be reused for anything
+ _xzm_chunk_reset_free(xz, chunk, false);
+
+ // Do not madvise again
+ xzm_segment_group_t sg = _xzm_segment_group_for_slice(zone,
+ chunk);
+ xzm_segment_group_free_chunk(sg, chunk, false, false);
+ }
+ }
+
+ if (sequestered) {
+ _malloc_lock_unlock(&iz->xziz_lock);
+ }
+}
+
+static xzm_block_t
+_xzm_xzone_free_to_chunk(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_chunk_t chunk, xzm_block_t block)
+{
+#if CONFIG_MTE
+ const bool memtag_enabled = xz->xz_tagged;
+#endif
+
+ switch(chunk->xzc_bits.xzcb_kind) {
+ case XZM_SLICE_KIND_SMALL_CHUNK:
+#if CONFIG_MTE
+ if (memtag_enabled) {
+ block = (xzm_block_t)memtag_strip_address((uint8_t *)block);
+ }
+#endif
+ chunk->xzc_free |= (1u << _xzm_chunk_block_index(zone, chunk, block));
+ break;
+ default:
+ xzm_abort_with_reason("Attempting to free to non-chunk slice",
+ (unsigned int)chunk->xzc_bits.xzcb_kind);
+ }
+
+ chunk->xzc_used--;
+
+ return block;
+}
+
+static void
+_xzm_xzone_chunk_free(xzm_malloc_zone_t zone, xzm_xzone_t xz, xzm_chunk_t chunk,
+ bool small_madvise_needed)
+{
+ xzm_segment_group_t sg = _xzm_segment_group_for_slice(zone, chunk);
+
+ // Reset mzone_idx first to "unpublish" the chunk for the enumerator
+ // protocol
+ //
+ // TODO: compiler barrier
+ chunk->xzc_mzone_idx = XZM_MZONE_INDEX_INVALID;
+
+ if (xz->xz_sequestered) {
+ xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
+ if (_xzm_chunk_should_defer_reclamation(main, chunk)) {
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ if (_xzm_segment_slice_is_deferred(_xzm_segment_for_slice(zone,
+ chunk), chunk)) {
+ // The only reason a chunk should already be in the reclaim
+ // buffer when passed to this function is if it's a tiny chunk
+ // and the zone is being destroyed, or it's a small chunk that's
+ // being freed from the batch
+ xzm_debug_assert(
+ ((chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_TINY_CHUNK ||
+ chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK) &&
+ chunk->xzc_atomic_meta.xca_alloc_head == XZM_FREE_MADVISED) ||
+ (chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_CHUNK &&
+ !chunk->xzc_bits.xzcb_on_partial_list));
+ } else {
+ xzm_chunk_mark_free(zone, chunk);
+ }
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+ } else {
+ bool should_madvise = true;
+ if (chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_TINY_CHUNK ||
+ chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK) {
+ if (chunk->xzc_atomic_meta.xca_alloc_head == XZM_FREE_MADVISED) {
+ // The chunk is already madvised, no point doing it again
+ should_madvise = false;
+ } else {
+ chunk->xzc_atomic_meta.xca_alloc_head = XZM_FREE_MADVISED;
+ chunk->xzc_atomic_meta.xca_free_count = 0;
+ }
+ } else if (chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_CHUNK) {
+ should_madvise = small_madvise_needed;
+ } else {
+ xzm_abort_with_reason("Unexpected chunk kind",
+ chunk->xzc_bits.xzcb_kind);
+ }
+
+ if (should_madvise) {
+ xzm_segment_group_segment_madvise_chunk(sg, chunk);
+ }
+ }
+
+ // Reset such that the chunk is ready for next use immediately
+ _xzm_chunk_reset_free(xz, chunk, true);
+
+ // chunk->xzc_used == 0
+ // xzc_xzone_idx unchanged
+ // xzcb_kind unchanged
+ // This chunk is no longer pristine, so bumping doesn't guarantee that
+ // the allocation is cleared
+ chunk->xzc_bits.xzcb_is_pristine = false;
+
+ if (chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_TINY_CHUNK ||
+ chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK) {
+ // Tiny chunks are only freed during zone destruction, so we
+ // can access the atomic metadata directly, since we're the
+ // only thread with access to this zone
+ chunk->xzc_atomic_meta.xca_on_partial_list = false;
+ chunk->xzc_atomic_meta.xca_on_empty_list = true;
+ chunk->xzc_atomic_meta.xca_alloc_idx = XZM_SLOT_INDEX_EMPTY;
+ }
+
+ xzm_isolation_zone_t iz = &main->xzmz_isolation_zones[xz->xz_idx];
+
+ _malloc_lock_lock(&iz->xziz_lock);
+ LIST_INSERT_HEAD(&iz->xziz_chunkq, chunk, xzc_entry);
+ _malloc_lock_unlock(&iz->xziz_lock);
+ } else {
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ xzm_debug_assert(
+ *_xzm_slice_meta_reclaim_id(zone, chunk) == VM_RECLAIM_ID_NULL);
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+
+ // Full reset - the chunk's span may be reused for anything
+ _xzm_chunk_reset_free(xz, chunk, false);
+
+ // will madvise if needed
+ xzm_segment_group_free_chunk(sg, chunk, false, small_madvise_needed);
+ }
+}
+
+// Attempt to madvise any slices now freed by a given block
+static void
+_xzm_xzone_chunk_madvise_free_slices(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_chunk_t chunk, xzm_block_t block)
+{
+ xzm_debug_assert(chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_CHUNK);
+
+ xzm_segment_group_t sg = _xzm_segment_group_for_slice(zone, chunk);
+
+ // Only multi-slice chunks that use a bitmap freelist may
+ // madvise individual body slices
+ if (chunk->xzc_bits.xzcb_kind != XZM_SLICE_KIND_SMALL_CHUNK ||
+ _xzm_segment_group_uses_deferred_reclamation(sg)) {
+ // TODO: Support for aggressive deferred reclamation of small
+ // slices (rdar://112088639)
+ return;
+ }
+
+#if CONFIG_MTE
+ if (xz->xz_tagged) {
+ block = (xzm_block_t)memtag_strip_address((uint8_t *)block);
+ }
+#endif // CONFIG_MTE
+
+ xzm_slice_count_t slice_idx, num_slices;
+ const xzm_segment_t segment = _xzm_segment_for_slice(zone, chunk);
+ const xzm_slice_count_t chunk_idx = _xzm_slice_index(segment, chunk);
+ const xzm_block_index_t block_idx = _xzm_chunk_block_index(zone, chunk,
+ block);
+ const size_t block_size = _xzm_chunk_block_size(zone, chunk);
+ _xzm_chunk_block_free_slices_on_deallocate(chunk, chunk_idx,
+ xz->xz_chunk_capacity, block_idx, block_size, &slice_idx,
+ &num_slices);
+ // At least one slice is madvisable
+ if (num_slices > 0) {
+ xzm_debug_assert(slice_idx >= chunk_idx);
+ xzm_segment_group_segment_madvise_span(sg, _xzm_segment_start(segment) +
+ slice_idx * XZM_SEGMENT_SLICE_SIZE, num_slices);
+ }
+}
+
+static void
+_xzm_xzone_batch_small_push(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_chunk_t chunk, size_t batch_size)
+{
+#ifdef DEBUG
+ _malloc_lock_assert_owner(&xz->xz_lock);
+#endif
+
+ xzm_chunk_t batch = NULL;
+ // Empty the batch if it is full
+ xzm_debug_assert(xz->xz_chunkq_batch_count <= batch_size);
+ if (os_unlikely(xz->xz_chunkq_batch_count >= batch_size)) {
+ batch = xz->xz_chunkq_batch;
+ xz->xz_chunkq_batch = NULL;
+ xz->xz_chunkq_batch_count = 0;
+ }
+
+ // Insert this chunk at the head of the batch
+ *_xzm_segment_slice_meta_batch_next(zone, chunk) = xz->xz_chunkq_batch;
+ xz->xz_chunkq_batch = chunk;
+ ++xz->xz_chunkq_batch_count;
+
+ // Perform the madvise outside the zone lock
+ _malloc_lock_unlock(&xz->xz_lock);
+
+ if (batch) {
+ _xzm_xzone_madvise_batch(zone, xz, batch);
+ }
+}
+
+MALLOC_NOINLINE
+static void
+_xzm_xzone_free_block_to_small_chunk(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_chunk_t chunk, xzm_block_t block)
+{
+ xzm_allocation_index_t alloc_idx = os_atomic_load(&chunk->xzc_alloc_idx,
+ relaxed);
+
+ while (true) {
+ if (alloc_idx == XZM_SLOT_INDEX_EMPTY) {
+ // This chunk does not appear to be installed in a slot, so we
+ // should first try to take its chunk lock.
+ _malloc_lock_lock(&chunk->xzc_lock);
+
+ // It's possible that this chunk was installed while we were waiting
+ // for the lock.
+ alloc_idx = os_atomic_load(&chunk->xzc_alloc_idx, relaxed);
+ if (alloc_idx != XZM_SLOT_INDEX_EMPTY) {
+ // We lost the race; this chunk was installed to a slot while
+ // we were taking its lock. Unlock it and try again.
+ _malloc_lock_unlock(&chunk->xzc_lock);
+ continue;
+ }
+
+ // The chunk is still not installed, so we hold the correct lock to
+ // free it.
+ bool chunk_was_full = _xzm_chunk_is_full(zone, xz, chunk);
+ block = _xzm_xzone_free_to_chunk(zone, xz, chunk, block);
+
+ // TODO: consider dropping the chunk lock while
+ // madvising (additional madvise synchronization
+ // required)
+ _xzm_xzone_chunk_madvise_free_slices(zone, xz, chunk, block);
+ if (_xzm_chunk_is_empty(zone, xz, chunk)) {
+ // This chunk has become empty, so we need to go to the xzone to
+ // remove it from the partial list.
+
+ // We've just freed the last block and this chunk wasn't
+ // installed, so it must be on the partial list (it can't be in
+ // the full -> partial transition because that's done under both
+ // the chunk and xzone locks).
+ xzm_debug_assert(chunk->xzc_bits.xzcb_on_partial_list);
+
+ // We need to acquire the xzone lock to remove this chunk from
+ // the partial list. We need to drop the chunk lock to avoid a
+ // lock inversion with code iterating the partial list looking
+ // for a chunk to reuse. Because this chunk is empty, it's
+ // guaranteed not to be reused, but it may be removed from the
+ // partial list by the time we get there.
+ _malloc_lock_unlock(&chunk->xzc_lock);
+
+ _malloc_lock_lock(&xz->xz_lock);
+
+ // Remove from the partial list, if it's still on it.
+ if (chunk->xzc_bits.xzcb_on_partial_list) {
+ LIST_REMOVE(chunk, xzc_entry);
+ chunk->xzc_bits.xzcb_on_partial_list = false;
+ }
+
+ xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
+ const uint8_t batch_size = main->xzmz_batch_size;
+ if (batch_size) {
+ // Enqueue the chunk into the batch, and drop the lock.
+ _xzm_xzone_batch_small_push(zone, xz, chunk, batch_size);
+ } else {
+ // We're done with everything we need from the xzone now.
+ _malloc_lock_unlock(&xz->xz_lock);
+
+ // madvise and sequester or release back to the segment
+ _xzm_xzone_chunk_free(zone, xz, chunk, false);
+ }
+ } else if (chunk_was_full) {
+ // This chunk was full, so we need to go to the xzone to put it
+ // on the partial list. Full chunks that aren't installed to a
+ // slot are guaranteed to be on the full list. Because this
+ // chunk is on the full list, no other threads will be trying to
+ // acquire its lock while holding the xzone lock. It is
+ // therefore safe to take the xzone lock here while holding the
+ // chunk lock.
+ _malloc_lock_lock(&xz->xz_lock);
+
+ // Remove from the full list
+ LIST_REMOVE(chunk, xzc_entry);
+ // Place on the partial list
+ LIST_INSERT_HEAD(&xz->xz_chunkq_partial, chunk, xzc_entry);
+ chunk->xzc_bits.xzcb_on_partial_list = true;
+ xzm_debug_assert(!_xzm_chunk_is_full(zone, xz, chunk));
+
+ _malloc_lock_unlock(&xz->xz_lock);
+ _malloc_lock_unlock(&chunk->xzc_lock);
+ } else {
+ // Neither empty nor newly unfull, so just unlock.
+ _malloc_lock_unlock(&chunk->xzc_lock);
+ }
+ break;
+ } else {
+ // This chunk appears to be installed in a slot, so we should first
+ // try to lock that slot.
+
+ xzm_xzone_allocation_slot_t xas =
+ _xzm_xzone_allocation_slot_for_index(zone, xz, alloc_idx - 1);
+
+ _malloc_lock_lock(&xas->xas_lock);
+
+ xzm_allocation_index_t orig_alloc_idx = alloc_idx;
+
+ // It's possible that this chunk was uninstalled (and maybe even
+ // installed somewhere else) while we were waiting for the lock.
+ alloc_idx = os_atomic_load(&chunk->xzc_alloc_idx, relaxed);
+ if (alloc_idx != orig_alloc_idx) {
+ // We lost the race; this chunk was uninstalled from this slot while
+ // we were taking its lock. Unlock it and try again.
+ _malloc_lock_unlock(&xas->xas_lock);
+ continue;
+ }
+
+ // The chunk is installed to the slot we locked, so we can proceed.
+ block = _xzm_xzone_free_to_chunk(zone, xz, chunk, block);
+
+ // TODO: consider dropping the AS lock while madvising
+ // (additional madvise synchronization required)
+
+ uint64_t thrash_threshold = zone->xzz_small_thrash_threshold;
+ uint64_t thrash_limit_size = zone->xzz_small_thrash_limit_size;
+ bool thrash_mitigation = (thrash_threshold &&
+ xz->xz_block_size < thrash_limit_size);
+ if (!thrash_mitigation) {
+ _xzm_xzone_chunk_madvise_free_slices(zone, xz, chunk, block);
+ }
+
+ if (_xzm_chunk_is_empty(zone, xz, chunk)) {
+ if (thrash_mitigation) {
+ uint64_t now = mach_absolute_time();
+ uint64_t time_elapsed = now - xas->xas_last_chunk_empty_ts;
+ xas->xas_last_chunk_empty_ts = now;
+
+ if (time_elapsed < thrash_threshold) {
+ // We seem to be thrashing, so leave the chunk in place
+ _malloc_lock_unlock(&xas->xas_lock);
+ break;
+ }
+ }
+
+ // This chunk has become empty and there are other partial
+ // pages available, so uninstall it from the slot.
+ xas->xas_chunk = NULL;
+
+ // Since we've just freed the last block and we hold the
+ // slot lock, nobody can have a reference to this chunk
+ // right now.
+ os_atomic_store(&chunk->xzc_alloc_idx, XZM_SLOT_INDEX_EMPTY,
+ relaxed);
+
+ // We're done with everything we need from the slot now.
+ _malloc_lock_unlock(&xas->xas_lock);
+
+ xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
+ const uint8_t batch_size = main->xzmz_batch_size;
+ if (batch_size) {
+ _malloc_lock_lock(&xz->xz_lock);
+ // Enqueue the chunk into the batch, and drop the lock.
+ _xzm_xzone_batch_small_push(zone, xz, chunk, batch_size);
+ } else {
+ bool small_madvise = thrash_mitigation;
+ // madvise and sequester or release back to the segment
+ _xzm_xzone_chunk_free(zone, xz, chunk, small_madvise);
+ }
+ } else {
+ // The chunk stays installed.
+ _malloc_lock_unlock(&xas->xas_lock);
+ }
+ break;
+ }
+ }
+}
+
+static bool
+_xzm_xzone_freelist_chunk_lock(xzm_chunk_t chunk,
+ xzm_chunk_atomic_meta_u *old_meta_out)
+{
+ xzm_chunk_atomic_meta_u old_meta = {
+ .xca_value = os_atomic_load_wide(
+ &chunk->xzc_atomic_meta.xca_value, dependency),
+ };
+ xzm_chunk_atomic_meta_u new_meta = { 0 };
+ while (true) {
+ xzm_debug_assert(!old_meta.xca_walk_locked);
+ if (old_meta.xca_alloc_head == XZM_FREE_MADVISING ||
+ old_meta.xca_alloc_head == XZM_FREE_MADVISED) {
+ return false;
+ }
+
+ new_meta = old_meta;
+ new_meta.xca_walk_locked = true;
+ bool success = os_atomic_cmpxchgv(&chunk->xzc_atomic_meta.xca_value,
+ old_meta.xca_value, new_meta.xca_value, &old_meta.xca_value,
+ dependency);
+ if (!success) {
+ continue;
+ }
+
+ xzm_trace(walk_lock, (uint64_t)chunk, old_meta.xca_value_lo,
+ new_meta.xca_value_lo, 0);
+ break;
+ }
+
+ *old_meta_out = old_meta;
+ return true;
+}
+
+static void
+_xzm_xzone_freelist_chunk_unlock(xzm_chunk_t chunk,
+ xzm_chunk_atomic_meta_u old_meta)
+{
+ xzm_chunk_atomic_meta_u new_meta = old_meta;
+ new_meta.xca_walk_locked = false;
+ bool success = os_atomic_cmpxchgv(&chunk->xzc_atomic_meta.xca_value,
+ old_meta.xca_value, new_meta.xca_value, &old_meta.xca_value,
+ relaxed);
+ xzm_assert(success);
+ xzm_trace(walk_unlock, (uint64_t)chunk, old_meta.xca_value_lo,
+ new_meta.xca_value_lo, 0);
+}
+
+MALLOC_NOINLINE
+static bool
+_xzm_xzone_freelist_chunk_block_is_free_slow(xzm_malloc_zone_t zone,
+ xzm_chunk_t chunk, xzm_block_t block)
+{
+ bool is_free = false;
+
+ // Re-compute the block information so it doesn't need to be passed
+ xzm_xzone_t xz = &zone->xzz_xzones[chunk->xzc_xzone_idx];
+ size_t granule = (xz->xz_block_size > XZM_TINY_BLOCK_SIZE_MAX) ?
+ XZM_SMALL_GRANULE : XZM_GRANULE;
+ uint8_t *start = (uint8_t *)_xzm_chunk_start(zone, chunk, NULL);
+ uint64_t block_granules = xz->xz_block_size / granule;
+ uint64_t block_offset = _xzm_chunk_block_offset(zone, chunk, block);
+ uint64_t max_offset = (xz->xz_chunk_capacity - 1) * block_granules;
+
+ uint64_t cur_block_offset = 0;
+ size_t freelist_count = 0;
+
+ bool cached = false;
+
+ xzm_chunk_atomic_meta_u old_meta = { 0 };
+
+#if CONFIG_XZM_THREAD_CACHE
+ if (!zone->xzz_thread_cache_enabled ||
+ xz->xz_block_size > XZM_THREAD_CACHE_THRESHOLD) {
+ goto not_cached;
+ }
+
+ old_meta.xca_value = os_atomic_load_wide(&chunk->xzc_atomic_meta.xca_value,
+ dependency);
+ if (old_meta.xca_alloc_idx != XZM_SLOT_INDEX_THREAD_INSTALLED) {
+ goto not_cached;
+ }
+
+ xzm_thread_cache_t tc = _xzm_get_thread_cache();
+ if (!tc) {
+ goto not_cached;
+ }
+
+ xzm_xzone_index_t xz_idx = xz->xz_idx;
+ xzm_xzone_thread_cache_t cache = &tc->xtc_xz_caches[xz_idx];
+
+ if (cache->xztc_head > XZM_FREE_LIMIT || cache->xztc_chunk != chunk) {
+ goto not_cached;
+ }
+
+ cached = true;
+
+ cur_block_offset = cache->xztc_head;
+ freelist_count = 0;
+ while (cur_block_offset != XZM_FREE_NULL &&
+ cur_block_offset % block_granules == 0 &&
+ cur_block_offset <= max_offset &&
+ freelist_count < cache->xztc_free_count) {
+ if (cur_block_offset == block_offset) {
+ // We found our block on the local freelist, so it's free
+ is_free = true;
+ goto unlocked;
+ }
+
+ freelist_count++;
+
+ xzm_block_t cur_block = (xzm_block_t)(
+ start + (cur_block_offset * granule));
+#if CONFIG_MTE
+ if (chunk->xzc_tagged) {
+ cur_block = (xzm_block_t)memtag_fixup_ptr((void *)cur_block);
+ }
+#endif
+ cur_block_offset = cur_block->xzb_linkage.xzbl_next_offset;
+ }
+
+ bool local_valid = (cur_block_offset == XZM_FREE_NULL &&
+ freelist_count <= cache->xztc_free_count);
+ if (os_unlikely(!local_valid)) {
+ xzm_client_abort_with_reason(
+ "corrupt tiny local freelist, client likely has a buffer"
+ " overflow or use-after-free bug", freelist_count);
+ }
+
+ // The local freelist has a bump offset. If the given block is above it, it
+ // must be free.
+ //
+ // N.B. Although we can check this here, if the block really is free because
+ // it's above the bump we might not get here in the first place, but it
+ // isn't guaranteed to have its free cookie set.
+ uint64_t local_bump_offset = (xz->xz_chunk_capacity -
+ (cache->xztc_free_count - freelist_count)) * block_granules;
+ if (block_offset >= local_bump_offset) {
+ is_free = true;
+ goto unlocked;
+ }
+
+not_cached:
+#endif // CONFIG_XZM_THREAD_CACHE
+
+ _malloc_lock_lock(&zone->xzz_lock);
+
+ bool active = _xzm_xzone_freelist_chunk_lock(chunk, &old_meta);
+ if (!active) {
+ is_free = true;
+ goto unlock;
+ }
+
+ cur_block_offset = old_meta.xca_alloc_head;
+ freelist_count = 0;
+ while (cur_block_offset != XZM_FREE_NULL &&
+ cur_block_offset % block_granules == 0 &&
+ cur_block_offset <= max_offset &&
+ freelist_count < old_meta.xca_free_count) {
+ if (cur_block_offset == block_offset) {
+ // We found our block on the freelist, so it's free.
+ is_free = true;
+ goto chunk_walk_unlock;
+ }
+
+ freelist_count++;
+
+ xzm_block_t cur_block = (xzm_block_t)(
+ start + (cur_block_offset * granule));
+#if CONFIG_MTE
+ if (chunk->xzc_tagged) {
+ cur_block = (xzm_block_t)memtag_fixup_ptr((void *)cur_block);
+ }
+#endif
+ cur_block_offset = cur_block->xzb_linkage.xzbl_next_offset;
+ }
+
+ if (cached) {
+ // If this chunk is cached, we should have fully walked the remote
+ // freelist
+ if (os_unlikely(freelist_count != old_meta.xca_free_count)) {
+ xzm_client_abort_with_reason(
+ "corrupt tiny remote freelist, client likely has a buffer"
+ " overflow or use-after-free bug", freelist_count);
+ }
+ } else {
+ // If this chunk isn't cached, we should have walked the remote freelist
+ // to its terminator, which may be the bump
+ if (os_unlikely(cur_block_offset != XZM_FREE_NULL)) {
+ xzm_client_abort_with_reason(
+ "corrupt tiny freelist, client likely has a buffer overflow"
+ " or use-after-free bug", freelist_count);
+ }
+
+ // If there's a bump and the block is above it, it's free.
+ //
+ // N.B. As above, although we can check this here, if the block really
+ // is free because it's above the bump we might not get here in the
+ // first place, but it isn't guaranteed to have its free cookie set.
+ xzm_debug_assert(freelist_count <= old_meta.xca_free_count);
+
+ uint64_t bump_offset = (xz->xz_chunk_capacity -
+ (old_meta.xca_free_count - freelist_count)) * block_granules;
+ if (block_offset >= bump_offset) {
+ is_free = true;
+ goto chunk_walk_unlock;
+ }
+ }
+
+chunk_walk_unlock:
+ old_meta.xca_walk_locked = true;
+ _xzm_xzone_freelist_chunk_unlock(chunk, old_meta);
+
+unlock:
+ _malloc_lock_unlock(&zone->xzz_lock);
+#if CONFIG_XZM_THREAD_CACHE
+unlocked:
+#endif
+ return is_free;
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static bool
+_xzm_xzone_chunk_block_is_free(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
+ xzm_slice_kind_t kind, size_t block_offset, size_t block_size,
+ xzm_block_t block)
+{
+ xzm_debug_assert(chunk->xzc_bits.xzcb_kind == kind);
+ switch (kind) {
+ case XZM_SLICE_KIND_TINY_CHUNK:
+ case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:;
+ // We optimize here for the case where the block is _not_ free, as that
+ // is what's expected on the fast path during free(). Quick heuristic
+ // check first: if the block does not have the free cookie we would
+ // expect it to if it were on the freelist, we assume that it's
+ // allocated. Note that this does _not_ work for pointers above the
+ // bump, so we'll report them as allocated too - nano also does this, so
+ // we have some evidence we can get away with it.
+ uint64_t cookie_value;
+
+#if CONFIG_MTE
+ if (chunk->xzc_tagged) {
+ // We need to TCO for this load because even though we just ldg'd
+ // the tag in the block pointer, if we're being asked about a block
+ // that doesn't belong to us it might race to get freed and
+ // re-tagged while we're looking at it.
+ memtag_disable_checking();
+ cookie_value = os_atomic_load(&block->xzb_cookie, relaxed);
+ memtag_enable_checking();
+ } else {
+#endif
+ cookie_value = os_atomic_load(&block->xzb_cookie, relaxed);
+#if CONFIG_MTE
+ }
+#endif
+
+ uint64_t free_cookie = zone->xzz_freelist_cookie ^ (uint64_t)block;
+
+#if CONFIG_MTE
+ if (kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK) {
+ // Small chunks do not encode the tag in the freelist cookie
+ free_cookie = (uint64_t)memtag_strip_address((uint8_t *)free_cookie);
+ }
+#endif
+ if (os_likely(cookie_value != free_cookie)) {
+ return false;
+ }
+
+ // If the block _does_ look free, it's still possible that the
+ // application just happened to have written the cookie value in the
+ // slot. We have a choice:
+ // - Nano chooses to just assume that the block really is free. This is
+ // a soundness problem, as it could cause free() to fail on a valid
+ // allocation.
+ // - We can walk the freelist to see if we can really find the block.
+ // This is sound but relatively costly.
+ //
+ // For now we take the latter choice, at the risk there's a workload
+ // that really hammers on malloc_size() of free blocks a lot. If we
+ // encounter such a workload, we could consider behaving like nano for
+ // compatibility, but most uses of malloc_size() on free blocks are
+ // incorrect so we would probably tell that workload to change.
+ bool is_free = _xzm_xzone_freelist_chunk_block_is_free_slow(zone, chunk,
+ block);
+ return is_free;
+ case XZM_SLICE_KIND_SMALL_CHUNK:;
+ size_t block_index = block_offset / block_size;
+ return _xzm_small_chunk_block_index_is_free(chunk,
+ (xzm_block_index_t)block_index);
+ default:
+ break;
+ }
+
+ return false;
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static xzm_chunk_t
+__xzm_ptr_lookup(xzm_malloc_zone_t zone, xzm_segment_table_entry_s entry,
+ const void *orig_ptr, const void *ptr, xzm_xzone_t *xz_out,
+ xzm_block_t *block_out, size_t *block_size_out)
+{
+ xzm_segment_t segment = _xzm_segment_table_entry_to_segment(entry);
+ if (!segment) {
+ return NULL;
+ }
+
+ xzm_chunk_t chunk = _xzm_segment_chunk_of(segment, (uintptr_t)ptr);
+ if (os_unlikely(!chunk)) {
+ return NULL;
+ }
+
+ if (os_unlikely(chunk->xzc_mzone_idx != zone->xzz_mzone_idx)) {
+ // The pointer belongs to one of the mzones, but not us. Depending on
+ // the context, we _could_ just go ahead and do what we were asked to
+ // (e.g. free) anyway, if we had a way to turn the mzone index back into
+ // the pointer, but that would probably be confusing to libmalloc and
+ // any wrapper zones, etc, so instead we'll just treat pointers
+ // belonging to other mzones as completely foreign.
+ return NULL;
+ }
+
+ xzm_block_t block = (xzm_block_t)ptr;
+
+ uintptr_t start = _xzm_chunk_start(zone, chunk, NULL);
+ size_t block_offset = (uintptr_t)block - start;
+
+ xzm_slice_kind_t kind = chunk->xzc_bits.xzcb_kind;
+ xzm_xzone_t xz = NULL;
+ size_t block_size = 0;
+ if (os_likely(_xzm_slice_kind_uses_xzones(kind))) {
+ xz = &zone->xzz_xzones[chunk->xzc_xzone_idx];
+ block_size = xz->xz_block_size;
+
+ if (os_unlikely(!XZM_FAST_ALIGNED(block_offset, block_size,
+ xz->xz_align_magic))) {
+ return NULL;
+ }
+ } else {
+ block_size = ((size_t)chunk->xzcs_slice_count) <<
+ XZM_SEGMENT_SLICE_SHIFT;
+ size_t ptr_offset = block_offset % block_size;
+ if (os_unlikely(ptr_offset)) {
+ return NULL;
+ }
+ }
+
+#ifdef DEBUG
+ size_t block_index = block_offset / block_size;
+ xzm_debug_assert(!_xzm_slice_kind_uses_xzones(chunk->xzc_bits.xzcb_kind) ||
+ block_index < xz->xz_chunk_capacity);
+#endif
+
+#if CONFIG_MTE
+ const bool memtag_enabled = _xzm_zone_memtag_enabled(zone);
+ if (memtag_enabled) {
+ // Load the tag for the block pointer we materialized
+ block = (xzm_block_t)memtag_fixup_ptr((void *)block);
+
+ // If the tag of the block doesn't match the tag we started with, we
+ // consider it not allocated
+ if (os_unlikely(!memtag_tags_match(orig_ptr, block))) {
+ return NULL;
+ }
+ }
+#endif
+
+ if (os_unlikely(_xzm_xzone_chunk_block_is_free(zone, chunk, kind,
+ block_offset, block_size, block))) {
+ return NULL;
+ }
+
+ if (xz_out) {
+ *xz_out = xz;
+ }
+
+ if (block_out) {
+ *block_out = block;
+ }
+ if (block_size_out) {
+ *block_size_out = block_size;
+ }
+
+ return chunk;
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static xzm_chunk_t
+_xzm_ptr_lookup(xzm_malloc_zone_t zone, const void *ptr, xzm_xzone_t *xz_out,
+ xzm_block_t *block_out, size_t *block_size_out)
+{
+ const void *orig_ptr = ptr;
+
+#if CONFIG_MTE
+ // Strip the pointer to use it in pointer arithmetic operations
+ ptr = memtag_strip_address((uint8_t *)ptr);
+#endif
+
+ xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
+ xzm_segment_table_entry_s *entry_p = _xzm_ptr_to_table_entry(ptr, main);
+ if (os_unlikely(!entry_p)) {
+ return NULL;
+ }
+
+ return __xzm_ptr_lookup(zone, *entry_p, orig_ptr, ptr, xz_out, block_out,
+ block_size_out);
+}
+
+MALLOC_NOINLINE
+static void
+_xzm_free_not_found(xzm_malloc_zone_t zone, void *ptr, bool try)
+{
+ if (_xzm_malloc_zone_is_main(zone) && mfm_claimed_address(ptr)) {
+ mfm_free(ptr);
+ return;
+ }
+
+ if (!try) {
+#if CONFIG_MTE
+ // When tagging is enabled (based on malloc_has_sec_transition), we will
+ // check if the logical tag of the pointer matches the tag stored in
+ // memory. We need to do this here, in xzone, to make sure that we
+ // validate the tag in the malloc_zone_free() path. All other paths
+ // should be handled at the dispatch layer (malloc proper).
+#endif // CONFIG_MTE
+ malloc_report_pointer_was_not_allocated(
+ MALLOC_REPORT_CRASH | MALLOC_REPORT_NOLOG, ptr);
+ }
+
+ find_zone_and_free(ptr, true);
+}
+
+MALLOC_NOINLINE
+static void
+_xzm_free_outlined(xzm_malloc_zone_t zone, void *ptr, bool try,
+ xzm_segment_table_entry_s entry)
+{
+ void *orig_ptr = ptr;
+
+#if CONFIG_MTE
+ // Strip the pointer to use it in pointer arithmetic operations
+ ptr = memtag_strip_address((uint8_t *)ptr);
+#endif
+
+ xzm_xzone_t xz = NULL;
+ xzm_block_t block = NULL;
+ xzm_chunk_t chunk = __xzm_ptr_lookup(zone, entry, orig_ptr, ptr, &xz,
+ &block, NULL);
+ if (os_unlikely(!chunk)) {
+ return _xzm_free_not_found(zone, orig_ptr, try);
+ }
+
+ if (chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_TINY_CHUNK ||
+ chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK) {
+ _xzm_xzone_free_freelist(zone, xz, chunk, block);
+ } else if (chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_CHUNK) {
+ _xzm_xzone_free_block_to_small_chunk(zone, xz, chunk, block);
+ } else {
+ xzm_debug_assert(!xz);
+ _xzm_free_large_huge(zone, chunk);
+ }
+ return;
+}
+
+#ifdef DEBUG
+static void
+_xzm_debug_validate_chunk_metadata(xzm_xzone_t xz, xzm_chunk_t chunk)
+{
+ xzm_debug_assert(xz->xz_block_size == chunk->xzc_freelist_block_size);
+ xzm_debug_assert(xz->xz_chunk_capacity == chunk->xzc_freelist_chunk_capacity);
+#if CONFIG_MTE
+ xzm_debug_assert(xz->xz_tagged == chunk->xzc_tagged);
+#endif
+}
+#else
+#define _xzm_debug_validate_chunk_metadata(...)
+#endif
+
+// mimalloc: mi_free
+static void
+_xzm_free(xzm_malloc_zone_t zone, void *ptr, bool try)
+{
+ if (os_unlikely(!ptr)) {
+ return;
+ }
+
+ void *orig_ptr = ptr;
+
+ // Inline fast path for tiny free
+
+#if CONFIG_MTE
+ // Strip the pointer to use it in pointer arithmetic operations
+ ptr = memtag_strip_address((uint8_t *)ptr);
+#endif
+
+ xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
+ xzm_segment_table_entry_s *entry_p = _xzm_ptr_to_table_entry(ptr, main);
+ if (os_unlikely(!entry_p)) {
+ goto not_found;
+ }
+
+ xzm_segment_table_entry_s entry = *entry_p;
+ if (os_unlikely(!entry.xste_normal)) {
+ goto outlined;
+ }
+
+ xzm_segment_t segment = _xzm_segment_table_entry_to_segment(entry);
+ uintptr_t offset_in_segment = (uintptr_t)orig_ptr & XZM_SEGMENT_MASK;
+ uint64_t slice_idx = offset_in_segment >> XZM_SEGMENT_SLICE_SHIFT;
+ xzm_slice_t slice = &segment->xzs_slices[slice_idx];
+
+ if (os_unlikely(slice->xzc_bits.xzcb_kind != XZM_SLICE_KIND_TINY_CHUNK)) {
+ goto outlined;
+ }
+
+ xzm_chunk_t chunk = (xzm_chunk_t)slice;
+ if (os_unlikely(chunk->xzc_mzone_idx != zone->xzz_mzone_idx)) {
+ goto not_found;
+ }
+
+ xzm_xzone_index_t xz_idx = chunk->xzc_xzone_idx;
+ __assert_only xzm_xzone_t xz = &zone->xzz_xzones[xz_idx];
+ _xzm_debug_validate_chunk_metadata(xz, chunk);
+
+ size_t block_size = chunk->xzc_freelist_block_size;
+ size_t block_offset = (uintptr_t)orig_ptr & XZM_SEGMENT_SLICE_MASK;
+ if (os_unlikely(block_offset % block_size)) {
+ goto not_found;
+ }
+
+ // Check if block is free
+
+ xzm_block_t block = (xzm_block_t)orig_ptr;
+
+#if CONFIG_MTE
+ const bool tagged = chunk->xzc_tagged;
+ if (tagged) {
+ block = (xzm_block_t)memtag_fixup_ptr((void *)ptr);
+
+ if (os_unlikely(!memtag_tags_match(orig_ptr, block))) {
+ goto not_found;
+ }
+ }
+#endif
+
+ uint64_t cookie_value = os_atomic_load(&block->xzb_cookie, relaxed);
+ uint64_t free_cookie = zone->xzz_freelist_cookie ^ (uint64_t)orig_ptr;
+ if (os_unlikely(cookie_value == free_cookie)) {
+ goto outlined;
+ }
+
+ if (block_size > sizeof(struct xzm_block_inline_meta_s) &&
+ block_size <= XZM_ZERO_ON_FREE_THRESHOLD) {
+ bzero(block, block_size);
+ }
+
+#if CONFIG_MTE
+ if (tagged) {
+ block = _xzm_xzone_block_memtag_retag(zone, block, block_size);
+ uint8_t tag = memtag_extract_tag((uint8_t *)block);
+ free_cookie = (uint64_t)memtag_mix_tag(
+ memtag_strip_address((uint8_t *)free_cookie), tag);
+ }
+#endif
+
+ // Mark it as free now
+ os_atomic_store(&block->xzb_cookie, free_cookie, relaxed);
+
+ uint16_t block_ref = block_offset / XZM_GRANULE;
+
+#if CONFIG_XZM_THREAD_CACHE
+ if (block_size > XZM_THREAD_CACHE_THRESHOLD ||
+ !zone->xzz_thread_cache_enabled) {
+ goto not_cached;
+ }
+
+ xzm_thread_cache_t tc = _xzm_get_thread_cache();
+ if (!tc) {
+ goto not_cached;
+ }
+
+ xzm_xzone_thread_cache_t cache = &tc->xtc_xz_caches[xz_idx];
+ if (cache->xztc_head > XZM_FREE_LIMIT || cache->xztc_chunk != chunk) {
+ goto not_cached;
+ }
+
+ return _xzm_xzone_thread_cache_free_tiny(zone, cache, block, block_ref);
+
+not_cached:
+#endif // CONFIG_XZM_THREAD_CACHE
+ return _xzm_xzone_free_freelist_inline(zone, xz_idx, chunk, block,
+ block_ref, block_size, chunk->xzc_freelist_chunk_capacity);
+
+outlined:
+ return _xzm_free_outlined(zone, orig_ptr, try, entry);
+
+not_found:
+ return _xzm_free_not_found(zone, orig_ptr, try);
+}
+
+MALLOC_NOINLINE
+static size_t
+_xzm_ptr_size_outlined(xzm_malloc_zone_t zone, const void *ptr)
+{
+ // TODO: As part of rdar://101779148, check the mzone that this early
+ // allocation came from
+ if (_xzm_malloc_zone_is_main(zone) && mfm_claimed_address((void *)ptr)) {
+ return mfm_alloc_size(ptr);
+ }
+
+ return 0;
+}
+
+// mimalloc: _mi_usable_size
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static size_t
+_xzm_ptr_size(xzm_malloc_zone_t zone, const void *ptr, xzm_chunk_t *out_chunk)
+{
+ xzm_block_t block;
+ size_t block_size;
+ xzm_chunk_t chunk = _xzm_ptr_lookup(zone, ptr, NULL, &block, &block_size);
+ if (out_chunk) {
+ *out_chunk = chunk;
+ }
+
+ if (!chunk) {
+ // Might be an early allocation, or a huge aligned allocation
+ return _xzm_ptr_size_outlined(zone, ptr);
+ }
+
+ if (os_likely((void *)block == ptr)) {
+ return block_size;
+ }
+
+ // Aligned allocation
+ size_t block_offset = (uint8_t *)ptr - (uint8_t *)block;
+ return block_size - block_offset;
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static void *
+_xzm_realloc_inner(xzm_malloc_zone_t zone, void *ptr, size_t new_size,
+ malloc_type_descriptor_t type_desc)
+{
+ xzm_debug_assert(ptr);
+ xzm_debug_assert(new_size);
+
+ xzm_chunk_t chunk;
+ size_t old_size = _xzm_ptr_size(zone, ptr, &chunk);
+ void *old_ptr = ptr;
+
+ if (old_size == 0) {
+#if CONFIG_MTE
+ // This implicitly includes validation of the pointer tag when memory
+ // tagging is enabled. We need this here to cover the realloc path of
+ // malloc_zone_realloc(), which doesn't go through _realloc().
+#endif
+ malloc_report_pointer_was_not_allocated(
+ MALLOC_REPORT_CRASH | MALLOC_REPORT_NOLOG, ptr);
+ }
+
+ if (chunk && os_unlikely(chunk->xzc_mzone_idx != zone->xzz_mzone_idx)) {
+ xzm_client_abort_with_reason(
+ "pointer zone mismatch, client may be passing the wrong malloc"
+ " zone", ptr);
+ }
+
+ if (chunk && old_size > XZM_SMALL_BLOCK_SIZE_MAX
+ && new_size > XZM_SMALL_BLOCK_SIZE_MAX) {
+ xzm_slice_count_t new_slice_count = (xzm_slice_count_t)
+ (roundup(new_size,
+ XZM_SEGMENT_SLICE_SIZE) / XZM_SEGMENT_SLICE_SIZE);
+
+ xzm_segment_t segment = _xzm_segment_for_slice(zone, chunk);
+ xzm_debug_assert(segment != NULL);
+
+ bool realloc_in_place = false;
+ if (old_size > XZM_LARGE_BLOCK_SIZE_MAX &&
+ new_size > XZM_LARGE_BLOCK_SIZE_MAX &&
+ os_likely((zone->xzz_flags & MALLOC_PURGEABLE) == 0)) {
+ realloc_in_place = xzm_segment_group_try_realloc_huge_chunk(
+ segment->xzs_segment_group, zone, segment, chunk,
+ new_slice_count);
+ } else if (old_size <= XZM_LARGE_BLOCK_SIZE_MAX &&
+ segment->xzs_kind == XZM_SEGMENT_KIND_NORMAL &&
+ new_size <= XZM_LARGE_BLOCK_SIZE_MAX &&
+ os_likely((zone->xzz_flags & MALLOC_PURGEABLE) == 0)) {
+ realloc_in_place = xzm_segment_group_try_realloc_large_chunk(
+ segment->xzs_segment_group, segment, chunk,
+ new_slice_count);
+ }
+
+ if (realloc_in_place) {
+ return old_ptr;
+ }
+ }
+
+ if (new_size <= old_size && new_size >= (old_size / 2)) {
+ // reallocation still fits and not more than 50% waste
+ // TODO: revisit?
+ return old_ptr;
+ }
+
+ void *new_ptr = xzm_malloc(zone, new_size, type_desc, 0);
+ if (os_unlikely(!new_ptr)) {
+ return NULL;
+ }
+
+ const size_t valid_size = MIN(old_size, new_size);
+#if CONFIG_REALLOC_CAN_USE_VMCOPY
+ // When supported, request a CoW mapping
+ if (valid_size > XZM_LARGE_BLOCK_SIZE_MAX) {
+ if (mach_vm_copy(mach_task_self(), (mach_vm_address_t)old_ptr,
+ valid_size, (mach_vm_address_t)new_ptr) == KERN_SUCCESS) {
+ return new_ptr;
+ }
+ }
+#endif
+ memcpy(new_ptr, old_ptr, valid_size);
+
+ return new_ptr;
+}
+
+void *
+xzm_realloc(xzm_malloc_zone_t zone, void *ptr, size_t new_size,
+ malloc_type_descriptor_t type_desc)
+{
+ if (!ptr) {
+ return xzm_malloc(zone, new_size, type_desc, 0);
+ } else if (new_size == 0) {
+ _xzm_free(zone, ptr, false);
+ return xzm_malloc(zone, new_size, type_desc, 0);
+ }
+
+ void *new_ptr = _xzm_realloc_inner(zone, ptr, new_size, type_desc);
+ if (new_ptr && new_ptr != ptr) {
+ _xzm_free(zone, ptr, false);
+ }
+
+ return new_ptr;
+}
+
+#pragma mark libmalloc zone
+
+static size_t
+xzm_malloc_zone_size(xzm_malloc_zone_t zone, const void *ptr)
+{
+ return _xzm_ptr_size(zone, ptr, NULL);
+}
+
+static boolean_t
+xzm_malloc_zone_claimed_address(xzm_malloc_zone_t zone, void *ptr)
+{
+ // Is it an early allocation?
+ if (mfm_claimed_address(ptr)) {
+ return true;
+ }
+
+#if CONFIG_MTE
+ // We strip the pointer here as xzm_segment_table_query operates on
+ // canonical pointers; however, since mfm_claimed_address validates
+ // the logical tag of the pointer, we need to ensure we don't strip
+ // the tag before passing it to MFM.
+ // We don't need to check if MTE is enabled, as this doesn't need MTE ISA.
+ ptr = memtag_strip_address(ptr);
+#endif
+
+ xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
+ return xzm_segment_table_query(main, ptr) != NULL;
+}
+
+static inline void *
+_xzm_malloc_zone_malloc_entry(xzm_malloc_zone_t zone, size_t size,
+ malloc_type_id_t type_id, xzm_malloc_options_t options)
+{
+ return xzm_malloc_inline(zone, size,
+ (malloc_type_descriptor_t){ .type_id = type_id, }, options);
+}
+
+static void *
+xzm_malloc_zone_malloc_type_malloc(xzm_malloc_zone_t zone, size_t size,
+ malloc_type_id_t type_id)
+{
+ return _xzm_malloc_zone_malloc_entry(zone, size, type_id, 0);
+}
+
+static void *
+xzm_malloc_zone_malloc(xzm_malloc_zone_t zone, size_t size)
+{
+ return _xzm_malloc_zone_malloc_entry(zone, size, malloc_get_tsd_type_id(),
+ _xzm_xzone_get_malloc_thread_options());
+}
+
+static inline void *
+_xzm_malloc_zone_malloc_type_calloc_entry(xzm_malloc_zone_t zone, size_t count,
+ size_t size, malloc_type_id_t type_id,
+ xzm_malloc_options_t options)
+{
+ size_t total_bytes;
+ if (os_unlikely(calloc_get_size(count, size, 0, &total_bytes))) {
+ malloc_set_errno_fast(MZ_POSIX, ENOMEM);
+ return NULL;
+ }
+
+ return xzm_malloc_inline(zone, total_bytes,
+ (malloc_type_descriptor_t){ .type_id = type_id, },
+ options | XZM_MALLOC_CLEAR);
+}
+
+static void *
+xzm_malloc_zone_malloc_type_calloc(xzm_malloc_zone_t zone, size_t count,
+ size_t size, malloc_type_id_t type_id)
+{
+ return _xzm_malloc_zone_malloc_type_calloc_entry(zone, count, size, type_id,
+ 0);
+}
+
+static void *
+xzm_malloc_zone_calloc(xzm_malloc_zone_t zone, size_t count, size_t size)
+{
+ return _xzm_malloc_zone_malloc_type_calloc_entry(zone, count, size,
+ malloc_get_tsd_type_id(), _xzm_xzone_get_malloc_thread_options());
+}
+
+static void * __alloc_size(2)
+xzm_malloc_zone_valloc(xzm_malloc_zone_t zone, size_t size)
+{
+ return _xzm_memalign(zone, vm_page_quanta_size, size,
+ malloc_get_tsd_type_descriptor(), 0);
+}
+
+static void
+xzm_malloc_zone_free(xzm_malloc_zone_t zone, void *ptr)
+{
+ _xzm_free(zone, ptr, false);
+}
+
+static void
+xzm_malloc_zone_try_free_default(xzm_malloc_zone_t zone, void *ptr)
+{
+ _xzm_free(zone, ptr, true);
+}
+
+static void
+xzm_malloc_zone_free_definite_size(xzm_malloc_zone_t zone, void *ptr,
+ size_t size)
+{
+ // Unfortunately, knowing the size doesn't really buy us anything.
+ (void)size;
+
+ _xzm_free(zone, ptr, false);
+}
+
+static void *
+xzm_malloc_zone_malloc_type_realloc(xzm_malloc_zone_t zone, void *ptr,
+ size_t new_size, malloc_type_id_t type_id)
+{
+ return xzm_realloc(zone, ptr, new_size,
+ (malloc_type_descriptor_t){ .type_id = type_id, });
+}
+
+static void *
+xzm_malloc_zone_realloc(xzm_malloc_zone_t zone, void *ptr, size_t new_size)
+{
+ return xzm_malloc_zone_malloc_type_realloc(zone, ptr, new_size,
+ malloc_get_tsd_type_id());
+}
+
+static inline void *
+_xzm_malloc_zone_malloc_type_memalign_entry(xzm_malloc_zone_t zone,
+ size_t alignment, size_t size, malloc_type_id_t type_id,
+ xzm_malloc_options_t options)
+{
+ return _xzm_memalign(zone, alignment, size,
+ (malloc_type_descriptor_t){ .type_id = type_id, }, options);
+}
+
+static void *
+xzm_malloc_zone_malloc_type_memalign(xzm_malloc_zone_t zone, size_t alignment,
+ size_t size, malloc_type_id_t type_id)
+{
+ return _xzm_malloc_zone_malloc_type_memalign_entry(zone, alignment, size,
+ type_id, 0);
+}
+
+static void *
+xzm_malloc_zone_memalign(xzm_malloc_zone_t zone, size_t alignment, size_t size)
+{
+ return _xzm_malloc_zone_malloc_type_memalign_entry(zone, alignment, size,
+ malloc_get_tsd_type_id(), _xzm_xzone_get_malloc_thread_options());
+}
+
+static void *
+xzm_malloc_zone_malloc_type_malloc_with_options(xzm_malloc_zone_t zone,
+ size_t align, size_t size, malloc_zone_malloc_options_t options,
+ malloc_type_id_t type_id)
+{
+ xzm_malloc_options_t opt = 0;
+ if (options & MALLOC_ZONE_MALLOC_OPTION_CLEAR) {
+ opt |= XZM_MALLOC_CLEAR;
+ }
+ if (options & MALLOC_NP_OPTION_CANONICAL_TAG) {
+ opt |= XZM_MALLOC_CANONICAL_TAG;
+ }
+
+ malloc_type_descriptor_t type_desc = { .type_id = type_id, };
+ if (align > MALLOC_ZONE_MALLOC_DEFAULT_ALIGN) {
+ return _xzm_memalign(zone, align, size, type_desc, opt);
+ } else {
+ return xzm_malloc_inline(zone, size, type_desc, opt);
+ }
+}
+
+static void *
+xzm_malloc_zone_malloc_with_options(xzm_malloc_zone_t zone, size_t align,
+ size_t size, malloc_zone_malloc_options_t options)
+{
+ return xzm_malloc_zone_malloc_type_malloc_with_options(zone, align, size,
+ options, malloc_get_tsd_type_id());
+}
+
+#pragma mark slowpath zone functions
+
+// nothing for size
+
+// nothing for claimed_address
+
+static void *
+xzm_malloc_zone_malloc_type_malloc_slow(xzm_malloc_zone_t zone, size_t size,
+ malloc_type_id_t type_id)
+{
+ void *ptr = NULL;
+ malloc_type_descriptor_t type_desc = { .type_id = type_id, };
+ xzm_malloc_options_t options = _xzm_xzone_get_malloc_thread_options();
+
+ if ((zone->xzz_flags & MALLOC_PURGEABLE) &&
+ (size <= XZM_SMALL_BLOCK_SIZE_MAX)) {
+ xzm_malloc_zone_t main = (xzm_malloc_zone_t)_xzm_malloc_zone_main(zone);
+ ptr = xzm_malloc_inline(main, size, type_desc, options);
+ } else {
+ ptr = xzm_malloc_inline(zone, size, type_desc, options);
+ }
+ if (ptr && (zone->xzz_flags & MALLOC_DO_SCRIBBLE)) {
+ memset(ptr, SCRIBBLE_BYTE, size);
+ }
+ return ptr;
+}
+
+static void *
+xzm_malloc_zone_malloc_slow(xzm_malloc_zone_t zone, size_t size)
+{
+ return xzm_malloc_zone_malloc_type_malloc_slow(zone, size,
+ malloc_get_tsd_type_id());
+}
+
+static void *
+xzm_malloc_zone_malloc_type_calloc_slow(xzm_malloc_zone_t zone, size_t count,
+ size_t size, malloc_type_id_t type_id)
+{
+ size_t total_bytes;
+ if (calloc_get_size(count, size, 0, &total_bytes)) {
+ return NULL;
+ }
+
+ malloc_type_descriptor_t type_desc = { .type_id = type_id, };
+ xzm_malloc_options_t options = _xzm_xzone_get_malloc_thread_options() |
+ XZM_MALLOC_CLEAR;
+ if ((zone->xzz_flags & MALLOC_PURGEABLE) == 0 ||
+ total_bytes > XZM_SMALL_BLOCK_SIZE_MAX) {
+ return xzm_malloc_inline(zone, total_bytes, type_desc, options);
+ } else {
+ xzm_malloc_zone_t main = (xzm_malloc_zone_t)_xzm_malloc_zone_main(zone);
+ return xzm_malloc_inline(main, total_bytes, type_desc, options);
+ }
+}
+
+static void *
+xzm_malloc_zone_calloc_slow(xzm_malloc_zone_t zone, size_t count,
+ size_t size)
+{
+ return xzm_malloc_zone_malloc_type_calloc_slow(zone, count, size,
+ malloc_get_tsd_type_id());
+}
+
+static void *
+xzm_malloc_zone_valloc_slow(xzm_malloc_zone_t zone, size_t size)
+{
+ void *ptr = NULL;
+ if ((zone->xzz_flags & MALLOC_PURGEABLE) &&
+ (size <= XZM_SMALL_BLOCK_SIZE_MAX)) {
+ xzm_malloc_zone_t main = (xzm_malloc_zone_t)_xzm_malloc_zone_main(zone);
+ ptr = xzm_malloc_zone_valloc(main, size);
+ } else {
+ ptr = xzm_malloc_zone_valloc(zone, size);
+ }
+ if (ptr && (zone->xzz_flags & MALLOC_DO_SCRIBBLE)) {
+ memset(ptr, SCRIBBLE_BYTE, size);
+ }
+ return ptr;
+}
+
+static void
+xzm_malloc_zone_free_slow(xzm_malloc_zone_t zone, void *ptr)
+{
+ if (!ptr) {
+ return;
+ }
+
+ size_t size = _xzm_ptr_size(zone, ptr, NULL);
+ // The main ref stays NULL if this allocation came from zone
+ xzm_malloc_zone_t main = NULL;
+ if (size == 0 && (zone->xzz_flags & MALLOC_PURGEABLE)) {
+ main = (xzm_malloc_zone_t)_xzm_malloc_zone_main(zone);
+ size = _xzm_ptr_size(main, ptr, NULL);
+ }
+ if (size == 0) {
+ xzm_client_abort_with_reason("pointer being freed was not allocated",
+ ptr);
+ }
+ if ((zone->xzz_flags & MALLOC_DO_SCRIBBLE) &&
+ (size > XZM_ZERO_ON_FREE_THRESHOLD)) {
+ memset(ptr, SCRABBLE_BYTE, size);
+ }
+
+ if (main) {
+ xzm_malloc_zone_free(main, ptr);
+ } else {
+ xzm_malloc_zone_free(zone, ptr);
+ }
+}
+
+static void
+xzm_malloc_zone_try_free_default_slow(xzm_malloc_zone_t zone, void *ptr)
+{
+ if (!ptr) {
+ return;
+ }
+
+ size_t size = _xzm_ptr_size(zone, ptr, NULL);
+ xzm_malloc_zone_t main = NULL;
+ if (size == 0 && (zone->xzz_flags & MALLOC_PURGEABLE)) {
+ main = (xzm_malloc_zone_t)_xzm_malloc_zone_main(zone);
+ size = _xzm_ptr_size(main, ptr, NULL);
+ }
+ if (size == 0) {
+ // pointer didn't come from this zone
+ find_zone_and_free(ptr, true);
+ return;
+ }
+ if (size && size > XZM_ZERO_ON_FREE_THRESHOLD &&
+ (zone->xzz_flags & MALLOC_DO_SCRIBBLE)) {
+ memset(ptr, SCRABBLE_BYTE, size);
+ }
+
+ if (main) {
+ xzm_malloc_zone_try_free_default(main, ptr);
+ } else {
+ xzm_malloc_zone_try_free_default(zone, ptr);
+ }
+}
+
+static void
+xzm_malloc_zone_free_definite_size_slow(xzm_malloc_zone_t zone, void *ptr,
+ size_t size)
+{
+ if (!ptr) {
+ return;
+ }
+
+ // TODO: Should we verify that the size the client passes in matches our
+ // size?
+ size_t our_size = _xzm_ptr_size(zone, ptr, NULL);
+ xzm_malloc_zone_t main = NULL;
+ if (our_size == 0 && (zone->xzz_flags & MALLOC_PURGEABLE)) {
+ main = (xzm_malloc_zone_t)_xzm_malloc_zone_main(zone);
+ our_size = _xzm_ptr_size(main, ptr, NULL);
+ xzm_debug_assert(our_size != 0);
+ }
+ if ((zone->xzz_flags & MALLOC_DO_SCRIBBLE) &&
+ (size > XZM_ZERO_ON_FREE_THRESHOLD)) {
+ memset(ptr, SCRABBLE_BYTE, size);
+ }
+
+ if (main) {
+ xzm_malloc_zone_free_definite_size(main, ptr, size);
+ } else {
+ xzm_malloc_zone_free_definite_size(zone, ptr, size);
+ }
+}
+
+static void *
+xzm_malloc_zone_malloc_type_realloc_slow(xzm_malloc_zone_t zone, void *ptr,
+ size_t new_size, malloc_type_id_t type_id)
+{
+ bool purgeable = zone->xzz_flags & MALLOC_PURGEABLE;
+ bool scribble = zone->xzz_flags & MALLOC_DO_SCRIBBLE;
+ if (!purgeable && !scribble) {
+ return xzm_malloc_zone_malloc_type_realloc(zone, ptr, new_size,
+ type_id);
+ }
+
+ if (!ptr) {
+ return xzm_malloc_zone_malloc_type_malloc_slow(zone, new_size, type_id);
+ } else if (new_size == 0) {
+ xzm_malloc_zone_free_slow(zone, ptr);
+ return xzm_malloc_zone_malloc_type_malloc_slow(zone, new_size, type_id);
+ }
+
+ malloc_type_descriptor_t type_desc = { .type_id = type_id, };
+ size_t old_size = _xzm_ptr_size(zone, ptr, NULL);
+ xzm_malloc_zone_t main = NULL;
+ if (old_size == 0 && purgeable) {
+ main = (xzm_malloc_zone_t)_xzm_malloc_zone_main(zone);
+ old_size = _xzm_ptr_size(main, ptr, NULL);
+ }
+ xzm_assert(old_size);
+
+ if ((old_size > new_size) && scribble) {
+ // Shrinking: no matter what, we'll definitely need to scrabble the
+ // tail, so do that now
+ memset((uint8_t *)ptr + new_size, SCRABBLE_BYTE, old_size - new_size);
+ }
+
+ void *new_ptr;
+ if (!purgeable || (!main && new_size > XZM_SMALL_BLOCK_SIZE_MAX) ||
+ (main && new_size <= XZM_SMALL_BLOCK_SIZE_MAX)) {
+ // The old pointer and new pointer will be from the same zone
+ new_ptr = _xzm_realloc_inner(zone, ptr, new_size, type_desc);
+ } else {
+ if (new_size <= XZM_SMALL_BLOCK_SIZE_MAX) {
+ // Need to serve the new allocation from the main zone, since it's
+ // too small to be purgeable
+ new_ptr = xzm_malloc_inline(&_xzm_malloc_zone_main(zone)->xzmz_base,
+ new_size, type_desc, 0);
+ } else {
+ // Serve the new allocation from the purgeable zone
+ new_ptr = xzm_malloc_inline(zone, new_size, type_desc, 0);
+ }
+ if (new_ptr) {
+ memcpy(new_ptr, ptr, MIN(old_size, new_size));
+ }
+ }
+
+ if (new_ptr) {
+ // Regardless of whether we reallocated in place or not, we always want
+ // to scribble the expanded space when growing
+ if ((new_size > old_size) && scribble) {
+ memset((uint8_t *)new_ptr + old_size, SCRIBBLE_BYTE,
+ new_size - old_size);
+ }
+ if (new_ptr != ptr) {
+ // If we didn't reallocate in place, we have some amount of
+ // scrabbling left to do on the old allocation:
+ if ((old_size > new_size) && scribble) {
+ // If shrinking, we already scrabbled the tail, so just scrabble
+ // the head now too
+ memset(ptr, SCRABBLE_BYTE, new_size);
+ } else if (scribble) {
+ // If growing, scrabble the whole previous allocation since we
+ // haven't done any of it yet
+ memset(ptr, SCRABBLE_BYTE, old_size);
+ }
+ if (main) {
+ _xzm_free(main, ptr, false);
+ } else {
+ _xzm_free(zone, ptr, false);
+ }
+ }
+ }
+
+ return new_ptr;
+}
+
+static void *
+xzm_malloc_zone_realloc_slow(xzm_malloc_zone_t zone, void *ptr, size_t new_size)
+{
+ return xzm_malloc_zone_malloc_type_realloc_slow(zone, ptr, new_size,
+ malloc_get_tsd_type_id());
+}
+
+static void *
+xzm_malloc_zone_malloc_type_memalign_slow(xzm_malloc_zone_t zone, size_t alignment,
+ size_t size, malloc_type_id_t type_id)
+{
+ void *ptr = NULL;
+ xzm_malloc_options_t options = _xzm_xzone_get_malloc_thread_options();
+ if ((zone->xzz_flags & MALLOC_PURGEABLE) &&
+ (size <= XZM_SMALL_BLOCK_SIZE_MAX)) {
+ xzm_malloc_zone_t main = (xzm_malloc_zone_t)_xzm_malloc_zone_main(zone);
+ ptr = _xzm_malloc_zone_malloc_type_memalign_entry(main, alignment, size,
+ type_id, options);
+ } else {
+ ptr = _xzm_malloc_zone_malloc_type_memalign_entry(zone, alignment, size,
+ type_id, options);
+ }
+ if (ptr && (zone->xzz_flags & MALLOC_DO_SCRIBBLE)) {
+ memset(ptr, SCRIBBLE_BYTE, size);
+ }
+ return ptr;
+}
+
+static void *
+xzm_malloc_zone_memalign_slow(xzm_malloc_zone_t zone, size_t alignment,
+ size_t size)
+{
+ return xzm_malloc_zone_malloc_type_memalign_slow(zone, alignment, size,
+ malloc_get_tsd_type_id());
+}
+
+static void *
+xzm_malloc_zone_malloc_type_malloc_with_options_slow(xzm_malloc_zone_t zone,
+ size_t align, size_t size, malloc_zone_malloc_options_t options,
+ malloc_type_id_t type_id)
+{
+ void *ptr = NULL;
+ if ((zone->xzz_flags & MALLOC_PURGEABLE) &&
+ (size <= XZM_SMALL_BLOCK_SIZE_MAX)) {
+ xzm_malloc_zone_t main = (xzm_malloc_zone_t)_xzm_malloc_zone_main(zone);
+ ptr = xzm_malloc_zone_malloc_type_malloc_with_options(main, align, size,
+ options, type_id);
+ } else {
+ ptr = xzm_malloc_zone_malloc_type_malloc_with_options(zone, align, size,
+ options, type_id);
+ }
+ if (ptr && !(options & MALLOC_ZONE_MALLOC_OPTION_CLEAR) &&
+ (zone->xzz_flags & MALLOC_DO_SCRIBBLE)) {
+ memset(ptr, SCRIBBLE_BYTE, size);
+ }
+ return ptr;
+}
+
+static void *
+xzm_malloc_zone_malloc_with_options_slow(xzm_malloc_zone_t zone, size_t align,
+ size_t size, malloc_zone_malloc_options_t options)
+{
+ return xzm_malloc_zone_malloc_type_malloc_with_options_slow(zone, align,
+ size, options, malloc_get_tsd_type_id());
+}
+
+#pragma mark libmalloc zone introspection
+size_t
+xzm_good_size(xzm_malloc_zone_t zone, size_t size)
+{
+ if (size <= XZM_SMALL_BLOCK_SIZE_MAX) {
+ xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
+ uint8_t bin = _xzm_bin(size);
+ return main->xzmz_xzone_bin_sizes[bin];
+ } else {
+ // MAX() handles overflow in roundup()
+ return MAX(roundup(size, XZM_SEGMENT_SLICE_SIZE), size);
+ }
+}
+
+boolean_t
+xzm_check(xzm_malloc_zone_t zone)
+{
+ // TODO: zone self-check
+ //
+ // Nano doesn't implement this - worth the trouble?
+ (void)zone;
+ return true;
+}
+
+void
+xzm_log(xzm_malloc_zone_t zone, void *log_address)
+{
+ // Not implemented
+ //
+ // Doesn't seem that useful? Nano doesn't implement this either.
+}
+
+OS_ENUM(xzm_lock_action, int,
+ XZM_LOCK_LOCK,
+ XZM_LOCK_UNLOCK,
+ XZM_LOCK_INIT,
+);
+
+static void
+_xzm_do_lock_action(_malloc_lock_s *lock, xzm_lock_action_t action)
+{
+ switch (action) {
+ case XZM_LOCK_LOCK:
+ _malloc_lock_lock(lock);
+ break;
+ case XZM_LOCK_UNLOCK:
+ _malloc_lock_unlock(lock);
+ break;
+ case XZM_LOCK_INIT:
+ _malloc_lock_init(lock);
+ break;
+ default:
+ xzm_debug_abort("invalid xzm lock action");
+ break;
+ }
+}
+
+static void
+_xzm_freelist_xzone_do_lock_action(xzm_malloc_zone_t zone, xzm_xzone_t xz,
+ xzm_lock_action_t action)
+{
+ if (action == XZM_LOCK_LOCK) {
+ for (xzm_allocation_index_t i = 0; i < zone->xzz_slot_count; i++) {
+ _xzm_chunk_list_fork_lock(&_xzm_xzone_chunk_list_for_index(zone,
+ xz, zone->xzz_partial_lists, i)->xcl_list);
+ }
+
+ _xzm_chunk_batch_list_fork_lock(&xz->xz_batch_list);
+ _xzm_chunk_list_fork_lock(&xz->xz_empty_list);
+ _xzm_chunk_list_fork_lock(&xz->xz_preallocated_list);
+ _xzm_chunk_list_fork_lock(&xz->xz_all_list);
+ }
+
+ xzm_chunk_list_head_u head = {
+ .xzch_value = os_atomic_load(&xz->xz_all_list.xzch_value, dependency),
+ };
+
+ xzm_chunk_t chunk = (xzm_chunk_t)head.xzch_ptr;
+ while (chunk) {
+ xzm_chunk_atomic_meta_u old_meta;
+ if (action == XZM_LOCK_LOCK) {
+ (void)_xzm_xzone_freelist_chunk_lock(chunk, &old_meta);
+ } else {
+ old_meta.xca_value = os_atomic_load_wide(
+ &chunk->xzc_atomic_meta.xca_value, dependency);
+ if (old_meta.xca_alloc_head != XZM_FREE_MADVISING &&
+ old_meta.xca_alloc_head != XZM_FREE_MADVISED) {
+ _xzm_xzone_freelist_chunk_unlock(chunk, old_meta);
+ }
+ }
+ chunk = chunk->xzc_linkages[XZM_CHUNK_LINKAGE_ALL];
+ }
+
+ if (action != XZM_LOCK_LOCK) {
+ _xzm_chunk_list_fork_unlock(&xz->xz_all_list);
+ _xzm_chunk_batch_list_fork_unlock(&xz->xz_batch_list);
+ _xzm_chunk_list_fork_unlock(&xz->xz_preallocated_list);
+ _xzm_chunk_list_fork_unlock(&xz->xz_empty_list);
+
+ for (xzm_allocation_index_t i = 0; i < zone->xzz_slot_count; i++) {
+ _xzm_chunk_list_fork_unlock(&_xzm_xzone_chunk_list_for_index(zone,
+ xz, zone->xzz_partial_lists, i)->xcl_list);
+ }
+
+ }
+}
+
+static void
+_xzm_small_xzone_lock_all(xzm_malloc_zone_t zone, xzm_xzone_t xz)
+{
+ xzm_debug_assert(xz->xz_block_size > XZM_TINY_BLOCK_SIZE_MAX);
+
+ while (true) {
+ _malloc_lock_lock(&xz->xz_lock);
+
+ xzm_chunk_t chunk = xz->xz_chunkq_batch;
+ while (chunk) {
+ _malloc_lock_lock(&chunk->xzc_lock);
+ chunk = *_xzm_segment_slice_meta_batch_next(zone, chunk);
+ xzm_debug_assert(_xzm_slice_meta_is_batch_pointer(zone, chunk));
+ }
+
+ LIST_FOREACH(chunk, &xz->xz_chunkq_partial, xzc_entry) {
+ _malloc_lock_lock(&chunk->xzc_lock);
+ }
+
+ LIST_FOREACH(chunk, &xz->xz_chunkq_full, xzc_entry) {
+ bool gotlock = _malloc_lock_trylock(&chunk->xzc_lock);
+ if (!gotlock) {
+ // Potential lock inversion - need to unwind and retry.
+ xzm_chunk_t chunk2 = xz->xz_chunkq_batch;
+
+ while (chunk2) {
+ _malloc_lock_unlock(&chunk2->xzc_lock);
+ chunk2 = *_xzm_segment_slice_meta_batch_next(zone, chunk2);
+ xzm_debug_assert(_xzm_slice_meta_is_batch_pointer(zone,
+ chunk2));
+ }
+
+ LIST_FOREACH(chunk2, &xz->xz_chunkq_partial, xzc_entry) {
+ _malloc_lock_unlock(&chunk2->xzc_lock);
+ }
+
+ LIST_FOREACH(chunk2, &xz->xz_chunkq_full, xzc_entry) {
+ if (chunk2 == chunk) {
+ break;
+ }
+
+ _malloc_lock_unlock(&chunk2->xzc_lock);
+ }
+ break;
+ }
+ }
+
+ if (chunk) {
+ // Encountered a trylock failure, so we need to unlock the xzone to
+ // let the holder of the chunk lock we need make progress.
+ //
+ // TODO: There's a priority inversion here because we're not doing
+ // anything to push on the holder of the chunk lock we need. We
+ // know exactly who we need to wait for, so we could give them an
+ // artificial push by doing a timed ulock wait on their thread ID
+ // after we release the lock - but before doing that we'd probably
+ // just replace this whole approach instead.
+ _malloc_lock_unlock(&xz->xz_lock);
+ yield();
+ } else {
+ // Made it to the end of the full list without any trylock failures,
+ // so we're done.
+ break;
+ }
+ }
+}
+
+static void
+_xzm_allocation_slots_do_lock_action(xzm_malloc_zone_t zone,
+ xzm_lock_action_t action)
+{
+ for (uint8_t i = XZM_XZONE_INDEX_FIRST; i < zone->xzz_xzone_count; i++) {
+ xzm_xzone_t xz = &zone->xzz_xzones[i];
+
+ __unused xzm_slice_kind_t kind;
+ if (xz->xz_block_size <= XZM_TINY_BLOCK_SIZE_MAX) {
+ kind = XZM_SLICE_KIND_TINY_CHUNK;
+ } else if (zone->xzz_small_freelist_enabled) {
+ kind = XZM_SLICE_KIND_SMALL_FREELIST_CHUNK;
+ } else {
+ kind = XZM_SLICE_KIND_SMALL_CHUNK;
+ }
+
+ for (uint8_t j = 0; j < zone->xzz_slot_count; j++) {
+ size_t slot_idx = (zone->xzz_xzone_count * j) + i;
+ xzm_xzone_allocation_slot_t xas =
+ &zone->xzz_xzone_allocation_slots[slot_idx];
+#if !CONFIG_TINY_ALLOCATION_SLOT_LOCK
+ if (kind == XZM_SLICE_KIND_TINY_CHUNK ||
+ kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK) {
+ if (action == XZM_LOCK_LOCK) {
+ _xzm_xzone_allocation_slot_fork_lock(xas);
+ } else {
+ _xzm_xzone_allocation_slot_fork_unlock(xas);
+ }
+ } else
+#endif // !CONFIG_TINY_ALLOCATION_SLOT_LOCK
+ {
+ _xzm_do_lock_action(&xas->xas_lock, action);
+ }
+ }
+ }
+}
+
+static void
+_xzm_foreach_lock(xzm_malloc_zone_t zone, xzm_lock_action_t action)
+{
+ // mzone-private locks
+ //
+ // There are 3 levels of locks:
+ // - the xzone slot locks
+ // - the xzone lock
+ // - the chunk locks
+ //
+ // The chunk locks are the most difficult to handle for fork, as there are
+ // so many of them. The simplest strategy that works is to just find and
+ // acquire them all.
+ //
+ // The listed order is the general locking order, but when full chunks
+ // transition back to partial their chunk lock is held while acquiring the
+ // xzone lock. It's therefore not safe to hold the xzone lock while trying
+ // to acquire the chunk lock of a full chunk. To deal with that, use
+ // trylock to acquire the locks of full chunks, and on failure back off,
+ // release the xzone lock and retry.
+ //
+ // It's not necessary to acquire the chunk locks of chunks that are empty
+ // and in the transition from the partial list to the isolation list,
+ // because they won't be allocated from or freed to on the child side of
+ // fork - they'll effectively leak when the thread responsible for them
+ // disappears as part of fork.
+ //
+ // TODO: this is not efficient, particularly in processes that have lots of
+ // chunks at the time of fork. Before we pursue general userspace
+ // enablement on macOS, where 3p code needs fork() to be reasonably fast, we
+ // should do something better, but this is good enough for the situations
+ // where it just needs to be correct.
+
+ // The fork-lock bits send waiters to the fork lock, so we need to ensure
+ // that we hold the fork lock whenever any of the fork-lock bits are set.
+ // That means we need to acquire it before setting any of the fork-lock bits
+ // and release it only after clearing all of them.
+ //
+ // The fork lock also provides mutual exclusion for mutation of the
+ // fork-lock bits. This is needed to prevent collisions between concurrent
+ // executions of foreach-lock: one thread un-setting the fork-lock bits must
+ // not trample another one making a pass setting them.
+ if (action == XZM_LOCK_LOCK) {
+ _xzm_do_lock_action(&zone->xzz_fork_lock, action);
+
+ // The zone lock (xzz_lock) is used for tiny chunk walk lock
+ // synchronization (among other things). It is acquired under the
+ // allocation slot lock when allocating from a partial chunk, and needs
+ // to be held before performing any of the fork walk-locking, so it
+ // needs to be taken here after the allocation slots are locked and
+ // before any of the tiny chunks are locked.
+ _xzm_do_lock_action(&zone->xzz_lock, action);
+
+ // The freelist allocation slots must be locked before the all-chunk
+ // lists are walked and unlocked after, so that no new chunks can be
+ // allocated and added in the middle. That's important because we need
+ // the set of chunks we walk during the lock phase to be the same as
+ // during the unlock phase.
+ _xzm_allocation_slots_do_lock_action(zone, action);
+ }
+
+ for (uint8_t i = XZM_XZONE_INDEX_FIRST; i < zone->xzz_xzone_count; i++) {
+ xzm_xzone_t xz = &zone->xzz_xzones[i];
+
+ xzm_slice_kind_t kind;
+ if (xz->xz_block_size <= XZM_TINY_BLOCK_SIZE_MAX) {
+ kind = XZM_SLICE_KIND_TINY_CHUNK;
+ } else if (zone->xzz_small_freelist_enabled) {
+ kind = XZM_SLICE_KIND_SMALL_FREELIST_CHUNK;
+ } else {
+ kind = XZM_SLICE_KIND_SMALL_CHUNK;
+ }
+
+ if (kind == XZM_SLICE_KIND_TINY_CHUNK ||
+ kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK) {
+ _xzm_freelist_xzone_do_lock_action(zone, xz, action);
+ } else if (kind == XZM_SLICE_KIND_SMALL_CHUNK) {
+ if (action == XZM_LOCK_LOCK) {
+ _xzm_small_xzone_lock_all(zone, xz);
+ } else {
+ xzm_chunk_t chunk = xz->xz_chunkq_batch;
+ while (chunk) {
+ _xzm_do_lock_action(&chunk->xzc_lock, action);
+ chunk = *_xzm_segment_slice_meta_batch_next(zone, chunk);
+ xzm_debug_assert(_xzm_slice_meta_is_batch_pointer(zone,
+ chunk));
+ }
+ LIST_FOREACH(chunk, &xz->xz_chunkq_partial, xzc_entry) {
+ _xzm_do_lock_action(&chunk->xzc_lock, action);
+ }
+ LIST_FOREACH(chunk, &xz->xz_chunkq_full, xzc_entry) {
+ _xzm_do_lock_action(&chunk->xzc_lock, action);
+ }
+
+ _xzm_do_lock_action(&xz->xz_lock, action);
+ }
+ }
+ }
+
+ if (action != XZM_LOCK_LOCK) {
+ _xzm_allocation_slots_do_lock_action(zone, action);
+ _xzm_do_lock_action(&zone->xzz_lock, action);
+ _xzm_do_lock_action(&zone->xzz_fork_lock, action);
+ }
+}
+
+void
+xzm_force_lock(xzm_malloc_zone_t zone)
+{
+ _xzm_foreach_lock(zone, XZM_LOCK_LOCK);
+}
+
+void
+xzm_force_unlock(xzm_malloc_zone_t zone)
+{
+ _xzm_foreach_lock(zone, XZM_LOCK_UNLOCK);
+}
+
+void
+xzm_reinit_lock(xzm_malloc_zone_t zone)
+{
+ _xzm_foreach_lock(zone, XZM_LOCK_INIT);
+}
+
+boolean_t
+xzm_locked(xzm_malloc_zone_t zone)
+{
+ // This mechanism is dead:
+ // - nanov2 doesn't implement it despite having locks that could lead to the
+ // deadlock this is meant to prevent
+ // - there's an obvious copy-paste bug in szone_locked() for medium
+ //
+ // There are no references to malloc_gdb_po_unsafe() anywhere. It should
+ // all be removed.
+ xzm_abort("xzm_locked not implemented");
+ return true;
+}
+
+static void
+_xzm_global_state_lock(xzm_main_malloc_zone_t main, xzm_lock_action_t action)
+{
+ // main zone shared locks
+ for (uint8_t i = XZM_XZONE_INDEX_FIRST; i < main->xzmz_base.xzz_xzone_count;
+ i++) {
+ xzm_isolation_zone_t iz = &main->xzmz_isolation_zones[i];
+ _xzm_do_lock_action(&iz->xziz_lock, action);
+ }
+
+ for (uint8_t i = 0; i < main->xzmz_segment_group_count; i++) {
+ xzm_segment_group_t sg = &main->xzmz_segment_groups[i];
+ _xzm_do_lock_action(&sg->xzsg_alloc_lock, action);
+ _xzm_do_lock_action(&sg->xzsg_lock, action);
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ if (sg->xzsg_id == XZM_SEGMENT_GROUP_DATA_LARGE) {
+ _xzm_do_lock_action(&sg->xzsg_cache.xzsc_lock, action);
+ }
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+ }
+
+ for (uint8_t i = 0; i < main->xzmz_range_group_count; i++) {
+ xzm_range_group_t rg = &main->xzmz_range_groups[i];
+ _xzm_do_lock_action(&rg->xzrg_lock, action);
+ }
+
+ // The extended segment table lock is held while allocating from the segment
+ // table metapool, so we need to acquire this lock before the metapool locks
+ _xzm_do_lock_action(&main->xzmz_extended_segment_table_lock, action);
+
+ for (int i = 0; i < main->xzmz_metapool_count; i++) {
+ xzm_metapool_t mp = &main->xzmz_metapools[i];
+ _xzm_do_lock_action(&mp->xzmp_lock, action);
+ }
+
+ _xzm_do_lock_action(&main->xzmz_mzones_lock, action);
+ _xzm_do_lock_action(&main->xzmz_thread_cache_list_lock, action);
+
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ if (main->xzmz_reclaim_buffer) {
+ _xzm_do_lock_action(&main->xzmz_reclaim_buffer->xrb_lock, action);
+ }
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+}
+
+void
+xzm_force_lock_global_state(malloc_zone_t *main_zone)
+{
+ xzm_debug_assert(_xzm_malloc_zone_is_xzm(main_zone));
+ xzm_debug_assert(_xzm_malloc_zone_is_main((xzm_malloc_zone_t)main_zone));
+ xzm_main_malloc_zone_t main_ref = (xzm_main_malloc_zone_t)main_zone;
+ _xzm_global_state_lock(main_ref, XZM_LOCK_LOCK);
+}
+
+void
+xzm_force_unlock_global_state(malloc_zone_t *main_zone)
+{
+ xzm_debug_assert(_xzm_malloc_zone_is_xzm(main_zone));
+ xzm_debug_assert(_xzm_malloc_zone_is_main((xzm_malloc_zone_t)main_zone));
+ xzm_main_malloc_zone_t main_ref = (xzm_main_malloc_zone_t)main_zone;
+ _xzm_global_state_lock(main_ref, XZM_LOCK_UNLOCK);
+}
+
+void
+xzm_force_reinit_lock_global_state(malloc_zone_t *main_zone)
+{
+ xzm_debug_assert(_xzm_malloc_zone_is_xzm(main_zone));
+ xzm_debug_assert(_xzm_malloc_zone_is_main((xzm_malloc_zone_t)main_zone));
+ xzm_main_malloc_zone_t main_ref = (xzm_main_malloc_zone_t)main_zone;
+ _xzm_global_state_lock(main_ref, XZM_LOCK_INIT);
+}
+
+static void
+xzm_malloc_zone_destroy(xzm_malloc_zone_t zone)
+{
+ xzm_main_malloc_zone_t main_ref = zone->xzz_main_ref;
+ // It is not sane to permit destroy on the main mzone in the general case:
+ // the libsystem initializer makes allocations that need to stay live. A
+ // sufficiently early interposing allocator (e.g. ASan) might plausibly be
+ // able to do it early enough, but it doesn't do any harm to keep the main
+ // zone around empty in that case.
+ if (_xzm_malloc_zone_is_main(zone)) {
+ return;
+ }
+
+ // Hold the zone lock while modifying the zone's chunk lists to guard
+ // against possible fork() problems (e.g. a tiny chunk's walk_locked bit
+ // being set)
+ _malloc_lock_lock(&zone->xzz_lock);
+
+ // During zone destroy, all the chunks belonging to this zone should exist
+ // on zone local lists:
+ // - tiny chunks should be on the xzone's ALL list
+ // - small chunks will either be in a slot or on the partial or full list
+ // - large and huge chunks will be on the zone's chunkq large
+ // We require that clients not be allocating from or freeing to zones that
+ // are mid destroy, so we don't need to acquire any locks while pulling
+ // chunks off of these lists
+ LIST_HEAD(, xzm_slice_s) chunks_to_free;
+ LIST_INIT(&chunks_to_free);
+
+ for (int i = XZM_XZONE_INDEX_FIRST; i < zone->xzz_xzone_count; i++) {
+ xzm_xzone_t xz = &zone->xzz_xzones[i];
+ if (xz->xz_block_size > XZM_TINY_BLOCK_SIZE_MAX &&
+ !zone->xzz_small_freelist_enabled) {
+ // small xzone, pull chunks off of the batch, partial, and full list
+ xzm_slice_t chunk = xz->xz_chunkq_batch;
+ xzm_slice_t temp_chunk = NULL;
+ while (chunk) {
+ xzm_debug_assert(!chunk->xzc_bits.xzcb_preallocated);
+ LIST_INSERT_HEAD(&chunks_to_free, chunk, xzc_entry);
+ temp_chunk = *_xzm_segment_slice_meta_batch_next(zone, chunk);
+ xzm_debug_assert(_xzm_slice_meta_is_batch_pointer(zone,
+ temp_chunk));
+ xzm_debug_assert(main_ref->xzmz_batch_size);
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ // Reset its reclaim id so it does not appear to have been deferred
+ *_xzm_slice_meta_reclaim_id(zone, chunk) = VM_RECLAIM_ID_NULL;
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+ chunk = temp_chunk;
+ }
+ LIST_FOREACH_SAFE(chunk, &xz->xz_chunkq_partial, xzc_entry,
+ temp_chunk) {
+ xzm_debug_assert(!chunk->xzc_bits.xzcb_preallocated);
+ LIST_REMOVE(chunk, xzc_entry);
+ LIST_INSERT_HEAD(&chunks_to_free, chunk, xzc_entry);
+ }
+ LIST_FOREACH_SAFE(chunk, &xz->xz_chunkq_full, xzc_entry,
+ temp_chunk) {
+ xzm_debug_assert(!chunk->xzc_bits.xzcb_preallocated);
+ LIST_REMOVE(chunk, xzc_entry);
+ LIST_INSERT_HEAD(&chunks_to_free, chunk, xzc_entry);
+ }
+ // Slots
+ for (xzm_allocation_index_t j = 0; j < zone->xzz_slot_count; j++) {
+ xzm_xzone_allocation_slot_t xas =
+ _xzm_xzone_allocation_slot_for_index(zone, xz, j);
+ chunk = xas->xas_chunk;
+ if (chunk) {
+ xzm_debug_assert(!chunk->xzc_bits.xzcb_preallocated);
+ xzm_debug_assert(chunk->xzc_alloc_idx == j+1);
+ xzm_debug_assert(!chunk->xzc_bits.xzcb_on_partial_list);
+ xas->xas_chunk = NULL;
+ LIST_INSERT_HEAD(&chunks_to_free, chunk, xzc_entry);
+ }
+ }
+ // Preallocated chunks
+ LIST_FOREACH_SAFE(chunk, &xz->xz_chunkq_preallocated, xzc_entry,
+ temp_chunk)
+ {
+ LIST_REMOVE(chunk, xzc_entry);
+ // This chunk won't have an mzone set, but otherwise will be
+ // initialized enough to be freed
+ _xzm_xzone_fresh_chunk_init(xz, chunk,
+ XZM_SLICE_KIND_SMALL_CHUNK);
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ xzm_debug_assert(*_xzm_slice_meta_reclaim_id(zone, chunk) ==
+ VM_RECLAIM_ID_NULL);
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+
+ LIST_INSERT_HEAD(&chunks_to_free, chunk, xzc_entry);
+ }
+ } else {
+ // freelist xzone, empty the batch list, which will reset reclaim ids
+ xzm_slice_t chunk = NULL;
+ while ((chunk = _xzm_chunk_list_pop(zone, &xz->xz_batch_list,
+ XZM_CHUNK_LINKAGE_BATCH, NULL))) {
+ xzm_debug_assert(main_ref->xzmz_batch_size);
+ }
+ // enumerate all chunks via the ALL list
+ while ((chunk = _xzm_chunk_list_pop(zone, &xz->xz_all_list,
+ XZM_CHUNK_LINKAGE_ALL, NULL))) {
+ xzm_assert(!chunk->xzc_bits.xzcb_preallocated);
+ chunk->xzc_atomic_meta.xca_on_partial_list = false;
+ chunk->xzc_atomic_meta.xca_on_empty_list = false;
+ chunk->xzc_atomic_meta.xca_alloc_idx = XZM_SLOT_INDEX_EMPTY;
+
+ LIST_INSERT_HEAD(&chunks_to_free, chunk, xzc_entry);
+ }
+ xzm_slice_kind_t kind;
+ if (xz->xz_block_size > XZM_TINY_BLOCK_SIZE_MAX) {
+ kind = XZM_SLICE_KIND_TINY_CHUNK;
+ } else {
+ kind = XZM_SLICE_KIND_SMALL_FREELIST_CHUNK;
+ }
+
+ // Preallocated freelist chunks
+ while ((chunk = _xzm_chunk_list_pop(zone, &xz->xz_preallocated_list,
+ XZM_CHUNK_LINKAGE_MAIN, NULL))) {
+ // chunk should be madvised and reset by the free path below
+ _xzm_xzone_fresh_chunk_init(xz, chunk, kind);
+ xzm_debug_assert(chunk->xzc_atomic_meta.xca_alloc_head !=
+ XZM_FREE_MADVISED);
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ xzm_debug_assert(*_xzm_slice_meta_reclaim_id(zone, chunk) ==
+ VM_RECLAIM_ID_NULL);
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+
+ LIST_INSERT_HEAD(&chunks_to_free, chunk, xzc_entry);
+ }
+ }
+ }
+
+ _malloc_lock_unlock(&zone->xzz_lock);
+
+ xzm_slice_t span = NULL;
+ xzm_slice_t temp_span = NULL;
+ LIST_FOREACH_SAFE(span, &chunks_to_free, xzc_entry, temp_span) {
+ xzm_xzone_t xz = &zone->xzz_xzones[span->xzc_xzone_idx];
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ // If this chunk is non-sequestered and has a reclaim index, we need to
+ // get it back from the reclaim buffer so it can be returned to the span
+ // queue in xzm_chunk_free
+ if (!xz->xz_sequestered && _xzm_chunk_should_defer_reclamation(main_ref, span)) {
+ int retries = 0;
+ while (_xzm_segment_slice_is_deferred(_xzm_segment_for_slice(zone,
+ span), span)) {
+ if (xzm_chunk_mark_used(zone, span, NULL)) {
+ xzm_debug_assert(span->xzc_bits.xzcb_kind ==
+ XZM_SLICE_KIND_TINY_CHUNK ||
+ span->xzc_bits.xzcb_kind ==
+ XZM_SLICE_KIND_SMALL_FREELIST_CHUNK);
+ // We don't know if the chunk was successfully reclaimed, so
+ // reset the allocation head so that it is synchronously
+ // madvised
+ span->xzc_atomic_meta.xca_alloc_head = XZM_FREE_NULL;
+ break;
+ }
+ xzm_reclaim_force_sync(main_ref->xzmz_reclaim_buffer);
+ xzm_assert(retries < 10);
+ retries++;
+ }
+ }
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+ if (span->xzc_bits.xzcb_kind == XZM_SLICE_KIND_TINY_CHUNK ||
+ span->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK) {
+ // If the chunk is already madvised, it should already be cleared if
+ // it's below the zero on free threshold
+ if (span->xzc_atomic_meta.xca_alloc_head != XZM_FREE_MADVISED) {
+ size_t size = 0;
+ void *chunk_start = _xzm_chunk_start_ptr(zone, span, &size);
+ xzm_debug_assert(size == XZM_TINY_CHUNK_SIZE ||
+ size == XZM_SMALL_FREELIST_CHUNK_SIZE);
+ if (xz->xz_block_size <= XZM_ZERO_ON_FREE_THRESHOLD) {
+#if CONFIG_MTE
+ if (xz->xz_tagged) {
+ memtag_disable_checking();
+ bzero(chunk_start, size);
+ memtag_enable_checking();
+ } else {
+#endif
+ bzero(chunk_start, size);
+#if CONFIG_MTE
+ }
+#endif
+ }
+
+ xzm_segment_group_t sg = _xzm_segment_group_for_slice(zone,
+ span);
+ xzm_segment_group_segment_madvise_chunk(sg, span);
+ span->xzc_atomic_meta.xca_free_count = 0;
+ span->xzc_atomic_meta.xca_alloc_head = XZM_FREE_MADVISED;
+ }
+ span->xzc_bits.xzcb_on_partial_list = false;
+ _xzm_xzone_chunk_free(zone, &zone->xzz_xzones[span->xzc_xzone_idx],
+ span, false);
+ } else if (span->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_CHUNK) {
+ // Small chunks are typically, in the non-MTE configuration,
+ // madvised when their individual blocks are freed. Since we're
+ // freeing the whole chunk, we cautiously madvise the whole chunk
+ bool madvise_needed = true;
+ span->xzc_bits.xzcb_on_partial_list = false;
+ _xzm_xzone_chunk_free(zone, &zone->xzz_xzones[span->xzc_xzone_idx],
+ span, madvise_needed);
+ } else {
+ xzm_abort_with_reason("Unexpected chunk kind",
+ span->xzc_bits.xzcb_kind);
+ }
+ }
+
+ LIST_FOREACH_SAFE(span, &zone->xzz_chunkq_large, xzc_entry, temp_span) {
+ // NB: _xzm_free_large_huge removes the chunk from chunkq_large, which
+ // is why we need to use LIST_FOREACH_SAFE here
+ _xzm_free_large_huge(zone, span);
+ }
+
+ xzm_metapool_t mp = &main_ref->xzmz_metapools[XZM_METAPOOL_MZONE_IDX];
+ xzm_reused_mzone_index_t reused = xzm_metapool_alloc(mp);
+ reused->xrmi_mzone_idx = zone->xzz_mzone_idx;
+ _malloc_lock_lock(&main_ref->xzmz_mzones_lock);
+ SLIST_INSERT_HEAD(&main_ref->xzmz_reusable_mzidxq, reused, xrmi_mzone_entry);
+ _malloc_lock_unlock(&main_ref->xzmz_mzones_lock);
+
+ mvm_deallocate_plat(zone, zone->xzz_total_size, 0, NULL);
+}
+
+#pragma mark Test helpers
+
+bool
+xzm_ptr_lookup_4test(xzm_malloc_zone_t zone, void *ptr,
+ xzm_slice_kind_t *kind_out, xzm_segment_group_id_t *sgid_out,
+ xzm_xzone_bucket_t *bucket_out)
+{
+ xzm_xzone_t xz = NULL;
+ xzm_chunk_t chunk = _xzm_ptr_lookup(zone, ptr, &xz, NULL, NULL);
+ if (!chunk) {
+ return false;
+ }
+
+ xzm_slice_kind_t kind = chunk->xzc_bits.xzcb_kind;
+ *kind_out = kind;
+
+ xzm_segment_t segment = _xzm_segment_for_slice(zone, chunk);
+ *sgid_out = segment->xzs_segment_group->xzsg_id;
+
+ if (_xzm_slice_kind_uses_xzones(kind)) {
+ xzm_assert(xz);
+ *bucket_out = xz->xz_bucket;
+ }
+
+ return true;
+}
+
+uint8_t
+xzm_type_choose_ptr_bucket_4test(const xzm_bucketing_keys_t *const keys,
+ uint8_t ptr_bucket_count, malloc_type_descriptor_t type_desc)
+{
+ return _xzm_type_choose_ptr_bucket(keys, ptr_bucket_count, type_desc);
+}
+
+#pragma mark Zone creation and initialization
+
+struct xzm_process_config_s {
+ xzm_slot_config_t xzpc_slot_config;
+ bool xzpc_madvise_workaround;
+ bool xzpc_disable_vm_user_ranges;
+};
+
+typedef const struct xzm_process_config_s *xzm_process_config_t;
+
+#if CONFIG_MALLOC_PROCESS_IDENTITY
+
+static const struct xzm_process_config_s _xzm_launchd_process_config = {
+ // almost everything in launchd happens on the event queue
+ .xzpc_slot_config = XZM_SLOT_SINGLE,
+};
+
+static const struct xzm_process_config_s _xzm_notifyd_process_config = {
+ // almost everything in notifyd happens on its global workloop
+ .xzpc_slot_config = XZM_SLOT_SINGLE,
+};
+
+static const xzm_process_config_t _xzm_process_configs[MALLOC_PROCESS_COUNT] = {
+ [MALLOC_PROCESS_LAUNCHD] = &_xzm_launchd_process_config,
+ [MALLOC_PROCESS_NOTIFYD] = &_xzm_notifyd_process_config,
+};
+
+#endif // CONFIG_MALLOC_PROCESS_IDENTITY
+
+#define XZM_FRONT_RANDOM_SIZE 32
+
+static xzm_front_index_t
+_xzm_random_front_index(uint8_t front_random[static XZM_FRONT_RANDOM_SIZE],
+ size_t allocation_front_count, xzm_xzone_index_t xz_idx)
+{
+ if (allocation_front_count == 1) {
+ return XZM_FRONT_INDEX_DEFAULT;
+ }
+
+ xzm_assert(allocation_front_count == 2);
+
+ size_t byte = xz_idx / CHAR_BIT;
+ size_t bit = xz_idx % CHAR_BIT;
+
+ xzm_assert(byte < XZM_FRONT_RANDOM_SIZE);
+ return (bool)(front_random[byte] & (1 << bit));
+}
+
+// mimalloc: MI_PAGE_QUEUES_EMPTY
+static const size_t _xzm_bin_sizes[] = {
+ 16,
+ 32,
+ 48,
+ 64,
+ 80,
+ 96,
+ 112,
+ 128,
+ 160,
+ 192,
+ 224,
+ 256,
+ 320,
+ 384,
+ 448,
+ 512,
+ 640,
+ 768,
+ 896,
+ 1024,
+ 1280,
+ 1536,
+ 1792,
+ 2048,
+ 2560,
+ 3072,
+ 3584,
+ 4096,
+ 5120,
+ 6144,
+ 7168,
+ 8192,
+ 10240,
+ 12288,
+ 14336,
+ 16384,
+ 20480,
+ 24576,
+ 28672,
+ 32768,
+};
+
+// mimalloc: MI_SEGMENT_SPAN_QUEUES_EMPTY
+static const size_t _xzm_span_queue_slice_counts[] = {
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 6,
+ 7,
+ 10,
+ 12,
+ 14,
+ 16,
+ 20,
+ 24,
+ 28,
+ 32,
+ 40,
+ 48,
+ 56,
+ 64,
+ 80,
+ 96,
+ 112,
+ 128,
+ 160,
+ 192,
+ 224,
+ 256,
+};
+
+MALLOC_STATIC_ASSERT(
+ countof_unsafe(_xzm_span_queue_slice_counts) == XZM_SPAN_QUEUE_COUNT,
+ "all span queues have slice counts");
+
+static const char ptr_buckets_boot_arg[] = "xzone_ptr_buckets";
+static const char xzone_slot_config_boot_arg[] = "malloc_xzone_slot_config";
+static const char xzone_guard_pages_boot_arg[] = "xzone_guard_pages";
+
+static void
+_xzm_initialize_const_zone_data(xzm_malloc_zone_t zone,
+ size_t size, xzm_mzone_index_t mzone_idx, size_t xzone_count,
+ size_t slot_count, xzm_xzone_t xzones,
+ xzm_xzone_allocation_slot_t slots, xzm_main_malloc_zone_t main_ref,
+ xzm_slot_config_t initial_slot_config, uint32_t initial_slot_threshold,
+ xzm_slot_config_t max_slot_config,
+ uint32_t list_upgrade_threshold_single,
+ uint32_t list_upgrade_threshold_cluster, uint32_t list_upgrade_period,
+ uint32_t slot_upgrade_threshold_single,
+ uint32_t slot_upgrade_threshold_cluster, uint32_t slot_upgrade_period,
+ uint64_t tiny_thrash_threshold, uint64_t small_thrash_threshold,
+ uint64_t small_thrash_limit_size, uint64_t debug_flags,
+ bool small_freelist, xzm_slot_config_t max_list_config,
+ xzm_chunk_list_t partial_lists)
+{
+ // We're making some assumptions about layout here, but I think that's fine
+ xzm_debug_assert((uintptr_t)zone + size >= (uintptr_t)slots +
+ sizeof(struct xzm_xzone_allocation_slot_s) * xzone_count * slot_count);
+ xzm_debug_assert((uintptr_t)slots >= (uintptr_t)xzones +
+ sizeof(struct xzm_xzone_s) * xzone_count);
+
+ *zone = (struct xzm_malloc_zone_s){
+ .xzz_basic_zone = {
+ .version = 16,
+ .size = (void *)xzm_malloc_zone_size,
+ .malloc = (void *)xzm_malloc_zone_malloc,
+ .calloc = (void *)xzm_malloc_zone_calloc,
+ .valloc = (void *)xzm_malloc_zone_valloc,
+ .free = (void *)xzm_malloc_zone_free,
+ .realloc = (void *)xzm_malloc_zone_realloc,
+ .destroy = (void *)xzm_malloc_zone_destroy,
+ // Note: batch_malloc and batch_free introduce way too much
+ // complexity and maintenance burden to be worth whatever
+ // marginal performance benefit they may offer, so we don't
+ // really want to implement them.
+ //
+ // Technically they're supposed to be optional, so ideally we'd
+ // just leave them NULL. However, we've never had a default
+ // zone that didn't have them, so we open ourselves up to
+ // compatibility risks with code that either isn't prepared to
+ // deal with getting 0 or isn't prepared to wrap a zone that
+ // doesn't have an implementation. Even within the project
+ // we depend on the default zone having an implementation: the
+ // virtual default zone calls through without checking for NULL. So,
+ // rather than leaving them NULL, we keep trivial implementations
+ // that just wrap plain malloc and free.
+ .batch_malloc = malloc_zone_batch_malloc_fallback,
+ .batch_free = malloc_zone_batch_free_fallback,
+ .introspect = (struct malloc_introspection_t
+ *)&xzm_malloc_zone_introspect,
+ .memalign = (void *)xzm_malloc_zone_memalign,
+ .free_definite_size = (void *)xzm_malloc_zone_free_definite_size,
+ .pressure_relief = (void *)malloc_zone_pressure_relief_fallback,
+ .claimed_address = (void *)xzm_malloc_zone_claimed_address,
+ .try_free_default = (void *)xzm_malloc_zone_try_free_default,
+ .malloc_with_options = (void *)xzm_malloc_zone_malloc_with_options,
+
+ .malloc_type_malloc = (void *)xzm_malloc_zone_malloc_type_malloc,
+ .malloc_type_calloc = (void *)xzm_malloc_zone_malloc_type_calloc,
+ .malloc_type_realloc = (void *)xzm_malloc_zone_malloc_type_realloc,
+ .malloc_type_memalign = (void *)xzm_malloc_zone_malloc_type_memalign,
+ .malloc_type_malloc_with_options =
+ (void *)xzm_malloc_zone_malloc_type_malloc_with_options,
+ },
+ .xzz_total_size = size,
+ .xzz_mzone_idx = mzone_idx,
+ .xzz_xzone_count = xzone_count,
+ .xzz_slot_count = slot_count,
+ .xzz_xzones = xzones,
+ .xzz_xzone_allocation_slots = slots,
+ .xzz_partial_lists = partial_lists,
+ .xzz_main_ref = main_ref,
+ .xzz_max_list_config = max_list_config,
+ .xzz_initial_slot_config = initial_slot_config,
+ .xzz_max_slot_config = max_slot_config,
+ .xzz_small_freelist_enabled = small_freelist,
+ .xzz_list_upgrade_threshold = {
+ list_upgrade_threshold_single,
+ list_upgrade_threshold_cluster,
+ },
+ .xzz_list_upgrade_period = list_upgrade_period,
+ .xzz_slot_initial_threshold = initial_slot_threshold,
+ .xzz_slot_upgrade_threshold = {
+ slot_upgrade_threshold_single,
+ slot_upgrade_threshold_cluster,
+ },
+ .xzz_slot_upgrade_period = slot_upgrade_period,
+ .xzz_tiny_thrash_threshold = tiny_thrash_threshold,
+ .xzz_small_thrash_threshold = small_thrash_threshold,
+ .xzz_small_thrash_limit_size = small_thrash_limit_size,
+ .xzz_lock = _MALLOC_LOCK_INIT,
+ .xzz_fork_lock = _MALLOC_LOCK_INIT,
+ .xzz_flags = debug_flags,
+ };
+
+ bool use_slowpath_zone_functions = false;
+ if (debug_flags & MALLOC_DO_SCRIBBLE ||
+ debug_flags & MALLOC_PURGEABLE) {
+ use_slowpath_zone_functions = true;
+ }
+
+ if (use_slowpath_zone_functions) {
+ zone->xzz_basic_zone.malloc = (void *)xzm_malloc_zone_malloc_slow;
+ zone->xzz_basic_zone.calloc = (void *)xzm_malloc_zone_calloc_slow;
+ zone->xzz_basic_zone.valloc = (void *)xzm_malloc_zone_valloc_slow;
+ zone->xzz_basic_zone.free = (void *)xzm_malloc_zone_free_slow;
+ zone->xzz_basic_zone.realloc = (void *)xzm_malloc_zone_realloc_slow;
+ zone->xzz_basic_zone.memalign = (void *)xzm_malloc_zone_memalign_slow;
+ zone->xzz_basic_zone.free_definite_size =
+ (void *)xzm_malloc_zone_free_definite_size_slow;
+ zone->xzz_basic_zone.try_free_default =
+ (void *)xzm_malloc_zone_try_free_default_slow;
+ zone->xzz_basic_zone.malloc_with_options =
+ (void *)xzm_malloc_zone_malloc_with_options_slow;
+ zone->xzz_basic_zone.malloc_type_malloc =
+ (void *)xzm_malloc_zone_malloc_type_malloc_slow;
+ zone->xzz_basic_zone.malloc_type_calloc =
+ (void *)xzm_malloc_zone_malloc_type_calloc_slow;
+ zone->xzz_basic_zone.malloc_type_realloc =
+ (void *)xzm_malloc_zone_malloc_type_realloc_slow;
+ zone->xzz_basic_zone.malloc_type_memalign =
+ (void *)xzm_malloc_zone_malloc_type_memalign_slow;
+ zone->xzz_basic_zone.malloc_type_malloc_with_options =
+ (void *)xzm_malloc_zone_malloc_type_malloc_with_options_slow;
+ }
+}
+
+static void
+_xzm_initialize_xzone_data(xzm_malloc_zone_t zone,
+ xzm_slot_config_t list_config, xzm_guard_page_config_t guard_config,
+ uint8_t front_random[XZM_FRONT_RANDOM_SIZE], bool all_data)
+{
+ xzm_main_malloc_zone_t main_ref = _xzm_malloc_zone_main(zone);
+ bool is_main = _xzm_malloc_zone_is_main(zone);
+
+ xzm_debug_assert((is_main && front_random) || (!is_main && !front_random));
+
+ // NOTE: although this is technically a layering violation with Libc and
+ // CoreCrypto, we and they believe it to be safe
+ //
+ // TODO: allow overriding with a fixed value for testing
+ uint64_t freelist_cookie = 0;
+ arc4random_buf(&freelist_cookie, sizeof(freelist_cookie));
+ if (!freelist_cookie) {
+ freelist_cookie = 0xdeaddeaddeaddeadull;
+ }
+#if CONFIG_MTE
+ if (zone->xzz_memtag_config.enabled) {
+ // Clear the tag bits from the freelist cookie to ensure they can always be
+ // used to store a block's tag
+ freelist_cookie = (uint64_t)memtag_strip_address(
+ (uint8_t *)freelist_cookie);
+ }
+#endif
+ zone->xzz_freelist_cookie = freelist_cookie;
+
+ xzm_xzone_index_t xzidx = XZM_XZONE_INDEX_FIRST;
+ for (size_t i = 0; i < countof(_xzm_bin_sizes); i++) {
+ if (is_main) {
+ main_ref->xzmz_xzone_bin_offsets[i] = xzidx;
+ }
+ size_t bin_buckets = main_ref->xzmz_xzone_bin_bucket_counts[i];
+ for (int bucket = 0; bucket < bin_buckets; xzidx++, bucket++) {
+ xzm_xzone_t xz = &zone->xzz_xzones[xzidx];
+ size_t block_size = main_ref->xzmz_xzone_bin_sizes[i];
+ size_t chunk_size;
+ if (block_size <= XZM_TINY_BLOCK_SIZE_MAX) {
+ chunk_size = XZM_TINY_CHUNK_SIZE;
+ } else if (zone->xzz_small_freelist_enabled) {
+ chunk_size = XZM_SMALL_FREELIST_CHUNK_SIZE;
+ } else {
+ chunk_size = XZM_SMALL_CHUNK_SIZE;
+ }
+
+ size_t chunk_capacity = chunk_size / block_size;
+
+ // The early budget is only in the main zone xzones, all other
+ // zones decrement those
+ uint16_t early_budget = 0;
+ if (is_main && main_ref->xzmz_mfm_address) {
+ if (block_size <= 256) {
+ early_budget = 2048 / block_size;
+ } else if (block_size <= 512) {
+ early_budget = 4096 / block_size;
+ } else if (block_size <= 2048) {
+ early_budget = 8192 / block_size;
+ } else if (block_size <= 8192) {
+ early_budget = 1;
+ }
+ }
+
+ xzm_segment_group_id_t sgid;
+ bool sequestered;
+ if ((bucket == XZM_XZONE_BUCKET_DATA || all_data) &&
+#if XZM_NARROW_BUCKETING
+ !main_ref->xzmz_narrow_bucketing &&
+#endif
+ !XZM_BUCKET_POINTER_ONLY) {
+ sgid = XZM_SEGMENT_GROUP_DATA;
+ sequestered = false;
+ } else {
+ sgid = XZM_SEGMENT_GROUP_POINTER_XZONES;
+ sequestered = true;
+ }
+
+ // Enable sequestering for small chunks to avoid coalescing with
+ // adjacent chunks that may not already be marked as free
+ if (block_size > XZM_TINY_BLOCK_SIZE_MAX &&
+ main_ref->xzmz_defer_small) {
+ sequestered = true;
+ }
+
+ xzm_front_index_t front;
+ if (is_main) {
+ if (sgid == XZM_SEGMENT_GROUP_POINTER_XZONES) {
+ front = _xzm_random_front_index(front_random,
+ main_ref->xzmz_allocation_front_count, xzidx);
+ } else {
+ xzm_debug_assert(sgid == XZM_SEGMENT_GROUP_DATA);
+ front = XZM_FRONT_INDEX_DEFAULT;
+ }
+ } else {
+ xzm_xzone_t main_xz = &main_ref->xzmz_base.xzz_xzones[xzidx];
+ front = main_xz->xz_front;
+ }
+
+ uint8_t run_length = 0;
+ uint8_t guard_density = 0;
+ if (guard_config->xgpc_enabled) {
+ if (guard_config->xgpc_enabled_for_data) {
+ // To avoid unbounded VA growth caused by unusable free
+ // spans in the data segments, all xzones are sequestered
+ // when guards are enabled
+ sequestered = true;
+ }
+
+ if (sequestered) {
+ if (block_size <= XZM_TINY_BLOCK_SIZE_MAX) {
+ run_length = guard_config->xgpc_max_run_tiny;
+ guard_density = guard_config->xgpc_tiny_guard_density;
+ } else {
+ run_length = guard_config->xgpc_max_run_small;
+ guard_density = guard_config->xgpc_small_guard_density;
+ }
+ }
+ }
+
+ *xz = (struct xzm_xzone_s){
+ .xz_segment_group_id = sgid,
+ .xz_front = front,
+ .xz_block_size = block_size,
+ .xz_quo_magic = XZM_MAGIC_QUO(block_size),
+ .xz_align_magic = XZM_MAGIC_ALIGNED(block_size),
+ .xz_chunk_capacity = (uint32_t)chunk_capacity,
+ .xz_lock = _MALLOC_LOCK_INIT,
+ .xz_early_budget = early_budget,
+ .xz_idx = xzidx,
+ .xz_mzone_idx = zone->xzz_mzone_idx,
+ .xz_bucket = (xzm_xzone_bucket_t)bucket,
+ .xz_sequestered = sequestered,
+ .xz_slot_config = XZM_SLOT_SINGLE,
+ .xz_guard_config = {
+ .xxgc_max_run_length = run_length,
+ .xxgc_density = guard_density,
+ }
+ };
+
+ // Initialize all possible slots with the initial slot config
+ xzm_debug_assert(zone->xzz_slot_count ==
+ _xzm_get_limit_allocation_index(zone->xzz_max_slot_config));
+ xzm_allocation_index_t limit_list =
+ _xzm_get_limit_allocation_index(zone->xzz_max_list_config);
+ for (xzm_allocation_index_t i = 0; i < zone->xzz_slot_count;
+ ++i) {
+ xzm_xzone_allocation_slot_t xas =
+ _xzm_xzone_allocation_slot_for_index(zone, xz, i);
+ xas->xas_counters.xsc_slot_config = xz->xz_slot_config;
+#if CONFIG_TINY_ALLOCATION_SLOT_LOCK
+ _malloc_lock_init(&xas->xas_lock);
+#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
+
+ if (i >= limit_list) {
+ continue;
+ }
+
+ xzm_chunk_list_t xcl =
+ _xzm_xzone_chunk_list_for_index(zone, xz,
+ zone->xzz_partial_lists, i);
+ xcl->xcl_counters.xsc_slot_config = xz->xz_list_config;
+ }
+
+#if CONFIG_MTE
+ xz->xz_tagged = _xzm_zone_memtag_block(zone, block_size,
+ /*data=*/(bucket == XZM_XZONE_BUCKET_DATA));
+
+ // Disable the early allocator for zones for which we don't
+ // enable tagging. This is because MFM will always provide
+ // tagged memory, and this might cause the client to violate
+ // MTE mapping policies that are expected to hold for
+ // data-only allocations, which are normally not tagged in
+ // the default configuration.
+ // `zone->xzz_memtag_config.enabled` is used as proxy for
+ // `mfm_memtag_enabled`.
+ if (zone->xzz_memtag_config.enabled && !xz->xz_tagged) {
+ xz->xz_early_budget = 0;
+ }
+#endif // CONFIG_MTE
+ }
+ }
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static const char *
+_xzm_development_only_getenv(const char **envp, const char *key)
+{
+ return malloc_internal_security_policy ? _simple_getenv(envp, key) : NULL;
+}
+
+MALLOC_ALWAYS_INLINE MALLOC_INLINE
+static const char *
+_xzm_production_getenv(const char **envp, const char *key)
+{
+ return _simple_getenv(envp, key);
+}
+
+#ifdef _simple_getenv
+#undef _simple_getenv
+#endif
+
+#define _simple_getenv(...) \
+ static_assert(0, "Use _xzm_(development_only|release)_getenv instead")
+
+
+malloc_zone_t *
+xzm_main_malloc_zone_create(unsigned debug_flags, const char **envp,
+ const char **apple, const char *bootargs)
+{
+ bool security_critical = false, security_critical_max_perf = false;
+#if CONFIG_MALLOC_PROCESS_IDENTITY
+ security_critical = malloc_process_is_security_critical(
+ malloc_process_identity);
+ security_critical_max_perf = malloc_process_is_security_critical_max_perf(
+ malloc_process_identity);
+#endif // CONFIG_MALLOC_PROCESS_IDENTITY
+
+ uint8_t front_random[XZM_FRONT_RANDOM_SIZE];
+
+ xzm_bucketing_keys_t bucketing_keys;
+ // executable_boothash is a salted hash of the concatenation of the current
+ // boot session UUID and cdhash of the main executable
+ const char *boothash = _xzm_production_getenv(apple, "executable_boothash");
+ if (!boothash) {
+ // executable_boothash isn't populated when the executable isn't
+ // codesigned (rdar://118451590). This should be very rare (only when
+ // the system is running with reduced security), so to guard against
+ // accidentally breaking the boothash, crash in known security-critical
+ // processes
+ if (security_critical) {
+ xzm_abort("couldn't find executable_boothash");
+ }
+
+ arc4random_buf(&bucketing_keys, sizeof(bucketing_keys));
+ } else {
+ size_t boothash_len = strlen(boothash);
+ if (boothash_len < 32) {
+ xzm_abort_with_reason("invalid executable_boothash length",
+ boothash_len);
+ }
+
+ const size_t part_len = 16;
+ char boothash_part[part_len + 1];
+ unsigned long long value;
+ const size_t keys_cnt = sizeof(bucketing_keys.xbk_key_data) /
+ sizeof(bucketing_keys.xbk_key_data[0]);
+
+ for (size_t i = 0; i < keys_cnt; i++) {
+ memcpy(boothash_part, &boothash[i * part_len], part_len);
+ boothash_part[part_len] = '\0';
+
+ value = strtoull(boothash_part, NULL, 16);
+ if ((value == 0 && errno == EINVAL) ||
+ (value == ULLONG_MAX && errno == ERANGE)) {
+ xzm_abort("invalid executable_boothash string");
+ }
+ bucketing_keys.xbk_key_data[i] = value;
+ }
+ }
+
+ // XXX We need a bunch of additional per-boot entropy for allocation front
+ // assignments. It may make sense to increase the amount of per-boot
+ // entropy supplied by the kernel, but for now we make do by deriving
+ // from what we've already got.
+ //
+ // TODO: If we do continue to derive, we should probably be seeding a PRNG
+ // instead
+ static_assert(sizeof(front_random) == CCSHA256_OUTPUT_SIZE,
+ "front_random size");
+
+ const struct ccdigest_info *di = ccsha256_di();
+ ccdigest_di_decl(di, dc);
+ ccdigest_init(di, dc);
+ char diversifier[] = "xzone malloc front random";
+ ccdigest_update(di, dc, sizeof(diversifier), diversifier);
+ ccdigest_update(di, dc, sizeof(bucketing_keys),
+ bucketing_keys.xbk_key_data);
+ ccdigest_final(di, dc, front_random);
+ ccdigest_di_clear(di, dc);
+
+
+ // TODO: scrub executable_boothash from the apple array like we do for
+ // malloc_entropy?
+
+ xzm_process_config_t process_config = NULL;
+#if CONFIG_MALLOC_PROCESS_IDENTITY
+ if (malloc_process_identity != MALLOC_PROCESS_NONE) {
+ process_config = _xzm_process_configs[malloc_process_identity];
+ }
+#endif // CONFIG_MALLOC_PROCESS_IDENTITY
+
+ bool use_slowpath_zone_functions = false;
+ if (debug_flags & MALLOC_DO_SCRIBBLE) {
+ use_slowpath_zone_functions = true;
+ }
+
+ xzm_slot_config_t max_slot_config = XZM_SLOT_CPU;
+
+ size_t override_ptr_bucket_count = 0;
+#if XZM_NARROW_BUCKETING
+ // Platforms that support narrow bucketing default to using it
+ bool narrow_bucketing = true;
+#endif
+
+ char value_buf[256];
+ const char *flag = malloc_common_value_for_key_copy(bootargs,
+ ptr_buckets_boot_arg, value_buf, sizeof(value_buf));
+ if (flag) {
+ const char *endp;
+ long value = malloc_common_convert_to_long(flag, &endp);
+ if (!*endp && value > 0 && value <= XZM_POINTER_BUCKETS_MAX) {
+ override_ptr_bucket_count = (size_t)value;
+ } else {
+ malloc_report(ASL_LEVEL_ERR,
+ "Invalid xzone_ptr_buckets value %ld - ignored.\n", value);
+ }
+ }
+
+#if MALLOC_TARGET_IOS_ONLY || TARGET_OS_WATCH
+ if (!security_critical) {
+#if MALLOC_TARGET_IOS_ONLY
+ override_ptr_bucket_count = 2;
+#elif TARGET_OS_WATCH
+ override_ptr_bucket_count = 1;
+#endif
+ }
+#endif
+
+ // There are a few ways MallocMaxMagazines might have been set:
+ // - JetsamProperties sets it for a number of daemons on iOS
+ // - A handful of projects found the envvar on their own and set it
+ // themselves in their launchd plists
+ //
+ // Pending an audit and rebalancing of bucketing by security sensitivity,
+ // give processes that have this set and that we don't otherwise have
+ // explicit policy for a reduced bucket configuration.
+ //
+ // If set to 1, we should also give them a reduced slot config.
+ bool allow_malloc_max_magazines = true;
+ // Note: this is load-bearing for wifip2pd
+ if (security_critical) {
+ allow_malloc_max_magazines = false;
+#if TARGET_OS_VISION
+ // On visionOS, don't clamp the security-critical-max-perf processes to
+ // per-cluster scaling
+ if (!security_critical_max_perf) {
+ max_slot_config = XZM_SLOT_CLUSTER;
+ }
+#else
+ // On the other platforms, since we're already clamping the
+ // security-critical-max-perf processes to per-cluster scaling without
+ // issue, clamp in all cases
+ max_slot_config = XZM_SLOT_CLUSTER;
+#endif
+ }
+ if (allow_malloc_max_magazines) {
+ flag = _xzm_production_getenv(envp, "MallocMaxMagazines");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 1) {
+ max_slot_config = XZM_SLOT_SINGLE;
+ } else if (value == 2 || value == UINT16_MAX) {
+ max_slot_config = XZM_SLOT_CLUSTER;
+ }
+
+#if MALLOC_TARGET_IOS_ONLY || MALLOC_TARGET_DK_IOS || \
+ TARGET_OS_OSX || MALLOC_TARGET_DK_OSX
+ if (value == 1 || value == 2 || value == UINT16_MAX) {
+ override_ptr_bucket_count = 1;
+ }
+#endif
+ }
+ }
+
+ flag = _xzm_development_only_getenv(envp, "MallocXzonePtrBucketCount");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value <= XZM_POINTER_BUCKETS_MAX) {
+ override_ptr_bucket_count = (size_t)value;
+#if XZM_NARROW_BUCKETING
+ narrow_bucketing = false;
+#endif
+ }
+ }
+
+ size_t ptr_bucket_count = override_ptr_bucket_count ?:
+ XZM_XZONE_DEFAULT_POINTER_BUCKET_COUNT;
+
+ size_t bucket_base = XZM_XZONE_BUCKET_POINTER_BASE;
+#if XZM_NARROW_BUCKETING
+ if (narrow_bucketing) {
+ bucket_base = 0;
+ }
+#endif
+
+ bool use_early_allocator = true;
+#if XZM_NARROW_BUCKETING
+ if (narrow_bucketing && ptr_bucket_count == 1) {
+ use_early_allocator = false;
+ }
+#endif
+
+ flag = _xzm_production_getenv(envp, "MallocXzoneEarlyAlloc");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 0 || value == 1) {
+ use_early_allocator = (bool)value;
+ } else {
+ malloc_report(ASL_LEVEL_ERR,
+ "MallocXzoneEarlyAlloc must be 0 or 1.\n");
+ }
+ }
+
+ if (use_early_allocator) {
+ mfm_initialize();
+ }
+
+ size_t buckets_per_bin = bucket_base + ptr_bucket_count;
+
+ size_t bin_count = countof(_xzm_bin_sizes);
+ size_t xzone_count = 1 + bin_count * buckets_per_bin;
+ xzm_assert(xzone_count <= UINT8_MAX);
+
+ bool madvise_workaround = false;
+ if (process_config && process_config->xzpc_madvise_workaround) {
+ madvise_workaround = true;
+ }
+
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneMadviseWorkaround");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 0 || value == 1) {
+ madvise_workaround = (bool)value;
+ }
+ }
+
+#if CONFIG_MTE
+ struct xzm_memtag_config_s memtag_config = {0};
+
+ // If the process has been spawned by setting has_sec_transition=1,
+ // load a default configuration for MTE support in xzone.
+ // Note that we still allow overriding the configuration through
+ // environment variables.
+ if (malloc_has_sec_transition) {
+ memtag_config.enabled = true;
+ memtag_config.tag_data = false;
+ memtag_config.max_block_size = XZM_SMALL_BLOCK_SIZE_MAX;
+ if (malloc_sec_transition_policy & /*TASK_SEC_POLICY_USER_DATA*/0x02) {
+ memtag_config.tag_data = true;
+ }
+ }
+
+ uint64_t max_supported_memtag_block_size = XZM_SMALL_BLOCK_SIZE_MAX;
+
+ // MTE debug mode is available on public builds
+ flag = _xzm_production_getenv(envp, "MallocTagAll");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 1) {
+ memtag_config.tag_data = true;
+ memtag_config.max_block_size = max_supported_memtag_block_size;
+
+ if (!malloc_has_sec_transition) {
+ malloc_report(MALLOC_REPORT_CRASH,
+ "Malloc MTE debug mode (MallocTagAll=1) requires the "
+ "process to be started with MTE enabled.\n");
+ }
+ }
+ }
+
+ flag = _xzm_development_only_getenv(envp, "MallocTagAllInternal");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 1) {
+ memtag_config.tag_data = true;
+ memtag_config.max_block_size = max_supported_memtag_block_size;
+ }
+ // Skip sanity check for general MTE enablement.
+ }
+
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneMemtagEnable");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 0 || value == 1) {
+ memtag_config.enabled = (bool)value;
+ }
+ }
+
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneMemtagTagData");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 0 || value == 1) {
+ memtag_config.tag_data = (bool)value;
+ }
+ }
+
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneMemtagMaxBlockSize");
+ if (flag) {
+ unsigned long long value = strtoull(flag, NULL, 10);
+ // TODO: allow all sizes?
+ if (value <= XZM_SMALL_BLOCK_SIZE_MAX && !(value & 0xf)) {
+ memtag_config.max_block_size = value;
+ }
+ }
+#endif // CONFIG_MTE
+
+ bool has_vm_user_ranges = true;
+ if (process_config && process_config->xzpc_disable_vm_user_ranges) {
+ has_vm_user_ranges = false;
+ }
+
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneHasRanges");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 0 || value == 1) {
+ has_vm_user_ranges = (bool)value;
+ }
+ }
+
+#if XZM_NARROW_BUCKETING
+ // Don't waste a PTE trying to separate data at the range level if it's
+ // mixed into the same xzones anyway
+ if (narrow_bucketing && !security_critical) {
+ has_vm_user_ranges = false;
+ }
+#endif
+
+ bool thread_caching = false;
+
+#if CONFIG_NANOZONE
+ bool nano_config = (_malloc_engaged_nano == NANO_V2);
+
+#if MALLOC_TARGET_DK_OSX
+ // TODO: clean up macOS DriverKit nano enablement config
+ nano_config = false;
+#endif
+
+ bool security_critical_allows_nano_config = false;
+#if CONFIG_MALLOC_PROCESS_IDENTITY
+ if (malloc_process_identity == MALLOC_PROCESS_HARDENED_HEAP_CONFIG) {
+ security_critical_allows_nano_config = true;
+ }
+#endif
+
+ if (security_critical && !security_critical_allows_nano_config) {
+ // Load-bearing for MTLCompilerService
+ nano_config = false;
+ }
+
+ if (nano_config) {
+ max_slot_config = XZM_SLOT_CPU;
+
+ // Not dealing with thread caching in the simulator yet
+#if !TARGET_OS_SIMULATOR
+#if CONFIG_FEATUREFLAGS_SIMPLE
+ thread_caching = os_feature_enabled_simple(libmalloc,
+ SecureAllocator_ThreadCaching, false);
+#else
+ // Not for DriverKit yet
+#endif // CONFIG_FEATUREFLAGS_SIMPLE
+#endif // !TARGET_OS_SIMULATOR
+ }
+
+ flag = _xzm_production_getenv(envp, "MallocXzoneThreadCaching");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 0 || value == 1) {
+ thread_caching = (value == 1);
+ } else {
+ malloc_report(ASL_LEVEL_ERR,
+ "MallocXzoneThreadCaching must be one of 0,1 - got %ld\n",
+ value);
+ }
+ }
+#endif // CONFIG_NANOZONE
+
+ if (process_config && process_config->xzpc_slot_config != XZM_SLOT_LAST) {
+ max_slot_config = process_config->xzpc_slot_config;
+ }
+
+ flag = malloc_common_value_for_key_copy(bootargs,
+ xzone_slot_config_boot_arg, value_buf, sizeof(value_buf));
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value < XZM_SLOT_LAST) {
+ max_slot_config = (xzm_slot_config_t)value;
+ }
+ }
+
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneSlotConfig");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value < XZM_SLOT_LAST) {
+ max_slot_config = (xzm_slot_config_t)value;
+ }
+ }
+
+ xzm_slot_config_t slot_config = XZM_SLOT_SINGLE;
+ uint32_t slot_threshold = 128;
+ uint8_t chunk_threshold = 1;
+#if TARGET_OS_OSX
+ // On macOS, not known processes get initial per-cluster
+ if (!(security_critical && !security_critical_max_perf) &&
+ !malloc_space_efficient_enabled) {
+ slot_config = XZM_SLOT_CLUSTER;
+ }
+#endif // TARGET_OS_OSX
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneInitialSlotConfig");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value < XZM_SLOT_LAST) {
+ slot_config = (xzm_slot_config_t)value;
+ }
+ }
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneInitialSlotThreshold");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value < UINT32_MAX) {
+ slot_threshold = (uint32_t)value;
+ }
+ }
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneInitialChunkThreshold");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value > 0 && value < UINT8_MAX) {
+ chunk_threshold = (uint8_t)value;
+ }
+ }
+
+ uint32_t list_upgrade_threshold_single = 32;
+ uint32_t list_upgrade_threshold_cluster = 128;
+ uint32_t slot_upgrade_threshold_single = 64;
+ uint32_t slot_upgrade_threshold_cluster = 256;
+ flag = _xzm_development_only_getenv(envp,
+ "MallocXzoneListUpgradeThreshold");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value < UINT32_MAX) {
+ list_upgrade_threshold_single = (uint32_t)value;
+ list_upgrade_threshold_cluster = (uint32_t)value;
+ }
+ }
+
+ flag = _xzm_development_only_getenv(envp,
+ "MallocXzoneListUpgradeThresholdSingle");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value < UINT32_MAX) {
+ list_upgrade_threshold_single = (uint32_t)value;
+ }
+ }
+
+ flag = _xzm_development_only_getenv(envp,
+ "MallocXzoneListUpgradeThresholdCluster");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value < UINT32_MAX) {
+ list_upgrade_threshold_cluster = (uint32_t)value;
+ }
+ }
+
+ flag = _xzm_development_only_getenv(envp,
+ "MallocXzoneSlotUpgradeThreshold");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value < UINT32_MAX) {
+ slot_upgrade_threshold_single = (uint32_t)value;
+ slot_upgrade_threshold_cluster = (uint32_t)value;
+ }
+ }
+
+ flag = _xzm_development_only_getenv(envp,
+ "MallocXzoneSlotUpgradeThresholdSingle");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value < UINT32_MAX) {
+ slot_upgrade_threshold_single = (uint32_t)value;
+ }
+ }
+
+ flag = _xzm_development_only_getenv(envp,
+ "MallocXzoneSlotUpgradeThresholdCluster");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value < UINT32_MAX) {
+ slot_upgrade_threshold_cluster = (uint32_t)value;
+ }
+ }
+
+ // Reset contention counters every 1K allocations to prevent long-lived
+ // processes from slowly accruing additional slots
+ uint32_t list_upgrade_period = 512;
+ uint32_t slot_upgrade_period = 1024;
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneListUpgradePeriod");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value < UINT32_MAX) {
+ list_upgrade_period = (uint32_t)value;
+ }
+ }
+
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneSlotUpgradePeriod");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value < UINT32_MAX) {
+ slot_upgrade_period = (uint32_t)value;
+ }
+ }
+
+ uint8_t slot_count = 0;
+ switch (max_slot_config) {
+ case XZM_SLOT_SINGLE:
+ slot_count = 1;
+ break;
+ case XZM_SLOT_CLUSTER:
+#if !CONFIG_XZM_CLUSTER_AWARE
+ slot_count = MIN(2, logical_ncpus);
+ break;
+#else // !CONFIG_XZM_CLUSTER_AWARE
+ if (ncpuclusters > 1) {
+ slot_count = (uint8_t)ncpuclusters;
+ break;
+ }
+ MALLOC_FALLTHROUGH;
+#endif // !CONFIG_XZM_CLUSTER_AWARE
+ case XZM_SLOT_CPU:
+ max_slot_config = XZM_SLOT_CPU; // handle fallthrough
+ slot_count = (uint8_t)logical_ncpus;
+ break;
+ default:
+ xzm_abort("Invalid xzone slot config");
+ }
+ if (slot_config > max_slot_config) {
+ slot_config = max_slot_config;
+ }
+
+ const uint64_t nsec_per_msec = 1000000ull;
+ mach_timebase_info_data_t tb_info;
+ kern_return_t kr = mach_timebase_info(&tb_info);
+ if (kr) {
+ xzm_abort_with_reason("mach_timebase_info failed", kr);
+ }
+
+ uint64_t small_thrash_threshold = 0;
+ uint64_t small_thrash_limit_size = 0;
+
+ uint64_t tiny_thrash_threshold_ns = 1 * nsec_per_msec;
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneTinyThrashThreshold");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value < UINT32_MAX) {
+ tiny_thrash_threshold_ns = (uint64_t)value * nsec_per_msec;
+ }
+ }
+ uint64_t tiny_thrash_threshold =
+ (tiny_thrash_threshold_ns * tb_info.denom / tb_info.numer);
+
+ uint64_t small_thrash_threshold_ns = 0;
+ uint64_t small_thrash_threshold_default_ns = 1 * nsec_per_msec;
+
+ // Cap to the size limit for szone SMALL
+ small_thrash_limit_size = KiB(16);
+
+#if CONFIG_NANOZONE
+ if (nano_config) {
+ small_thrash_threshold_ns = small_thrash_threshold_default_ns;
+ }
+#else
+ (void)small_thrash_threshold_default_ns;
+#endif
+
+#if CONFIG_MTE
+ if (memtag_config.enabled) {
+ small_thrash_threshold_ns = small_thrash_threshold_default_ns;
+ }
+#endif
+
+ flag = _xzm_development_only_getenv(envp,
+ "MallocXzoneSmallThrashThreshold");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value < UINT32_MAX) {
+ small_thrash_threshold_ns = (uint64_t)value * nsec_per_msec;
+ }
+ }
+
+ small_thrash_threshold =
+ (small_thrash_threshold_ns * tb_info.denom / tb_info.numer);
+
+ flag = _xzm_development_only_getenv(envp,
+ "MallocXzoneSmallThrashLimitSize");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value <= XZM_SMALL_BLOCK_SIZE_MAX) {
+ small_thrash_limit_size = (uint64_t)value;
+ }
+ }
+
+ uint32_t thread_cache_activation_period = 16384;
+ uint32_t thread_cache_activation_contentions = 256;
+ uint64_t thread_cache_activation_time = 0;
+#if CONFIG_XZM_THREAD_CACHE
+ flag = _xzm_production_getenv(envp, "MallocXzoneThreadCacheActivationPeriod");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value < UINT32_MAX) {
+ thread_cache_activation_period = (uint32_t)value;
+ }
+ }
+
+ flag = _xzm_production_getenv(envp,
+ "MallocXzoneThreadCacheActivationContentions");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value < UINT32_MAX) {
+ thread_cache_activation_contentions = (uint32_t)value;
+ }
+ }
+
+ uint64_t thread_cache_activation_time_ns = 1000 * nsec_per_msec;
+ flag = _xzm_production_getenv(envp, "MallocXzoneThreadCacheActivationTime");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value < UINT64_MAX) {
+ thread_cache_activation_time_ns = (uint64_t)value * nsec_per_msec;
+ }
+ }
+
+ thread_cache_activation_time =
+ (thread_cache_activation_time_ns * tb_info.denom / tb_info.numer);
+#endif // CONFIG_XZM_THREAD_CACHE
+
+#if TARGET_OS_VISION
+ bool vision_max_perf = !aggressive_madvise_enabled;
+#endif
+
+ uint32_t huge_cache_max_entry_bytes = 0;
+
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ mach_vm_reclaim_count_t reclaim_buffer_count =
+ XZM_RECLAIM_BUFFER_COUNT_DEFAULT;
+ mach_vm_reclaim_count_t max_reclaim_buffer_count =
+ XZM_RECLAIM_BUFFER_MAX_COUNT_DEFAULT;
+ uint16_t huge_cache_size = XZM_HUGE_CACHE_SIZE_DEFAULT; // number of entries
+ huge_cache_max_entry_bytes = XZM_HUGE_CACHE_MAX_ENTRY_BYTES_DEFAULT;
+ bool defer_tiny = XZM_DEFERRED_RECLAIM_ENABLED_DEFAULT;
+ bool defer_small = XZM_DEFERRED_RECLAIM_ENABLED_DEFAULT;
+ bool defer_large = XZM_DEFERRED_RECLAIM_ENABLED_DEFAULT;
+
+#if TARGET_OS_VISION
+ if (vision_max_perf) {
+ defer_tiny = true;
+ defer_small = true;
+ defer_large = true;
+ huge_cache_size = XZM_HUGE_CACHE_SIZE_ENABLED;
+ }
+#endif
+
+ flag = _xzm_development_only_getenv(envp, "MallocDeferredReclaim");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 0 || value == 1) {
+ defer_tiny = (value == 1);
+ defer_small = (value == 1);
+ defer_large = (value == 1);
+ huge_cache_size = (value == 1) ? XZM_HUGE_CACHE_SIZE_ENABLED : 0;
+ } else {
+ malloc_report(ASL_LEVEL_ERR, "MallocDeferredReclaim must be one of 0,1 - got %ld\n", value);
+ }
+ }
+
+ flag = _xzm_development_only_getenv(envp, "MallocDeferredReclaimBufferCount");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value <= UINT32_MAX) {
+ reclaim_buffer_count = (mach_vm_reclaim_count_t)value;
+ }
+ }
+ flag = _xzm_development_only_getenv(envp, "MallocDeferredReclaimBufferMaxCount");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value <= UINT32_MAX) {
+ max_reclaim_buffer_count = (mach_vm_reclaim_count_t)value;
+ }
+ }
+
+ // Round capacities up to page-alignment, or down to the maximum
+ reclaim_buffer_count = mach_vm_reclaim_round_capacity(reclaim_buffer_count);
+ max_reclaim_buffer_count = mach_vm_reclaim_round_capacity(max_reclaim_buffer_count);
+
+ // MallocLargeCache enables both the huge cache and deferred reclamation
+ // for all large allocations
+ flag = _xzm_production_getenv(envp, "MallocLargeCache");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 0 || value == 1) {
+ huge_cache_size = (value == 1) ? XZM_HUGE_CACHE_SIZE_ENABLED : 0;
+ defer_large = (value == 1);
+ defer_tiny = (value == 1);
+ defer_small = (value == 1);
+ } else {
+ malloc_report(ASL_LEVEL_ERR, "MallocLargeCache must be 0 or 1.\n");
+ }
+ }
+
+#if CONFIG_NANOZONE && !TARGET_OS_OSX
+ // MallocLargeCache is only supported for processes with Nano
+ if (!nano_config) {
+ defer_large = false;
+ huge_cache_size = 0;
+#if MALLOC_TARGET_IOS_ONLY
+ defer_tiny = false;
+ defer_small = false;
+#endif // MALLOC_TARGET_IOS_ONLY
+ }
+#endif // CONFIG_NANOZONE && !TARGET_OS_OSX
+
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneHugeCacheSize");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value <= UINT16_MAX) {
+ huge_cache_size = (uint16_t)value;
+ } else {
+ malloc_report(ASL_LEVEL_ERR,
+ "xzm: unsupported value for MallocXzoneHugeCacheSize (%ld)",
+ value);
+ }
+ }
+
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneHugeCacheMaxEntryBytes");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value <= UINT32_MAX) {
+ huge_cache_max_entry_bytes = (uint32_t)value;
+ } else {
+ malloc_report(ASL_LEVEL_ERR,
+ "xzm: unsupported value for MallocXzoneHugeCacheMaxEntryBytes (%ld)",
+ value);
+ }
+ }
+
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneDeferTiny");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 0 || value == 1) {
+ defer_tiny = (value == 1);
+ } else {
+ malloc_report(ASL_LEVEL_ERR, "MallocXzoneDeferTiny must be one of 0,1 - got %ld\n", value);
+ }
+ }
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneDeferSmall");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 0 || value == 1) {
+ defer_small = (value == 1);
+ } else {
+ malloc_report(ASL_LEVEL_ERR, "MallocXzoneDeferSmall must be one of 0,1 - got %ld\n", value);
+ }
+ }
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneDeferLarge");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 0 || value == 1) {
+ defer_large = (value == 1);
+ } else {
+ malloc_report(ASL_LEVEL_ERR, "MallocXzoneDeferLarge must be one of "
+ "0,1 - got %ld\n", value);
+ }
+ }
+
+ // Known processes do not get deferred reclamation
+ if ((security_critical && !security_critical_max_perf) ||
+ malloc_space_efficient_enabled) {
+ defer_tiny = false;
+ defer_small = false;
+ defer_large = false;
+ huge_cache_size = 0;
+ }
+
+ if (huge_cache_size && !defer_large) {
+ // Deferral of xzones is only supported in conjunction with large/huge
+ malloc_report(ASL_LEVEL_ERR, "Huge cache requires deferred reclamation "
+ "for large.\n");
+ defer_large = true;
+ }
+
+ if ((defer_tiny || defer_small) && !defer_large) {
+ // Deferral of xzones is only supported in conjunction with large/huge
+ malloc_report(ASL_LEVEL_ERR, "Deferred reclamation cannot be used for "
+ "xzones without large\n");
+ defer_large = true;
+ }
+#else // CONFIG_XZM_DEFERRED_RECLAIM
+ const uint16_t huge_cache_size = 0;
+ const bool defer_tiny = false;
+ const bool defer_small = false;
+ const bool defer_large = false;
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+
+ bool small_freelist = false;
+ bool guards_enabled = false;
+ bool guards_enabled_data = false;
+ uint8_t tiny_run_size = 0;
+ uint8_t tiny_guard_density = 0;
+ uint8_t small_run_size = 0;
+ uint8_t small_guard_density = 0;
+ uint8_t batch_size = 0;
+#if TARGET_OS_OSX
+ batch_size = !malloc_space_efficient_enabled ? 10 : 0;
+ small_freelist = (!security_critical || security_critical_max_perf)
+ && !malloc_space_efficient_enabled;
+#elif TARGET_OS_VISION
+ batch_size = vision_max_perf ? 10 : 0;
+#endif
+
+ // Default config:
+ // - Guards enabled and batching disabled for security critical (read,
+ // known) processes
+ // - Data guarded
+ // - Tiny max run size = 8
+ // - Tiny density = 64 (20% guard density)
+ // - Small max run size = 3
+ // - Small density = 32 (11% on 4-page small chunk config)
+
+ if (security_critical) {
+ guards_enabled = true;
+ }
+
+ if (security_critical && !security_critical_max_perf) {
+ batch_size = 0;
+ }
+
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneGuarded");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 0 || value == 1) {
+ guards_enabled = (bool)value;
+ }
+ }
+
+ flag = malloc_common_value_for_key_copy(bootargs,
+ xzone_guard_pages_boot_arg, value_buf, sizeof(value_buf));
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 0 || value == 1) {
+ guards_enabled = (bool)value;
+ } else {
+ malloc_report(ASL_LEVEL_ERR, "%s must be 0 or 1.\n",
+ xzone_guard_pages_boot_arg);
+ }
+ }
+
+ if (guards_enabled) {
+ guards_enabled_data = true;
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneGuardedData");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 0 || value == 1) {
+ guards_enabled_data = (bool)value;
+ }
+ }
+
+ tiny_run_size = 8;
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneGuardTinyRun");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value > 0) {
+ tiny_run_size = value;
+ }
+ }
+
+ tiny_guard_density = 64;
+ flag = _xzm_development_only_getenv(envp,
+ "MallocXzoneGuardTinyDensity");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value > 0) {
+ tiny_guard_density = value;
+ }
+ }
+
+ small_run_size = 3;
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneGuardSmallRun");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value > 0) {
+ small_run_size = value;
+ }
+ }
+
+ small_guard_density = 32;
+ flag = _xzm_development_only_getenv(envp,
+ "MallocXzoneGuardSmallDensity");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value > 0) {
+ small_guard_density = value;
+ }
+ }
+ }
+
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneBatchSize");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value < (1u << XZM_BATCH_SIZE_BITS)) {
+ batch_size = value;
+ }
+ }
+
+ flag = _xzm_development_only_getenv(envp, "MallocSmallFreelist");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 0 || value == 1) {
+ small_freelist = (bool)value;
+ }
+ }
+
+#if CONFIG_XZM_CLUSTER_AWARE
+ bool per_cluster_segment_groups = false;
+#endif // CONFIG_XZM_CLUSTER_AWARE
+#if TARGET_OS_OSX
+ // On macOS, use per-cluster segment groups by default when not space
+ // efficient
+ per_cluster_segment_groups = !malloc_space_efficient_enabled;
+#elif TARGET_OS_VISION && CONFIG_XZM_CLUSTER_AWARE
+ per_cluster_segment_groups = vision_max_perf;
+#endif
+ size_t segment_group_ids_count = XZM_SEGMENT_GROUP_IDS_COUNT;
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneDataOnly");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 1) {
+ segment_group_ids_count = XZM_SEGMENT_GROUP_IDS_COUNT_DATA_ONLY;
+ } else if (value != 0) {
+ malloc_report(ASL_LEVEL_ERR,
+ "MallocXzoneDataOnly must be 0 or 1.\n");
+ }
+ }
+
+# if CONFIG_XZM_CLUSTER_AWARE
+ flag = _xzm_development_only_getenv(envp, "MallocXzonePerClusterSegmentGroups");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 0 || value == 1) {
+ per_cluster_segment_groups = value;
+ } else {
+ malloc_report(ASL_LEVEL_ERR,
+ "MallocXzonePerClusterSegmentGroups must be 0 or 1.\n");
+ }
+ }
+# endif // CONFIG_XZM_CLUSTER_AWARE
+
+ // By default, have two allocation fronts
+ size_t allocation_front_count = 2;
+
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneAllocationFronts");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 1 || value == 2) {
+ allocation_front_count = (size_t)value;
+ } else {
+ malloc_report(ASL_LEVEL_ERR,
+ "Unsupported MallocXzoneAllocationFronts\n");
+ }
+ }
+
+ xzm_slot_config_t max_list_config = XZM_SLOT_SINGLE;
+#if TARGET_OS_OSX
+ // On macOS, upgrade the max list config to the max slot config unless it
+ // is a known process
+ if ((!security_critical || security_critical_max_perf) &&
+ !malloc_space_efficient_enabled) {
+ max_list_config = max_slot_config;
+ }
+#endif // TARGET_OS_OSX
+ xzm_slot_config_t list_config = XZM_SLOT_SINGLE;
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneListConfig");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value < XZM_SLOT_LAST) {
+ list_config = (xzm_slot_config_t)value;
+ }
+ }
+
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneMaxListConfig");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value >= 0 && value < XZM_SLOT_LAST && value < max_slot_config) {
+ max_list_config = (xzm_slot_config_t)value;
+ }
+ }
+
+ bool segment_deallocate = true;
+#if TARGET_OS_OSX
+ // On macOS, skip deallocation unless it is a known process
+ if ((!security_critical || security_critical_max_perf) &&
+ !malloc_space_efficient_enabled) {
+ segment_deallocate = false;
+ }
+#endif // TARGET_OS_OSX
+ flag = _xzm_development_only_getenv(envp, "MallocXzoneSegmentDeallocate");
+ if (flag) {
+ long value = strtol(flag, NULL, 10);
+ if (value == 0 || value == 1) {
+ segment_deallocate = (bool)value;
+ }
+ }
+
+#if CONFIG_XZM_CLUSTER_AWARE
+ // Known processes do not get per-cluster segment groups
+ if (security_critical && !security_critical_max_perf) {
+ per_cluster_segment_groups = false;
+ }
+#endif // CONFIG_XZM_CLUSTER_AWARE
+
+ // There is one PTR range group per allocation front, and one PTR_LARGE
+ // (when applicable) and DATA each globally
+ size_t range_group_count = allocation_front_count +
+ (XZM_RANGE_GROUP_COUNT - 1);
+#if !CONFIG_XZM_CLUSTER_AWARE
+ size_t segment_group_cluster_count = 1;
+#else
+ size_t segment_group_cluster_count = per_cluster_segment_groups ? ncpuclusters : 1;
+#endif // !CONFIG_XZM_CLUSTER_AWARE
+
+ size_t segment_group_front_count = segment_group_ids_count;
+ if (segment_group_ids_count > XZM_SEGMENT_GROUP_IDS_COUNT_DATA_ONLY &&
+ allocation_front_count > 1) {
+ // If POINTER_XZONES is active it has N fronts
+ segment_group_front_count += allocation_front_count - 1;
+ }
+
+ size_t segment_group_count = segment_group_front_count *
+ segment_group_cluster_count;
+
+ size_t metapool_count = XZM_METAPOOL_COUNT;
+
+ size_t tail_allocation_offset = sizeof(struct xzm_main_malloc_zone_s);
+
+ size_t main_xzones_offset = tail_allocation_offset;
+ tail_allocation_offset += sizeof(struct xzm_xzone_s) * xzone_count;
+
+ size_t main_xzone_slots_offset = tail_allocation_offset;
+ size_t total_slot_count = xzone_count * slot_count;
+ tail_allocation_offset +=
+ sizeof(struct xzm_xzone_allocation_slot_s) * total_slot_count;
+
+ size_t main_xzone_partial_offset = tail_allocation_offset;
+ tail_allocation_offset += sizeof(struct xzm_chunk_list_s) * total_slot_count;
+
+ size_t bin_sizes_offset = tail_allocation_offset;
+ tail_allocation_offset += sizeof(uint64_t) * bin_count;
+
+ size_t bin_bucket_counts_offset = tail_allocation_offset;
+ tail_allocation_offset += sizeof(uint8_t) * bin_count;
+
+ size_t xzone_bin_offsets_offset = tail_allocation_offset;
+ tail_allocation_offset += sizeof(uint8_t) * bin_count;
+
+ size_t isolation_zones_offset = tail_allocation_offset;
+ tail_allocation_offset += sizeof(struct xzm_isolation_zone_s) * xzone_count;
+
+ size_t range_groups_offset = tail_allocation_offset;
+ tail_allocation_offset +=
+ sizeof(struct xzm_range_group_s) * range_group_count;
+
+ size_t segment_groups_offset = tail_allocation_offset;
+ tail_allocation_offset +=
+ sizeof(struct xzm_segment_group_s) * segment_group_count;
+
+ size_t metapools_offset = tail_allocation_offset;
+ tail_allocation_offset += sizeof(struct xzm_metapool_s) * metapool_count;
+
+ size_t segment_table_offset = tail_allocation_offset;
+ tail_allocation_offset +=
+ sizeof(xzm_segment_table_entry_s) * XZM_SEGMENT_TABLE_ENTRIES;
+
+#if CONFIG_EXTERNAL_METADATA_LARGE
+ size_t extended_segment_table_offset = tail_allocation_offset;
+ tail_allocation_offset += sizeof(xzm_extended_segment_table_entry_s) *
+ XZM_EXTENDED_SEGMENT_TABLE_ENTRIES;
+#endif // CONFIG_EXTERNAL_METADATA_LARGE
+
+ size_t total_size = tail_allocation_offset;
+
+ int flags = VM_FLAGS_ANYWHERE;
+ int tag = 0;
+ plat_map_t *plat_map_ptr = NULL;
+ tag = VM_MEMORY_MALLOC;
+ mach_vm_address_t vm_addr = (mach_vm_address_t)mvm_allocate_plat(0,
+ total_size, 0, flags, MALLOC_GUARDED_METADATA, tag, plat_map_ptr);
+ if (!vm_addr) {
+ xzm_abort("Failed to allocate xzm zone");
+ }
+
+ xzm_main_malloc_zone_t main = (xzm_main_malloc_zone_t)vm_addr;
+ xzm_xzone_t main_xzones_ptr = (xzm_xzone_t)((uintptr_t)main + main_xzones_offset);
+ xzm_xzone_allocation_slot_t main_xzone_slots_ptr = (xzm_xzone_allocation_slot_t)
+ ((uint8_t*)main + main_xzone_slots_offset);
+
+ xzm_chunk_list_t partial_lists = (xzm_chunk_list_t)((uint8_t *)main +
+ main_xzone_partial_offset);
+
+ xzm_mzone_index_t mzone_idx = XZM_MZONE_INDEX_MAIN;
+
+ *main = (struct xzm_main_malloc_zone_s){
+ .xzmz_total_size = total_size,
+ .xzmz_bucketing_keys = bucketing_keys,
+#if XZM_NARROW_BUCKETING
+ .xzmz_narrow_bucketing = narrow_bucketing,
+#endif
+ .xzmz_madvise_workaround = madvise_workaround,
+ .xzmz_defer_tiny = defer_tiny,
+ .xzmz_defer_small = defer_small,
+ .xzmz_defer_large = defer_large,
+ .xzmz_deallocate_segment = segment_deallocate,
+ .xzmz_range_group_count = (uint8_t)range_group_count,
+ .xzmz_segment_group_ids_count = segment_group_ids_count,
+ .xzmz_segment_group_front_count = segment_group_front_count,
+ .xzmz_segment_group_count = segment_group_count,
+ .xzmz_metapool_count = (uint8_t)metapool_count,
+ .xzmz_allocation_front_count = (uint8_t)allocation_front_count,
+ .xzmz_mfm_address = mfm_zone_address(),
+ .xzmz_batch_size = batch_size,
+ .xzmz_bin_count = (uint8_t)bin_count,
+ .xzmz_ptr_bucket_count = (uint8_t)ptr_bucket_count,
+ .xzmz_xzone_chunk_threshold = (uint8_t)chunk_threshold,
+ .xzmz_xzone_bin_sizes = (uint64_t *)
+ ((uintptr_t)main + bin_sizes_offset),
+ .xzmz_xzone_bin_bucket_counts = (uint8_t *)
+ ((uintptr_t)main + bin_bucket_counts_offset),
+ .xzmz_xzone_bin_offsets = (uint8_t *)
+ ((uintptr_t)main + xzone_bin_offsets_offset),
+ .xzmz_isolation_zones = (struct xzm_isolation_zone_s *)
+ ((uintptr_t)main + isolation_zones_offset),
+ .xzmz_range_groups = (struct xzm_range_group_s *)
+ ((uintptr_t)main + range_groups_offset),
+ .xzmz_segment_groups = (struct xzm_segment_group_s *)
+ ((uintptr_t)main + segment_groups_offset),
+ .xzmz_metapools = (struct xzm_metapool_s *)(
+ (uintptr_t)main + metapools_offset),
+ .xzmz_segment_table = (xzm_segment_table_entry_s *)
+ ((uintptr_t)main + segment_table_offset),
+#if CONFIG_EXTERNAL_METADATA_LARGE
+ .xzmz_extended_segment_table_entries =
+ XZM_EXTENDED_SEGMENT_TABLE_ENTRIES,
+ .xzmz_extended_segment_table = (xzm_extended_segment_table_entry_s *)
+ ((uintptr_t)main + extended_segment_table_offset),
+#endif // CONFIG_EXTERNAL_METADATA_LARGE
+ .xzmz_max_mzone_idx = mzone_idx,
+ .xzmz_mzones_lock = _MALLOC_LOCK_INIT,
+ .xzmz_guard_config = {
+ .xgpc_enabled = guards_enabled,
+ .xgpc_enabled_for_data = guards_enabled_data,
+ .xgpc_max_run_tiny = tiny_run_size,
+ .xgpc_tiny_guard_density = tiny_guard_density,
+ .xgpc_max_run_small = small_run_size,
+ .xgpc_small_guard_density = small_guard_density,
+ },
+ .xzmz_thread_cache_list =
+ LIST_HEAD_INITIALIZER(main->xzmz_thread_cache_list),
+ };
+ _xzm_initialize_const_zone_data(&main->xzmz_base, total_size, mzone_idx,
+ xzone_count, slot_count, main_xzones_ptr, main_xzone_slots_ptr,
+ NULL, slot_config, slot_threshold, max_slot_config,
+ list_upgrade_threshold_single, list_upgrade_threshold_cluster,
+ list_upgrade_period, slot_upgrade_threshold_single,
+ slot_upgrade_threshold_cluster, slot_upgrade_period,
+ tiny_thrash_threshold, small_thrash_threshold,
+ small_thrash_limit_size, debug_flags, small_freelist,
+ max_list_config, partial_lists);
+#if CONFIG_MTE
+ main->xzmz_base.xzz_memtag_config = memtag_config;
+#endif // CONFIG_MTE
+
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ if (defer_tiny || defer_small || defer_large || huge_cache_size) {
+ bool success = xzm_reclaim_init(main, reclaim_buffer_count,
+ max_reclaim_buffer_count);
+ if (!success) {
+ huge_cache_size = 0;
+ main->xzmz_defer_tiny = defer_tiny = false;
+ main->xzmz_defer_small = defer_small = false;
+ main->xzmz_defer_large = defer_large = false;
+ }
+ }
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+
+ main->xzmz_base.xzz_thread_cache_enabled = thread_caching;
+ main->xzmz_base.xzz_thread_cache_xzone_activation_period =
+ thread_cache_activation_period;
+ main->xzmz_base.xzz_thread_cache_xzone_activation_contentions =
+ thread_cache_activation_contentions;
+ main->xzmz_base.xzz_thread_cache_xzone_activation_time =
+ thread_cache_activation_time;
+
+#if CONFIG_XZM_THREAD_CACHE
+ if (thread_caching) {
+ int rc = pthread_key_init_np(__TSD_MALLOC_XZONE_THREAD_CACHE,
+ _xzm_xzone_thread_cache_destructor);
+ if (os_unlikely(rc)) {
+ xzm_abort_with_reason("pthread_key_init_np failed", rc);
+ }
+ }
+#endif // CONFIG_XZM_THREAD_CACHE
+
+ // Initialize the main mzone structures
+ for (size_t i = 0; i < bin_count; i++) {
+ main->xzmz_xzone_bin_sizes[i] = _xzm_bin_sizes[i];
+ }
+
+ for (size_t i = 0; i < bin_count; i++) {
+ main->xzmz_xzone_bin_bucket_counts[i] = buckets_per_bin;
+ }
+
+ for (size_t i = 0; i < xzone_count; i++) {
+ xzm_isolation_zone_t iz = &main->xzmz_isolation_zones[i];
+ _malloc_lock_init(&iz->xziz_lock);
+ }
+
+ size_t rg_idx = 0;
+ for (size_t i = 0; i < XZM_RANGE_GROUP_COUNT; i++) {
+ xzm_range_group_id_t rgid = (xzm_range_group_id_t)i;
+ size_t rg_fronts = (rgid == XZM_RANGE_GROUP_PTR) ?
+ allocation_front_count : 1;
+ for (size_t j = 0; j < rg_fronts; j++) {
+ xzm_range_group_t rg = &main->xzmz_range_groups[rg_idx];
+ rg->xzrg_id = rgid;
+ rg->xzrg_front = (xzm_front_index_t)j;
+ rg->xzrg_main_ref = main;
+ _malloc_lock_init(&rg->xzrg_lock);
+
+ rg_idx++;
+ }
+ }
+
+ if (has_vm_user_ranges) {
+ xzm_main_malloc_zone_init_range_groups(main);
+ }
+
+ if (!main->xzmz_use_ranges) {
+ // If we don't actually have ranges we can't do allocation fronts
+ allocation_front_count = 1;
+ main->xzmz_allocation_front_count = 1;
+ }
+
+ for (size_t i = 0; i < segment_group_count; i++) {
+ xzm_segment_group_t sg = &main->xzmz_segment_groups[i];
+ size_t sg_front_idx = i % segment_group_front_count;
+ sg->xzsg_id = sg_front_idx < XZM_SEGMENT_GROUP_POINTER_XZONES ?
+ (xzm_segment_group_id_t)sg_front_idx :
+ XZM_SEGMENT_GROUP_POINTER_XZONES;
+
+ _malloc_lock_init(&sg->xzsg_lock);
+ _malloc_lock_init(&sg->xzsg_alloc_lock);
+
+ if (_xzm_segment_group_id_is_data(sg->xzsg_id)) {
+ sg->xzsg_range_group =
+ &main->xzmz_range_groups[XZM_RANGE_GROUP_DATA];
+ } else if (sg->xzsg_id == XZM_SEGMENT_GROUP_POINTER_LARGE) {
+
+ if (!sg->xzsg_range_group) {
+ xzm_front_index_t front = _xzm_random_front_index(front_random,
+ allocation_front_count, XZM_XZONE_INDEX_INVALID);
+ size_t rg_idx = XZM_RANGE_GROUP_PTR + front;
+ sg->xzsg_range_group = &main->xzmz_range_groups[rg_idx];
+ }
+ } else {
+ xzm_assert(sg->xzsg_id == XZM_SEGMENT_GROUP_POINTER_XZONES);
+ size_t ptr_front_idx =
+ sg_front_idx - XZM_SEGMENT_GROUP_POINTER_XZONES;
+ size_t rg_idx = XZM_RANGE_GROUP_PTR + ptr_front_idx;
+ xzm_debug_assert(rg_idx < main->xzmz_range_group_count);
+ sg->xzsg_range_group = &main->xzmz_range_groups[rg_idx];
+ }
+
+ sg->xzsg_front = sg->xzsg_range_group->xzrg_front;
+ sg->xzsg_main_ref = main;
+
+ if (i == XZM_SEGMENT_GROUP_DATA_LARGE) {
+ // Huge allocations are all routed to the same segment group,
+ // so only initialize the huge cache for that one segment group
+ sg->xzsg_cache = (struct xzm_segment_cache_s){
+ .xzsc_max_count = huge_cache_size,
+ .xzsc_count = 0,
+ // TODO: consider limiting the maximum entry size based on the
+ // reclaim threshold
+ .xzsc_max_entry_slices =
+ (huge_cache_max_entry_bytes / XZM_SEGMENT_SLICE_SIZE),
+ };
+ TAILQ_INIT(&sg->xzsg_cache.xzsc_head);
+ _malloc_lock_init(&sg->xzsg_cache.xzsc_lock);
+ }
+
+ for (size_t j = 0; j < XZM_SPAN_QUEUE_COUNT; j++) {
+ sg->xzsg_spans[j].xzsq_slice_count =
+ (xzm_slice_count_t)_xzm_span_queue_slice_counts[j];
+ }
+ }
+
+ // NOTE: The order of these metapool allocators matters for the fork lock,
+ // we have to grab the metadata metapool last, since the other metapools
+ // acquire its lock after taking their own
+ xzm_metapool_t metadata_mp = &main->xzmz_metapools[XZM_METAPOOL_METADATA];
+ uint32_t mp_metadata_size = MAX(sizeof(struct xzm_metapool_block_s),
+ sizeof(struct xzm_metapool_slab_s));
+ uint32_t mp_metadata_slab_size = KiB(16); // arbitrary slab size
+ xzm_metapool_init(metadata_mp, XZM_METAPOOL_METADATA, VM_MEMORY_MALLOC,
+ mp_metadata_slab_size, mp_metadata_size, mp_metadata_size, NULL);
+
+ xzm_metapool_t segment_mp = &main->xzmz_metapools[XZM_METAPOOL_SEGMENT];
+ xzm_metapool_init(segment_mp, XZM_METAPOOL_SEGMENT, VM_MEMORY_MALLOC,
+ XZM_METAPOOL_SEGMENT_SLAB_SIZE, XZM_METAPOOL_SEGMENT_ALIGN,
+ XZM_METAPOOL_SEGMENT_BLOCK_SIZE, metadata_mp);
+
+ xzm_metapool_t leaf_table_mp = &main->xzmz_metapools[XZM_METAPOOL_SEGMENT_TABLE];
+ xzm_metapool_init(leaf_table_mp, XZM_METAPOOL_SEGMENT_TABLE,
+ VM_MEMORY_MALLOC, XZM_METAPOOL_SEGMENT_TABLE_SLAB_SIZE,
+ XZM_SEGMENT_TABLE_ALIGN, XZM_SEGMENT_TABLE_SIZE, metadata_mp);
+
+ xzm_metapool_t mzone_idx_mp = &main->xzmz_metapools[XZM_METAPOOL_MZONE_IDX];
+ xzm_metapool_init(mzone_idx_mp, XZM_METAPOOL_MZONE_IDX, VM_MEMORY_MALLOC,
+ XZM_METAPOOL_MZIDX_SLAB_SIZE, XZM_METAPOOL_MZIDX_BLOCK_ALIGN,
+ XZM_METAPOOL_MZIDX_BLOCK_SIZE, NULL);
+
+ xzm_metapool_t tc_mp = &main->xzmz_metapools[XZM_METAPOOL_THREAD_CACHE];
+
+ // XXX Needs to be re-worked if we allow per-bin bucket counts
+ xzm_debug_assert(_xzm_bin_sizes[XZM_THREAD_CACHE_BINS - 1] ==
+ XZM_THREAD_CACHE_THRESHOLD);
+ size_t thread_cache_xzone_count =
+ 1 + (XZM_THREAD_CACHE_BINS * buckets_per_bin);
+
+ main->xzmz_base.xzz_thread_cache_xzone_count =
+ (uint8_t)thread_cache_xzone_count;
+
+ size_t thread_cache_block_size = sizeof(struct xzm_thread_cache_s) +
+ (thread_cache_xzone_count * sizeof(xzm_xzone_thread_cache_u));
+ xzm_metapool_init(tc_mp, XZM_METAPOOL_THREAD_CACHE,
+ VM_MEMORY_MALLOC_NANO,
+ XZM_METAPOOL_THREAD_CACHE_SLAB_SIZE, 0,
+ (uint32_t)thread_cache_block_size, metadata_mp);
+
+ // Initialize the per-mzone structures
+ bool data_only = (main->xzmz_segment_group_ids_count ==
+ XZM_SEGMENT_GROUP_IDS_COUNT_DATA_ONLY);
+ _xzm_initialize_xzone_data(&main->xzmz_base, list_config, &main->xzmz_guard_config, front_random, data_only);
+
+ flag = _xzm_production_getenv(envp, "MallocReportConfig");
+ if (flag) {
+ // Report our config to stderr for debugging purposes
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ int vm_reclaim_enabled;
+ size_t vm_reclaim_enabled_size = sizeof(vm_reclaim_enabled);
+ int kr = sysctlbyname("vm.reclaim.enabled",
+ &vm_reclaim_enabled, &vm_reclaim_enabled_size, 0, 0);
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+ malloc_report(ASL_LEVEL_INFO,
+ "XZM Config:\n"
+ "\tData Only: %d\n"
+ "\tAllocation Fronts: %d\n"
+#if XZM_NARROW_BUCKETING
+ "\tNarrow Bucketing: %d\n"
+#endif
+ "\tGuards Enabled: %d\n"
+ "\tScribble: %d\n"
+ "\tTiny/Small Batch Max: %d\n"
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ "\tDefer Tiny: %d\n"
+ "\tDefer Small: %d\n"
+ "\tDefer Large: %d\n"
+ "\tHuge Cache Size: %d\n"
+ "\tHuge Cache Max Entry Bytes: %u\n"
+ "\tReclaim Buffer Count: %u/%u (%s)\n"
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+ "\tSmall Freelist: %u\n"
+#if CONFIG_VM_USER_RANGES || CONFIG_MACOS_RANGES
+ "\tData Range: 0x%llx/%lu\n"
+ "\tPointer Range 1: 0x%llx/%lu\n"
+ "\tPointer Range 2: 0x%llx/%lu\n"
+#endif // CONFIG_VM_USER_RANGES || CONFIG_MACOS_RANGES
+ "\tEarly Allocator: %s\n"
+ "\tSegment Deallocate: %u\n"
+#if CONFIG_MTE
+ "\tMTE (enabled/data/max size): %d/%d/%llu\n"
+#endif
+ "\tInitial Slot Config: %s/%s (Chunk, Size Thresholds: %u, %u)\n"
+ "\tInitial List Config: %s/%s\n"
+ "\tList Upgrade Thresholds: %d/%d, %d/%d\n"
+ "\tSlot Upgrade Thresholds: %d/%d, %d/%d\n"
+ "\tTiny Thrash Threshold: %llu ms\n"
+ "\tSmall Thrash Threshold: %llu ms, %llu bytes\n"
+#if CONFIG_XZM_THREAD_CACHE
+ "\tThread Caching: %s (%u allocs, %u contentions, %llu ms)\n"
+#endif
+ "\tPointer Bucket Count: %lu\n",
+ data_only,
+ (int)allocation_front_count,
+#if XZM_NARROW_BUCKETING
+ main->xzmz_narrow_bucketing,
+#endif
+ guards_enabled, !!(debug_flags & MALLOC_DO_SCRIBBLE),
+ batch_size,
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ defer_tiny, defer_small, defer_large,
+ huge_cache_size, huge_cache_max_entry_bytes,
+ reclaim_buffer_count, max_reclaim_buffer_count,
+ (kr || !vm_reclaim_enabled) ? "DISABLED": "ENABLED",
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+ small_freelist,
+#if CONFIG_VM_USER_RANGES || CONFIG_MACOS_RANGES
+ !malloc_internal_security_policy ? 0 :
+ main->xzmz_range_groups[XZM_RANGE_GROUP_DATA].xzrg_base,
+ main->xzmz_range_groups[XZM_RANGE_GROUP_DATA].xzrg_size,
+ !malloc_internal_security_policy ? 0 :
+ main->xzmz_range_groups[XZM_RANGE_GROUP_PTR + 0].xzrg_base,
+ main->xzmz_range_groups[XZM_RANGE_GROUP_PTR + 0].xzrg_size,
+ !malloc_internal_security_policy ||
+ main->xzmz_allocation_front_count < 2 ? 0 :
+ main->xzmz_range_groups[XZM_RANGE_GROUP_PTR + 1].xzrg_base,
+ main->xzmz_allocation_front_count < 2 ? 0 :
+ main->xzmz_range_groups[XZM_RANGE_GROUP_PTR + 1].xzrg_size,
+#endif // CONFIG_VM_USER_RANGES || CONFIG_MACOS_RANGES
+ main->xzmz_mfm_address ? "enabled" : "disabled",
+ segment_deallocate,
+#if CONFIG_MTE
+ memtag_config.enabled, memtag_config.tag_data,
+ memtag_config.max_block_size,
+#endif
+ _xzm_slot_config_to_string(slot_config),
+ _xzm_slot_config_to_string(max_slot_config),
+ chunk_threshold, slot_threshold,
+ _xzm_slot_config_to_string(list_config),
+ _xzm_slot_config_to_string(max_list_config),
+ list_upgrade_threshold_single,
+ list_upgrade_period, list_upgrade_threshold_cluster,
+ list_upgrade_period, slot_upgrade_threshold_single,
+ slot_upgrade_period, slot_upgrade_threshold_cluster,
+ slot_upgrade_period,
+ tiny_thrash_threshold_ns / nsec_per_msec,
+ small_thrash_threshold_ns / nsec_per_msec,
+ small_thrash_limit_size,
+#if CONFIG_XZM_THREAD_CACHE
+ thread_caching ? "enabled" : "disabled",
+ thread_cache_activation_period,
+ thread_cache_activation_contentions,
+ thread_cache_activation_time_ns / nsec_per_msec,
+#endif // CONFIG_XZM_THREAD_CACHE
+ (unsigned long)ptr_bucket_count);
+ }
+
+ return &main->xzmz_base.xzz_basic_zone;
+}
+
+MALLOC_NOEXPORT
+malloc_zone_t *
+xzm_malloc_zone_create(unsigned debug_flags, xzm_main_malloc_zone_t main_ref)
+{
+ size_t tail_allocation_offset = sizeof(struct xzm_malloc_zone_s);
+ size_t xzones_offset = tail_allocation_offset;
+
+ uint8_t xzone_count = main_ref->xzmz_base.xzz_xzone_count;
+ uint8_t slot_count = main_ref->xzmz_base.xzz_slot_count;
+ tail_allocation_offset += sizeof(struct xzm_xzone_s) * xzone_count;
+ size_t slots_offset = tail_allocation_offset;
+
+ size_t total_slot_count = xzone_count * slot_count;
+ tail_allocation_offset += sizeof(struct xzm_xzone_allocation_slot_s) *
+ total_slot_count;
+
+ size_t xzone_partial_offset = tail_allocation_offset;
+ tail_allocation_offset += sizeof(struct xzm_chunk_list_s) * total_slot_count;
+
+ // Try to pop a previously destroyed mzone index from the reuse list,
+ // otherwise use the next unused value
+ _malloc_lock_lock(&main_ref->xzmz_mzones_lock);
+ xzm_mzone_index_t mzone_idx;
+ xzm_reused_mzone_index_t reuse = SLIST_FIRST(&main_ref->xzmz_reusable_mzidxq);
+ if (reuse) {
+ SLIST_REMOVE_HEAD(&main_ref->xzmz_reusable_mzidxq, xrmi_mzone_entry);
+ mzone_idx = reuse->xrmi_mzone_idx;
+ xzm_metapool_t mp = &main_ref->xzmz_metapools[XZM_METAPOOL_MZONE_IDX];
+ xzm_metapool_free(mp, reuse);
+ } else if (main_ref->xzmz_max_mzone_idx == XZM_MZONE_INDEX_MAX) {
+ mzone_idx = XZM_MZONE_INDEX_INVALID;
+ } else {
+ main_ref->xzmz_max_mzone_idx += 1;
+ mzone_idx = main_ref->xzmz_max_mzone_idx;
+ }
+ _malloc_lock_unlock(&main_ref->xzmz_mzones_lock);
+
+ if (os_unlikely(mzone_idx == XZM_MZONE_INDEX_INVALID)) {
+ return NULL;
+ }
+
+ // allocate a new mzone
+ int flags = VM_FLAGS_ANYWHERE;
+ int tag = 0;
+ plat_map_t *plat_map_ptr = NULL;
+ tag = VM_MEMORY_MALLOC;
+
+ mach_vm_address_t vm_addr = (mach_vm_address_t)mvm_allocate_plat(0,
+ tail_allocation_offset, 0, flags, MALLOC_GUARDED_METADATA, tag,
+ plat_map_ptr);
+ if (!vm_addr) {
+ return NULL;
+ }
+ xzm_malloc_zone_t new_zone = (xzm_malloc_zone_t)vm_addr;
+
+ xzm_chunk_list_t partial_lists = (xzm_chunk_list_t)((uint8_t *)new_zone +
+ xzone_partial_offset);
+
+ // Currently we copy the slot upgrade parameters from the main zone into
+ // newly created zones. To limit fragmentation, we could cap the new
+ // zones at single/per cluster. Pre-xzm, new malloc zones were all
+ // scalable zones (you could only have one nano zone), so capping
+ // scalability of non-default zones has prior art
+ _xzm_initialize_const_zone_data(new_zone, tail_allocation_offset, mzone_idx,
+ xzone_count, slot_count,
+ (xzm_xzone_t)((uintptr_t)new_zone + xzones_offset),
+ (xzm_xzone_allocation_slot_t)((uintptr_t)new_zone + slots_offset),
+ main_ref, main_ref->xzmz_base.xzz_initial_slot_config,
+ main_ref->xzmz_base.xzz_slot_initial_threshold,
+ main_ref->xzmz_base.xzz_max_slot_config,
+ main_ref->xzmz_base.xzz_list_upgrade_threshold[0],
+ main_ref->xzmz_base.xzz_list_upgrade_threshold[1],
+ main_ref->xzmz_base.xzz_list_upgrade_period,
+ main_ref->xzmz_base.xzz_slot_upgrade_threshold[0],
+ main_ref->xzmz_base.xzz_slot_upgrade_threshold[1],
+ main_ref->xzmz_base.xzz_slot_upgrade_period,
+ main_ref->xzmz_base.xzz_tiny_thrash_threshold,
+ /* small_thrash_threshold */ 0, /* small_thrash_limit_size */ 0,
+ debug_flags, main_ref->xzmz_base.xzz_small_freelist_enabled,
+ main_ref->xzmz_base.xzz_max_list_config, partial_lists);
+#if CONFIG_MTE
+ // TODO: Do we want a different MTE config in new zones?
+ new_zone->xzz_memtag_config = main_ref->xzmz_base.xzz_memtag_config;
+#endif // CONFIG_MTE
+
+ xzm_debug_assert(new_zone != NULL);
+
+ bool data_only = (main_ref->xzmz_segment_group_ids_count ==
+ XZM_SEGMENT_GROUP_IDS_COUNT_DATA_ONLY);
+ // Currently new zones are never nano, so start all new xzone slot configs
+ // at single.
+ // TODO: if the envvar is set (MallocXzoneInitialSlotConfig), use
+ // that slot config in non-default zones
+ _xzm_initialize_xzone_data(new_zone, XZM_SLOT_SINGLE,
+ &main_ref->xzmz_guard_config, NULL, data_only);
+ LIST_INIT(&new_zone->xzz_chunkq_large);
+
+ return &new_zone->xzz_basic_zone;
+}
+
+#endif // CONFIG_XZONE_MALLOC