Loading...
--- libmalloc/libmalloc-792.41.1/src/xzone_malloc/xzone_malloc.c
+++ /dev/null
@@ -1,8025 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2022, Microsoft Research, Daan Leijen
-Copyright © 2025 Apple Inc.
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" in the same directory as this file.
------------------------------------------------------------------------------*/
-
-#include "../internal.h"
-
-#if CONFIG_XZONE_MALLOC
-
-#if defined(DEBUG) && !MALLOC_TARGET_EXCLAVES
-#define xzm_trace(name, ...) MALLOC_TRACE(TRACE_xzone_##name, __VA_ARGS__)
-#else
-#define xzm_trace(...)
-#endif
-
-#pragma mark xzone lookup
-
-// mimalloc: mi_bin
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static uint8_t
-_xzm_bin(size_t size)
-{
- if (size == 0) {
- // TODO: fancy unmapped page
- return 0;
- } else if (size <= 128) {
- return howmany(size, XZM_GRANULE) - 1;
- } else {
- xzm_debug_assert(size <= XZM_SMALL_BLOCK_SIZE_MAX);
-
- // The bin calculation rounds up, so subtract to handle perfect fits
- size--;
-
- int msb = 63 - __builtin_clzl(size);
- return ((msb << 2) + ((size >> (msb - 2)) & 0x3)) - 20;
- }
-}
-
-MALLOC_STATIC_ASSERT(
- XZM_POINTER_BUCKETS_MAX <= 4,
- "The bucketing function supports up to 4 pointer buckets");
-
-MALLOC_STATIC_ASSERT(
- sizeof(xzm_bucketing_keys_t) == 2 * sizeof(uint64_t),
- "Invalid size for struct xzm_bucketing_keys_t");
-
-// This function assigns a type descriptor to a pointer bucket.
-// It takes as input the type descriptor itself and a set of keys, which are
-// obtained from the executable_boothash provided to the process by the kernel.
-// It is both security and performance sensitive.
-//
-// The function is designed to fulfill three requirements:
-// 1. It is stable with respect to identical type hashes, for executions of
-// the same process in the same boot session: this is required to prevent
-// an attacker from brute-forcing the bucketing assignment by repeatedly
-// crashing a given process.
-// 2. It distributes the same type hash pseudo-uniformly across buckets, when
-// the execution hash changes: this is required to prevent an attacker from
-// predicting the bucket assignment across processes, or to statically
-// determine the assignment for a given binary across boot sessions.
-// 3. It distributes different type hashes pseudo-uniformly across buckets,
-// given a fixed execution hash: it should not be possible for any
-// random pair of type hashes to always be assigned to the same bucket,
-// independently of the execution hash.
-//
-// The first requirement only holds when the key material is derived from the
-// executable_boothash. It does not hold otherwise.
-MALLOC_ALWAYS_INLINE MALLOC_INLINE MALLOC_USED
-static uint8_t
-_xzm_type_choose_ptr_bucket(const xzm_bucketing_keys_t *const keys,
- uint8_t ptr_bucket_count, malloc_type_descriptor_t type_desc)
-{
- xzm_debug_assert(ptr_bucket_count <= XZM_POINTER_BUCKETS_MAX);
- uint8_t bucket = 0;
-
- if (ptr_bucket_count > 1) {
- const uint32_t type_hash = type_desc.hash;
- const uint64_t key_a = keys->xbk_key_data[0];
- const uint64_t key_b = keys->xbk_key_data[1];
- const uint32_t hash = ((key_a * (uint64_t)type_hash) + key_b) >> 32;
-
- switch (ptr_bucket_count) {
- case 2:
- bucket = hash & 0x1;
- break;
- case 3:
- bucket = hash % 0x3;
- break;
- case 4:
- bucket = hash & 0x3;
- break;
- default:
- __builtin_unreachable();
- }
- }
-
- return bucket;
-}
-
-// mimalloc: mi_page_queue
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_xzone_index_t
-_xzm_xzone_lookup(xzm_malloc_zone_t zone, size_t size,
- malloc_type_descriptor_t type_desc)
-{
- xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
- uint8_t bin = _xzm_bin(size);
- uint8_t bin_bucket_count = main->xzmz_xzone_bin_bucket_counts[bin];
- uint8_t bin_offset = main->xzmz_xzone_bin_offsets[bin];
-
- uint8_t bucket;
-#if XZM_BUCKET_DATA_ONLY || XZM_BUCKET_POINTER_ONLY
- (void)bin_bucket_count;
- bucket = 0;
-#else // XZM_BUCKET_DATA_ONLY || XZM_BUCKET_POINTER_ONLY
-
-#if XZM_NARROW_BUCKETING
- if (main->xzmz_narrow_bucketing) {
- if (type_desc.summary.type_kind == MALLOC_TYPE_KIND_OBJC) {
- // - There are pure-data callsites that provide a 0 type hash
- // because they assume that nothing is done with it.
- // - The ObjC runtime callsite also provides a 0 type hash, for the
- // same reason.
- //
- // To prevent these callsites from deterministically bucketing
- // together, pick a different arbitrary type hash for ObjC.
- //
- // TODO: update the callsite in the ObjC runtime directly so that
- // this is no longer necessary
- type_desc.hash = 1;
- }
- bucket = _xzm_type_choose_ptr_bucket(&main->xzmz_bucketing_keys,
- bin_bucket_count, type_desc);
- goto bucket_computed;
- }
-#endif // XZM_NARROW_BUCKETING
-
- bool pure_data = malloc_type_descriptor_is_pure_data(type_desc);
- if (pure_data) {
- bucket = XZM_XZONE_BUCKET_DATA;
- } else if (type_desc.summary.type_kind == MALLOC_TYPE_KIND_OBJC) {
- bucket = XZM_XZONE_BUCKET_OBJC;
-#if XZM_BUCKET_VISIBILITY
- } else if (type_desc.type_id == MALLOC_TYPE_ID_NONE) {
- bucket = XZM_XZONE_BUCKET_PLAIN;
- } else if (malloc_type_descriptor_is_uninferred(type_desc)) {
- bucket = XZM_XZONE_BUCKET_UNINFERRED;
-#endif // XZM_BUCKET_VISIBILITY
- } else {
- bool fallback_hash = (type_desc.type_id == MALLOC_TYPE_ID_NONE);
- if (fallback_hash) {
- type_desc.hash = malloc_entropy[0] >> 32;
- }
- uint8_t ptr_bucket = _xzm_type_choose_ptr_bucket(
- &main->xzmz_bucketing_keys,
- (bin_bucket_count - XZM_XZONE_BUCKET_POINTER_BASE), type_desc);
- bucket = XZM_XZONE_BUCKET_POINTER_BASE + ptr_bucket;
- }
-
-#if XZM_NARROW_BUCKETING
-bucket_computed:
-#endif // XZM_NARROW_BUCKETING
-
-#endif // XZM_BUCKET_DATA_ONLY || XZM_BUCKET_POINTER_ONLY
- xzm_debug_assert(bucket < bin_bucket_count);
-
- return (xzm_xzone_index_t)(bin_offset + bucket);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_allocation_index_t
-_xzm_get_allocation_index(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_slot_config_t *cur_slot_config, bool is_xas)
-{
- xzm_slot_config_t *slot = is_xas ? &xz->xz_slot_config :
- &xz->xz_list_config;
- xzm_slot_config_t slot_config = os_atomic_load(slot, relaxed);
- xzm_debug_assert(slot_config <= zone->xzz_max_slot_config);
-
- if (cur_slot_config) {
- *cur_slot_config = slot_config;
- }
-
- switch (slot_config) {
- case XZM_SLOT_CPU:
- return _malloc_cpu_number();
- case XZM_SLOT_CLUSTER:
-#if CONFIG_XZM_CLUSTER_AWARE
- return _malloc_cpu_cluster_number();
-#else // CONFIG_XZM_CLUSTER_AWARE
- return _malloc_cpu_number() % 2;
-#endif // CONFIG_XZM_CLUSTER_AWARE
- case XZM_SLOT_SINGLE:
- default:
- return 0;
- }
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_allocation_index_t
-_xzm_get_limit_allocation_index(xzm_slot_config_t slot_config)
-{
- switch (slot_config) {
- case XZM_SLOT_CPU:
- return logical_ncpus;
- case XZM_SLOT_CLUSTER:
-#if CONFIG_XZM_CLUSTER_AWARE
- return ncpuclusters;
-#else // CONFIG_XZM_CLUSTER_AWARE
- return MIN(2, logical_ncpus);
-#endif // CONFIG_XZM_CLUSTER_AWARE
- case XZM_SLOT_SINGLE:
- default:
- return 1;
- }
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_chunk_list_t
-_xzm_xzone_chunk_list_for_index(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_chunk_list_t lists, xzm_allocation_index_t alloc_idx)
-{
- xzm_xzone_index_t xz_idx = xz->xz_idx;
- size_t alloc_base_idx = alloc_idx * zone->xzz_xzone_count;
- xzm_debug_assert(alloc_base_idx + xz_idx <
- zone->xzz_slot_count * zone->xzz_xzone_count);
- return &lists[alloc_base_idx + xz_idx];
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_segment_group_t
-_xzm_segment_group_for_id_and_front(xzm_malloc_zone_t zone,
- xzm_segment_group_id_t sgid, xzm_front_index_t front, bool huge)
-{
- xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
-#if CONFIG_XZM_CLUSTER_AWARE
- const uint32_t clusterid = _malloc_cpu_cluster_number();
- xzm_debug_assert(clusterid < ncpuclusters);
-#else
- const uint32_t clusterid = _malloc_cpu_number() % 2;
- xzm_debug_assert(clusterid < logical_ncpus);
-#endif // CONFIG_XZM_CLUSTER_AWARE
-
- xzm_debug_assert(sgid < main->xzmz_segment_group_ids_count);
- if (sgid < XZM_SEGMENT_GROUP_POINTER_XZONES) {
- xzm_debug_assert(front == XZM_FRONT_INDEX_DEFAULT);
- } else {
- xzm_debug_assert(front < main->xzmz_allocation_front_count);
- }
-
- uint8_t sg_front_index = sgid + front;
-
- if (main->xzmz_segment_group_front_count < main->xzmz_segment_group_count) {
- uint8_t sg_index;
- bool use_data_large = false;
-#if CONFIG_XZM_CLUSTER_AWARE
- // Route all huge allocations to the same global segment group that
- // has the huge cache enabled
- xzm_debug_assert(main->xzmz_defer_large);
- use_data_large = huge;
-#endif // CONFIG_XZM_CLUSTER_AWARE
- if (use_data_large) {
- sg_index = XZM_SEGMENT_GROUP_DATA_LARGE;
- } else {
- sg_index = main->xzmz_segment_group_front_count * clusterid +
- sg_front_index;
- }
- xzm_debug_assert(sg_index < main->xzmz_segment_group_count);
- return &main->xzmz_segment_groups[sg_index];
- } else {
- return &main->xzmz_segment_groups[sg_front_index];
- }
-}
-
-MALLOC_NOINLINE
-static void
-_xzm_fork_lock_wait(xzm_malloc_zone_t zone)
-{
- // This lock is taken first during fork, so that anything that needs to be
- // locked during fork that otherwise doesn't have one can be sent here.
- _malloc_lock_lock(&zone->xzz_fork_lock);
- _malloc_lock_unlock(&zone->xzz_fork_lock);
-}
-
-MALLOC_NOINLINE MALLOC_PRESERVE_MOST
-static void
-_xzm_walk_lock_wait(xzm_malloc_zone_t zone)
-{
- // We take this lock prior to walking any chunk freelist. A chunk that is
- // marked as being walked will send any allocating and deallocating threads
- // here.
- _malloc_lock_lock(&zone->xzz_lock);
- _malloc_lock_unlock(&zone->xzz_lock);
-}
-
-#pragma mark Large allocation and deallocation
-
-// mimalloc: mi_large_huge_page_alloc
-static void * __alloc_size(2)
-_xzm_malloc_large_huge(xzm_malloc_zone_t zone, size_t size, size_t alignment,
- malloc_type_descriptor_t type_desc, xzm_malloc_options_t opt)
-{
- bool clear = (opt & XZM_MALLOC_CLEAR);
- void *ptr = NULL;
-
- xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
-
- // Consider: _mi_os_good_alloc_size()?
- size_t rounded_size = roundup(size, XZM_SEGMENT_SLICE_SIZE);
- xzm_slice_kind_t kind;
- if (rounded_size > XZM_LARGE_BLOCK_SIZE_MAX ||
- alignment > XZM_ALIGNMENT_MAX) {
- kind = XZM_SLICE_KIND_HUGE_CHUNK;
- } else {
- kind = XZM_SLICE_KIND_LARGE_CHUNK;
- }
- xzm_slice_count_t slice_count;
- if (os_convert_overflow(rounded_size / XZM_SEGMENT_SLICE_SIZE,
- &slice_count)) {
- goto out;
- }
-
- bool use_data_for_large = true;
-#if XZM_NARROW_BUCKETING
- if (main->xzmz_narrow_bucketing && !main->xzmz_use_ranges) {
- use_data_for_large = false;
- }
-#endif
-
- xzm_segment_group_id_t sg_id;
- if ((use_data_for_large && malloc_type_descriptor_is_pure_data(type_desc)) ||
- kind == XZM_SLICE_KIND_HUGE_CHUNK ||
- main->xzmz_segment_group_ids_count == XZM_SEGMENT_GROUP_IDS_COUNT_DATA_ONLY) {
- // Use a separate segment group for large data only when using deferred
- // reclamation
- bool use_data_large = main->xzmz_defer_large;
-
-#if CONFIG_MTE
- struct xzm_memtag_config_s *memtag_config = &zone->xzz_memtag_config;
- if (memtag_config->tag_data &&
- memtag_config->max_block_size <= XZM_SMALL_BLOCK_SIZE_MAX) {
- use_data_large = true;
- }
-#endif
-
- sg_id = use_data_large ? XZM_SEGMENT_GROUP_DATA_LARGE :
- XZM_SEGMENT_GROUP_DATA;
- } else {
- sg_id = XZM_SEGMENT_GROUP_POINTER_LARGE;
- }
- xzm_debug_assert(sg_id < main->xzmz_segment_group_ids_count);
- xzm_segment_group_t sg = _xzm_segment_group_for_id_and_front(zone, sg_id,
- XZM_FRONT_INDEX_DEFAULT, kind == XZM_SLICE_KIND_HUGE_CHUNK);
-
- bool purgeable = (zone->xzz_flags & MALLOC_PURGEABLE);
- xzm_chunk_t chunk = xzm_segment_group_alloc_chunk(sg, kind, NULL,
- slice_count, NULL, alignment, clear, purgeable);
- if (os_unlikely(!chunk)) {
- goto out;
- }
-
- xzm_debug_assert(!clear || chunk->xzc_bits.xzcb_is_pristine);
-
- // Set the mzone_idx last to publish this chunk for the enumerator protocol
- chunk->xzc_mzone_idx = zone->xzz_mzone_idx;
-
- _malloc_lock_lock(&zone->xzz_lock);
- LIST_INSERT_HEAD(&zone->xzz_chunkq_large, chunk, xzc_entry);
- _malloc_lock_unlock(&zone->xzz_lock);
-
- size_t chunk_size = 0;
- ptr = (uint8_t *)_xzm_chunk_start(zone, chunk, &chunk_size);
-
-#if CONFIG_MTE
- bool memtag_enabled = _xzm_segment_group_memtag_block(sg, chunk_size);
- bool canonical_tag = (opt & XZM_MALLOC_CANONICAL_TAG);
- if (memtag_enabled) {
- if (canonical_tag) {
- ptr = memtag_tag_canonical(ptr, chunk_size);
- } else {
- ptr = memtag_retag(ptr, chunk_size);
- }
- }
-#endif
-
-
-out:
- if (os_unlikely(!ptr)) {
- malloc_set_errno_fast(MZ_POSIX, ENOMEM);
- }
- return ptr;
-}
-
-// mimalloc: _mi_segment_huge_page_free
-MALLOC_NOINLINE
-static void
-_xzm_free_large_huge(xzm_malloc_zone_t zone, xzm_chunk_t chunk)
-{
- _malloc_lock_lock(&zone->xzz_lock);
-
- // Unset mzone_idx first to unpublish it for the enumerator protocol
- chunk->xzc_mzone_idx = XZM_MZONE_INDEX_INVALID;
- LIST_REMOVE(chunk, xzc_entry);
-
- _malloc_lock_unlock(&zone->xzz_lock);
-
- xzm_segment_group_t sg = _xzm_segment_group_for_slice(zone, chunk);
- xzm_segment_group_free_chunk(sg, chunk, zone->xzz_flags & MALLOC_PURGEABLE,
- /* small_madvise_needed */ false);
-}
-
-#if CONFIG_MTE
-
-#pragma mark MTE
-
-// FIXME: `zone` parameter of the following functions is unused.
-
-// Initialize the tags for the given chunk.
-static void *
-_xzm_xzone_chunk_memtag_init(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_chunk_t chunk)
-{
- size_t chunk_size = 0;
- uint8_t *chunk_start = _xzm_chunk_start_ptr(zone, chunk, &chunk_size);
- return memtag_init_chunk(chunk_start, chunk_size, xz->xz_block_size);
-}
-
-// Retag a single block.
-static xzm_block_t
-_xzm_xzone_block_memtag_retag(xzm_malloc_zone_t zone, xzm_block_t block,
- size_t block_size)
-{
- return (xzm_block_t)memtag_retag((uint8_t *)block, block_size);
-}
-#endif // CONFIG_MTE
-
-static inline xzm_malloc_options_t
-_xzm_xzone_get_malloc_thread_options(void)
-{
-#if CONFIG_MTE
- // rdar://140822174
- // Check the flag in the TSD to see if canonical tagging was requested.
- malloc_thread_options_t options = malloc_get_thread_options();
- if (options.ReservedFlag) {
- return XZM_MALLOC_CANONICAL_TAG;
- }
-#endif // CONFIG_MTE
-
- return 0;
-}
-
-#pragma mark chunk list
-
-static void
-_xzm_chunk_list_fork_lock(xzm_chunk_list_head_t list)
-{
- xzm_chunk_list_head_u locked = {
- .xzch_fork_locked = true,
- };
-
- __assert_only xzm_chunk_list_head_u old_head = {
- .xzch_value = os_atomic_or_orig(&list->xzch_value, locked.xzch_value,
- relaxed),
- };
- xzm_debug_assert(!old_head.xzch_fork_locked);
-}
-
-static void
-_xzm_chunk_list_fork_unlock(xzm_chunk_list_head_t list)
-{
- xzm_chunk_list_head_u locked = {
- .xzch_fork_locked = true,
- };
- uint64_t unlocked = ~locked.xzch_value;
-
- __assert_only xzm_chunk_list_head_u old_head = {
- .xzch_value = os_atomic_and_orig(&list->xzch_value, unlocked, relaxed),
- };
- xzm_debug_assert(old_head.xzch_fork_locked);
-}
-
-static xzm_chunk_t
-_xzm_chunk_list_pop(xzm_malloc_zone_t zone, xzm_chunk_list_head_t list,
- xzm_chunk_linkage_t linkage, bool *contended_out)
-{
- xzm_chunk_t chunk = NULL;
- const bool is_batch = (linkage == XZM_CHUNK_LINKAGE_BATCH);
- xzm_chunk_list_head_u head = {
- .xzch_value = os_atomic_load(&list->xzch_value, dependency),
- };
-
- while (true) {
- if (os_unlikely(is_batch ?
- head.xzch_batch_fork_locked : head.xzch_fork_locked)) {
- _xzm_fork_lock_wait(zone);
- head.xzch_value = os_atomic_load(&list->xzch_value, dependency);
- continue;
- }
-
- chunk = (xzm_chunk_t)head.xzch_ptr;
- if (!chunk) {
- break;
- }
-
- xzm_chunk_list_head_u new_head;
- xzm_chunk_t next;
- if (!is_batch) {
- next = chunk->xzc_linkages[linkage];
- new_head = (xzm_chunk_list_head_u){
- .xzch_ptr = (uint64_t)next,
- .xzch_gen = head.xzch_gen + 1,
- };
- } else {
- xzm_debug_assert(head.xzch_batch_count);
- next = *_xzm_segment_slice_meta_batch_next(zone, chunk);
- new_head = (xzm_chunk_list_head_u){
- .xzch_batch_ptr = (uint64_t)next,
- .xzch_batch_gen = head.xzch_batch_gen + 1,
- .xzch_batch_count = head.xzch_batch_count - 1,
- };
- xzm_debug_assert(new_head.xzch_batch_count < head.xzch_batch_count);
- }
- xzm_debug_assert(chunk != next);
- bool success = os_atomic_cmpxchgv(&list->xzch_value, head.xzch_value,
- new_head.xzch_value, &head.xzch_value, dependency);
- if (success) {
- if (is_batch) {
- xzm_debug_assert(_xzm_slice_meta_is_batch_pointer(zone,
- (xzm_chunk_t)new_head.xzch_batch_ptr));
-#if CONFIG_XZM_DEFERRED_RECLAIM
- *_xzm_slice_meta_reclaim_id(zone, chunk) = VM_RECLAIM_ID_NULL;
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
- }
- break;
- } else if (contended_out) {
- *contended_out = true;
- }
- }
-
- return chunk;
-}
-
-static void
-_xzm_chunk_list_push(xzm_malloc_zone_t zone, xzm_chunk_list_head_t list,
- xzm_chunk_t chunk, xzm_chunk_linkage_t linkage, bool *contended_out)
-{
- xzm_debug_assert(linkage != XZM_CHUNK_LINKAGE_BATCH);
-
- xzm_chunk_list_head_u head = {
- .xzch_value = os_atomic_load(&list->xzch_value, dependency),
- };
-
- while (true) {
- if (os_unlikely(head.xzch_fork_locked)) {
- _xzm_fork_lock_wait(zone);
- head.xzch_value = os_atomic_load(&list->xzch_value, dependency);
- continue;
- }
-
- xzm_chunk_t head_chunk = (xzm_chunk_t)head.xzch_ptr;
- xzm_debug_assert(head_chunk != chunk);
- xzm_chunk_list_head_u new_head = {
- .xzch_ptr = (uint64_t)chunk,
- .xzch_gen = head.xzch_gen + 1,
- };
- chunk->xzc_linkages[linkage] = head_chunk;
-
- bool success = os_atomic_cmpxchgv(&list->xzch_value, head.xzch_value,
- new_head.xzch_value, &head.xzch_value, release);
- if (success) {
- break;
- } else if (contended_out) {
- *contended_out = true;
- }
- }
-}
-
-static xzm_chunk_list_t
-_xzm_chunk_list_get(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_slot_config_t *slot_config, xzm_chunk_list_t lists)
-{
- xzm_allocation_index_t alloc_idx = _xzm_get_allocation_index(zone, xz,
- slot_config, false);
- return _xzm_xzone_chunk_list_for_index(zone, xz, lists, alloc_idx);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static void
-_xzm_xzone_slot_record_contention(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_xzone_slot_counters_u *xsc, xzm_slot_config_t slot_config,
- bool is_xas, bool contended);
-
-static xzm_chunk_t
-_xzm_chunk_list_slot_pop(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_chunk_list_t lists)
-{
- xzm_slot_config_t slot_config;
- xzm_chunk_list_t xcl = _xzm_chunk_list_get(zone, xz, &slot_config, lists);
-
- bool contended = false;
- xzm_chunk_t chunk = _xzm_chunk_list_pop(zone, &xcl->xcl_list,
- XZM_CHUNK_LINKAGE_MAIN, &contended);
- if (chunk) {
- _xzm_xzone_slot_record_contention(zone, xz, &xcl->xcl_counters,
- slot_config, false, contended);
- }
-
- return chunk;
-}
-
-static void
-_xzm_chunk_list_slot_push(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_chunk_list_t lists, xzm_chunk_t chunk)
-{
- xzm_slot_config_t slot_config;
- xzm_chunk_list_t xcl = _xzm_chunk_list_get(zone, xz, &slot_config, lists);
- bool contended = false;
- _xzm_chunk_list_push(zone, &xcl->xcl_list, chunk, XZM_CHUNK_LINKAGE_MAIN,
- &contended);
- _xzm_xzone_slot_record_contention(zone, xz, &xcl->xcl_counters, slot_config,
- false, contended);
-}
-
-static void
-_xzm_chunk_batch_list_fork_lock(xzm_chunk_list_head_t list)
-{
- xzm_chunk_list_head_u locked = {
- .xzch_batch_fork_locked = true,
- };
-
- __assert_only xzm_chunk_list_head_u old_head = {
- .xzch_value = os_atomic_or_orig(&list->xzch_value, locked.xzch_value,
- relaxed),
- };
- xzm_debug_assert(!old_head.xzch_batch_fork_locked);
-}
-
-static void
-_xzm_chunk_batch_list_fork_unlock(xzm_chunk_list_head_t list)
-{
- xzm_chunk_list_head_u locked = {
- .xzch_batch_fork_locked = true,
- };
- uint64_t unlocked = ~locked.xzch_value;
-
- __assert_only xzm_chunk_list_head_u old_head = {
- .xzch_value = os_atomic_and_orig(&list->xzch_value, unlocked, relaxed),
- };
- xzm_debug_assert(old_head.xzch_batch_fork_locked);
-}
-
-static void
-_xzm_xzone_madvise_batch(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_chunk_t chunk);
-
-static void
-_xzm_chunk_batch_list_push(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_chunk_t chunk, size_t max)
-{
- xzm_chunk_list_head_t list = &xz->xz_batch_list;
- xzm_chunk_list_head_u head = {
- .xzch_value = os_atomic_load(&list->xzch_value, dependency),
- };
-
- bool was_full;
- while (true) {
- if (os_unlikely(head.xzch_batch_fork_locked)) {
- _xzm_fork_lock_wait(zone);
- head.xzch_value = os_atomic_load(&list->xzch_value, dependency);
- continue;
- }
-
- xzm_debug_assert(head.xzch_batch_count <= max);
- was_full = (head.xzch_batch_count >= max);
-
- xzm_chunk_list_head_u new_head = {
- .xzch_batch_ptr = (uint64_t)chunk,
- .xzch_batch_gen = head.xzch_batch_gen + 1,
- .xzch_batch_count = os_likely(!was_full) ?
- head.xzch_batch_count + 1 : 1,
- };
- xzm_debug_assert(was_full ||
- new_head.xzch_batch_count > head.xzch_batch_count);
- xzm_chunk_t next = os_likely(!was_full) ?
- (xzm_chunk_t)head.xzch_ptr : NULL;
- xzm_debug_assert(chunk != next);
- *_xzm_segment_slice_meta_batch_next(zone, chunk) = next;
-
- bool success = os_atomic_cmpxchgv(&list->xzch_value, head.xzch_value,
- new_head.xzch_value, &head.xzch_value, release);
- if (success) {
- break;
- }
- }
-
- // Perform the batch madvise on the old head
- if (os_unlikely(was_full)) {
- _xzm_xzone_madvise_batch(zone, xz, (xzm_chunk_t)head.xzch_batch_ptr);
- }
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_chunk_list_empty(xzm_chunk_list_head_t list)
-{
- xzm_chunk_list_head_u head = {
- .xzch_value = os_atomic_load(&list->xzch_value, relaxed),
- };
-
- return !head.xzch_ptr;
-}
-
-#pragma mark Tiny allocation
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static void *
-_xzm_xzone_malloc_from_freelist_chunk_inline(xzm_malloc_zone_t zone,
- xzm_xzone_t xz, xzm_allocation_index_t alloc_idx,
- xzm_xzone_thread_cache_t cache, xzm_chunk_t chunk, bool small,
- bool walk_wait, bool *corrupt_out, bool *contended_out,
- bool *install_empty_out)
-{
-#if CONFIG_MTE
- bool memtag_enabled = chunk->xzc_tagged;
-#endif
-
- bool install = !!install_empty_out;
- size_t granule = small ? XZM_SMALL_GRANULE : XZM_GRANULE;
-
- uint8_t *start = (uint8_t *)_xzm_chunk_start(zone, chunk, NULL);
- if (small) {
- xzm_debug_assert(chunk->xzc_bits.xzcb_kind ==
- XZM_SLICE_KIND_SMALL_FREELIST_CHUNK);
- } else {
- xzm_debug_assert(chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_TINY_CHUNK);
- }
-
- // If we're trying to cache this chunk, it's important for the enumerator
- // that we initialize the cache to point to it before we mark it as
- // installed to a thread
- if (cache) {
- if (install) {
- cache->xztc_chunk = chunk;
- cache->xztc_chunk_start = start;
- } else {
- xzm_debug_assert(cache->xztc_chunk == chunk);
- xzm_debug_assert(cache->xztc_chunk_start == start);
- }
- }
-
- void *ptr = NULL;
- bool from_free_list = false;
- struct xzm_block_inline_meta_s block_meta;
-
- uint32_t contentions = 0;
-
- // Dependency order to observe initialization and zeroing of the block we
- // allocated
- xzm_chunk_atomic_meta_u old_meta = {
- .xca_value = os_atomic_load_wide(
- &chunk->xzc_atomic_meta.xca_value, dependency),
- };
- while (true) {
- if (os_unlikely(old_meta.xca_walk_locked)) {
- if (walk_wait) {
- _xzm_walk_lock_wait(zone);
- old_meta.xca_value = os_atomic_load_wide(
- &chunk->xzc_atomic_meta.xca_value, dependency);
- continue;
- } else {
- return NULL;
- }
- }
-
- xzm_chunk_atomic_meta_u new_meta = old_meta;
-
- if (install) {
- // We're considering this chunk as a candidate to install. The
- // chunk we're looking at should not be installed in any slot, and
- // should be marked as being on the partial list (though in fact
- // we'll have just popped it).
- xzm_debug_assert(old_meta.xca_alloc_idx == XZM_SLOT_INDEX_EMPTY);
- xzm_debug_assert(old_meta.xca_on_partial_list);
- xzm_debug_assert(!old_meta.xca_on_empty_list);
-
- new_meta.xca_on_partial_list = false;
-
- // Does the chunk have any blocks?
- if (old_meta.xca_free_count == 0) {
- // No, no blocks to allocate.
- if (old_meta.xca_alloc_head == XZM_FREE_MADVISED) {
- // If the chunk is fully madvised and it was on the partial
- // list, it's the caller's responsibility to move it to the
- // empty list.
- new_meta.xca_on_empty_list = true;
- *install_empty_out = true;
- } else {
- // It shouldn't be possible for us to have gotten a full
- // chunk from the partial list, so the only remaining
- // possibility is an in-progress madvise.
- xzm_debug_assert(
- old_meta.xca_alloc_head == XZM_FREE_MADVISING);
- }
-
- // We may have initialized our thread cache entry to this chunk
- // on a previous iteration of the loop. It's important that we
- // invalidate it now, before it can potentially be reused by
- // another thread cache, so that there's no ambiguity to the
- // enumerator about which cache owns the chunk.
- if (cache) {
- cache->xztc_state = XZM_XZONE_CACHE_EMPTY;
- }
-
- bool success = os_atomic_cmpxchgv(
- &chunk->xzc_atomic_meta.xca_value, old_meta.xca_value,
- new_meta.xca_value, &old_meta.xca_value, dependency);
- if (!success) {
- // The only way this can happen is MADVISING -> MADVISED
- xzm_debug_assert(!(*install_empty_out));
- continue;
- }
-
- xzm_trace(malloc_install_skip, (uint64_t)chunk,
- old_meta.xca_value_lo, new_meta.xca_value_lo,
- cache ? cache->xztc_freelist_state : 0);
- return NULL;
- } else {
- // Yes, we want to proceed with the allocation attempt and mark
- // the chunk as installed to our slot if successful.
- new_meta.xca_alloc_idx = alloc_idx + 1;
-
-#if CONFIG_TINY_ALLOCATION_SLOT_LOCK
- // In this configuration, we actually can't proceed with the
- // attempt until after we've marked the chunk as installed to
- // our slot, as that's what allows a thread depopulating the
- // chunk to be able to wait until we've exited the critical
- // section. So, we're going to mark the chunk as installed
- // directly, and then transition to the non-install logic to try
- // to actually allocate a block.
- bool success = os_atomic_cmpxchgv(
- &chunk->xzc_atomic_meta.xca_value, old_meta.xca_value,
- new_meta.xca_value, &old_meta.xca_value, dependency);
- if (!success) {
- continue;
- }
-
- // Update the chunk state for the next loop iteration
- old_meta = new_meta;
-
- install = false;
- continue;
-#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
- }
- } else {
- // We found this chunk as the one currently installed in our slot,
- // and only want to allocate from it if it still is.
- if (old_meta.xca_alloc_idx != alloc_idx + 1) {
- // Shouldn't be possible when the chunk is installed to the
- // thread cache
- xzm_debug_assert(!cache);
-
- // No. Go back to try to get a different one.
- return NULL;
- }
-
- // Does the chunk have any blocks?
- if (old_meta.xca_free_count == 0) {
- // We should try to uninstall this chunk.
- new_meta.xca_alloc_idx = XZM_SLOT_INDEX_EMPTY;
-
- // It's marked as installed, so it should be full.
- xzm_debug_assert(old_meta.xca_alloc_head == XZM_FREE_NULL);
-
- // It shouldn't be on any chunk lists either, and doesn't need
- // to be. If it eventually becomes unfull, it's the
- // responsibility of that thread to move it back to the partial
- // list.
- xzm_debug_assert(!old_meta.xca_on_partial_list);
- xzm_debug_assert(!old_meta.xca_on_empty_list);
-
- if (cache) {
- // We need to mark our thread cache entry as invalid before
- // we mark the chunk as uninstalled, so that there's no
- // ambiguity about which cache owns the chunk if we get
- // pre-empted and that chunk gets installed to another cache
- cache->xztc_state = XZM_XZONE_CACHE_EMPTY;
- }
-
- bool success = os_atomic_cmpxchgv(
- &chunk->xzc_atomic_meta.xca_value, old_meta.xca_value,
- new_meta.xca_value, &old_meta.xca_value, dependency);
- if (!success) {
- continue;
- }
-
- xzm_trace(malloc_full_skip, (uint64_t)chunk,
- old_meta.xca_value_lo, new_meta.xca_value_lo,
- cache ? cache->xztc_freelist_state : 0);
- return NULL;
- }
- }
-
- // It looks like we should be able to allocate.
- from_free_list = false;
-
- if (cache) {
- // Allocating all blocks for this cache
-
- // We initialize the cache entry to the freelist _before_ we mark
- // the chunk as installed so that the enumerator can reliably walk
- // the local freelist in the cache if it sees the chunk marked as
- // installed.
-
- new_meta.xca_alloc_head = XZM_FREE_NULL;
- new_meta.xca_free_count = 0;
-
- cache->xztc_free_count = old_meta.xca_free_count - 1;
-
- if (old_meta.xca_alloc_head < XZM_FREE_LIMIT) {
- // Pop from the alloc head
- ptr = start + (old_meta.xca_alloc_head * granule);
-#if CONFIG_MTE
- if (memtag_enabled) {
- memtag_disable_checking();
- block_meta = *(struct xzm_block_inline_meta_s *)ptr;
- memtag_enable_checking();
- } else {
-#endif
- block_meta = *(struct xzm_block_inline_meta_s *)ptr;
-#if CONFIG_MTE
- }
-#endif
-
- cache->xztc_head = block_meta.xzb_linkage.xzbl_next_offset;
- cache->xztc_head_seqno = block_meta.xzb_linkage.xzbl_next_seqno;
-
- from_free_list = true;
- } else {
- xzm_debug_assert(old_meta.xca_alloc_head == XZM_FREE_NULL);
- xzm_debug_assert(old_meta.xca_free_count);
-
- // Note: it's important that we compute ptr in the loop, before
- // we pop, for the benefit of the memory tools: if we don't,
- // then the block will appear allocated but the tools won't be
- // able to find any references to it. Getting it into a
- // register first will ensure that there's always a reference.
- size_t capacity = xz->xz_chunk_capacity;
- size_t n_allocated = capacity - old_meta.xca_free_count;
- ptr = start + (n_allocated * xz->xz_block_size);
-
- cache->xztc_head = XZM_FREE_NULL;
- }
- } else {
- // Allocating one block
- new_meta.xca_free_count--;
- if (old_meta.xca_alloc_head < XZM_FREE_LIMIT) {
- // Pop from the alloc head
- ptr = start + (old_meta.xca_alloc_head * granule);
-
-#if CONFIG_MTE
- if (memtag_enabled) {
- memtag_disable_checking();
- block_meta = *(struct xzm_block_inline_meta_s *)ptr;
- memtag_enable_checking();
- } else {
-#endif
- block_meta = *(struct xzm_block_inline_meta_s *)ptr;
-#if CONFIG_MTE
- }
-#endif
-
- new_meta.xca_alloc_head = block_meta.xzb_linkage.xzbl_next_offset;
- new_meta.xca_head_seqno = block_meta.xzb_linkage.xzbl_next_seqno;
- from_free_list = true;
- } else {
- xzm_debug_assert(old_meta.xca_free_count);
- xzm_debug_assert(old_meta.xca_alloc_head == XZM_FREE_NULL);
-
- // Note: it's important that we compute ptr in the loop, before we
- // pop, for the benefit of the memory tools: if we don't, then the
- // block will appear allocated but the tools won't be able to find
- // any references to it. Getting it into a register first will
- // ensure that there's always a reference.
- size_t capacity = xz->xz_chunk_capacity;
- ptr = start + ((capacity - old_meta.xca_free_count) *
- xz->xz_block_size);
- }
- }
-
- bool success = os_atomic_cmpxchgv(&chunk->xzc_atomic_meta.xca_value,
- old_meta.xca_value, new_meta.xca_value, &old_meta.xca_value,
- dependency);
- if (!success) {
- // XXX: possible madvise race - record? Retry?
-
- // Record the contention
- *contended_out = true;
-
- contentions++;
-
- // Try again.
- continue;
- }
-
- // We got it.
- xzm_trace(malloc_success, (uint64_t)chunk, old_meta.xca_value_lo,
- new_meta.xca_value_lo,
- cache ? cache->xztc_freelist_state : 0);
- break;
- }
-
-#if CONFIG_MTE
- if (memtag_enabled) {
- if (small) {
- // Small blocks are tagged only on alloc
- ptr = memtag_retag(ptr, xz->xz_block_size);
- } else {
- if (from_free_list) {
- uint8_t tag = memtag_extract_tag((uint8_t *)block_meta.xzb_cookie);
- xzm_debug_assert(tag != 0);
- ptr = (void *)memtag_mix_tag(ptr, tag);
- } else {
- ptr = memtag_fixup_ptr(ptr);
- }
- }
- }
-#endif
-
- if (from_free_list) {
- uint64_t expected_cookie = (uint64_t)ptr ^ zone->xzz_freelist_cookie;
-#if CONFIG_MTE
- if (small) {
- // Small chunks do not encode the tag in the freelist cookie
- expected_cookie = (uint64_t)memtag_strip_address(
- (uint8_t *)expected_cookie);
- }
-#endif
- if (expected_cookie != block_meta.xzb_cookie) {
- *corrupt_out = true;
- goto freelist_done;
- }
-
- // Check the PAC by adding the seqno back
- union xzm_block_linkage_u linkage = {
- .xzbl_next_offset = block_meta.xzb_linkage.xzbl_next_offset,
- .xzbl_next_seqno = block_meta.xzb_linkage.xzbl_next_seqno,
- .xzbl_seqno = old_meta.xca_head_seqno,
- };
-
- linkage.xzbl_next_value = (uintptr_t)ptrauth_sign_unauthenticated(
- linkage.xzbl_next, ptrauth_key_process_dependent_data,
- ptrauth_blend_discriminator(ptr,
- ptrauth_string_discriminator("xzb_linkage")));
-
- // Take the seqno back out of the signed value for the comparison with
- // the stored one
- linkage.xzbl_seqno = 0;
-
- if (block_meta.xzb_linkage.xzbl_next_value != linkage.xzbl_next_value) {
- *corrupt_out = true;
- goto freelist_done;
- }
-
-freelist_done:
- ;
- }
-
- return ptr;
-}
-
-static void *
-_xzm_xzone_malloc_from_freelist_chunk(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_allocation_index_t alloc_idx, xzm_xzone_thread_cache_t cache,
- xzm_chunk_t chunk, bool small, bool *contended_out,
- bool *install_empty_out)
-{
- const bool walk_wait = true;
- bool corrupt = false;
- void *ptr = _xzm_xzone_malloc_from_freelist_chunk_inline(zone, xz,
- alloc_idx, cache, chunk, small, walk_wait, &corrupt, contended_out,
- install_empty_out);
- if (corrupt) {
- _xzm_corruption_detected(ptr);
- }
-
- return ptr;
-}
-
-static void *
-_xzm_xzone_malloc_from_empty_freelist_chunk(xzm_malloc_zone_t zone,
- xzm_xzone_t xz, xzm_allocation_index_t alloc_idx,
- xzm_xzone_thread_cache_t cache, xzm_chunk_t chunk,
- bool is_fresh)
-{
- xzm_debug_assert(chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_TINY_CHUNK ||
- chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK);
-#if CONFIG_MTE
- bool small = chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK;
-#endif
-
- // acquire to make sure the reclaim index is visible
- // TODO make this load relaxed. the address of the reclaim index is derived
- // from the chunk pointer that we popped from the list, and stores to the
- // list are done with a release barrier
- xzm_chunk_atomic_meta_u old_meta = {
- .xca_value = os_atomic_load_wide(
- &chunk->xzc_atomic_meta.xca_value, acquire),
- };
-
- // We found this on the batch list or the empty list, so it is either
- // enqueued for madvising or has been madvised
- xzm_debug_assert(old_meta.xca_alloc_head == XZM_FREE_MADVISED ||
- old_meta.xca_alloc_head == XZM_FREE_MADVISING);
-
- // A chunk is unusable if it is both on the partial list and the batch list;
- // e.g., if its insertion raced with our emptying of the partial list
- if (os_unlikely(old_meta.xca_on_partial_list)) {
- xzm_debug_assert(old_meta.xca_alloc_head == XZM_FREE_MADVISING);
- return NULL;
- }
-
-
- // We found this chunk on the batch or isolation list. It is not installed
- // anywhere.
- xzm_debug_assert(!old_meta.xca_on_partial_list);
- xzm_debug_assert(old_meta.xca_alloc_idx == XZM_SLOT_INDEX_EMPTY);
-
- uint8_t *start = (uint8_t *)_xzm_chunk_start(zone, chunk, NULL);
- void *ptr = (void *)start;
-#if CONFIG_MTE
- const bool memtag_enabled = xz->xz_tagged;
- if (memtag_enabled) {
- if (small) {
- // For small, retag blocks lazily instead of for the entire chunk
- // up front
- ptr = memtag_retag(start, xz->xz_block_size);
- } else {
- if (is_fresh) {
- ptr = _xzm_xzone_chunk_memtag_init(zone, xz, chunk);
- } else {
- ptr = memtag_fixup_ptr(ptr);
- }
- }
- }
-#else
- (void)is_fresh;
-#endif
-
- // Preserve the seqno across reuses - this is important for anti-replay
- xzm_chunk_atomic_meta_u new_meta = old_meta;
-
- // We've taken it off the isolation list and won't be putting it back.
- new_meta.xca_on_empty_list = false;
-
- // In the absence of deferred reclaim, we're guaranteed to be able to bring
- // this chunk back into use.
- new_meta.xca_alloc_idx = alloc_idx + 1;
- if (cache) {
- new_meta.xca_alloc_head = XZM_FREE_NULL;
- new_meta.xca_free_count = 0;
-
- cache->xztc_chunk = chunk;
- cache->xztc_chunk_start = start;
- cache->xztc_head = XZM_FREE_NULL;
- cache->xztc_free_count = xz->xz_chunk_capacity - 1;
- // preserve xztc_seqno
- } else {
- new_meta.xca_alloc_head = XZM_FREE_NULL;
- new_meta.xca_free_count = xz->xz_chunk_capacity - 1;
- }
-
- // This needs an acquire barrier because the pointer we're returning is not
- // derived from the metadata, so the dependency ordering we usually rely on
- // doesn't work.
- bool success = os_atomic_cmpxchgv(&chunk->xzc_atomic_meta.xca_value,
- old_meta.xca_value, new_meta.xca_value, &old_meta.xca_value,
- acquire);
- xzm_assert(success);
-
- xzm_trace(malloc_from_empty, (uint64_t)chunk, old_meta.xca_value_lo,
- new_meta.xca_value_lo, old_meta.xca_seqno);
-
- return ptr;
-}
-
-// Called whenever a tiny or small chunk is received from the span queues. This
-// function needs to perform all the initialization needed to allocate a block
-// from this chunk, or to free the chunk during zone destruction.
-static void
-_xzm_xzone_fresh_chunk_init(xzm_xzone_t xz, xzm_chunk_t chunk,
- xzm_slice_kind_t kind)
-{
- xzm_debug_assert(chunk->xzc_xzone_idx == xz->xz_idx);
-
- chunk->xzc_bits.xzcb_preallocated = false;
-
- switch (kind) {
- case XZM_SLICE_KIND_SMALL_CHUNK:
- _xzm_chunk_reset_free(xz, chunk, true);
- break;
- case XZM_SLICE_KIND_TINY_CHUNK:
- case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
- xzm_debug_assert(xz->xz_block_size <= UINT16_MAX);
- chunk->xzc_freelist_block_size = (uint16_t)xz->xz_block_size;
-
- xzm_debug_assert(xz->xz_chunk_capacity <= UINT16_MAX);
- chunk->xzc_freelist_chunk_capacity = (uint16_t)xz->xz_chunk_capacity;
-#if CONFIG_MTE
- chunk->xzc_tagged = xz->xz_tagged;
-#endif
- break;
- default:
- xzm_abort("Unexpected chunk kind");
- break;
- }
-}
-
-static void *
-_xzm_xzone_malloc_from_fresh_freelist_chunk(xzm_malloc_zone_t zone,
- xzm_xzone_t xz, xzm_allocation_index_t alloc_idx,
- xzm_xzone_thread_cache_t cache, xzm_chunk_t chunk, bool small)
-{
- xzm_debug_assert(chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_TINY_CHUNK ||
- chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK);
-
- _xzm_xzone_fresh_chunk_init(xz, chunk, chunk->xzc_bits.xzcb_kind);
-
- xzm_chunk_atomic_meta_u new_meta;
-
- uint8_t *start = (uint8_t *)_xzm_chunk_start(zone, chunk, NULL);
-
- if (cache) {
- new_meta = (xzm_chunk_atomic_meta_u){
- .xca_alloc_head = XZM_FREE_NULL,
- .xca_free_count = 0,
- .xca_alloc_idx = alloc_idx + 1,
- };
-
- cache->xztc_chunk = chunk;
- cache->xztc_chunk_start = start;
- cache->xztc_head = XZM_FREE_NULL;
- cache->xztc_free_count = xz->xz_chunk_capacity - 1;
- // preserve xztc_seqno
- } else {
- new_meta = (xzm_chunk_atomic_meta_u){
- .xca_alloc_head = XZM_FREE_NULL,
- .xca_free_count = xz->xz_chunk_capacity - 1,
- .xca_alloc_idx = alloc_idx + 1,
- };
- }
-
- xzm_trace(malloc_from_fresh, (uint64_t)chunk, xz->xz_idx,
- new_meta.xca_value_lo, 0);
- os_atomic_store_wide(&chunk->xzc_atomic_meta.xca_value, new_meta.xca_value,
- relaxed);
-
- // Enumerator protocol: mzone_idx is initialized last to "publish" the
- // chunk. Because there are no references to this chunk yet, this store is
- // guaranteed by dependency ordering to be visible to all future threads
- // that observe it when the chunk is store-released to its initial slot.
- chunk->xzc_mzone_idx = xz->xz_mzone_idx;
-
-#if CONFIG_MTE
- if (chunk->xzc_tagged) {
- void *ptr;
- if (!small) {
- ptr = _xzm_xzone_chunk_memtag_init(zone, xz, chunk);
- } else {
- // For small, retag blocks lazily instead of for the entire chunk
- // up front
- ptr = (void *)_xzm_xzone_block_memtag_retag(zone, (xzm_block_t)start,
- xz->xz_block_size);
- }
- return ptr;
- }
-#endif
- return (void *)start;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_slot_config_t
-_xzm_next_slot_config(xzm_slot_config_t slot_config)
-{
- switch (slot_config) {
- case XZM_SLOT_SINGLE:
-#if CONFIG_XZM_CLUSTER_AWARE
- if (ncpuclusters > 1) {
- return XZM_SLOT_CLUSTER;
- } else {
- return XZM_SLOT_CPU;
- }
-#else // CONFIG_XZM_CLUSTER_AWARE
- return XZM_SLOT_CLUSTER;
-#endif // CONFIG_XZM_CLUSTER_AWARE
- case XZM_SLOT_CLUSTER:
- return XZM_SLOT_CPU;
- case XZM_SLOT_CPU:
- xzm_debug_abort("Can't upgrade from XZM_SLOT_CPU");
- return XZM_SLOT_CPU;
- default:
- xzm_debug_abort("Invalid xzone slot config");
- return XZM_SLOT_CPU;
- }
-}
-
-#if CONFIG_XZM_THREAD_CACHE
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static void
-_xzm_xzone_thread_cache_record_contention(xzm_malloc_zone_t zone,
- xzm_xzone_t xz, xzm_xzone_thread_cache_t cache);
-
-#endif // CONFIG_XZM_THREAD_CACHE
-
-static void
-_xzm_xzone_upgrade_freelist_slot_config(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_xzone_slot_counters_u *xsc, xzm_xzone_slot_counters_u counter,
- xzm_slot_config_t next_slot_config, bool is_xas)
-{
- if (is_xas) {
- xzm_trace(slot_upgrade, xz->xz_idx, (uint64_t)xsc,
- next_slot_config, counter.xsc_value);
- } else {
- xzm_trace(list_upgrade, xz->xz_idx, (uint64_t)xsc,
- next_slot_config, counter.xsc_value);
- }
-
- xzm_slot_config_t *slot_config_p = is_xas ? &xz->xz_slot_config :
- &xz->xz_list_config;
- os_atomic_store(slot_config_p, next_slot_config, relaxed);
-
- // Try to upgrade/reset the counters in all the slots,
- // regardless of whether or not we actually upgraded the xzone
- counter = (xzm_xzone_slot_counters_u){
- .xsc_slot_config = next_slot_config,
- };
- xzm_allocation_index_t limit_idx =
- _xzm_get_limit_allocation_index(next_slot_config);
- for (xzm_allocation_index_t alloc_idx = 0; alloc_idx < limit_idx;
- alloc_idx++) {
- xsc = is_xas ?
- &_xzm_xzone_allocation_slot_for_index(zone, xz,
- alloc_idx)->xas_counters :
- &_xzm_xzone_chunk_list_for_index(zone, xz,
- zone->xzz_partial_lists, alloc_idx)->xcl_counters;
- os_atomic_store(&xsc->xsc_value, counter.xsc_value, relaxed);
- }
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static void
-_xzm_xzone_slot_record_contention(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_xzone_slot_counters_u *xsc, xzm_slot_config_t slot_config,
- bool is_xas, bool contended)
-{
-#if CONFIG_XZM_THREAD_CACHE
- // XXX Somewhat wasteful to re-check this condition here - consider plumbing
- // thread cache loaded earlier through to here
- if (is_xas &&
- zone->xzz_thread_cache_enabled &&
- xz->xz_block_size <= XZM_THREAD_CACHE_THRESHOLD) {
- if (contended) {
- xzm_thread_cache_t tc = _xzm_get_thread_cache();
- if (os_likely(tc)) {
- xzm_xzone_index_t xz_idx = xz->xz_idx;
- xzm_xzone_thread_cache_t cache = &tc->xtc_xz_caches[xz_idx];
- _xzm_xzone_thread_cache_record_contention(zone, xz, cache);
- }
- }
- return;
- }
-#endif // CONFIG_XZM_THREAD_CACHE
-
- xzm_xzone_slot_counters_u old_counters = {
- .xsc_value = os_atomic_load(&xsc->xsc_value,
- relaxed),
- };
-
- xzm_slot_config_t max_config = is_xas ? zone->xzz_max_slot_config :
- zone->xzz_max_list_config;
- if (old_counters.xsc_slot_config == max_config) {
- return;
- }
-
- // Only need to record if:
- // - We detected a contention, or
- // - Some contention has already been detected, so we need to record the
- // operation to assess the rate
- if (contended || old_counters.xsc_contentions) {
- uint64_t inc = 1; // inc xsc_ops
- if (contended) {
- inc |= (1ull << 32); // inc xsc_contentions
- }
-
- xzm_xzone_slot_counters_u new_counters = {
- .xsc_value = os_atomic_add(&xsc->xsc_value, inc,
- relaxed),
- };
-
- // The "current" slot config that we observe in the slot
- // after incrementing the counters may be different than the one we
- // observed originally on the xzone:
- // - It could have a lower slot config if it was just upgraded
- //
- // Regardless, we'll use the slot config that we saw in the counters as
- // the one that dictates what we should do next from here on.
- xzm_slot_config_t current_slot_config = new_counters.xsc_slot_config;
-
- if (current_slot_config == max_config) {
- return;
- }
-
- uint32_t period = is_xas ? zone->xzz_slot_upgrade_period :
- zone->xzz_list_upgrade_period;
- uint32_t *thresholds = is_xas ? zone->xzz_slot_upgrade_threshold :
- zone->xzz_list_upgrade_threshold;
- if (os_unlikely(new_counters.xsc_contentions >=
- thresholds[current_slot_config])) {
- if (new_counters.xsc_contentions >
- thresholds[current_slot_config]) {
- // Someone else is upgrading all the slots, do nothing
- return;
- }
-
- // If we've detected contention above the threshold, try to
- // upgrade the xzone
- xzm_slot_config_t next_slot_config = _xzm_next_slot_config(
- new_counters.xsc_slot_config);
- xzm_debug_assert(new_counters.xsc_slot_config < next_slot_config);
-
- _xzm_xzone_upgrade_freelist_slot_config(zone, xz, xsc,
- new_counters, next_slot_config, is_xas);
- } else if (os_unlikely(new_counters.xsc_ops >= period)) {
- if (new_counters.xsc_ops > period) {
- // Someone else is resetting the counter, do nothing
- return;
- }
- // If the detection period has elapsed without meeting the
- // threshold, reset the counters for the next period.
- xzm_xzone_slot_counters_u orig_counters = new_counters;
-
- new_counters = (xzm_xzone_slot_counters_u){
- .xsc_slot_config = current_slot_config,
- };
-
- os_atomic_rmw_loop(&xsc->xsc_value,
- old_counters.xsc_value, new_counters.xsc_value, relaxed, {
- if (old_counters.xsc_value < orig_counters.xsc_value ||
- old_counters.xsc_slot_config >
- orig_counters.xsc_slot_config) {
- os_atomic_rmw_loop_give_up(break);
- }
- });
- }
- }
-}
-
-#if !CONFIG_TINY_ALLOCATION_SLOT_LOCK
-// This inlines badly: the compiler unconditionally hoists the mrs out of the
-// loop, which is exactly what we don't want since we normally won't need it
-MALLOC_NOINLINE
-static uint32_t
-_malloc_ulock_self_owner_value(void)
-{
- mach_port_t self_port = _pthread_mach_thread_self_direct();
- return (uint32_t)self_port >> 2;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_allocation_slot_atomic_meta_u
-_xzm_allocation_slot_gate_wait(xzm_xzone_allocation_slot_t xas,
- xzm_allocation_slot_atomic_meta_u slot_meta)
-{
- xzm_allocation_slot_atomic_meta_u *xasa = &xas->xas_atomic;
-
- xzm_debug_assert(slot_meta.xasa_gate.xsg_locked);
-
- if (!slot_meta.xasa_gate.xsg_waiters) {
- // We're the first waiter, so we first need to mark ourselves as
- // waiting.
- xzm_allocation_slot_atomic_meta_u new_slot_meta = slot_meta;
- new_slot_meta.xasa_gate.xsg_waiters = true;
- bool success = os_atomic_cmpxchgv(&xasa->xasa_value,
- slot_meta.xasa_value, new_slot_meta.xasa_value,
- &slot_meta.xasa_value, dependency);
- if (!success) {
- return slot_meta;
- }
- slot_meta.xasa_gate.xsg_waiters = true;
- }
-
- uint32_t wait_op = UL_UNFAIR_LOCK | ULF_NO_ERRNO |
- ULF_WAIT_ADAPTIVE_SPIN | ULF_WAIT_WORKQ_DATA_CONTENTION;
- int rc = __ulock_wait(wait_op, &xasa->xasa_ulock,
- slot_meta.xasa_ulock, 0);
- if (os_unlikely(rc < 0)) {
- switch (-rc) {
- case EINTR:
- case EFAULT:
- break;
- default:
- xzm_abort_with_reason("ulock_wait failure", -rc);
- }
- } else {
- // We should only wake as part of a wake-all broadcast, but
- // there's no way to assert that. rc indicates the number of
- // waiters still in the ulock, but we can't assert anything
- // useful about that.
- }
-
- slot_meta = (xzm_allocation_slot_atomic_meta_u){
- .xasa_value = os_atomic_load(&xasa->xasa_value, dependency),
- };
- return slot_meta;
-}
-
-static void
-_xzm_xzone_allocation_slot_fork_lock(xzm_xzone_allocation_slot_t xas)
-{
- xzm_allocation_slot_atomic_meta_u *xasa = &xas->xas_atomic;
-
- xzm_allocation_slot_atomic_meta_u slot_meta = {
- .xasa_value = os_atomic_load(&xasa->xasa_value, dependency),
- };
- while (true) {
- if (slot_meta.xasa_gate.xsg_locked) {
- slot_meta = _xzm_allocation_slot_gate_wait(xas, slot_meta);
- continue;
- }
-
- xzm_allocation_slot_atomic_meta_u new_slot_meta = slot_meta;
- new_slot_meta.xasa_chunk.xsc_fork_locked = true;
-
- bool success = os_atomic_cmpxchgv(&xasa->xasa_value,
- slot_meta.xasa_value, new_slot_meta.xasa_value,
- &slot_meta.xasa_value, relaxed);
- if (!success) {
- continue;
- }
-
- break;
- }
-}
-
-static void
-_xzm_xzone_allocation_slot_fork_unlock(xzm_xzone_allocation_slot_t xas)
-{
- xzm_allocation_slot_atomic_meta_u *xasa = &xas->xas_atomic;
-
- xzm_allocation_slot_atomic_meta_u slot_meta = {
- .xasa_value = os_atomic_load(&xasa->xasa_value, dependency),
- };
-
- xzm_debug_assert(slot_meta.xasa_chunk.xsc_fork_locked);
-
- xzm_allocation_slot_atomic_meta_u new_slot_meta = slot_meta;
- new_slot_meta.xasa_chunk.xsc_fork_locked = false;
-
- uint64_t prev_slot_value = os_atomic_xchg(&xasa->xasa_value,
- new_slot_meta.xasa_value, release);
- xzm_assert(prev_slot_value == slot_meta.xasa_value);
-}
-#endif // !CONFIG_TINY_ALLOCATION_SLOT_LOCK
-
-static xzm_chunk_t
-_xzm_xzone_allocate_chunk_from_isolation(xzm_main_malloc_zone_t main,
- xzm_xzone_t xz)
-{
- xzm_debug_assert(xz->xz_sequestered);
- xzm_isolation_zone_t iz = &main->xzmz_isolation_zones[xz->xz_idx];
-
- xzm_chunk_t chunk = NULL;
-
- // Don't bother trying to take the iz lock if it doesn't look like
- // there's anything in there
- //
- // XXX TODO: this should be a proper atomic relaxed load
- if (LIST_FIRST(&iz->xziz_chunkq)) {
-#if CONFIG_XZM_DEFERRED_RECLAIM
- LIST_HEAD(, xzm_slice_s) busy_list = LIST_HEAD_INITIALIZER(busy_list);
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
- _malloc_lock_lock(&iz->xziz_lock);
- chunk = LIST_FIRST(&iz->xziz_chunkq);
- while (chunk) {
- xzm_debug_assert(_xzm_chunk_is_empty(&main->xzmz_base, xz, chunk));
- LIST_REMOVE(chunk, xzc_entry);
-#if CONFIG_XZM_DEFERRED_RECLAIM
- // Chunks stay in the reclaim buffer when removed from the
- // isolation zone, and must be removed from the buffer
- if ((xz->xz_block_size > XZM_TINY_BLOCK_SIZE_MAX &&
- main->xzmz_defer_small) ||
- ((xz->xz_block_size <= XZM_TINY_BLOCK_SIZE_MAX &&
- main->xzmz_defer_tiny))) {
- if (!xzm_chunk_mark_used(&main->xzmz_base, chunk, NULL)) {
- // this chunk is busy being reclaimed by the kernel
- LIST_INSERT_HEAD(&busy_list, chunk, xzc_entry);
- chunk = LIST_FIRST(&iz->xziz_chunkq);
- continue;
- }
- }
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
- // chunk is already initialized, other than the mzone idx
- chunk->xzc_mzone_idx = xz->xz_mzone_idx;
- break;
- }
-#if CONFIG_XZM_DEFERRED_RECLAIM
- if (!LIST_EMPTY(&busy_list)) {
- // place any busy chunks back on the sequester list
- xzm_chunk_t busy_chunk, tmp_chunk;
- LIST_FOREACH_SAFE(busy_chunk, &busy_list, xzc_entry,
- tmp_chunk) {
- LIST_REMOVE(busy_chunk, xzc_entry);
- LIST_INSERT_HEAD(&iz->xziz_chunkq, busy_chunk, xzc_entry);
- }
- }
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
- _malloc_lock_unlock(&iz->xziz_lock);
- }
-
-
- return chunk;
-}
-
-static void *
-_xzm_xzone_find_and_malloc_from_freelist_chunk(xzm_malloc_zone_t zone,
- xzm_xzone_t xz, xzm_allocation_index_t alloc_idx,
- xzm_xzone_thread_cache_t cache, xzm_chunk_t *chunk_out,
- bool *contended_out)
-{
- void *ptr = NULL;
- xzm_chunk_t chunk = NULL;
- bool small = (xz->xz_block_size > XZM_TINY_BLOCK_SIZE_MAX);
-
- // The first place to try to get a new chunk from is the partial list.
- while ((chunk = _xzm_chunk_list_slot_pop(zone, xz,
- zone->xzz_partial_lists))) {
- xzm_debug_assert(chunk->xzc_atomic_meta.xca_on_partial_list &&
- !chunk->xzc_atomic_meta.xca_on_empty_list);
- bool empty = false;
- ptr = _xzm_xzone_malloc_from_freelist_chunk(zone, xz, alloc_idx, cache,
- chunk, small, contended_out, &empty);
- if (ptr) {
- // Great, we allocated from this chunk and can install it.
- goto done;
- } else if (empty) {
- xzm_debug_assert(!chunk->xzc_atomic_meta.xca_on_partial_list &&
- chunk->xzc_atomic_meta.xca_on_empty_list);
- // It's our responsibility to move this chunk to the empty list.
- _xzm_chunk_list_push(zone, &xz->xz_empty_list, chunk,
- XZM_CHUNK_LINKAGE_MAIN, NULL);
- }
- }
-
- xzm_debug_assert(!chunk);
-
- // Try the batch list. Some chunks may not be usable, so we need to build a
- // list of the busy ones that can be reinserted. Since some of these chunks
- // may still be on the partial list, we must reuse the batch linkage instead
- // of the inline linkage (c.f. empty list busy chunks below).
- xzm_chunk_t busy_chunk = NULL;
- while ((chunk = _xzm_chunk_list_pop(zone, &xz->xz_batch_list,
- XZM_CHUNK_LINKAGE_BATCH, NULL))) {
- xzm_debug_assert(!chunk->xzc_atomic_meta.xca_on_empty_list);
- ptr = _xzm_xzone_malloc_from_empty_freelist_chunk(zone, xz, alloc_idx,
- cache, chunk, false);
- if (ptr) {
- break;
- }
-
- *_xzm_segment_slice_meta_batch_next(zone, chunk) = busy_chunk;
- busy_chunk = chunk;
- }
-
- // Push unusable chunks back onto the batch list
- xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
- while (busy_chunk) {
- xzm_debug_assert(!busy_chunk->xzc_atomic_meta.xca_on_empty_list);
- xzm_chunk_t next = *_xzm_segment_slice_meta_batch_next(zone, busy_chunk);
- _xzm_chunk_batch_list_push(zone, xz, busy_chunk, main->xzmz_batch_size);
- busy_chunk = next;
- }
-
- if (ptr) {
- // Great, we allocated from this chunk and can install it.
- goto done;
- }
-
-#if CONFIG_XZM_DEFERRED_RECLAIM
- SLIST_HEAD(, xzm_slice_s) busy_list = SLIST_HEAD_INITIALIZER(busy_list);
-#endif
- // Next, try the empty list. Some chunks may not be usable, so we need to
- // build a list of the busy ones that can be reinserted. Note that this list
- // is intrusive, and reuses the same linkage as that used by the partial and
- // empty lists.
- while ((chunk = _xzm_chunk_list_pop(zone, &xz->xz_empty_list,
- XZM_CHUNK_LINKAGE_MAIN, NULL))) {
- xzm_debug_assert(chunk->xzc_atomic_meta.xca_on_empty_list);
-
- bool was_reclaimed = true;
-#if CONFIG_XZM_DEFERRED_RECLAIM
- // This relies on dependency ordering from the pop of the chunk
- // list to obtain the correct reclaim buffer index
- if (((main->xzmz_defer_tiny && !small) ||
- (main->xzmz_defer_small && small)) &&
- !xzm_chunk_mark_used(zone, chunk, &was_reclaimed)) {
- // this chunk is busy so we can't use it
- goto empty_busy;
- }
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
-
- ptr = _xzm_xzone_malloc_from_empty_freelist_chunk(zone, xz, alloc_idx,
- cache, chunk, was_reclaimed);
- if (ptr) {
- break;
- }
-
-#if CONFIG_XZM_DEFERRED_RECLAIM
-empty_busy:
- SLIST_INSERT_HEAD(&busy_list, chunk, xzc_slist_entry);
-#else // CONFIG_XZM_DEFERRED_RECLAIM
- xzm_debug_abort("_xzm_xzone_malloc_from_empty_freelist_chunk failed");
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
- }
-
-#if CONFIG_XZM_DEFERRED_RECLAIM
- // Push unusable chunks back onto the empty list
- while ((busy_chunk = SLIST_FIRST(&busy_list))) {
- xzm_debug_assert(busy_chunk->xzc_atomic_meta.xca_on_empty_list);
- SLIST_REMOVE_HEAD(&busy_list, xzc_slist_entry);
- _xzm_chunk_list_push(zone, &xz->xz_empty_list, busy_chunk,
- XZM_CHUNK_LINKAGE_MAIN, NULL);
- }
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
-
- if (ptr) {
- // Great, we allocated from this chunk and can install it.
- goto done;
- }
-
- // try the global isolation list
- if (xz->xz_sequestered) {
- xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
- chunk = _xzm_xzone_allocate_chunk_from_isolation(main, xz);
- if (chunk) {
- xzm_debug_assert(_xzm_chunk_is_empty(zone, xz, chunk));
- ptr = _xzm_xzone_malloc_from_empty_freelist_chunk(zone, xz, alloc_idx,
- cache, chunk, true);
- xzm_debug_assert(ptr);
- goto push_to_all_list_and_unlock;
- }
- }
-
- bool zero_on_free = (xz->xz_block_size <= XZM_ZERO_ON_FREE_THRESHOLD);
- if (!(chunk = _xzm_chunk_list_pop(zone, &xz->xz_preallocated_list,
- XZM_CHUNK_LINKAGE_MAIN, NULL))) {
- // Failed to get a chunk from the preallocated list, need to go to the
- // segment group to get a fresh chunk.
- bool clear_chunk = zero_on_free;
- bool purgeable = false; // TINY chunks can't be purgeable
- xzm_preallocate_list_s preallocated;
- SLIST_INIT(&preallocated);
- xzm_preallocate_list_s *list = &preallocated;
- xzm_segment_group_t sg = _xzm_segment_group_for_id_and_front(zone,
- xz->xz_segment_group_id, xz->xz_front, false);
-
- xzm_slice_kind_t kind = XZM_SLICE_KIND_TINY_CHUNK;
- xzm_slice_count_t slice_count = 1;
- if (small) {
- kind = XZM_SLICE_KIND_SMALL_FREELIST_CHUNK;
- slice_count =
- XZM_SMALL_FREELIST_CHUNK_SIZE / XZM_SEGMENT_SLICE_SIZE;
- }
-
- chunk = xzm_segment_group_alloc_chunk(sg, kind,
- &xz->xz_guard_config, slice_count, list, 0, clear_chunk,
- purgeable);
- if (!chunk) {
- // Not able to get anything from the segment group, so give up
- // entirely.
- xzm_debug_assert(!list || !SLIST_FIRST(list));
- goto done;
- } else {
- chunk->xzc_xzone_idx = xz->xz_idx;
- xzm_chunk_t c, t;
- SLIST_FOREACH_SAFE(c, list, xzc_slist_entry, t) {
- SLIST_REMOVE_HEAD(list, xzc_slist_entry);
- c->xzc_xzone_idx = xz->xz_idx;
- c->xzc_bits.xzcb_preallocated = true;
- _xzm_chunk_list_push(zone, &xz->xz_preallocated_list, c,
- XZM_CHUNK_LINKAGE_MAIN, NULL);
- }
-
- if (xz->xz_slot_config < zone->xzz_initial_slot_config &&
- xz->xz_block_size < zone->xzz_slot_initial_threshold) {
- xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
- uint64_t count = os_atomic_inc(&xz->xz_chunk_count, relaxed);
- if (count > main->xzmz_xzone_chunk_threshold) {
- // Upgrade/reset the counters in all the slots
- xzm_xzone_slot_counters_u counter = {
- .xsc_slot_config = xz->xz_slot_config,
- };
- _xzm_xzone_upgrade_freelist_slot_config(zone, xz, NULL,
- counter, zone->xzz_initial_slot_config, true);
- }
- }
- }
- } else {
- // We got a chunk from the preallocated list, which is not guaranteed to
- // be zero'd
- if (zero_on_free && !chunk->xzc_bits.xzcb_is_pristine) {
- size_t chunk_size = 0;
- void *ptr = _xzm_chunk_start_ptr(zone, chunk, &chunk_size);
- xzm_debug_assert(ptr);
- xzm_debug_assert(chunk_size == XZM_SEGMENT_SLICE_SIZE);
- bzero(ptr, chunk_size);
- }
- }
-
- ptr = _xzm_xzone_malloc_from_fresh_freelist_chunk(zone, xz, alloc_idx,
- cache, chunk, small);
- xzm_debug_assert(ptr);
-
-push_to_all_list_and_unlock:
- // Put the chunk on the "all" list for the xzone so that it can be found
- // during fork
- _xzm_chunk_list_push(zone, &xz->xz_all_list, chunk, XZM_CHUNK_LINKAGE_ALL,
- NULL);
-
-done:
- xzm_debug_assert((ptr && chunk) || (!ptr && !chunk));
-
- *chunk_out = chunk;
- return ptr;
-}
-
-MALLOC_NOINLINE
-static void *
-_xzm_xzone_malloc_freelist_outlined(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_allocation_index_t alloc_idx, xzm_xzone_allocation_slot_t xas,
- void *corrupt_block, xzm_malloc_options_t opt)
-{
- if (os_unlikely(corrupt_block)) {
- _xzm_corruption_detected(corrupt_block);
- }
-
- bool clear = (opt & XZM_MALLOC_CLEAR);
- bool zero_on_free = (xz->xz_block_size <= XZM_ZERO_ON_FREE_THRESHOLD);
-#if !CONFIG_TINY_ALLOCATION_SLOT_LOCK
- bool small = (xz->xz_block_size > XZM_TINY_BLOCK_SIZE_MAX);
-#endif
-#if CONFIG_MTE
- const bool memtag_enabled = xz->xz_tagged;
- bool needs_canonical_tagging = (opt & XZM_MALLOC_CANONICAL_TAG);
-#endif
-
-#if CONFIG_TINY_ALLOCATION_SLOT_LOCK
-#ifdef DEBUG
- _malloc_lock_assert_owner(&xas->xas_lock);
-#endif // DEBUG
-#else // CONFIG_TINY_ALLOCATION_SLOT_LOCK
- uint32_t self_owner_value = 0;
-#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
-
- xzm_allocation_slot_atomic_meta_u *xasa = &xas->xas_atomic;
-
- xzm_allocation_slot_atomic_meta_u slot_meta = {
- .xasa_value = os_atomic_load(&xasa->xasa_value, dependency),
- };
-
- void *ptr = NULL;
- xzm_chunk_t chunk = NULL;
- bool contended = false;
-
-#if !CONFIG_TINY_ALLOCATION_SLOT_LOCK
- while (true) {
- if (slot_meta.xasa_gate.xsg_locked) {
- // No chunk is currently installed, and another thread is in the
- // middle of fetching and installing one. Wait for them and then
- // try again.
- slot_meta = _xzm_allocation_slot_gate_wait(xas, slot_meta);
- continue;
- } else if (os_unlikely(slot_meta.xasa_chunk.xsc_fork_locked)) {
- // We can't acquire the gate because a fork is in progress that we
- // need to wait out.
- _xzm_fork_lock_wait(zone);
-
- slot_meta = (xzm_allocation_slot_atomic_meta_u){
- .xasa_value = os_atomic_load(&xasa->xasa_value, dependency),
- };
- continue;
- }
-
- // The gate isn't locked, so the next question is whether a chunk is
- // installed.
- chunk = (xzm_chunk_t)slot_meta.xasa_chunk.xsc_ptr;
- if (chunk) {
- // A chunk is installed, so let's try to allocate from it.
- ptr = _xzm_xzone_malloc_from_freelist_chunk(zone, xz, alloc_idx,
- NULL, chunk, small, &contended,
- /* install_empty_out */ NULL);
- if (ptr) {
- // Success!
- goto done;
- }
- }
-
- if (!self_owner_value) {
- self_owner_value = _malloc_ulock_self_owner_value();
- }
-
- // Either we had no chunk reference, or the one we had couldn't be
- // allocated from. Let's try to become an allocator for our slot to go
- // get a new one.
- xzm_allocation_slot_atomic_meta_u new_slot_meta = {
- .xasa_gate = {
- .xsg_locked = true,
- .xsg_owner = self_owner_value,
- .xsg_gen = slot_meta.xasa_gate.xsg_gen,
- },
- };
- bool success = os_atomic_cmpxchgv(&xasa->xasa_value,
- slot_meta.xasa_value, new_slot_meta.xasa_value,
- &slot_meta.xasa_value, dependency);
- if (!success) {
- continue;
- }
-
- break;
- }
-
- xzm_debug_assert(self_owner_value);
-#else // CONFIG_TINY_ALLOCATION_SLOT_LOCK
- xzm_debug_assert(!slot_meta.xasa_gate.xsg_locked);
-#if CONFIG_MTE
- chunk = (xzm_chunk_t)slot_meta.xasa_chunk.xsc_ptr;
- if (chunk && (xz->xz_tagged && (opt & XZM_MALLOC_CANONICAL_TAG))) {
- // If we're serving a canonically-tagged request, we won't have actually
- // attempted to allocate from the chunk yet, so we need to do that first
- ptr = _xzm_xzone_malloc_from_freelist_chunk(zone, xz, alloc_idx,
- NULL, chunk, /* small */ false, &contended,
- /* install_empty_out */ NULL);
- if (ptr) {
- // Success!
- goto unlock;
- }
- }
-#endif
-#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
-
- // It's now up to us to get a usable chunk and install it.
-
- xzm_debug_assert(!ptr);
-
- ptr = _xzm_xzone_find_and_malloc_from_freelist_chunk(zone, xz, alloc_idx,
- NULL, &chunk, &contended);
-
- // Unlock the gate and publish our new chunk (or lack thereof)
- xzm_allocation_slot_atomic_meta_u new_slot_meta = {
- .xasa_chunk = {
- .xsc_ptr = (uint64_t)chunk,
- .xsc_gen = slot_meta.xasa_gate.xsg_gen + 1,
- },
- };
-
- uint64_t prev_slot_value = os_atomic_xchg(&xasa->xasa_value,
- new_slot_meta.xasa_value, release);
- xzm_allocation_slot_atomic_meta_u prev_slot_meta = {
- .xasa_value = prev_slot_value,
- };
-#if CONFIG_TINY_ALLOCATION_SLOT_LOCK
- (void)prev_slot_meta;
- xzm_debug_assert(prev_slot_meta.xasa_value == slot_meta.xasa_value);
-#else // CONFIG_TINY_ALLOCATION_SLOT_LOCK
- xzm_debug_assert(prev_slot_meta.xasa_gate.xsg_locked);
- xzm_debug_assert(prev_slot_meta.xasa_gate.xsg_owner == self_owner_value);
- xzm_debug_assert(prev_slot_meta.xasa_gate.xsg_unused == 0);
- xzm_debug_assert(prev_slot_meta.xasa_gate.xsg_gen ==
- slot_meta.xasa_gate.xsg_gen);
-
- if (prev_slot_meta.xasa_gate.xsg_waiters) {
- uint32_t wake_op = UL_UNFAIR_LOCK | ULF_WAKE_ALL | ULF_NO_ERRNO;
- int rc = __ulock_wake(wake_op, &xasa->xasa_ulock, 0);
- if (rc && rc != -ENOENT) {
- xzm_abort_with_reason("ulock_wake failure", -rc);
- }
- }
-#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
-
-#if CONFIG_TINY_ALLOCATION_SLOT_LOCK
-#if CONFIG_MTE
-unlock:
-#endif
- _malloc_lock_unlock(&xas->xas_lock);
-#else // CONFIG_TINY_ALLOCATION_SLOT_LOCK
-done:
-#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
-
- // Now that we've unlocked the gate, we can safely touch the chunk to
- // perform any necessary zeroing to the block we allocated. It's very
- // important that we not do this inside the gate because taking a fault in
- // there makes it much more likely for other threads to contend and get
- // stuck waiting for us.
- if (ptr) {
- // The true contention information we'd want to record in the locking
- // case would be from the trylock rather than the list pop, but we
- // didn't forward that information so there's nothing to do here
-#if !CONFIG_TINY_ALLOCATION_SLOT_LOCK
- xzm_slot_config_t slot_config = os_atomic_load(&xz->xz_slot_config,
- relaxed);
- _xzm_xzone_slot_record_contention(zone, xz, &xas->xas_counters,
- slot_config, true, contended);
-#endif // !CONFIG_TINY_ALLOCATION_SLOT_LOCK
-
-#if CONFIG_MTE
- if (memtag_enabled) {
- if (needs_canonical_tagging) {
- ptr = memtag_tag_canonical(ptr, xz->xz_block_size);
- } else {
- xzm_debug_assert(ptr == memtag_fixup_ptr(ptr));
- }
- }
-#endif
-
- xzm_block_t block = ptr;
- *block = (struct xzm_block_inline_meta_s){ 0 };
-
- if (clear && !zero_on_free) {
- bzero(ptr, xz->xz_block_size);
- }
- } else {
- malloc_set_errno_fast(MZ_POSIX, ENOMEM);
- }
- return ptr;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static void *
-_xzm_xzone_malloc_freelist(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_malloc_options_t opt, const bool small)
-{
- bool clear = (opt & XZM_MALLOC_CLEAR);
-
- xzm_slot_config_t slot_config;
- xzm_allocation_index_t alloc_idx = _xzm_get_allocation_index(zone, xz,
- &slot_config, true);
- xzm_xzone_allocation_slot_t xas =
- _xzm_xzone_allocation_slot_for_index(zone, xz, alloc_idx);
-
- bool contended = false;
-#if CONFIG_TINY_ALLOCATION_SLOT_LOCK
- bool contended_dummy = false;
- if (!_malloc_lock_trylock(&xas->xas_lock)) {
- contended = true;
- _malloc_lock_lock(&xas->xas_lock);
- }
-#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
-
- xzm_allocation_slot_atomic_meta_u *xasa = &xas->xas_atomic;
-
- xzm_allocation_slot_atomic_meta_u slot_meta = {
- .xasa_value = os_atomic_load(&xasa->xasa_value, dependency),
- };
-
- void *ptr = NULL;
-#if CONFIG_MTE
- const bool fast_path =
- !slot_meta.xasa_gate.xsg_locked && slot_meta.xasa_chunk.xsc_ptr &&
- !(xz->xz_tagged && (opt & XZM_MALLOC_CANONICAL_TAG));
-#else
- const bool fast_path =
- !slot_meta.xasa_gate.xsg_locked && slot_meta.xasa_chunk.xsc_ptr;
-#endif
- // Fast path: the slot is not locked and a chunk is installed
- if (fast_path) {
- xzm_chunk_t chunk = (xzm_chunk_t)slot_meta.xasa_chunk.xsc_ptr;
-
- // Try to allocate from the installed chunk
- const bool walk_wait = false;
- bool corrupt = false;
- ptr = _xzm_xzone_malloc_from_freelist_chunk_inline(zone, xz, alloc_idx,
- NULL, chunk, small, walk_wait, &corrupt,
-#if CONFIG_TINY_ALLOCATION_SLOT_LOCK
- &contended_dummy,
-#else // CONFIG_TINY_ALLOCATION_SLOT_LOCK
- &contended,
-#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
- /* install_empty_out */ NULL);
- if (ptr && !corrupt) {
- // Success!
-
-#if CONFIG_TINY_ALLOCATION_SLOT_LOCK
- _malloc_lock_unlock(&xas->xas_lock);
-#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
-
- _xzm_xzone_slot_record_contention(zone, xz, &xas->xas_counters,
- slot_config, true, contended);
-
- xzm_block_t block = ptr;
- *block = (struct xzm_block_inline_meta_s){ 0 };
-
- if (clear && xz->xz_block_size > XZM_ZERO_ON_FREE_THRESHOLD) {
- return memset(ptr, 0, xz->xz_block_size);
- }
-
- return ptr;
- }
- }
-
- return _xzm_xzone_malloc_freelist_outlined(zone, xz, alloc_idx, xas, ptr,
- opt);
-}
-
-MALLOC_NOINLINE
-static void *
-_xzm_xzone_malloc_tiny(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_malloc_options_t opt)
-{
- return _xzm_xzone_malloc_freelist(zone, xz, opt, false);
-}
-
-MALLOC_NOINLINE
-static void *
-_xzm_xzone_malloc_small_freelist(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_malloc_options_t opt)
-{
- return _xzm_xzone_malloc_freelist(zone, xz, opt, true);
-}
-
-#pragma mark Tiny deallocation
-
-static void
-_xzm_xzone_freelist_chunks_mark_empty(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_chunk_t *chunks, size_t chunk_count)
-{
- for (size_t i = 0; i < chunk_count; ++i) {
- xzm_chunk_t chunk = chunks[i];
- xzm_chunk_atomic_meta_u old_meta = {
- .xca_value = os_atomic_load_wide(
- &chunk->xzc_atomic_meta.xca_value, relaxed),
- };
- while (true) {
- xzm_chunk_atomic_meta_u new_meta = old_meta;
-
- xzm_debug_assert(old_meta.xca_alloc_head == XZM_FREE_MADVISING);
- new_meta.xca_alloc_head = XZM_FREE_MADVISED;
- if (!old_meta.xca_on_partial_list) {
- new_meta.xca_on_empty_list = true;
- }
- // release to publish the reclaim index if we stored one
- // TODO make this store relaxed. the reclaim index is no longer
- // set in this function
- bool success = os_atomic_cmpxchgv(&chunk->xzc_atomic_meta.xca_value,
- old_meta.xca_value, new_meta.xca_value, &old_meta.xca_value,
- release);
- if (!success) {
- continue;
- }
-
- xzm_trace(free_madvise, (uint64_t)chunk, old_meta.xca_value_lo,
- new_meta.xca_value_lo, old_meta.xca_seqno);
- break;
- }
-
- // because the partial list is atomically singly-linked, an empty chunk
- // could remain there until it is popped off the head, so we cannot put
- // it onto the empty list until after that happens
- if (!old_meta.xca_on_partial_list) {
- _xzm_chunk_list_push(zone, &xz->xz_empty_list, chunk,
- XZM_CHUNK_LINKAGE_MAIN, NULL);
- }
- }
-}
-
-static void
-_xzm_xzone_madvise_freelist_chunk(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_chunk_t chunk)
-{
-#if CONFIG_XZM_DEFERRED_RECLAIM
- xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
- if ((chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_TINY_CHUNK &&
- main->xzmz_defer_tiny) ||
- (chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK &&
- main->xzmz_defer_small)) {
- xzm_chunk_mark_free(zone, chunk);
- } else
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
- {
- xzm_segment_group_t sg = _xzm_segment_group_for_slice(zone, chunk);
- xzm_segment_group_segment_madvise_chunk(sg, chunk);
- }
-
- _xzm_xzone_freelist_chunks_mark_empty(zone, xz, &chunk, 1);
-}
-
-static void
-_xzm_xzone_small_chunks_mark_empty(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_chunk_t *chunk, size_t chunk_count);
-
-static void
-_xzm_xzone_madvise_batch(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_chunk_t chunk)
-{
- const xzm_slice_kind_t kind = chunk->xzc_bits.xzcb_kind;
- // We need to store the local list of entries in the batch. Otherwise,
- // the list pointers could be replaced by the deferred reclaim id, and
- // attempting to update metadata while under the reclaim buffer lock could
- // deadlock against a concurrent allocator that holds the isolation zone
- // lock and wants the reclaim buffer lock.
- xzm_chunk_t batch_list[1u << XZM_BATCH_SIZE_BITS];
- unsigned batch_list_size = 0;
-#if CONFIG_XZM_DEFERRED_RECLAIM
- xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
- xzm_reclaim_buffer_t buffer = main->xzmz_reclaim_buffer;
- const bool deferred_reclaim = (kind == XZM_SLICE_KIND_TINY_CHUNK) ?
- main->xzmz_defer_tiny : main->xzmz_defer_small;
- bool should_update_kernel_accounting = false;
-
- if (deferred_reclaim) {
- _malloc_lock_lock(&buffer->xrb_lock);
- }
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
-
- // Walk the batch, make a local copy, and perform the reclaim
- while (chunk) {
-#if CONFIG_XZM_DEFERRED_RECLAIM
- xzm_debug_assert(batch_list_size < main->xzmz_batch_size &&
- batch_list_size < (1u << XZM_BATCH_SIZE_BITS));
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
- batch_list[batch_list_size++] = chunk;
-
- xzm_debug_assert(kind == chunk->xzc_bits.xzcb_kind);
- // Fetch the pointer for the next chunk first, because madvising it
- // may replace the linkage with the deferred reclaim id
- xzm_chunk_t next = *_xzm_segment_slice_meta_batch_next(zone, chunk);
- xzm_debug_assert(_xzm_slice_meta_is_batch_pointer(zone, next));
-
-#if CONFIG_XZM_DEFERRED_RECLAIM
- if (deferred_reclaim) {
- size_t size;
- uint8_t *addr = _xzm_chunk_start_ptr(zone, chunk, &size);
- uint64_t *reclaim_id = _xzm_slice_meta_reclaim_id(zone, chunk);
-
- bool should_update_kernel_chunk = false;
- // This relies on release ordering from the push of the batch
- // list to obtain the correct reclaim buffer index
- *reclaim_id = xzm_reclaim_mark_free_locked(buffer, addr, size, true,
- &should_update_kernel_chunk);
- should_update_kernel_accounting |= should_update_kernel_chunk;
- } else
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
- {
- xzm_segment_group_t sg = _xzm_segment_group_for_slice(zone,
- chunk);
- xzm_segment_group_segment_madvise_chunk(sg, chunk);
- }
-
- chunk = next;
- }
-
-#if CONFIG_XZM_DEFERRED_RECLAIM
- if (deferred_reclaim) {
- _malloc_lock_unlock(&buffer->xrb_lock);
-
- if (should_update_kernel_accounting) {
- mach_vm_reclaim_update_kernel_accounting(buffer->xrb_ringbuffer);
- }
- }
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
-
- if (kind == XZM_SLICE_KIND_TINY_CHUNK ||
- kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK) {
- _xzm_xzone_freelist_chunks_mark_empty(zone, xz, batch_list,
- batch_list_size);
- } else if (kind == XZM_SLICE_KIND_SMALL_CHUNK) {
- _xzm_xzone_small_chunks_mark_empty(zone, xz, batch_list, batch_list_size);
- } else {
- xzm_abort_with_reason("Unexpected chunk kind", kind);
- }
-}
-
-MALLOC_NOINLINE
-static void
-_xzm_free_abort(void *ptr)
-{
- xzm_client_abort_with_reason(
- "free to empty or invalid chunk detected (likely double-free)",
- ptr);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static void
-_xzm_xzone_free_freelist_inline(xzm_malloc_zone_t zone,
- xzm_xzone_index_t xz_idx, xzm_chunk_t chunk, xzm_block_t block,
- uint16_t block_ref, size_t block_size, size_t chunk_capacity)
-{
- xzm_xzone_t xz = NULL;
- union xzm_block_linkage_u linkage = { 0 };
-
- uint64_t now = 0;
- uint64_t last_empty_ts = 0;
- bool should_madvise = true;
- bool madvise_considered = false;
-
- uint32_t contentions = 0;
-
- xzm_xzone_allocation_slot_t xas = NULL;
-
- xzm_chunk_atomic_meta_u old_meta = {
- .xca_value = os_atomic_load_wide(
- &chunk->xzc_atomic_meta.xca_value, dependency),
- };
-
- xzm_chunk_atomic_meta_u new_meta;
-
- bool push_to_partial = false;
- while (true) {
- if (os_unlikely(old_meta.xca_walk_locked)) {
- _xzm_walk_lock_wait(zone);
- old_meta.xca_value = os_atomic_load_wide(
- &chunk->xzc_atomic_meta.xca_value, dependency);
- continue;
- }
-
- new_meta = old_meta;
- new_meta.xca_seqno++;
-
- xzm_debug_assert(!push_to_partial);
-
- if (old_meta.xca_free_count + 1 == chunk_capacity) {
- // We're freeing the last block.
-
- // Evaluate the madvise heuristic the first time through.
- if (!madvise_considered) {
- xz = &zone->xzz_xzones[xz_idx];
- if (old_meta.xca_alloc_idx &&
- old_meta.xca_alloc_idx != XZM_SLOT_INDEX_THREAD_INSTALLED) {
- xzm_chunk_list_t partial_list = _xzm_chunk_list_get(zone,
- xz, NULL, zone->xzz_partial_lists);
- bool last_chunk = _xzm_chunk_list_empty(&partial_list->xcl_list);
- if (last_chunk) {
- now = mach_absolute_time();
- xas = _xzm_xzone_allocation_slot_for_index(zone, xz,
- old_meta.xca_alloc_idx - 1);
- last_empty_ts = os_atomic_load(
- &xas->xas_last_chunk_empty_ts, relaxed);
- if (now - last_empty_ts < zone->xzz_tiny_thrash_threshold) {
- should_madvise = false;
- }
- }
- }
- madvise_considered = true;
- }
-
- if (should_madvise &&
- old_meta.xca_alloc_idx != XZM_SLOT_INDEX_THREAD_INSTALLED) {
- new_meta.xca_alloc_head = XZM_FREE_MADVISING;
- new_meta.xca_free_count = 0;
- new_meta.xca_alloc_idx = XZM_SLOT_INDEX_EMPTY;
- goto do_cmpxchg;
- }
- } else if (old_meta.xca_free_count == 0) {
- if (os_unlikely(old_meta.xca_alloc_head != XZM_FREE_NULL)) {
- return _xzm_free_abort((void *)block);
- }
-
- if (old_meta.xca_alloc_idx == XZM_SLOT_INDEX_EMPTY) {
- // This chunk is not currently installed anywhere. We
- // should move it to the partial list so it can be
- // considered for future allocations.
- xzm_debug_assert(!old_meta.xca_on_partial_list &&
- !old_meta.xca_on_empty_list);
- new_meta.xca_on_partial_list = true;
- push_to_partial = true;
-
- xz = &zone->xzz_xzones[xz_idx];
- } else {
- xzm_debug_assert(!old_meta.xca_on_partial_list);
- }
- }
-
- uint32_t seqno = (uint32_t)old_meta.xca_seqno & XZM_SEQNO_COUNTER_MASK;
-
- // Link the new block to the current head of the free list.
- linkage = (union xzm_block_linkage_u){
- .xzbl_next_offset = old_meta.xca_alloc_head,
- .xzbl_next_seqno = old_meta.xca_head_seqno,
- .xzbl_seqno = seqno,
- };
- linkage.xzbl_next_value = (uintptr_t)ptrauth_sign_unauthenticated(
- linkage.xzbl_next, ptrauth_key_process_dependent_data,
- ptrauth_blend_discriminator(block,
- ptrauth_string_discriminator("xzb_linkage")));
- linkage.xzbl_seqno = 0;
-
- os_atomic_store(&block->xzb_linkage.xzbl_next_value,
- linkage.xzbl_next_value, relaxed);
-
- // Install the new block as the head.
- new_meta.xca_alloc_head = block_ref;
- new_meta.xca_free_count++;
- new_meta.xca_head_seqno = seqno;
-
-do_cmpxchg:;
- // Release ordering to publish our zeroing and initialization of the
- // block - pairs with the dependency order loads/cmpxchgs on allocation
- bool success = os_atomic_cmpxchgv(&chunk->xzc_atomic_meta.xca_value,
- old_meta.xca_value, new_meta.xca_value, &old_meta.xca_value,
- release);
- if (!success) {
- // Try again.
- push_to_partial = false;
- contentions++;
- continue;
- }
-
- xzm_trace(free, (uint64_t)chunk, old_meta.xca_value_lo,
- new_meta.xca_value_lo,
- (uint32_t)old_meta.xca_seqno | ((uint64_t)contentions) << 32);
- break;
- }
-
- // If we checked the time while evaluating the madvise heuristic, update it
- // in the allocation slot.
- if (now) {
- xzm_debug_assert(xas);
- os_atomic_store(&xas->xas_last_chunk_empty_ts, now, relaxed);
- }
-
- if (new_meta.xca_alloc_head == XZM_FREE_MADVISING) {
- xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
- xzm_debug_assert(xz);
-
-#if CONFIG_TINY_ALLOCATION_SLOT_LOCK
- // We need to ensure that there are no threads in the critical section
- // of the allocation slot that we uninstalled the chunk from
- if (old_meta.xca_alloc_idx != XZM_SLOT_INDEX_EMPTY) {
- xas = _xzm_xzone_allocation_slot_for_index(zone, xz,
- old_meta.xca_alloc_idx - 1);
- _malloc_lock_lock(&xas->xas_lock);
- _malloc_lock_unlock(&xas->xas_lock);
- }
-#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
-
- if (!main->xzmz_batch_size) {
- // Batching is disabled, do the madvise we committed to
- _xzm_xzone_madvise_freelist_chunk(zone, xz, chunk);
- } else {
- // Batching is enabled, enqueue the chunk. Note that the batch list
- // is not exclusive, the chunk could also still be on the partial
- // list as well
- _xzm_chunk_batch_list_push(zone, xz, chunk,
- main->xzmz_batch_size);
- }
- } else if (push_to_partial) {
- xzm_debug_assert(xz);
- _xzm_chunk_list_slot_push(zone, xz, zone->xzz_partial_lists, chunk);
- }
-}
-
-MALLOC_NOINLINE
-static void
-_xzm_xzone_free_freelist(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_chunk_t chunk, xzm_block_t block)
-{
- size_t block_size = xz->xz_block_size;
- if (block_size <= XZM_ZERO_ON_FREE_THRESHOLD) {
- bzero(block, block_size);
- }
-
- void *ptr = block;
-#if CONFIG_MTE
- ptr = memtag_strip_address((uint8_t *)ptr);
-#endif
-
-#if CONFIG_MTE
- if (xz->xz_tagged && block_size <= XZM_TINY_BLOCK_SIZE_MAX) {
- // Only tiny chunks are tagged on free
- block = _xzm_xzone_block_memtag_retag(zone, block, block_size);
- }
-#endif
-
- uint64_t free_cookie = zone->xzz_freelist_cookie ^ (uint64_t)block;
-#if CONFIG_MTE
- if (block_size > XZM_TINY_BLOCK_SIZE_MAX) {
- // Small blocks do not encode the tag in the freelist cookie
- free_cookie = (uint64_t)memtag_strip_address((uint8_t *)free_cookie);
- }
-#endif
- os_atomic_store(&block->xzb_cookie, free_cookie, relaxed);
-
- uint16_t block_ref;
- if (xz->xz_block_size > XZM_TINY_BLOCK_SIZE_MAX) {
- size_t block_offset = _xzm_chunk_block_offset(zone, chunk, block);
- block_ref = block_offset / XZM_SMALL_GRANULE;
- } else {
- size_t block_offset = (uintptr_t)ptr & XZM_SEGMENT_SLICE_MASK;
- block_ref = block_offset / XZM_GRANULE;
- }
-
- _xzm_xzone_free_freelist_inline(zone, xz->xz_idx, chunk, block, block_ref,
- block_size, xz->xz_chunk_capacity);
-}
-
-#if CONFIG_XZM_THREAD_CACHE
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static void
-_xzm_xzone_thread_cache_free_tiny(xzm_malloc_zone_t zone,
- xzm_xzone_thread_cache_t cache, xzm_block_t block, uint16_t block_ref)
-{
- uint16_t seqno = cache->xztc_seqno & XZM_SEQNO_COUNTER_MASK;
- seqno |= XZM_SEQNO_THREAD_LOCAL;
-
- union xzm_block_linkage_u linkage = {
- .xzbl_next_offset = cache->xztc_head,
- .xzbl_next_seqno = cache->xztc_head_seqno,
- .xzbl_seqno = seqno,
- };
- linkage.xzbl_next_value = (uintptr_t)ptrauth_sign_unauthenticated(
- linkage.xzbl_next, ptrauth_key_process_dependent_data,
- ptrauth_blend_discriminator(block,
- ptrauth_string_discriminator("xzb_linkage")));
- linkage.xzbl_seqno = 0;
-
- os_atomic_store(&block->xzb_linkage.xzbl_next_value,
- linkage.xzbl_next_value, relaxed);
-
-#ifdef DEBUG
- uint64_t orig_state = cache->xztc_freelist_state;
-#endif
-
- xzm_xzone_thread_cache_atomic_meta_u tc_meta = {
- .xztcam_head = block_ref,
- .xztcam_free_count = cache->xztc_free_count + 1,
- };
- os_atomic_store(&cache->xztc_atomic_meta.xztcam_value,
- tc_meta.xztcam_value, relaxed);
- cache->xztc_head_seqno = seqno;
- cache->xztc_seqno++;
-
-#ifdef DEBUG
- uint64_t new_state = cache->xztc_freelist_state;
-#endif
-
- xzm_trace(thread_cache_free, (uint64_t)cache->xztc_chunk, orig_state,
- new_state, 0);
-}
-
-static void
-_xzm_xzone_thread_cache_detach_link_to_remote(xzm_malloc_zone_t zone,
- xzm_xzone_thread_cache_t cache, xzm_chunk_atomic_meta_u old_meta)
-{
- xzm_chunk_t chunk = cache->xztc_chunk;
-
- uint8_t *start = cache->xztc_chunk_start;
- uint64_t block_granules = chunk->xzc_freelist_block_size / XZM_GRANULE;
- uint64_t max_offset = (chunk->xzc_freelist_chunk_capacity - 1) * block_granules;
-
- uint64_t zone_freelist_cookie = zone->xzz_freelist_cookie;
-
- xzm_debug_assert(old_meta.xca_alloc_head < XZM_FREE_LIMIT);
- xzm_debug_assert(old_meta.xca_free_count);
-
- xzm_block_t cur_block = NULL;
- uint64_t cur_block_offset = old_meta.xca_alloc_head;
- uint64_t cur_seqno = old_meta.xca_head_seqno;
- size_t freelist_count = 1;
- while (true) {
- cur_block = (xzm_block_t)(
- start + (cur_block_offset * XZM_GRANULE));
-
-#if CONFIG_MTE
- if (chunk->xzc_tagged) {
- cur_block = (xzm_block_t)memtag_fixup_ptr((void *)cur_block);
- }
-#endif
- uint64_t expected_cookie = zone_freelist_cookie ^ (uintptr_t)cur_block;
-
- struct xzm_block_inline_meta_s block_meta = *cur_block;
- if (os_unlikely(block_meta.xzb_cookie != expected_cookie)) {
- xzm_client_abort_with_reason(
- "corrupt tiny freelist - cookie, client likely has a buffer"
- " overflow or use-after-free bug", block_meta.xzb_cookie);
- }
-
- union xzm_block_linkage_u linkage = {
- .xzbl_next_offset = block_meta.xzb_linkage.xzbl_next_offset,
- .xzbl_next_seqno = block_meta.xzb_linkage.xzbl_next_seqno,
- .xzbl_seqno = cur_seqno,
- };
-
- linkage.xzbl_next_value = (uintptr_t)ptrauth_sign_unauthenticated(
- linkage.xzbl_next, ptrauth_key_process_dependent_data,
- ptrauth_blend_discriminator(cur_block,
- ptrauth_string_discriminator("xzb_linkage")));
-
- linkage.xzbl_seqno = 0;
-
- if (block_meta.xzb_linkage.xzbl_next_value != linkage.xzbl_next_value) {
- xzm_client_abort_with_reason(
- "corrupt tiny freelist - linkage, client likely has a "
- "buffer overflow or use-after-free bug",
- block_meta.xzb_linkage.xzbl_next_value);
- }
-
- uint64_t next_seqno = (uint16_t)block_meta.xzb_linkage.xzbl_next_seqno;
- uint64_t next_block_offset = block_meta.xzb_linkage.xzbl_next_offset;
-
- if (next_block_offset == XZM_FREE_NULL) {
- // freelist walk should be complete
- if (os_unlikely(freelist_count != old_meta.xca_free_count)) {
- xzm_client_abort_with_reason(
- "corrupt tiny freelist - free count, client likely has a"
- " buffer overflow or use-after-free bug",
- freelist_count);
- }
-
- break;
- }
-
- if (os_unlikely(next_block_offset % block_granules != 0 ||
- next_block_offset > max_offset ||
- freelist_count >= old_meta.xca_free_count)) {
- xzm_client_abort_with_reason(
- "corrupt tiny freelist - inconsistent walk, client likely"
- " has a buffer overflow or use-after-free bug",
- freelist_count);
- }
-
- cur_block_offset = next_block_offset;
- cur_seqno = next_seqno;
- freelist_count++;
- }
-
- xzm_block_t tail_block = cur_block;
- uint64_t tail_seqno = cur_seqno;
-
- union xzm_block_linkage_u linkage = {
- .xzbl_next_offset = cache->xztc_head,
- .xzbl_next_seqno = cache->xztc_head_seqno,
- .xzbl_seqno = tail_seqno,
- };
- linkage.xzbl_next_value = (uintptr_t)ptrauth_sign_unauthenticated(
- linkage.xzbl_next, ptrauth_key_process_dependent_data,
- ptrauth_blend_discriminator(tail_block,
- ptrauth_string_discriminator("xzb_linkage")));
- linkage.xzbl_seqno = 0;
-
- os_atomic_store(&tail_block->xzb_linkage.xzbl_next_value,
- linkage.xzbl_next_value, relaxed);
-}
-
-static void
-_xzm_xzone_thread_cache_detach(xzm_main_malloc_zone_t main, xzm_xzone_t xz,
- xzm_xzone_thread_cache_t cache)
-{
- xzm_malloc_zone_t zone = &main->xzmz_base;
- xzm_chunk_t chunk = cache->xztc_chunk;
-
- size_t chunk_capacity = xz->xz_chunk_capacity;
- uint16_t local_free_count = cache->xztc_free_count;
-
- xzm_chunk_atomic_meta_u old_meta = {
- .xca_value = os_atomic_load_wide(
- &chunk->xzc_atomic_meta.xca_value, dependency),
- };
-
- xzm_chunk_atomic_meta_u new_meta;
-
- bool needs_link = (cache->xztc_head < XZM_FREE_LIMIT);
- bool push_to_partial = false;
- while (true) {
- if (os_unlikely(old_meta.xca_walk_locked)) {
- _xzm_walk_lock_wait(zone);
- old_meta.xca_value = os_atomic_load_wide(
- &chunk->xzc_atomic_meta.xca_value, dependency);
- continue;
- }
-
- new_meta = old_meta;
- new_meta.xca_alloc_idx = XZM_SLOT_INDEX_EMPTY;
-
- xzm_debug_assert(!push_to_partial);
-
- uint16_t new_free_count = old_meta.xca_free_count + local_free_count;
- if (new_free_count == chunk_capacity) {
- // The entire chunk will now be free
-
- // TODO madvise heuristic? Doesn't seem like it would really make
- // sense here
- new_meta.xca_free_count = 0;
- new_meta.xca_alloc_head = XZM_FREE_MADVISING;
- } else if (new_free_count) {
- xzm_debug_assert(!old_meta.xca_on_partial_list &&
- !old_meta.xca_on_empty_list);
- new_meta.xca_free_count = new_free_count;
- new_meta.xca_on_partial_list = true;
- push_to_partial = true;
-
- if (old_meta.xca_alloc_head == XZM_FREE_NULL) {
- new_meta.xca_alloc_head = cache->xztc_head;
- new_meta.xca_head_seqno = cache->xztc_head_seqno;
- } else {
- xzm_debug_assert(old_meta.xca_alloc_head < XZM_FREE_LIMIT);
- if (needs_link) {
- _xzm_xzone_thread_cache_detach_link_to_remote(zone, cache,
- old_meta);
- needs_link = false;
- }
- }
- }
-
- // Release ordering pairs with the dependency order loads/cmpxchgs on
- // allocation
- bool success = os_atomic_cmpxchgv(&chunk->xzc_atomic_meta.xca_value,
- old_meta.xca_value, new_meta.xca_value, &old_meta.xca_value,
- release);
- if (!success) {
- // Try again.
- push_to_partial = false;
- continue;
- }
-
- xzm_trace(thread_detach, (uint64_t)chunk, old_meta.xca_value_lo,
- new_meta.xca_value_lo, cache->xztc_freelist_state);
- break;
- }
-
- if (new_meta.xca_alloc_head == XZM_FREE_MADVISING) {
-#if CONFIG_TINY_ALLOCATION_SLOT_LOCK
-#error "not compatible with thread caching"
-#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
-
- // Do the madvise we committed to.
- _xzm_xzone_madvise_freelist_chunk(zone, xz, chunk);
- } else if (push_to_partial) {
- _xzm_chunk_list_slot_push(zone, xz, zone->xzz_partial_lists, chunk);
- }
-}
-
-static void
-_xzm_xzone_thread_cache_destructor(void *arg)
-{
- xzm_thread_cache_t tc = arg;
- xzm_main_malloc_zone_t main = tc->xtc_main;
- xzm_malloc_zone_t zone = &main->xzmz_base;
-
- tc->xtc_teardown_gen = os_atomic_inc(
- &main->xzmz_thread_cache_teardown_gen, relaxed);
-
- for (size_t i = XZM_XZONE_INDEX_FIRST;
- i < zone->xzz_thread_cache_xzone_count; i++) {
- xzm_xzone_thread_cache_t cache = &tc->xtc_xz_caches[i];
- if (cache->xztc_head <= XZM_FREE_LIMIT) {
- xzm_xzone_t xz = &zone->xzz_xzones[i];
- _xzm_xzone_thread_cache_detach(main, xz, cache);
- }
- }
-
- _malloc_lock_lock(&main->xzmz_thread_cache_list_lock);
- LIST_REMOVE(tc, xtc_linkage);
- _malloc_lock_unlock(&main->xzmz_thread_cache_list_lock);
-
- xzm_metapool_t mp = &main->xzmz_metapools[XZM_METAPOOL_THREAD_CACHE];
- xzm_metapool_free(mp, tc);
-}
-
-#endif // CONFIG_XZM_THREAD_CACHE
-
-#pragma mark Small allocation
-
-// mimalloc: mi_page_init
-static void
-_xzm_xzone_small_chunk_init(xzm_xzone_t xz, xzm_chunk_t chunk)
-{
- // Precondition: xzcb_kind, xzcs_slice_count and xzsl_slice_offset are
- // already initialized
- xzm_debug_assert(chunk->xzc_free == 0);
- xzm_debug_assert(chunk->xzc_used == 0);
- xzm_debug_assert(chunk->xzc_mzone_idx == XZM_MZONE_INDEX_INVALID);
-
- _xzm_xzone_fresh_chunk_init(xz, chunk, XZM_SLICE_KIND_SMALL_CHUNK);
-
- // Enumerator protocol: mzone_idx is initialized last to "publish" the chunk
- chunk->xzc_mzone_idx = xz->xz_mzone_idx;
-}
-
-// mimalloc: mi_page_fresh_alloc
-static xzm_chunk_t
-_xzm_xzone_small_chunk_alloc(xzm_malloc_zone_t zone, xzm_xzone_t xz)
-{
- // First, check the preallocated list
- xzm_chunk_t chunk = NULL;
- if (LIST_FIRST(&xz->xz_chunkq_preallocated)) {
- _malloc_lock_lock(&xz->xz_lock);
- if ((chunk = LIST_FIRST(&xz->xz_chunkq_preallocated))) {
- LIST_REMOVE(chunk, xzc_entry);
- }
- _malloc_lock_unlock(&xz->xz_lock);
- }
-
- // If none are present on the preallocated list, request a new run from the
- // segment group, and enqueue the spares into the preallocated list
- if (!chunk) {
- xzm_slice_kind_t kind = XZM_SLICE_KIND_SMALL_CHUNK;
- size_t span_size = XZM_SMALL_CHUNK_SIZE;
- xzm_slice_count_t slice_count =
- (xzm_slice_count_t)(span_size / XZM_SEGMENT_SLICE_SIZE);
- bool clear = false;
- bool purgeable = false; // Purgeable only applies to LARGE/HUGE chunks
- xzm_preallocate_list_s preallocated;
- SLIST_INIT(&preallocated);
- xzm_preallocate_list_s *list = &preallocated;
- xzm_segment_group_t sg = _xzm_segment_group_for_id_and_front(zone,
- xz->xz_segment_group_id, xz->xz_front, false);
- chunk = xzm_segment_group_alloc_chunk(sg, kind, &xz->xz_guard_config,
- slice_count, list, 0, clear, purgeable);
- if (!chunk) {
- xzm_debug_assert(!list || !SLIST_FIRST(list));
- return NULL;
- }
- chunk->xzc_xzone_idx = xz->xz_idx;
- // Most processes don't get guard pages, so will only allocate one chunk
- // from the segment group. Avoid taking the lock in this common case
- if (SLIST_FIRST(list)) {
- xzm_chunk_t c, t;
- _malloc_lock_lock(&xz->xz_lock);
- SLIST_FOREACH_SAFE(c, list, xzc_slist_entry, t) {
- SLIST_REMOVE_HEAD(list, xzc_slist_entry);
- c->xzc_xzone_idx = xz->xz_idx;
- c->xzc_bits.xzcb_preallocated = true;
- LIST_INSERT_HEAD(&xz->xz_chunkq_preallocated, c, xzc_entry);
- }
- _malloc_lock_unlock(&xz->xz_lock);
- }
- }
-
- xzm_debug_assert(chunk);
- _xzm_xzone_small_chunk_init(xz, chunk);
-
- return chunk;
-}
-
-static xzm_slice_t
-_xzm_chunk_find_dirtiest_slice(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
- uint32_t *slice_free_bitmap)
-{
- // Attempt to find the dirtiest slice with at least one free block
- xzm_slice_count_t slice_count = _xzm_chunk_slice_count(chunk);
- struct xzm_slice_s *slices = _xzm_chunk_slices_of(chunk, slice_count);
- uint32_t lowest_free = UINT32_MAX;
- xzm_slice_t best_slice = NULL;
- for (xzm_slice_count_t i = 0; i < slice_count; ++i) {
- xzm_slice_t slice = &slices[i];
- uint32_t slice_mask = _xzm_xzone_slice_free_mask(zone, slice);
- uint32_t slice_free = chunk->xzc_free & slice_mask;
- if (slice_free && slice_free != slice_mask) {
- // This slice contains at least one used and one free block
- // TODO: consider replacing popcount with table (max blocks per
- // slice is bounded)
- uint32_t free_count = __builtin_popcount(slice_free);
- if (free_count < lowest_free) {
- lowest_free = free_count;
- best_slice = slice;
- *slice_free_bitmap = slice_free;
- }
- }
- }
- return best_slice;
-}
-
-// mimalloc: _mi_page_malloc
-static void *
-_xzm_xzone_alloc_from_chunk(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_chunk_t chunk, bool *is_zero_out)
-{
- xzm_debug_assert(!_xzm_chunk_is_full(zone, xz, chunk));
-
- *is_zero_out = false;
-
- uintptr_t block;
- xzm_block_index_t block_index = UINT32_MAX;
- uintptr_t start = _xzm_chunk_start(zone, chunk, NULL);
- const xzm_segment_t segment = _xzm_segment_for_slice(zone, chunk);
- const size_t block_size = _xzm_chunk_block_size(zone, chunk);
-
- switch (chunk->xzc_bits.xzcb_kind) {
- case XZM_SLICE_KIND_SMALL_CHUNK:
- // TODO: Find a way to determine if this block is zeroed to
- // avoid excessive zeroing
- if (_xzm_chunk_is_empty(zone, xz, chunk) ||
- (block_size % XZM_SEGMENT_SLICE_SIZE) == 0) {
- // It is impossible for a free block to exist on a dirty slice for
- // this chunk -- don't bother searching for one
- block_index = __builtin_ffs(chunk->xzc_free) - 1;
- } else {
- uint32_t slice_free;
- xzm_slice_t best_slice = _xzm_chunk_find_dirtiest_slice(zone, chunk,
- &slice_free);
-
- if (best_slice != NULL) {
- block_index = __builtin_ffs(slice_free) - 1;
-
- if (slice_free && !powerof2(slice_free)) {
- // This slice has more than one free block; check
- // if the one we selected would dirty multiple slices
- uintptr_t block_start = start +
- (block_index + xz->xz_block_size);
- xzm_slice_t start_slice = _xzm_segment_slice_of(segment,
- block_start);
- if (start_slice < best_slice) {
- // This block would dirty multiple slices; choose the
- // next free block instead. Note that the next free
- // block may also dirty multiple slices; this implies
- // that there are no free blocks in between that might
- // dirty fewer slices.
- block_index = __builtin_ffs(slice_free &
- ~(1u << block_index)) - 1;
- }
- }
- } else {
- // Unable to find a dirty slice, fall back to the first
- // available free block
- block_index = __builtin_ffs(chunk->xzc_free) - 1;
- }
- }
-
- xzm_debug_assert(block_index <= xz->xz_chunk_capacity);
- block = start + (block_index * xz->xz_block_size);
-
-
- // Mark the current block as not free
- chunk->xzc_free &= ~(1u << block_index);
-
- break;
- default:
- xzm_abort_with_reason("attempting to allocate from chunk of bad kind",
- (unsigned int)chunk->xzc_bits.xzcb_kind);
- }
-
- chunk->xzc_used++;
-
- return (void *)block;
-}
-
-static void
-_xzm_xzone_upgrade_small_slot_config(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_xzone_allocation_slot_t xas, xzm_slot_config_t slot_config)
-{
- // This allocation slot is hot, upgrade this xzone's slot config
- // for subsequent allocations and reset its contention counters
- xas->xas_contentions = 0;
-
- // Walk all the other allocation slots and reset their contention counters
- xzm_allocation_index_t limit_idx = _xzm_get_limit_allocation_index(slot_config);
- if (limit_idx > 1) {
- _malloc_lock_unlock(&xas->xas_lock);
- for (xzm_allocation_index_t alloc_idx = 0;
- alloc_idx < limit_idx;
- alloc_idx++) {
- xzm_xzone_allocation_slot_t xas_other =
- _xzm_xzone_allocation_slot_for_index(zone, xz,
- alloc_idx);
- if (xas_other != xas) {
- _malloc_lock_lock(&xas_other->xas_lock);
- xas_other->xas_contentions = 0;
- _malloc_lock_unlock(&xas_other->xas_lock);
- }
- }
- _malloc_lock_lock(&xas->xas_lock);
- }
-
- _malloc_lock_lock(&xz->xz_lock);
- // Another thread may have already upgraded the slot config
- if (xz->xz_slot_config == slot_config) {
- switch (xz->xz_slot_config) {
- case XZM_SLOT_SINGLE:
-#if CONFIG_XZM_CLUSTER_AWARE
- if (ncpuclusters > 1) {
- xz->xz_slot_config = XZM_SLOT_CLUSTER;
- } else {
- xz->xz_slot_config = XZM_SLOT_CPU;
- }
-#else // CONFIG_XZM_CLUSTER_AWARE
- xz->xz_slot_config = XZM_SLOT_CLUSTER;
-#endif // CONFIG_XZM_CLUSTER_AWARE
- break;
- case XZM_SLOT_CLUSTER:
- xz->xz_slot_config = XZM_SLOT_CPU;
- break;
- case XZM_SLOT_CPU:
- xzm_abort("Can't upgrade from XZM_SLOT_CPU");
- break;
- default:
- xzm_abort("Invalid xzone slot config");
- break;
- }
- xzm_debug_assert(xz->xz_slot_config <= zone->xzz_max_slot_config);
- }
- _malloc_lock_unlock(&xz->xz_lock);
-}
-
-MALLOC_NOINLINE
-static void *
-_xzm_xzone_malloc_small(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_malloc_options_t opt)
-{
- bool clear = (opt & XZM_MALLOC_CLEAR);
- void *ptr = NULL;
- bool is_zero = false;
- xzm_slot_config_t slot_config;
-
- xzm_allocation_index_t alloc_idx = _xzm_get_allocation_index(zone, xz,
- &slot_config, true);
- xzm_xzone_allocation_slot_t xas =
- _xzm_xzone_allocation_slot_for_index(zone, xz, alloc_idx);
-
- if (slot_config < zone->xzz_max_slot_config) {
- // This xzone's slot config is upgradable -- evaluate for contention
- if (!_malloc_lock_trylock(&xas->xas_lock)) {
- // Contended for this allocation slot
- _malloc_lock_lock(&xas->xas_lock);
- xas->xas_contentions++;
- if (xas->xas_contentions > zone->xzz_slot_upgrade_threshold[slot_config]) {
- _xzm_xzone_upgrade_small_slot_config(zone, xz, xas, slot_config);
- }
- }
- xas->xas_allocs++;
- // Reset the contention counter every T allocations -- this prevents
- // low-traffic xzones in long-lived processes from being upgraded over
- // time
- if ((xas->xas_allocs % zone->xzz_slot_upgrade_period) == 0) {
- xas->xas_contentions = 0;
- }
- } else {
- _malloc_lock_lock(&xas->xas_lock);
-#ifdef DEBUG
- xas->xas_allocs++;
-#endif
- }
-
- xzm_chunk_t chunk = xas->xas_chunk;
- if (!chunk || _xzm_chunk_is_full(zone, xz, chunk)) {
- // We need to go to the xzone to get a new chunk
- _malloc_lock_lock(&xz->xz_lock);
-
- if (chunk) {
- // If we have an existing chunk that's full, we need to uninstall it and
- // move it to the full list
- LIST_INSERT_HEAD(&xz->xz_chunkq_full, chunk, xzc_entry);
-
- // This is the final step of uninstallation - as soon as the chunk
- // transitions to the uninstalled state, other threads freeing to it
- // will go to its lock instead of the lock for this slot. Store
- // release to publish all of the changes made to the chunk under the
- // slot lock, which will pair with the acquire when they take the
- // chunk lock
- os_atomic_store(&chunk->xzc_alloc_idx, XZM_SLOT_INDEX_EMPTY,
- release);
- }
-
- while ((chunk = LIST_FIRST(&xz->xz_chunkq_partial))) {
- _malloc_lock_lock(&chunk->xzc_lock);
- xzm_debug_assert(chunk->xzc_bits.xzcb_on_partial_list);
- LIST_REMOVE(chunk, xzc_entry);
- chunk->xzc_bits.xzcb_on_partial_list = false;
- if (chunk->xzc_used) {
- xzm_debug_assert(!_xzm_chunk_is_full(zone, xz, chunk));
- // We can use this chunk! Install it. We hold both the slot
- // and chunk locks now so threads that are freeing to this chunk
- // are guaranteed visibility of our modifications regardless of
- // what they observe for the slot index.
- xas->xas_chunk = chunk;
- os_atomic_store(&chunk->xzc_alloc_idx, alloc_idx + 1, relaxed);
- _malloc_lock_unlock(&chunk->xzc_lock);
- break;
- } else {
- // This chunk is fully free, and the thread that freed it is on
- // its way to remove it, so we don't want to use it. We've
- // pulled it off the partial list so nobody else spends time on
- // it and marked it accordingly so that the freeing thread
- // knows.
- _malloc_lock_unlock(&chunk->xzc_lock);
- }
- }
-
- // Attempt to reuse a chunk from the batch list
- if (!chunk && xz->xz_chunkq_batch_count) {
- chunk = xz->xz_chunkq_batch;
- xzm_debug_assert(chunk);
- xz->xz_chunkq_batch = *_xzm_segment_slice_meta_batch_next(zone,
- chunk);
- xzm_debug_assert(_xzm_slice_meta_is_batch_pointer(zone,
- xz->xz_chunkq_batch));
-#if CONFIG_XZM_DEFERRED_RECLAIM
- *_xzm_slice_meta_reclaim_id(zone, chunk) = VM_RECLAIM_ID_NULL;
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
- --xz->xz_chunkq_batch_count;
- xas->xas_chunk = chunk;
- os_atomic_store(&chunk->xzc_alloc_idx, alloc_idx + 1, relaxed);
- }
-
- // We're done with the xzone now - either we got a partial chunk from
- // it, or we didn't and we need to get a sequestered or fresh one
- _malloc_lock_unlock(&xz->xz_lock);
-
- if (!chunk && xz->xz_sequestered) {
- xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
- chunk = _xzm_xzone_allocate_chunk_from_isolation(main, xz);
-
- if (chunk) {
- // Now install the chunk to the slot (TODO: does this need to be
- // done under the iz lock? TODO: does this need to be done
- // before populating in exclaves?)
- xas->xas_chunk = chunk;
- os_atomic_store(&chunk->xzc_alloc_idx, alloc_idx + 1, relaxed);
- }
- }
-
- if (!chunk) {
- chunk = _xzm_xzone_small_chunk_alloc(zone, xz);
- if (chunk) {
- xzm_debug_assert(!_xzm_chunk_is_full(zone, xz, chunk));
- // Succeeded at allocating a chunk. Install it to the slot.
- xas->xas_chunk = chunk;
- os_atomic_store(&chunk->xzc_alloc_idx, alloc_idx + 1, relaxed);
- } else {
- // There's no other way for us to get a chunk, so we need to
- // give up.
- xas->xas_chunk = NULL;
- goto out;
- }
- }
- }
-
- xzm_debug_assert(xas->xas_chunk == chunk);
- xzm_debug_assert(!_xzm_chunk_is_full(zone, xz, chunk));
-
- ptr = _xzm_xzone_alloc_from_chunk(zone, xz, chunk, &is_zero);
- xzm_debug_assert(ptr);
-
-out:
- _malloc_lock_unlock(&xas->xas_lock);
-
- if (os_likely(ptr)) {
-#if CONFIG_MTE
- bool memtag_enabled = xz->xz_tagged;
- bool needs_canonical_tagging = (opt & XZM_MALLOC_CANONICAL_TAG);
- if (memtag_enabled) {
- if (needs_canonical_tagging) {
- // We need this specific block to have canonical tagging (i.e. tag = 0)
- ptr = memtag_tag_canonical(ptr, xz->xz_block_size);
- } else {
- ptr = _xzm_xzone_block_memtag_retag(zone, ptr,
- xz->xz_block_size);
- }
- }
-#endif
-
- // TODO: MallocCheckZeroOnFreeCorruption support
- if (!is_zero && clear &&
- xz->xz_block_size > XZM_ZERO_ON_FREE_THRESHOLD) {
- bzero(ptr, xz->xz_block_size);
- }
- } else {
- malloc_set_errno_fast(MZ_POSIX, ENOMEM);
- }
- return ptr;
-}
-
-#pragma mark General allocation
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_xzone_try_reserve_early_budget(xzm_malloc_zone_t zone, xzm_xzone_t xz)
-{
- uint16_t early_budget = os_atomic_load(&xz->xz_early_budget, relaxed);
- uint16_t updated_budget;
- do {
- if (os_likely(!early_budget)) {
- return false;
- }
-
- updated_budget = early_budget - 1;
- } while (!os_atomic_cmpxchgv(&xz->xz_early_budget, early_budget,
- updated_budget, &early_budget, relaxed));
-
- return true;
-}
-
-#ifdef DEBUG
-// mimalloc: mi_mem_is_zero
-static bool
-_xzm_mem_is_zero(uint8_t *ptr, size_t size)
-{
- return !_malloc_memcmp_zero_aligned8(ptr, size);
-}
-#endif // DEBUG
-
-#if CONFIG_XZM_THREAD_CACHE
-
-static void *
-_xzm_xzone_malloc_tiny_or_early(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_malloc_options_t opt)
-{
- if (!(opt & XZM_MALLOC_NO_MFM) &&
- _xzm_xzone_try_reserve_early_budget(zone, xz)) {
- void *ptr = mfm_alloc(xz->xz_block_size);
- xzm_debug_assert(ptr);
-#ifdef DEBUG
- if (opt & XZM_MALLOC_CLEAR) {
- xzm_debug_assert(_xzm_mem_is_zero(ptr, xz->xz_block_size));
- }
-#endif
- return ptr;
- }
-
- void *ptr = _xzm_xzone_malloc_tiny(zone, xz, 0);
-#ifdef DEBUG
- if (opt & XZM_MALLOC_CLEAR) {
- xzm_debug_assert(_xzm_mem_is_zero(ptr, xz->xz_block_size));
- }
-#endif
- return ptr;
-}
-
-MALLOC_NOINLINE
-static void *
-_xzm_xzone_thread_cache_malloc_corrupt(void *corrupt_block)
-{
- // This looks kind of weird, but I couldn't convince the optimizer not to
- // push a frame in _xzm_xzone_malloc() if I put the abort here directly, so
- // we need to launder it apparently
- return _xzm_xzone_malloc_freelist_outlined(NULL, NULL, 0, NULL, corrupt_block,
- 0);
-}
-
-MALLOC_NOINLINE
-static void *
-_xzm_xzone_thread_cache_fill_and_malloc(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_xzone_thread_cache_t cache)
-{
- void *ptr = NULL;
- bool contended = false;
- if (cache->xztc_state != XZM_XZONE_CACHE_EMPTY) {
- xzm_debug_assert(cache->xztc_head == XZM_FREE_NULL);
-
- ptr = _xzm_xzone_malloc_from_freelist_chunk(zone, xz,
- XZM_SLOT_INDEX_THREAD, cache, cache->xztc_chunk, false,
- &contended, NULL);
- if (ptr) {
- goto done;
- }
- }
-
- xzm_debug_assert(cache->xztc_state == XZM_XZONE_CACHE_EMPTY);
-
- xzm_chunk_t chunk = NULL;
- ptr = _xzm_xzone_find_and_malloc_from_freelist_chunk(zone, xz,
- XZM_SLOT_INDEX_THREAD, cache, &chunk, &contended);
-
-done:
- if (ptr) {
- xzm_block_t block = ptr;
- *block = (struct xzm_block_inline_meta_s){ 0 };
- } else {
- xzm_debug_assert(cache->xztc_state == XZM_XZONE_CACHE_EMPTY);
- malloc_set_errno_fast(MZ_POSIX, ENOMEM);
- }
-
- return ptr;
-}
-
-static void
-_xzm_thread_cache_create(xzm_malloc_zone_t zone)
-{
- xzm_main_malloc_zone_t main = (xzm_main_malloc_zone_t)zone;
- xzm_metapool_t mp = &main->xzmz_metapools[XZM_METAPOOL_THREAD_CACHE];
- xzm_thread_cache_t tc = xzm_metapool_alloc(mp);
-
- *tc = (struct xzm_thread_cache_s){
- .xtc_main = main,
- .xtc_thread = pthread_self(),
- };
-
- uint64_t now = mach_absolute_time();
- for (size_t i = 0; i < zone->xzz_thread_cache_xzone_count; i++) {
- tc->xtc_xz_caches[i] = (xzm_xzone_thread_cache_u){
- .xztc_state = XZM_XZONE_NOT_CACHED,
- .xztc_timestamp = now,
- };
- }
-
- _malloc_lock_lock(&main->xzmz_thread_cache_list_lock);
- LIST_INSERT_HEAD(&main->xzmz_thread_cache_list, tc, xtc_linkage);
- _malloc_lock_unlock(&main->xzmz_thread_cache_list_lock);
-
- _pthread_setspecific_direct(__TSD_MALLOC_XZONE_THREAD_CACHE, tc);
-}
-
-MALLOC_NOINLINE
-static void *
-_xzm_thread_cache_create_and_malloc(xzm_malloc_zone_t zone,
- xzm_xzone_index_t xz_idx, xzm_xzone_t xz, xzm_malloc_options_t opt)
-{
- _xzm_thread_cache_create(zone);
-
- return _xzm_xzone_malloc_tiny_or_early(zone, xz, opt);
-}
-
-MALLOC_NOINLINE
-static void *
-_xzm_xzone_thread_cache_record_and_malloc_outlined(xzm_malloc_zone_t zone,
- xzm_xzone_t xz, xzm_malloc_options_t opt,
- xzm_xzone_thread_cache_t cache)
-{
- xzm_debug_assert(cache->xztc_head == XZM_XZONE_NOT_CACHED);
-
- uint64_t now = mach_absolute_time();
- uint64_t activation_time = zone->xzz_thread_cache_xzone_activation_time;
- if (now - cache->xztc_timestamp < activation_time) {
- // We reached the threshold number of allocations within the
- // activation threshold time, so activate caching for this xzone
- cache->xztc_head = XZM_XZONE_CACHE_EMPTY;
-
- // Initialize the seqno for this cache with the low bits of the
- // timestamp to add some non-determinism to its base
- cache->xztc_seqno = cache->xztc_timestamp & XZM_SEQNO_COUNTER_MASK;
-
- return _xzm_xzone_thread_cache_fill_and_malloc(zone, xz, cache);
- }
-
- // reset for the next period
- cache->xztc_timestamp = now;
- cache->xztc_allocs = 0;
- cache->xztc_contentions = 0;
-
- return _xzm_xzone_malloc_tiny_or_early(zone, xz, opt);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static void *
-_xzm_xzone_thread_cache_record_and_malloc(xzm_malloc_zone_t zone,
- xzm_xzone_t xz, xzm_malloc_options_t opt,
- xzm_xzone_thread_cache_t cache)
-{
- xzm_debug_assert(cache->xztc_head == XZM_XZONE_NOT_CACHED);
-
- cache->xztc_allocs++;
-
- uint32_t activation_period = zone->xzz_thread_cache_xzone_activation_period;
- if (os_unlikely(cache->xztc_allocs == activation_period)) {
- return _xzm_xzone_thread_cache_record_and_malloc_outlined(zone, xz, opt,
- cache);
- }
-
- return _xzm_xzone_malloc_tiny_or_early(zone, xz, opt);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static void
-_xzm_xzone_thread_cache_record_contention(xzm_malloc_zone_t zone,
- xzm_xzone_t xz, xzm_xzone_thread_cache_t cache)
-{
- // We can end up here after caching is already engaged if serving an
- // allocation that can't be handled from the cache
- if (cache->xztc_head != XZM_XZONE_NOT_CACHED) {
- return;
- }
-
- cache->xztc_contentions++;
-
- uint32_t activation_contentions =
- zone->xzz_thread_cache_xzone_activation_contentions;
- if (os_unlikely(cache->xztc_contentions == activation_contentions)) {
- // We reached the threshold number of contentions within the threshold
- // period, so activate caching for this xzone
- cache->xztc_head = XZM_XZONE_CACHE_EMPTY;
-
- // Initialize the seqno for this cache with the low bits of the
- // timestamp to add some non-determinism to its base
- cache->xztc_seqno = cache->xztc_timestamp & XZM_SEQNO_COUNTER_MASK;
- }
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static void *
-_xzm_xzone_thread_cache_malloc(xzm_malloc_zone_t zone, xzm_xzone_index_t xz_idx,
- xzm_xzone_t xz, xzm_malloc_options_t opt)
-{
- xzm_thread_cache_t tc = _xzm_get_thread_cache();
- if (os_unlikely(!tc)) {
- return _xzm_thread_cache_create_and_malloc(zone, xz_idx, xz, opt);
- }
-
- xzm_debug_assert(xz_idx < zone->xzz_thread_cache_xzone_count);
- xzm_xzone_thread_cache_t cache = &tc->xtc_xz_caches[xz_idx];
- if (cache->xztc_head < XZM_FREE_LIMIT) {
- uint8_t *start = cache->xztc_chunk_start;
- void *ptr = start + (cache->xztc_head * XZM_GRANULE);
-
-#if CONFIG_MTE
- if (_xzm_zone_memtag_enabled(zone)) {
- ptr = memtag_fixup_ptr(ptr);
- }
-#endif
- struct xzm_block_inline_meta_s *block_meta_p =
- (struct xzm_block_inline_meta_s *)ptr;
- struct xzm_block_inline_meta_s block_meta = *block_meta_p;
-
- // Check the PAC by adding the seqno back
- union xzm_block_linkage_u linkage = {
- .xzbl_next_offset = block_meta.xzb_linkage.xzbl_next_offset,
- .xzbl_next_seqno = block_meta.xzb_linkage.xzbl_next_seqno,
- .xzbl_seqno = cache->xztc_head_seqno,
- };
-
- linkage.xzbl_next_value = (uintptr_t)ptrauth_sign_unauthenticated(
- linkage.xzbl_next, ptrauth_key_process_dependent_data,
- ptrauth_blend_discriminator(ptr,
- ptrauth_string_discriminator("xzb_linkage")));
-
- // Take the seqno back out of the signed value for the comparison with
- // the stored one
- linkage.xzbl_seqno = 0;
-
- if (os_unlikely(block_meta.xzb_linkage.xzbl_next_value !=
- linkage.xzbl_next_value)) {
- return _xzm_xzone_thread_cache_malloc_corrupt(ptr);
- }
-
- *block_meta_p = (struct xzm_block_inline_meta_s){ 0 };
-
-#ifdef DEBUG
- uint64_t orig_state = cache->xztc_freelist_state;
-#endif
-
- // We update the local freelist head and count together with an atomic
- // store so that the enumerator gets a consistent picture of the
- // freelist. Having the head and the count agree is required in order
- // to handle the bump correctly:
- // - If the head were updated before the count, we'd think the bump
- // offset was higher than its true value, so we'd falsely report the
- // block at the bump offset as allocated when it isn't
- // - If the head were updated after the count, we'd think the bump
- // offset was lower than its true value, so we'd falsely report the
- // last block before the bump as free when it may not be
- xzm_xzone_thread_cache_atomic_meta_u tc_meta = {
- .xztcam_head = block_meta.xzb_linkage.xzbl_next_offset,
- .xztcam_free_count = cache->xztc_free_count - 1,
- };
- os_atomic_store(&cache->xztc_atomic_meta.xztcam_value,
- tc_meta.xztcam_value, relaxed);
- cache->xztc_head_seqno = block_meta.xzb_linkage.xzbl_next_seqno;
-
-#ifdef DEBUG
- uint64_t new_state = cache->xztc_freelist_state;
-#endif
-
- xzm_trace(thread_cache_malloc, (uint64_t)cache->xztc_chunk, orig_state,
- new_state, 0);
-
- return ptr;
- } else if (cache->xztc_head == XZM_FREE_NULL && cache->xztc_free_count) {
- uint8_t *start = cache->xztc_chunk_start;
- size_t capacity = xz->xz_chunk_capacity;
- void *ptr = start + ((capacity - cache->xztc_free_count) *
- xz->xz_block_size);
-#if CONFIG_MTE
- if (_xzm_zone_memtag_enabled(zone)) {
- ptr = memtag_fixup_ptr(ptr);
- }
-#endif
-
- struct xzm_block_inline_meta_s *block_meta_p =
- (struct xzm_block_inline_meta_s *)ptr;
- *block_meta_p = (struct xzm_block_inline_meta_s){ 0 };
-
-#ifdef DEBUG
- uint64_t orig_state = cache->xztc_freelist_state;
-#endif
-
- cache->xztc_free_count--;
-
-#ifdef DEBUG
- uint64_t new_state = cache->xztc_freelist_state;
-#endif
-
- xzm_trace(thread_cache_malloc, (uint64_t)cache->xztc_chunk, orig_state,
- new_state, 0);
-
- return ptr;
- } else if (cache->xztc_head == XZM_XZONE_NOT_CACHED) {
- return _xzm_xzone_thread_cache_record_and_malloc(zone, xz, opt, cache);
- } else {
- xzm_debug_assert(cache->xztc_head == XZM_FREE_NULL ||
- cache->xztc_head == XZM_XZONE_CACHE_EMPTY);
- return _xzm_xzone_thread_cache_fill_and_malloc(zone, xz, cache);
- }
-}
-
-#endif // CONFIG_XZM_THREAD_CACHE
-
-static void *
-_xzm_xzone_malloc(xzm_malloc_zone_t zone, size_t size,
- xzm_xzone_index_t xz_idx, xzm_malloc_options_t opt)
-{
- void *ptr = NULL;
-
- xzm_xzone_t xz = &zone->xzz_xzones[xz_idx];
-
-#if CONFIG_XZM_THREAD_CACHE
- // First, try thread cache
- if (zone->xzz_thread_cache_enabled &&
- size <= XZM_THREAD_CACHE_THRESHOLD
-#if CONFIG_MTE
- && !(opt & XZM_MALLOC_CANONICAL_TAG)
-#endif
- ) {
- return _xzm_xzone_thread_cache_malloc(zone, xz_idx, xz, opt);
- }
-#endif // CONFIG_XZM_THREAD_CACHE
-
- if (!(opt & XZM_MALLOC_NO_MFM) && _xzm_malloc_zone_is_main(zone) &&
-#if CONFIG_MTE
- // With memory tagging enabled, we shouldn't call into the early
- // allocator if we're being asked for a canonically tagged
- // allocation, as we don't support that in MFM.
- !((opt & XZM_MALLOC_CANONICAL_TAG) && zone->xzz_memtag_config.enabled) &&
-#endif // CONFIG_MTE
- _xzm_xzone_try_reserve_early_budget(zone, xz)) {
- // To maintain compatibility with code expecting
- // malloc_size(malloc(size)) >= malloc_good_size(size), just always use
- // the block size
- ptr = mfm_alloc(xz->xz_block_size);
- // The way that we use the early allocator, it can't ever legitimately
- // fail, and we're depending on that here (to tail-call)
- xzm_debug_assert(ptr);
- return ptr;
- }
-
- if (size <= XZM_TINY_BLOCK_SIZE_MAX) {
- return _xzm_xzone_malloc_tiny(zone, xz, opt);
- } else if (zone->xzz_small_freelist_enabled) {
- return _xzm_xzone_malloc_small_freelist(zone, xz, opt);
- }
-
- xzm_debug_assert(size <= XZM_SMALL_BLOCK_SIZE_MAX);
- return _xzm_xzone_malloc_small(zone, xz, opt);
-}
-
-static size_t
-xzm_malloc_zone_size(xzm_malloc_zone_t zone, const void *ptr);
-
-void *
-xzm_malloc_inline(xzm_malloc_zone_t zone, size_t size,
- malloc_type_descriptor_t type_desc, xzm_malloc_options_t opt)
-{
- void *ptr;
- if (size > XZM_SMALL_BLOCK_SIZE_MAX) {
- ptr = _xzm_malloc_large_huge(zone, size, 0, type_desc, opt);
- } else {
- xzm_debug_assert((zone->xzz_flags & MALLOC_PURGEABLE) == 0);
- xzm_xzone_index_t xz_idx = _xzm_xzone_lookup(zone, size, type_desc);
- ptr = _xzm_xzone_malloc(zone, size, xz_idx, opt);
- }
-
-#ifdef DEBUG
- xzm_debug_assert(ptr);
- if (opt & XZM_MALLOC_CLEAR) {
- xzm_debug_assert(_xzm_mem_is_zero(ptr, size));
- }
-#if CONFIG_MTE
- // The `size` parameter means "requested size", but we make tagging decisions
- // based on block size. Note that in the special case of allocations with
- // large alignments we may get an incorrect answer from
- // _xzm_zone_memtag_block(), since it can't account for the segment group
- // min block size, but that case doesn't take this path so the assert here
- // is still safe.
- size_t block_size = xzm_malloc_zone_size(zone, ptr);
- bool data = malloc_type_descriptor_is_pure_data(type_desc);
- bool memtag = _xzm_zone_memtag_block(zone, block_size, data);
- bool canonical_tag = (opt & XZM_MALLOC_CANONICAL_TAG);
- if (!mfm_claimed_address(ptr)) {
- if (memtag && !canonical_tag) {
- xzm_debug_assert(memtag_strip_address(ptr) != ptr);
- } else {
- xzm_debug_assert(memtag_strip_address(ptr) == ptr);
- }
- }
-#endif // CONFIG_MTE
-#endif // DEBUG
- return ptr;
-}
-
-MALLOC_NOINLINE
-void *
-xzm_malloc(xzm_malloc_zone_t zone, size_t size,
- malloc_type_descriptor_t type_desc, xzm_malloc_options_t opt)
-{
- return xzm_malloc_inline(zone, size, type_desc, opt);
-}
-
-// mimalloc: mi_heap_malloc_aligned
-static void * __alloc_align(2) __alloc_size(3)
-_xzm_memalign(xzm_malloc_zone_t zone, size_t alignment, size_t size,
- malloc_type_descriptor_t type_desc, xzm_malloc_options_t opt)
-{
- // - Assumption: alignment is a power of 2
- // - Assumption: all xzm size classes are aligned to their MSb/4
- // (e.g. the 3548-byte size class is aligned to 2048/4=512)
- // - Assumption: all power-of-2 size classes less than or equal to page size
- // are naturally aligned
- //
- // With those, the following definition of a memalign function will always
- // return properly aligned memory (for values of size/align <= page size)
- // memalign(align, size) =
- // { alloc(align) , size <= align
- // { alloc(2 * align) , align < size <= 2 * align
- // { alloc(roundup(size, 4 * align)) , 2 * align < size < 4 * align
- // { alloc(size) , 4 * align <= size
- //
- // The mimalloc memalign implementation (roundup(alloc(size + align - 1)))
- // has a fixed internal fragmentation of align - 1. For the definition
- // above, the internal fragmentation is:
- // { align - size , size <= align
- // { 2 * align - size , align < size <= 2 * align
- // { roundup(size, 4 * align) <= 2 * align , 2 * align < size < 4 * align
- // { 0 , 4 * align <= size
- //
- // In all but the third case, the internal fragmentation should be strictly
- // better than the original implementation. An example of the third case
- // being worse is memalign(2k, 5k), which will require an 8k allocation
- // under the new scheme, but a 7k allocation under the original.
- // Empirically this case seems rare enough that wins from the other three
- // cases make up for it.
-
- xzm_debug_assert(powerof2(alignment)); // should be guaranteed
-
- // The early allocator doesn't support alignment
- opt |= XZM_MALLOC_NO_MFM;
-
- void *ptr;
- if (size > XZM_SMALL_BLOCK_SIZE_MAX ||
- alignment > XZM_SEGMENT_SLICE_SIZE) {
- ptr = _xzm_malloc_large_huge(zone, size, alignment, type_desc, opt);
- } else if (size <= alignment) {
- xzm_debug_assert(alignment <= XZM_SMALL_BLOCK_SIZE_MAX);
- ptr = xzm_malloc(zone, alignment, type_desc, opt);
- } else if (size <= 2 * alignment) {
- xzm_debug_assert(2 * alignment <= XZM_SMALL_BLOCK_SIZE_MAX);
- ptr = xzm_malloc(zone, 2 * alignment, type_desc, opt);
- } else if (size < 4 * alignment) {
- xzm_debug_assert(roundup(size, 4 * alignment) <=
- XZM_SMALL_BLOCK_SIZE_MAX);
- ptr = xzm_malloc(zone, roundup(size, 4 * alignment), type_desc, opt);
- } else {
- xzm_debug_assert(size <= XZM_SMALL_BLOCK_SIZE_MAX);
- ptr = xzm_malloc(zone, size, type_desc, opt);
- }
-
- xzm_debug_assert(ptr);
- xzm_debug_assert((uintptr_t)ptr % alignment == 0);
- return ptr;
-}
-
-void *
-xzm_memalign(xzm_malloc_zone_t zone, size_t alignment, size_t size,
- malloc_type_descriptor_t type_desc, xzm_malloc_options_t opt)
-{
- return _xzm_memalign(zone, alignment, size, type_desc, opt);
-}
-
-#pragma mark Small deallocation
-
-static void
-_xzm_xzone_chunk_free(xzm_malloc_zone_t zone, xzm_xzone_t xz, xzm_chunk_t chunk,
- bool small_madvise_needed);
-
-static void
-_xzm_xzone_small_chunks_mark_empty(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_chunk_t *chunks, size_t chunk_count)
-{
- xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
- xzm_isolation_zone_t iz = &main->xzmz_isolation_zones[xz->xz_idx];
- const bool sequestered = xz->xz_sequestered;
-
- if (sequestered) {
- _malloc_lock_lock(&iz->xziz_lock);
- }
-
- for (size_t i = 0; i < chunk_count; ++i) {
- xzm_chunk_t chunk = chunks[i];
-
- // See _xzm_xzone_chunk_free
- chunk->xzc_mzone_idx = XZM_MZONE_INDEX_INVALID;
-
- if (sequestered) {
- // Reset such that the chunk is ready for next use immediately
- _xzm_chunk_reset_free(xz, chunk, true);
-
- // This chunk is no longer pristine
- chunk->xzc_bits.xzcb_is_pristine = false;
-
- LIST_INSERT_HEAD(&iz->xziz_chunkq, chunk, xzc_entry);
- } else {
- // Full reset - the chunk's span may be reused for anything
- _xzm_chunk_reset_free(xz, chunk, false);
-
- // Do not madvise again
- xzm_segment_group_t sg = _xzm_segment_group_for_slice(zone,
- chunk);
- xzm_segment_group_free_chunk(sg, chunk, false, false);
- }
- }
-
- if (sequestered) {
- _malloc_lock_unlock(&iz->xziz_lock);
- }
-}
-
-static xzm_block_t
-_xzm_xzone_free_to_chunk(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_chunk_t chunk, xzm_block_t block)
-{
-#if CONFIG_MTE
- const bool memtag_enabled = xz->xz_tagged;
-#endif
-
- switch(chunk->xzc_bits.xzcb_kind) {
- case XZM_SLICE_KIND_SMALL_CHUNK:
-#if CONFIG_MTE
- if (memtag_enabled) {
- block = (xzm_block_t)memtag_strip_address((uint8_t *)block);
- }
-#endif
- chunk->xzc_free |= (1u << _xzm_chunk_block_index(zone, chunk, block));
- break;
- default:
- xzm_abort_with_reason("Attempting to free to non-chunk slice",
- (unsigned int)chunk->xzc_bits.xzcb_kind);
- }
-
- chunk->xzc_used--;
-
- return block;
-}
-
-static void
-_xzm_xzone_chunk_free(xzm_malloc_zone_t zone, xzm_xzone_t xz, xzm_chunk_t chunk,
- bool small_madvise_needed)
-{
- xzm_segment_group_t sg = _xzm_segment_group_for_slice(zone, chunk);
-
- // Reset mzone_idx first to "unpublish" the chunk for the enumerator
- // protocol
- //
- // TODO: compiler barrier
- chunk->xzc_mzone_idx = XZM_MZONE_INDEX_INVALID;
-
- if (xz->xz_sequestered) {
- xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
- if (_xzm_chunk_should_defer_reclamation(main, chunk)) {
-#if CONFIG_XZM_DEFERRED_RECLAIM
- if (_xzm_segment_slice_is_deferred(_xzm_segment_for_slice(zone,
- chunk), chunk)) {
- // The only reason a chunk should already be in the reclaim
- // buffer when passed to this function is if it's a tiny chunk
- // and the zone is being destroyed, or it's a small chunk that's
- // being freed from the batch
- xzm_debug_assert(
- ((chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_TINY_CHUNK ||
- chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK) &&
- chunk->xzc_atomic_meta.xca_alloc_head == XZM_FREE_MADVISED) ||
- (chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_CHUNK &&
- !chunk->xzc_bits.xzcb_on_partial_list));
- } else {
- xzm_chunk_mark_free(zone, chunk);
- }
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
- } else {
- bool should_madvise = true;
- if (chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_TINY_CHUNK ||
- chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK) {
- if (chunk->xzc_atomic_meta.xca_alloc_head == XZM_FREE_MADVISED) {
- // The chunk is already madvised, no point doing it again
- should_madvise = false;
- } else {
- chunk->xzc_atomic_meta.xca_alloc_head = XZM_FREE_MADVISED;
- chunk->xzc_atomic_meta.xca_free_count = 0;
- }
- } else if (chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_CHUNK) {
- should_madvise = small_madvise_needed;
- } else {
- xzm_abort_with_reason("Unexpected chunk kind",
- chunk->xzc_bits.xzcb_kind);
- }
-
- if (should_madvise) {
- xzm_segment_group_segment_madvise_chunk(sg, chunk);
- }
- }
-
- // Reset such that the chunk is ready for next use immediately
- _xzm_chunk_reset_free(xz, chunk, true);
-
- // chunk->xzc_used == 0
- // xzc_xzone_idx unchanged
- // xzcb_kind unchanged
- // This chunk is no longer pristine, so bumping doesn't guarantee that
- // the allocation is cleared
- chunk->xzc_bits.xzcb_is_pristine = false;
-
- if (chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_TINY_CHUNK ||
- chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK) {
- // Tiny chunks are only freed during zone destruction, so we
- // can access the atomic metadata directly, since we're the
- // only thread with access to this zone
- chunk->xzc_atomic_meta.xca_on_partial_list = false;
- chunk->xzc_atomic_meta.xca_on_empty_list = true;
- chunk->xzc_atomic_meta.xca_alloc_idx = XZM_SLOT_INDEX_EMPTY;
- }
-
- xzm_isolation_zone_t iz = &main->xzmz_isolation_zones[xz->xz_idx];
-
- _malloc_lock_lock(&iz->xziz_lock);
- LIST_INSERT_HEAD(&iz->xziz_chunkq, chunk, xzc_entry);
- _malloc_lock_unlock(&iz->xziz_lock);
- } else {
-#if CONFIG_XZM_DEFERRED_RECLAIM
- xzm_debug_assert(
- *_xzm_slice_meta_reclaim_id(zone, chunk) == VM_RECLAIM_ID_NULL);
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
-
- // Full reset - the chunk's span may be reused for anything
- _xzm_chunk_reset_free(xz, chunk, false);
-
- // will madvise if needed
- xzm_segment_group_free_chunk(sg, chunk, false, small_madvise_needed);
- }
-}
-
-// Attempt to madvise any slices now freed by a given block
-static void
-_xzm_xzone_chunk_madvise_free_slices(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_chunk_t chunk, xzm_block_t block)
-{
- xzm_debug_assert(chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_CHUNK);
-
- xzm_segment_group_t sg = _xzm_segment_group_for_slice(zone, chunk);
-
- // Only multi-slice chunks that use a bitmap freelist may
- // madvise individual body slices
- if (chunk->xzc_bits.xzcb_kind != XZM_SLICE_KIND_SMALL_CHUNK ||
- _xzm_segment_group_uses_deferred_reclamation(sg)) {
- // TODO: Support for aggressive deferred reclamation of small
- // slices (rdar://112088639)
- return;
- }
-
-#if CONFIG_MTE
- if (xz->xz_tagged) {
- block = (xzm_block_t)memtag_strip_address((uint8_t *)block);
- }
-#endif // CONFIG_MTE
-
- xzm_slice_count_t slice_idx, num_slices;
- const xzm_segment_t segment = _xzm_segment_for_slice(zone, chunk);
- const xzm_slice_count_t chunk_idx = _xzm_slice_index(segment, chunk);
- const xzm_block_index_t block_idx = _xzm_chunk_block_index(zone, chunk,
- block);
- const size_t block_size = _xzm_chunk_block_size(zone, chunk);
- _xzm_chunk_block_free_slices_on_deallocate(chunk, chunk_idx,
- xz->xz_chunk_capacity, block_idx, block_size, &slice_idx,
- &num_slices);
- // At least one slice is madvisable
- if (num_slices > 0) {
- xzm_debug_assert(slice_idx >= chunk_idx);
- xzm_segment_group_segment_madvise_span(sg, _xzm_segment_start(segment) +
- slice_idx * XZM_SEGMENT_SLICE_SIZE, num_slices);
- }
-}
-
-static void
-_xzm_xzone_batch_small_push(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_chunk_t chunk, size_t batch_size)
-{
-#ifdef DEBUG
- _malloc_lock_assert_owner(&xz->xz_lock);
-#endif
-
- xzm_chunk_t batch = NULL;
- // Empty the batch if it is full
- xzm_debug_assert(xz->xz_chunkq_batch_count <= batch_size);
- if (os_unlikely(xz->xz_chunkq_batch_count >= batch_size)) {
- batch = xz->xz_chunkq_batch;
- xz->xz_chunkq_batch = NULL;
- xz->xz_chunkq_batch_count = 0;
- }
-
- // Insert this chunk at the head of the batch
- *_xzm_segment_slice_meta_batch_next(zone, chunk) = xz->xz_chunkq_batch;
- xz->xz_chunkq_batch = chunk;
- ++xz->xz_chunkq_batch_count;
-
- // Perform the madvise outside the zone lock
- _malloc_lock_unlock(&xz->xz_lock);
-
- if (batch) {
- _xzm_xzone_madvise_batch(zone, xz, batch);
- }
-}
-
-MALLOC_NOINLINE
-static void
-_xzm_xzone_free_block_to_small_chunk(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_chunk_t chunk, xzm_block_t block)
-{
- xzm_allocation_index_t alloc_idx = os_atomic_load(&chunk->xzc_alloc_idx,
- relaxed);
-
- while (true) {
- if (alloc_idx == XZM_SLOT_INDEX_EMPTY) {
- // This chunk does not appear to be installed in a slot, so we
- // should first try to take its chunk lock.
- _malloc_lock_lock(&chunk->xzc_lock);
-
- // It's possible that this chunk was installed while we were waiting
- // for the lock.
- alloc_idx = os_atomic_load(&chunk->xzc_alloc_idx, relaxed);
- if (alloc_idx != XZM_SLOT_INDEX_EMPTY) {
- // We lost the race; this chunk was installed to a slot while
- // we were taking its lock. Unlock it and try again.
- _malloc_lock_unlock(&chunk->xzc_lock);
- continue;
- }
-
- // The chunk is still not installed, so we hold the correct lock to
- // free it.
- bool chunk_was_full = _xzm_chunk_is_full(zone, xz, chunk);
- block = _xzm_xzone_free_to_chunk(zone, xz, chunk, block);
-
- // TODO: consider dropping the chunk lock while
- // madvising (additional madvise synchronization
- // required)
- _xzm_xzone_chunk_madvise_free_slices(zone, xz, chunk, block);
- if (_xzm_chunk_is_empty(zone, xz, chunk)) {
- // This chunk has become empty, so we need to go to the xzone to
- // remove it from the partial list.
-
- // We've just freed the last block and this chunk wasn't
- // installed, so it must be on the partial list (it can't be in
- // the full -> partial transition because that's done under both
- // the chunk and xzone locks).
- xzm_debug_assert(chunk->xzc_bits.xzcb_on_partial_list);
-
- // We need to acquire the xzone lock to remove this chunk from
- // the partial list. We need to drop the chunk lock to avoid a
- // lock inversion with code iterating the partial list looking
- // for a chunk to reuse. Because this chunk is empty, it's
- // guaranteed not to be reused, but it may be removed from the
- // partial list by the time we get there.
- _malloc_lock_unlock(&chunk->xzc_lock);
-
- _malloc_lock_lock(&xz->xz_lock);
-
- // Remove from the partial list, if it's still on it.
- if (chunk->xzc_bits.xzcb_on_partial_list) {
- LIST_REMOVE(chunk, xzc_entry);
- chunk->xzc_bits.xzcb_on_partial_list = false;
- }
-
- xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
- const uint8_t batch_size = main->xzmz_batch_size;
- if (batch_size) {
- // Enqueue the chunk into the batch, and drop the lock.
- _xzm_xzone_batch_small_push(zone, xz, chunk, batch_size);
- } else {
- // We're done with everything we need from the xzone now.
- _malloc_lock_unlock(&xz->xz_lock);
-
- // madvise and sequester or release back to the segment
- _xzm_xzone_chunk_free(zone, xz, chunk, false);
- }
- } else if (chunk_was_full) {
- // This chunk was full, so we need to go to the xzone to put it
- // on the partial list. Full chunks that aren't installed to a
- // slot are guaranteed to be on the full list. Because this
- // chunk is on the full list, no other threads will be trying to
- // acquire its lock while holding the xzone lock. It is
- // therefore safe to take the xzone lock here while holding the
- // chunk lock.
- _malloc_lock_lock(&xz->xz_lock);
-
- // Remove from the full list
- LIST_REMOVE(chunk, xzc_entry);
- // Place on the partial list
- LIST_INSERT_HEAD(&xz->xz_chunkq_partial, chunk, xzc_entry);
- chunk->xzc_bits.xzcb_on_partial_list = true;
- xzm_debug_assert(!_xzm_chunk_is_full(zone, xz, chunk));
-
- _malloc_lock_unlock(&xz->xz_lock);
- _malloc_lock_unlock(&chunk->xzc_lock);
- } else {
- // Neither empty nor newly unfull, so just unlock.
- _malloc_lock_unlock(&chunk->xzc_lock);
- }
- break;
- } else {
- // This chunk appears to be installed in a slot, so we should first
- // try to lock that slot.
-
- xzm_xzone_allocation_slot_t xas =
- _xzm_xzone_allocation_slot_for_index(zone, xz, alloc_idx - 1);
-
- _malloc_lock_lock(&xas->xas_lock);
-
- xzm_allocation_index_t orig_alloc_idx = alloc_idx;
-
- // It's possible that this chunk was uninstalled (and maybe even
- // installed somewhere else) while we were waiting for the lock.
- alloc_idx = os_atomic_load(&chunk->xzc_alloc_idx, relaxed);
- if (alloc_idx != orig_alloc_idx) {
- // We lost the race; this chunk was uninstalled from this slot while
- // we were taking its lock. Unlock it and try again.
- _malloc_lock_unlock(&xas->xas_lock);
- continue;
- }
-
- // The chunk is installed to the slot we locked, so we can proceed.
- block = _xzm_xzone_free_to_chunk(zone, xz, chunk, block);
-
- // TODO: consider dropping the AS lock while madvising
- // (additional madvise synchronization required)
-
- uint64_t thrash_threshold = zone->xzz_small_thrash_threshold;
- uint64_t thrash_limit_size = zone->xzz_small_thrash_limit_size;
- bool thrash_mitigation = (thrash_threshold &&
- xz->xz_block_size < thrash_limit_size);
- if (!thrash_mitigation) {
- _xzm_xzone_chunk_madvise_free_slices(zone, xz, chunk, block);
- }
-
- if (_xzm_chunk_is_empty(zone, xz, chunk)) {
- if (thrash_mitigation) {
- uint64_t now = mach_absolute_time();
- uint64_t time_elapsed = now - xas->xas_last_chunk_empty_ts;
- xas->xas_last_chunk_empty_ts = now;
-
- if (time_elapsed < thrash_threshold) {
- // We seem to be thrashing, so leave the chunk in place
- _malloc_lock_unlock(&xas->xas_lock);
- break;
- }
- }
-
- // This chunk has become empty and there are other partial
- // pages available, so uninstall it from the slot.
- xas->xas_chunk = NULL;
-
- // Since we've just freed the last block and we hold the
- // slot lock, nobody can have a reference to this chunk
- // right now.
- os_atomic_store(&chunk->xzc_alloc_idx, XZM_SLOT_INDEX_EMPTY,
- relaxed);
-
- // We're done with everything we need from the slot now.
- _malloc_lock_unlock(&xas->xas_lock);
-
- xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
- const uint8_t batch_size = main->xzmz_batch_size;
- if (batch_size) {
- _malloc_lock_lock(&xz->xz_lock);
- // Enqueue the chunk into the batch, and drop the lock.
- _xzm_xzone_batch_small_push(zone, xz, chunk, batch_size);
- } else {
- bool small_madvise = thrash_mitigation;
- // madvise and sequester or release back to the segment
- _xzm_xzone_chunk_free(zone, xz, chunk, small_madvise);
- }
- } else {
- // The chunk stays installed.
- _malloc_lock_unlock(&xas->xas_lock);
- }
- break;
- }
- }
-}
-
-static bool
-_xzm_xzone_freelist_chunk_lock(xzm_chunk_t chunk,
- xzm_chunk_atomic_meta_u *old_meta_out)
-{
- xzm_chunk_atomic_meta_u old_meta = {
- .xca_value = os_atomic_load_wide(
- &chunk->xzc_atomic_meta.xca_value, dependency),
- };
- xzm_chunk_atomic_meta_u new_meta = { 0 };
- while (true) {
- xzm_debug_assert(!old_meta.xca_walk_locked);
- if (old_meta.xca_alloc_head == XZM_FREE_MADVISING ||
- old_meta.xca_alloc_head == XZM_FREE_MADVISED) {
- return false;
- }
-
- new_meta = old_meta;
- new_meta.xca_walk_locked = true;
- bool success = os_atomic_cmpxchgv(&chunk->xzc_atomic_meta.xca_value,
- old_meta.xca_value, new_meta.xca_value, &old_meta.xca_value,
- dependency);
- if (!success) {
- continue;
- }
-
- xzm_trace(walk_lock, (uint64_t)chunk, old_meta.xca_value_lo,
- new_meta.xca_value_lo, 0);
- break;
- }
-
- *old_meta_out = old_meta;
- return true;
-}
-
-static void
-_xzm_xzone_freelist_chunk_unlock(xzm_chunk_t chunk,
- xzm_chunk_atomic_meta_u old_meta)
-{
- xzm_chunk_atomic_meta_u new_meta = old_meta;
- new_meta.xca_walk_locked = false;
- bool success = os_atomic_cmpxchgv(&chunk->xzc_atomic_meta.xca_value,
- old_meta.xca_value, new_meta.xca_value, &old_meta.xca_value,
- relaxed);
- xzm_assert(success);
- xzm_trace(walk_unlock, (uint64_t)chunk, old_meta.xca_value_lo,
- new_meta.xca_value_lo, 0);
-}
-
-MALLOC_NOINLINE
-static bool
-_xzm_xzone_freelist_chunk_block_is_free_slow(xzm_malloc_zone_t zone,
- xzm_chunk_t chunk, xzm_block_t block)
-{
- bool is_free = false;
-
- // Re-compute the block information so it doesn't need to be passed
- xzm_xzone_t xz = &zone->xzz_xzones[chunk->xzc_xzone_idx];
- size_t granule = (xz->xz_block_size > XZM_TINY_BLOCK_SIZE_MAX) ?
- XZM_SMALL_GRANULE : XZM_GRANULE;
- uint8_t *start = (uint8_t *)_xzm_chunk_start(zone, chunk, NULL);
- uint64_t block_granules = xz->xz_block_size / granule;
- uint64_t block_offset = _xzm_chunk_block_offset(zone, chunk, block);
- uint64_t max_offset = (xz->xz_chunk_capacity - 1) * block_granules;
-
- uint64_t cur_block_offset = 0;
- size_t freelist_count = 0;
-
- bool cached = false;
-
- xzm_chunk_atomic_meta_u old_meta = { 0 };
-
-#if CONFIG_XZM_THREAD_CACHE
- if (!zone->xzz_thread_cache_enabled ||
- xz->xz_block_size > XZM_THREAD_CACHE_THRESHOLD) {
- goto not_cached;
- }
-
- old_meta.xca_value = os_atomic_load_wide(&chunk->xzc_atomic_meta.xca_value,
- dependency);
- if (old_meta.xca_alloc_idx != XZM_SLOT_INDEX_THREAD_INSTALLED) {
- goto not_cached;
- }
-
- xzm_thread_cache_t tc = _xzm_get_thread_cache();
- if (!tc) {
- goto not_cached;
- }
-
- xzm_xzone_index_t xz_idx = xz->xz_idx;
- xzm_xzone_thread_cache_t cache = &tc->xtc_xz_caches[xz_idx];
-
- if (cache->xztc_head > XZM_FREE_LIMIT || cache->xztc_chunk != chunk) {
- goto not_cached;
- }
-
- cached = true;
-
- cur_block_offset = cache->xztc_head;
- freelist_count = 0;
- while (cur_block_offset != XZM_FREE_NULL &&
- cur_block_offset % block_granules == 0 &&
- cur_block_offset <= max_offset &&
- freelist_count < cache->xztc_free_count) {
- if (cur_block_offset == block_offset) {
- // We found our block on the local freelist, so it's free
- is_free = true;
- goto unlocked;
- }
-
- freelist_count++;
-
- xzm_block_t cur_block = (xzm_block_t)(
- start + (cur_block_offset * granule));
-#if CONFIG_MTE
- if (chunk->xzc_tagged) {
- cur_block = (xzm_block_t)memtag_fixup_ptr((void *)cur_block);
- }
-#endif
- cur_block_offset = cur_block->xzb_linkage.xzbl_next_offset;
- }
-
- bool local_valid = (cur_block_offset == XZM_FREE_NULL &&
- freelist_count <= cache->xztc_free_count);
- if (os_unlikely(!local_valid)) {
- xzm_client_abort_with_reason(
- "corrupt tiny local freelist, client likely has a buffer"
- " overflow or use-after-free bug", freelist_count);
- }
-
- // The local freelist has a bump offset. If the given block is above it, it
- // must be free.
- //
- // N.B. Although we can check this here, if the block really is free because
- // it's above the bump we might not get here in the first place, but it
- // isn't guaranteed to have its free cookie set.
- uint64_t local_bump_offset = (xz->xz_chunk_capacity -
- (cache->xztc_free_count - freelist_count)) * block_granules;
- if (block_offset >= local_bump_offset) {
- is_free = true;
- goto unlocked;
- }
-
-not_cached:
-#endif // CONFIG_XZM_THREAD_CACHE
-
- _malloc_lock_lock(&zone->xzz_lock);
-
- bool active = _xzm_xzone_freelist_chunk_lock(chunk, &old_meta);
- if (!active) {
- is_free = true;
- goto unlock;
- }
-
- cur_block_offset = old_meta.xca_alloc_head;
- freelist_count = 0;
- while (cur_block_offset != XZM_FREE_NULL &&
- cur_block_offset % block_granules == 0 &&
- cur_block_offset <= max_offset &&
- freelist_count < old_meta.xca_free_count) {
- if (cur_block_offset == block_offset) {
- // We found our block on the freelist, so it's free.
- is_free = true;
- goto chunk_walk_unlock;
- }
-
- freelist_count++;
-
- xzm_block_t cur_block = (xzm_block_t)(
- start + (cur_block_offset * granule));
-#if CONFIG_MTE
- if (chunk->xzc_tagged) {
- cur_block = (xzm_block_t)memtag_fixup_ptr((void *)cur_block);
- }
-#endif
- cur_block_offset = cur_block->xzb_linkage.xzbl_next_offset;
- }
-
- if (cached) {
- // If this chunk is cached, we should have fully walked the remote
- // freelist
- if (os_unlikely(freelist_count != old_meta.xca_free_count)) {
- xzm_client_abort_with_reason(
- "corrupt tiny remote freelist, client likely has a buffer"
- " overflow or use-after-free bug", freelist_count);
- }
- } else {
- // If this chunk isn't cached, we should have walked the remote freelist
- // to its terminator, which may be the bump
- if (os_unlikely(cur_block_offset != XZM_FREE_NULL)) {
- xzm_client_abort_with_reason(
- "corrupt tiny freelist, client likely has a buffer overflow"
- " or use-after-free bug", freelist_count);
- }
-
- // If there's a bump and the block is above it, it's free.
- //
- // N.B. As above, although we can check this here, if the block really
- // is free because it's above the bump we might not get here in the
- // first place, but it isn't guaranteed to have its free cookie set.
- xzm_debug_assert(freelist_count <= old_meta.xca_free_count);
-
- uint64_t bump_offset = (xz->xz_chunk_capacity -
- (old_meta.xca_free_count - freelist_count)) * block_granules;
- if (block_offset >= bump_offset) {
- is_free = true;
- goto chunk_walk_unlock;
- }
- }
-
-chunk_walk_unlock:
- old_meta.xca_walk_locked = true;
- _xzm_xzone_freelist_chunk_unlock(chunk, old_meta);
-
-unlock:
- _malloc_lock_unlock(&zone->xzz_lock);
-#if CONFIG_XZM_THREAD_CACHE
-unlocked:
-#endif
- return is_free;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_xzone_chunk_block_is_free(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
- xzm_slice_kind_t kind, size_t block_offset, size_t block_size,
- xzm_block_t block)
-{
- xzm_debug_assert(chunk->xzc_bits.xzcb_kind == kind);
- switch (kind) {
- case XZM_SLICE_KIND_TINY_CHUNK:
- case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:;
- // We optimize here for the case where the block is _not_ free, as that
- // is what's expected on the fast path during free(). Quick heuristic
- // check first: if the block does not have the free cookie we would
- // expect it to if it were on the freelist, we assume that it's
- // allocated. Note that this does _not_ work for pointers above the
- // bump, so we'll report them as allocated too - nano also does this, so
- // we have some evidence we can get away with it.
- uint64_t cookie_value;
-
-#if CONFIG_MTE
- if (chunk->xzc_tagged) {
- // We need to TCO for this load because even though we just ldg'd
- // the tag in the block pointer, if we're being asked about a block
- // that doesn't belong to us it might race to get freed and
- // re-tagged while we're looking at it.
- memtag_disable_checking();
- cookie_value = os_atomic_load(&block->xzb_cookie, relaxed);
- memtag_enable_checking();
- } else {
-#endif
- cookie_value = os_atomic_load(&block->xzb_cookie, relaxed);
-#if CONFIG_MTE
- }
-#endif
-
- uint64_t free_cookie = zone->xzz_freelist_cookie ^ (uint64_t)block;
-
-#if CONFIG_MTE
- if (kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK) {
- // Small chunks do not encode the tag in the freelist cookie
- free_cookie = (uint64_t)memtag_strip_address((uint8_t *)free_cookie);
- }
-#endif
- if (os_likely(cookie_value != free_cookie)) {
- return false;
- }
-
- // If the block _does_ look free, it's still possible that the
- // application just happened to have written the cookie value in the
- // slot. We have a choice:
- // - Nano chooses to just assume that the block really is free. This is
- // a soundness problem, as it could cause free() to fail on a valid
- // allocation.
- // - We can walk the freelist to see if we can really find the block.
- // This is sound but relatively costly.
- //
- // For now we take the latter choice, at the risk there's a workload
- // that really hammers on malloc_size() of free blocks a lot. If we
- // encounter such a workload, we could consider behaving like nano for
- // compatibility, but most uses of malloc_size() on free blocks are
- // incorrect so we would probably tell that workload to change.
- bool is_free = _xzm_xzone_freelist_chunk_block_is_free_slow(zone, chunk,
- block);
- return is_free;
- case XZM_SLICE_KIND_SMALL_CHUNK:;
- size_t block_index = block_offset / block_size;
- return _xzm_small_chunk_block_index_is_free(chunk,
- (xzm_block_index_t)block_index);
- default:
- break;
- }
-
- return false;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_chunk_t
-__xzm_ptr_lookup(xzm_malloc_zone_t zone, xzm_segment_table_entry_s entry,
- const void *orig_ptr, const void *ptr, xzm_xzone_t *xz_out,
- xzm_block_t *block_out, size_t *block_size_out)
-{
- xzm_segment_t segment = _xzm_segment_table_entry_to_segment(entry);
- if (!segment) {
- return NULL;
- }
-
- xzm_chunk_t chunk = _xzm_segment_chunk_of(segment, (uintptr_t)ptr);
- if (os_unlikely(!chunk)) {
- return NULL;
- }
-
- if (os_unlikely(chunk->xzc_mzone_idx != zone->xzz_mzone_idx)) {
- // The pointer belongs to one of the mzones, but not us. Depending on
- // the context, we _could_ just go ahead and do what we were asked to
- // (e.g. free) anyway, if we had a way to turn the mzone index back into
- // the pointer, but that would probably be confusing to libmalloc and
- // any wrapper zones, etc, so instead we'll just treat pointers
- // belonging to other mzones as completely foreign.
- return NULL;
- }
-
- xzm_block_t block = (xzm_block_t)ptr;
-
- uintptr_t start = _xzm_chunk_start(zone, chunk, NULL);
- size_t block_offset = (uintptr_t)block - start;
-
- xzm_slice_kind_t kind = chunk->xzc_bits.xzcb_kind;
- xzm_xzone_t xz = NULL;
- size_t block_size = 0;
- if (os_likely(_xzm_slice_kind_uses_xzones(kind))) {
- xz = &zone->xzz_xzones[chunk->xzc_xzone_idx];
- block_size = xz->xz_block_size;
-
- if (os_unlikely(!XZM_FAST_ALIGNED(block_offset, block_size,
- xz->xz_align_magic))) {
- return NULL;
- }
- } else {
- block_size = ((size_t)chunk->xzcs_slice_count) <<
- XZM_SEGMENT_SLICE_SHIFT;
- size_t ptr_offset = block_offset % block_size;
- if (os_unlikely(ptr_offset)) {
- return NULL;
- }
- }
-
-#ifdef DEBUG
- size_t block_index = block_offset / block_size;
- xzm_debug_assert(!_xzm_slice_kind_uses_xzones(chunk->xzc_bits.xzcb_kind) ||
- block_index < xz->xz_chunk_capacity);
-#endif
-
-#if CONFIG_MTE
- const bool memtag_enabled = _xzm_zone_memtag_enabled(zone);
- if (memtag_enabled) {
- // Load the tag for the block pointer we materialized
- block = (xzm_block_t)memtag_fixup_ptr((void *)block);
-
- // If the tag of the block doesn't match the tag we started with, we
- // consider it not allocated
- if (os_unlikely(!memtag_tags_match(orig_ptr, block))) {
- return NULL;
- }
- }
-#endif
-
- if (os_unlikely(_xzm_xzone_chunk_block_is_free(zone, chunk, kind,
- block_offset, block_size, block))) {
- return NULL;
- }
-
- if (xz_out) {
- *xz_out = xz;
- }
-
- if (block_out) {
- *block_out = block;
- }
- if (block_size_out) {
- *block_size_out = block_size;
- }
-
- return chunk;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_chunk_t
-_xzm_ptr_lookup(xzm_malloc_zone_t zone, const void *ptr, xzm_xzone_t *xz_out,
- xzm_block_t *block_out, size_t *block_size_out)
-{
- const void *orig_ptr = ptr;
-
-#if CONFIG_MTE
- // Strip the pointer to use it in pointer arithmetic operations
- ptr = memtag_strip_address((uint8_t *)ptr);
-#endif
-
- xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
- xzm_segment_table_entry_s *entry_p = _xzm_ptr_to_table_entry(ptr, main);
- if (os_unlikely(!entry_p)) {
- return NULL;
- }
-
- return __xzm_ptr_lookup(zone, *entry_p, orig_ptr, ptr, xz_out, block_out,
- block_size_out);
-}
-
-MALLOC_NOINLINE
-static void
-_xzm_free_not_found(xzm_malloc_zone_t zone, void *ptr, bool try)
-{
- if (_xzm_malloc_zone_is_main(zone) && mfm_claimed_address(ptr)) {
- mfm_free(ptr);
- return;
- }
-
- if (!try) {
-#if CONFIG_MTE
- // When tagging is enabled (based on malloc_has_sec_transition), we will
- // check if the logical tag of the pointer matches the tag stored in
- // memory. We need to do this here, in xzone, to make sure that we
- // validate the tag in the malloc_zone_free() path. All other paths
- // should be handled at the dispatch layer (malloc proper).
-#endif // CONFIG_MTE
- malloc_report_pointer_was_not_allocated(
- MALLOC_REPORT_CRASH | MALLOC_REPORT_NOLOG, ptr);
- }
-
- find_zone_and_free(ptr, true);
-}
-
-MALLOC_NOINLINE
-static void
-_xzm_free_outlined(xzm_malloc_zone_t zone, void *ptr, bool try,
- xzm_segment_table_entry_s entry)
-{
- void *orig_ptr = ptr;
-
-#if CONFIG_MTE
- // Strip the pointer to use it in pointer arithmetic operations
- ptr = memtag_strip_address((uint8_t *)ptr);
-#endif
-
- xzm_xzone_t xz = NULL;
- xzm_block_t block = NULL;
- xzm_chunk_t chunk = __xzm_ptr_lookup(zone, entry, orig_ptr, ptr, &xz,
- &block, NULL);
- if (os_unlikely(!chunk)) {
- return _xzm_free_not_found(zone, orig_ptr, try);
- }
-
- if (chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_TINY_CHUNK ||
- chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK) {
- _xzm_xzone_free_freelist(zone, xz, chunk, block);
- } else if (chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_CHUNK) {
- _xzm_xzone_free_block_to_small_chunk(zone, xz, chunk, block);
- } else {
- xzm_debug_assert(!xz);
- _xzm_free_large_huge(zone, chunk);
- }
- return;
-}
-
-#ifdef DEBUG
-static void
-_xzm_debug_validate_chunk_metadata(xzm_xzone_t xz, xzm_chunk_t chunk)
-{
- xzm_debug_assert(xz->xz_block_size == chunk->xzc_freelist_block_size);
- xzm_debug_assert(xz->xz_chunk_capacity == chunk->xzc_freelist_chunk_capacity);
-#if CONFIG_MTE
- xzm_debug_assert(xz->xz_tagged == chunk->xzc_tagged);
-#endif
-}
-#else
-#define _xzm_debug_validate_chunk_metadata(...)
-#endif
-
-// mimalloc: mi_free
-static void
-_xzm_free(xzm_malloc_zone_t zone, void *ptr, bool try)
-{
- if (os_unlikely(!ptr)) {
- return;
- }
-
- void *orig_ptr = ptr;
-
- // Inline fast path for tiny free
-
-#if CONFIG_MTE
- // Strip the pointer to use it in pointer arithmetic operations
- ptr = memtag_strip_address((uint8_t *)ptr);
-#endif
-
- xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
- xzm_segment_table_entry_s *entry_p = _xzm_ptr_to_table_entry(ptr, main);
- if (os_unlikely(!entry_p)) {
- goto not_found;
- }
-
- xzm_segment_table_entry_s entry = *entry_p;
- if (os_unlikely(!entry.xste_normal)) {
- goto outlined;
- }
-
- xzm_segment_t segment = _xzm_segment_table_entry_to_segment(entry);
- uintptr_t offset_in_segment = (uintptr_t)orig_ptr & XZM_SEGMENT_MASK;
- uint64_t slice_idx = offset_in_segment >> XZM_SEGMENT_SLICE_SHIFT;
- xzm_slice_t slice = &segment->xzs_slices[slice_idx];
-
- if (os_unlikely(slice->xzc_bits.xzcb_kind != XZM_SLICE_KIND_TINY_CHUNK)) {
- goto outlined;
- }
-
- xzm_chunk_t chunk = (xzm_chunk_t)slice;
- if (os_unlikely(chunk->xzc_mzone_idx != zone->xzz_mzone_idx)) {
- goto not_found;
- }
-
- xzm_xzone_index_t xz_idx = chunk->xzc_xzone_idx;
- __assert_only xzm_xzone_t xz = &zone->xzz_xzones[xz_idx];
- _xzm_debug_validate_chunk_metadata(xz, chunk);
-
- size_t block_size = chunk->xzc_freelist_block_size;
- size_t block_offset = (uintptr_t)orig_ptr & XZM_SEGMENT_SLICE_MASK;
- if (os_unlikely(block_offset % block_size)) {
- goto not_found;
- }
-
- // Check if block is free
-
- xzm_block_t block = (xzm_block_t)orig_ptr;
-
-#if CONFIG_MTE
- const bool tagged = chunk->xzc_tagged;
- if (tagged) {
- block = (xzm_block_t)memtag_fixup_ptr((void *)ptr);
-
- if (os_unlikely(!memtag_tags_match(orig_ptr, block))) {
- goto not_found;
- }
- }
-#endif
-
- uint64_t cookie_value = os_atomic_load(&block->xzb_cookie, relaxed);
- uint64_t free_cookie = zone->xzz_freelist_cookie ^ (uint64_t)orig_ptr;
- if (os_unlikely(cookie_value == free_cookie)) {
- goto outlined;
- }
-
- if (block_size > sizeof(struct xzm_block_inline_meta_s) &&
- block_size <= XZM_ZERO_ON_FREE_THRESHOLD) {
- bzero(block, block_size);
- }
-
-#if CONFIG_MTE
- if (tagged) {
- block = _xzm_xzone_block_memtag_retag(zone, block, block_size);
- uint8_t tag = memtag_extract_tag((uint8_t *)block);
- free_cookie = (uint64_t)memtag_mix_tag(
- memtag_strip_address((uint8_t *)free_cookie), tag);
- }
-#endif
-
- // Mark it as free now
- os_atomic_store(&block->xzb_cookie, free_cookie, relaxed);
-
- uint16_t block_ref = block_offset / XZM_GRANULE;
-
-#if CONFIG_XZM_THREAD_CACHE
- if (block_size > XZM_THREAD_CACHE_THRESHOLD ||
- !zone->xzz_thread_cache_enabled) {
- goto not_cached;
- }
-
- xzm_thread_cache_t tc = _xzm_get_thread_cache();
- if (!tc) {
- goto not_cached;
- }
-
- xzm_xzone_thread_cache_t cache = &tc->xtc_xz_caches[xz_idx];
- if (cache->xztc_head > XZM_FREE_LIMIT || cache->xztc_chunk != chunk) {
- goto not_cached;
- }
-
- return _xzm_xzone_thread_cache_free_tiny(zone, cache, block, block_ref);
-
-not_cached:
-#endif // CONFIG_XZM_THREAD_CACHE
- return _xzm_xzone_free_freelist_inline(zone, xz_idx, chunk, block,
- block_ref, block_size, chunk->xzc_freelist_chunk_capacity);
-
-outlined:
- return _xzm_free_outlined(zone, orig_ptr, try, entry);
-
-not_found:
- return _xzm_free_not_found(zone, orig_ptr, try);
-}
-
-MALLOC_NOINLINE
-static size_t
-_xzm_ptr_size_outlined(xzm_malloc_zone_t zone, const void *ptr)
-{
- // TODO: As part of rdar://101779148, check the mzone that this early
- // allocation came from
- if (_xzm_malloc_zone_is_main(zone) && mfm_claimed_address((void *)ptr)) {
- return mfm_alloc_size(ptr);
- }
-
- return 0;
-}
-
-// mimalloc: _mi_usable_size
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static size_t
-_xzm_ptr_size(xzm_malloc_zone_t zone, const void *ptr, xzm_chunk_t *out_chunk)
-{
- xzm_block_t block;
- size_t block_size;
- xzm_chunk_t chunk = _xzm_ptr_lookup(zone, ptr, NULL, &block, &block_size);
- if (out_chunk) {
- *out_chunk = chunk;
- }
-
- if (!chunk) {
- // Might be an early allocation, or a huge aligned allocation
- return _xzm_ptr_size_outlined(zone, ptr);
- }
-
- if (os_likely((void *)block == ptr)) {
- return block_size;
- }
-
- // Aligned allocation
- size_t block_offset = (uint8_t *)ptr - (uint8_t *)block;
- return block_size - block_offset;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static void *
-_xzm_realloc_inner(xzm_malloc_zone_t zone, void *ptr, size_t new_size,
- malloc_type_descriptor_t type_desc)
-{
- xzm_debug_assert(ptr);
- xzm_debug_assert(new_size);
-
- xzm_chunk_t chunk;
- size_t old_size = _xzm_ptr_size(zone, ptr, &chunk);
- void *old_ptr = ptr;
-
- if (old_size == 0) {
-#if CONFIG_MTE
- // This implicitly includes validation of the pointer tag when memory
- // tagging is enabled. We need this here to cover the realloc path of
- // malloc_zone_realloc(), which doesn't go through _realloc().
-#endif
- malloc_report_pointer_was_not_allocated(
- MALLOC_REPORT_CRASH | MALLOC_REPORT_NOLOG, ptr);
- }
-
- if (chunk && os_unlikely(chunk->xzc_mzone_idx != zone->xzz_mzone_idx)) {
- xzm_client_abort_with_reason(
- "pointer zone mismatch, client may be passing the wrong malloc"
- " zone", ptr);
- }
-
- if (chunk && old_size > XZM_SMALL_BLOCK_SIZE_MAX
- && new_size > XZM_SMALL_BLOCK_SIZE_MAX) {
- xzm_slice_count_t new_slice_count = (xzm_slice_count_t)
- (roundup(new_size,
- XZM_SEGMENT_SLICE_SIZE) / XZM_SEGMENT_SLICE_SIZE);
-
- xzm_segment_t segment = _xzm_segment_for_slice(zone, chunk);
- xzm_debug_assert(segment != NULL);
-
- bool realloc_in_place = false;
- if (old_size > XZM_LARGE_BLOCK_SIZE_MAX &&
- new_size > XZM_LARGE_BLOCK_SIZE_MAX &&
- os_likely((zone->xzz_flags & MALLOC_PURGEABLE) == 0)) {
- realloc_in_place = xzm_segment_group_try_realloc_huge_chunk(
- segment->xzs_segment_group, zone, segment, chunk,
- new_slice_count);
- } else if (old_size <= XZM_LARGE_BLOCK_SIZE_MAX &&
- segment->xzs_kind == XZM_SEGMENT_KIND_NORMAL &&
- new_size <= XZM_LARGE_BLOCK_SIZE_MAX &&
- os_likely((zone->xzz_flags & MALLOC_PURGEABLE) == 0)) {
- realloc_in_place = xzm_segment_group_try_realloc_large_chunk(
- segment->xzs_segment_group, segment, chunk,
- new_slice_count);
- }
-
- if (realloc_in_place) {
- return old_ptr;
- }
- }
-
- if (new_size <= old_size && new_size >= (old_size / 2)) {
- // reallocation still fits and not more than 50% waste
- // TODO: revisit?
- return old_ptr;
- }
-
- void *new_ptr = xzm_malloc(zone, new_size, type_desc, 0);
- if (os_unlikely(!new_ptr)) {
- return NULL;
- }
-
- const size_t valid_size = MIN(old_size, new_size);
-#if CONFIG_REALLOC_CAN_USE_VMCOPY
- // When supported, request a CoW mapping
- if (valid_size > XZM_LARGE_BLOCK_SIZE_MAX) {
- if (mach_vm_copy(mach_task_self(), (mach_vm_address_t)old_ptr,
- valid_size, (mach_vm_address_t)new_ptr) == KERN_SUCCESS) {
- return new_ptr;
- }
- }
-#endif
- memcpy(new_ptr, old_ptr, valid_size);
-
- return new_ptr;
-}
-
-void *
-xzm_realloc(xzm_malloc_zone_t zone, void *ptr, size_t new_size,
- malloc_type_descriptor_t type_desc)
-{
- if (!ptr) {
- return xzm_malloc(zone, new_size, type_desc, 0);
- } else if (new_size == 0) {
- _xzm_free(zone, ptr, false);
- return xzm_malloc(zone, new_size, type_desc, 0);
- }
-
- void *new_ptr = _xzm_realloc_inner(zone, ptr, new_size, type_desc);
- if (new_ptr && new_ptr != ptr) {
- _xzm_free(zone, ptr, false);
- }
-
- return new_ptr;
-}
-
-#pragma mark libmalloc zone
-
-static size_t
-xzm_malloc_zone_size(xzm_malloc_zone_t zone, const void *ptr)
-{
- return _xzm_ptr_size(zone, ptr, NULL);
-}
-
-static boolean_t
-xzm_malloc_zone_claimed_address(xzm_malloc_zone_t zone, void *ptr)
-{
- // Is it an early allocation?
- if (mfm_claimed_address(ptr)) {
- return true;
- }
-
-#if CONFIG_MTE
- // We strip the pointer here as xzm_segment_table_query operates on
- // canonical pointers; however, since mfm_claimed_address validates
- // the logical tag of the pointer, we need to ensure we don't strip
- // the tag before passing it to MFM.
- // We don't need to check if MTE is enabled, as this doesn't need MTE ISA.
- ptr = memtag_strip_address(ptr);
-#endif
-
- xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
- return xzm_segment_table_query(main, ptr) != NULL;
-}
-
-static inline void *
-_xzm_malloc_zone_malloc_entry(xzm_malloc_zone_t zone, size_t size,
- malloc_type_id_t type_id, xzm_malloc_options_t options)
-{
- return xzm_malloc_inline(zone, size,
- (malloc_type_descriptor_t){ .type_id = type_id, }, options);
-}
-
-static void *
-xzm_malloc_zone_malloc_type_malloc(xzm_malloc_zone_t zone, size_t size,
- malloc_type_id_t type_id)
-{
- return _xzm_malloc_zone_malloc_entry(zone, size, type_id, 0);
-}
-
-static void *
-xzm_malloc_zone_malloc(xzm_malloc_zone_t zone, size_t size)
-{
- return _xzm_malloc_zone_malloc_entry(zone, size, malloc_get_tsd_type_id(),
- _xzm_xzone_get_malloc_thread_options());
-}
-
-static inline void *
-_xzm_malloc_zone_malloc_type_calloc_entry(xzm_malloc_zone_t zone, size_t count,
- size_t size, malloc_type_id_t type_id,
- xzm_malloc_options_t options)
-{
- size_t total_bytes;
- if (os_unlikely(calloc_get_size(count, size, 0, &total_bytes))) {
- malloc_set_errno_fast(MZ_POSIX, ENOMEM);
- return NULL;
- }
-
- return xzm_malloc_inline(zone, total_bytes,
- (malloc_type_descriptor_t){ .type_id = type_id, },
- options | XZM_MALLOC_CLEAR);
-}
-
-static void *
-xzm_malloc_zone_malloc_type_calloc(xzm_malloc_zone_t zone, size_t count,
- size_t size, malloc_type_id_t type_id)
-{
- return _xzm_malloc_zone_malloc_type_calloc_entry(zone, count, size, type_id,
- 0);
-}
-
-static void *
-xzm_malloc_zone_calloc(xzm_malloc_zone_t zone, size_t count, size_t size)
-{
- return _xzm_malloc_zone_malloc_type_calloc_entry(zone, count, size,
- malloc_get_tsd_type_id(), _xzm_xzone_get_malloc_thread_options());
-}
-
-static void * __alloc_size(2)
-xzm_malloc_zone_valloc(xzm_malloc_zone_t zone, size_t size)
-{
- return _xzm_memalign(zone, vm_page_quanta_size, size,
- malloc_get_tsd_type_descriptor(), 0);
-}
-
-static void
-xzm_malloc_zone_free(xzm_malloc_zone_t zone, void *ptr)
-{
- _xzm_free(zone, ptr, false);
-}
-
-static void
-xzm_malloc_zone_try_free_default(xzm_malloc_zone_t zone, void *ptr)
-{
- _xzm_free(zone, ptr, true);
-}
-
-static void
-xzm_malloc_zone_free_definite_size(xzm_malloc_zone_t zone, void *ptr,
- size_t size)
-{
- // Unfortunately, knowing the size doesn't really buy us anything.
- (void)size;
-
- _xzm_free(zone, ptr, false);
-}
-
-static void *
-xzm_malloc_zone_malloc_type_realloc(xzm_malloc_zone_t zone, void *ptr,
- size_t new_size, malloc_type_id_t type_id)
-{
- return xzm_realloc(zone, ptr, new_size,
- (malloc_type_descriptor_t){ .type_id = type_id, });
-}
-
-static void *
-xzm_malloc_zone_realloc(xzm_malloc_zone_t zone, void *ptr, size_t new_size)
-{
- return xzm_malloc_zone_malloc_type_realloc(zone, ptr, new_size,
- malloc_get_tsd_type_id());
-}
-
-static inline void *
-_xzm_malloc_zone_malloc_type_memalign_entry(xzm_malloc_zone_t zone,
- size_t alignment, size_t size, malloc_type_id_t type_id,
- xzm_malloc_options_t options)
-{
- return _xzm_memalign(zone, alignment, size,
- (malloc_type_descriptor_t){ .type_id = type_id, }, options);
-}
-
-static void *
-xzm_malloc_zone_malloc_type_memalign(xzm_malloc_zone_t zone, size_t alignment,
- size_t size, malloc_type_id_t type_id)
-{
- return _xzm_malloc_zone_malloc_type_memalign_entry(zone, alignment, size,
- type_id, 0);
-}
-
-static void *
-xzm_malloc_zone_memalign(xzm_malloc_zone_t zone, size_t alignment, size_t size)
-{
- return _xzm_malloc_zone_malloc_type_memalign_entry(zone, alignment, size,
- malloc_get_tsd_type_id(), _xzm_xzone_get_malloc_thread_options());
-}
-
-static void *
-xzm_malloc_zone_malloc_type_malloc_with_options(xzm_malloc_zone_t zone,
- size_t align, size_t size, malloc_zone_malloc_options_t options,
- malloc_type_id_t type_id)
-{
- xzm_malloc_options_t opt = 0;
- if (options & MALLOC_ZONE_MALLOC_OPTION_CLEAR) {
- opt |= XZM_MALLOC_CLEAR;
- }
- if (options & MALLOC_NP_OPTION_CANONICAL_TAG) {
- opt |= XZM_MALLOC_CANONICAL_TAG;
- }
-
- malloc_type_descriptor_t type_desc = { .type_id = type_id, };
- if (align > MALLOC_ZONE_MALLOC_DEFAULT_ALIGN) {
- return _xzm_memalign(zone, align, size, type_desc, opt);
- } else {
- return xzm_malloc_inline(zone, size, type_desc, opt);
- }
-}
-
-static void *
-xzm_malloc_zone_malloc_with_options(xzm_malloc_zone_t zone, size_t align,
- size_t size, malloc_zone_malloc_options_t options)
-{
- return xzm_malloc_zone_malloc_type_malloc_with_options(zone, align, size,
- options, malloc_get_tsd_type_id());
-}
-
-#pragma mark slowpath zone functions
-
-// nothing for size
-
-// nothing for claimed_address
-
-static void *
-xzm_malloc_zone_malloc_type_malloc_slow(xzm_malloc_zone_t zone, size_t size,
- malloc_type_id_t type_id)
-{
- void *ptr = NULL;
- malloc_type_descriptor_t type_desc = { .type_id = type_id, };
- xzm_malloc_options_t options = _xzm_xzone_get_malloc_thread_options();
-
- if ((zone->xzz_flags & MALLOC_PURGEABLE) &&
- (size <= XZM_SMALL_BLOCK_SIZE_MAX)) {
- xzm_malloc_zone_t main = (xzm_malloc_zone_t)_xzm_malloc_zone_main(zone);
- ptr = xzm_malloc_inline(main, size, type_desc, options);
- } else {
- ptr = xzm_malloc_inline(zone, size, type_desc, options);
- }
- if (ptr && (zone->xzz_flags & MALLOC_DO_SCRIBBLE)) {
- memset(ptr, SCRIBBLE_BYTE, size);
- }
- return ptr;
-}
-
-static void *
-xzm_malloc_zone_malloc_slow(xzm_malloc_zone_t zone, size_t size)
-{
- return xzm_malloc_zone_malloc_type_malloc_slow(zone, size,
- malloc_get_tsd_type_id());
-}
-
-static void *
-xzm_malloc_zone_malloc_type_calloc_slow(xzm_malloc_zone_t zone, size_t count,
- size_t size, malloc_type_id_t type_id)
-{
- size_t total_bytes;
- if (calloc_get_size(count, size, 0, &total_bytes)) {
- return NULL;
- }
-
- malloc_type_descriptor_t type_desc = { .type_id = type_id, };
- xzm_malloc_options_t options = _xzm_xzone_get_malloc_thread_options() |
- XZM_MALLOC_CLEAR;
- if ((zone->xzz_flags & MALLOC_PURGEABLE) == 0 ||
- total_bytes > XZM_SMALL_BLOCK_SIZE_MAX) {
- return xzm_malloc_inline(zone, total_bytes, type_desc, options);
- } else {
- xzm_malloc_zone_t main = (xzm_malloc_zone_t)_xzm_malloc_zone_main(zone);
- return xzm_malloc_inline(main, total_bytes, type_desc, options);
- }
-}
-
-static void *
-xzm_malloc_zone_calloc_slow(xzm_malloc_zone_t zone, size_t count,
- size_t size)
-{
- return xzm_malloc_zone_malloc_type_calloc_slow(zone, count, size,
- malloc_get_tsd_type_id());
-}
-
-static void *
-xzm_malloc_zone_valloc_slow(xzm_malloc_zone_t zone, size_t size)
-{
- void *ptr = NULL;
- if ((zone->xzz_flags & MALLOC_PURGEABLE) &&
- (size <= XZM_SMALL_BLOCK_SIZE_MAX)) {
- xzm_malloc_zone_t main = (xzm_malloc_zone_t)_xzm_malloc_zone_main(zone);
- ptr = xzm_malloc_zone_valloc(main, size);
- } else {
- ptr = xzm_malloc_zone_valloc(zone, size);
- }
- if (ptr && (zone->xzz_flags & MALLOC_DO_SCRIBBLE)) {
- memset(ptr, SCRIBBLE_BYTE, size);
- }
- return ptr;
-}
-
-static void
-xzm_malloc_zone_free_slow(xzm_malloc_zone_t zone, void *ptr)
-{
- if (!ptr) {
- return;
- }
-
- size_t size = _xzm_ptr_size(zone, ptr, NULL);
- // The main ref stays NULL if this allocation came from zone
- xzm_malloc_zone_t main = NULL;
- if (size == 0 && (zone->xzz_flags & MALLOC_PURGEABLE)) {
- main = (xzm_malloc_zone_t)_xzm_malloc_zone_main(zone);
- size = _xzm_ptr_size(main, ptr, NULL);
- }
- if (size == 0) {
- xzm_client_abort_with_reason("pointer being freed was not allocated",
- ptr);
- }
- if ((zone->xzz_flags & MALLOC_DO_SCRIBBLE) &&
- (size > XZM_ZERO_ON_FREE_THRESHOLD)) {
- memset(ptr, SCRABBLE_BYTE, size);
- }
-
- if (main) {
- xzm_malloc_zone_free(main, ptr);
- } else {
- xzm_malloc_zone_free(zone, ptr);
- }
-}
-
-static void
-xzm_malloc_zone_try_free_default_slow(xzm_malloc_zone_t zone, void *ptr)
-{
- if (!ptr) {
- return;
- }
-
- size_t size = _xzm_ptr_size(zone, ptr, NULL);
- xzm_malloc_zone_t main = NULL;
- if (size == 0 && (zone->xzz_flags & MALLOC_PURGEABLE)) {
- main = (xzm_malloc_zone_t)_xzm_malloc_zone_main(zone);
- size = _xzm_ptr_size(main, ptr, NULL);
- }
- if (size == 0) {
- // pointer didn't come from this zone
- find_zone_and_free(ptr, true);
- return;
- }
- if (size && size > XZM_ZERO_ON_FREE_THRESHOLD &&
- (zone->xzz_flags & MALLOC_DO_SCRIBBLE)) {
- memset(ptr, SCRABBLE_BYTE, size);
- }
-
- if (main) {
- xzm_malloc_zone_try_free_default(main, ptr);
- } else {
- xzm_malloc_zone_try_free_default(zone, ptr);
- }
-}
-
-static void
-xzm_malloc_zone_free_definite_size_slow(xzm_malloc_zone_t zone, void *ptr,
- size_t size)
-{
- if (!ptr) {
- return;
- }
-
- // TODO: Should we verify that the size the client passes in matches our
- // size?
- size_t our_size = _xzm_ptr_size(zone, ptr, NULL);
- xzm_malloc_zone_t main = NULL;
- if (our_size == 0 && (zone->xzz_flags & MALLOC_PURGEABLE)) {
- main = (xzm_malloc_zone_t)_xzm_malloc_zone_main(zone);
- our_size = _xzm_ptr_size(main, ptr, NULL);
- xzm_debug_assert(our_size != 0);
- }
- if ((zone->xzz_flags & MALLOC_DO_SCRIBBLE) &&
- (size > XZM_ZERO_ON_FREE_THRESHOLD)) {
- memset(ptr, SCRABBLE_BYTE, size);
- }
-
- if (main) {
- xzm_malloc_zone_free_definite_size(main, ptr, size);
- } else {
- xzm_malloc_zone_free_definite_size(zone, ptr, size);
- }
-}
-
-static void *
-xzm_malloc_zone_malloc_type_realloc_slow(xzm_malloc_zone_t zone, void *ptr,
- size_t new_size, malloc_type_id_t type_id)
-{
- bool purgeable = zone->xzz_flags & MALLOC_PURGEABLE;
- bool scribble = zone->xzz_flags & MALLOC_DO_SCRIBBLE;
- if (!purgeable && !scribble) {
- return xzm_malloc_zone_malloc_type_realloc(zone, ptr, new_size,
- type_id);
- }
-
- if (!ptr) {
- return xzm_malloc_zone_malloc_type_malloc_slow(zone, new_size, type_id);
- } else if (new_size == 0) {
- xzm_malloc_zone_free_slow(zone, ptr);
- return xzm_malloc_zone_malloc_type_malloc_slow(zone, new_size, type_id);
- }
-
- malloc_type_descriptor_t type_desc = { .type_id = type_id, };
- size_t old_size = _xzm_ptr_size(zone, ptr, NULL);
- xzm_malloc_zone_t main = NULL;
- if (old_size == 0 && purgeable) {
- main = (xzm_malloc_zone_t)_xzm_malloc_zone_main(zone);
- old_size = _xzm_ptr_size(main, ptr, NULL);
- }
- xzm_assert(old_size);
-
- if ((old_size > new_size) && scribble) {
- // Shrinking: no matter what, we'll definitely need to scrabble the
- // tail, so do that now
- memset((uint8_t *)ptr + new_size, SCRABBLE_BYTE, old_size - new_size);
- }
-
- void *new_ptr;
- if (!purgeable || (!main && new_size > XZM_SMALL_BLOCK_SIZE_MAX) ||
- (main && new_size <= XZM_SMALL_BLOCK_SIZE_MAX)) {
- // The old pointer and new pointer will be from the same zone
- new_ptr = _xzm_realloc_inner(zone, ptr, new_size, type_desc);
- } else {
- if (new_size <= XZM_SMALL_BLOCK_SIZE_MAX) {
- // Need to serve the new allocation from the main zone, since it's
- // too small to be purgeable
- new_ptr = xzm_malloc_inline(&_xzm_malloc_zone_main(zone)->xzmz_base,
- new_size, type_desc, 0);
- } else {
- // Serve the new allocation from the purgeable zone
- new_ptr = xzm_malloc_inline(zone, new_size, type_desc, 0);
- }
- if (new_ptr) {
- memcpy(new_ptr, ptr, MIN(old_size, new_size));
- }
- }
-
- if (new_ptr) {
- // Regardless of whether we reallocated in place or not, we always want
- // to scribble the expanded space when growing
- if ((new_size > old_size) && scribble) {
- memset((uint8_t *)new_ptr + old_size, SCRIBBLE_BYTE,
- new_size - old_size);
- }
- if (new_ptr != ptr) {
- // If we didn't reallocate in place, we have some amount of
- // scrabbling left to do on the old allocation:
- if ((old_size > new_size) && scribble) {
- // If shrinking, we already scrabbled the tail, so just scrabble
- // the head now too
- memset(ptr, SCRABBLE_BYTE, new_size);
- } else if (scribble) {
- // If growing, scrabble the whole previous allocation since we
- // haven't done any of it yet
- memset(ptr, SCRABBLE_BYTE, old_size);
- }
- if (main) {
- _xzm_free(main, ptr, false);
- } else {
- _xzm_free(zone, ptr, false);
- }
- }
- }
-
- return new_ptr;
-}
-
-static void *
-xzm_malloc_zone_realloc_slow(xzm_malloc_zone_t zone, void *ptr, size_t new_size)
-{
- return xzm_malloc_zone_malloc_type_realloc_slow(zone, ptr, new_size,
- malloc_get_tsd_type_id());
-}
-
-static void *
-xzm_malloc_zone_malloc_type_memalign_slow(xzm_malloc_zone_t zone, size_t alignment,
- size_t size, malloc_type_id_t type_id)
-{
- void *ptr = NULL;
- xzm_malloc_options_t options = _xzm_xzone_get_malloc_thread_options();
- if ((zone->xzz_flags & MALLOC_PURGEABLE) &&
- (size <= XZM_SMALL_BLOCK_SIZE_MAX)) {
- xzm_malloc_zone_t main = (xzm_malloc_zone_t)_xzm_malloc_zone_main(zone);
- ptr = _xzm_malloc_zone_malloc_type_memalign_entry(main, alignment, size,
- type_id, options);
- } else {
- ptr = _xzm_malloc_zone_malloc_type_memalign_entry(zone, alignment, size,
- type_id, options);
- }
- if (ptr && (zone->xzz_flags & MALLOC_DO_SCRIBBLE)) {
- memset(ptr, SCRIBBLE_BYTE, size);
- }
- return ptr;
-}
-
-static void *
-xzm_malloc_zone_memalign_slow(xzm_malloc_zone_t zone, size_t alignment,
- size_t size)
-{
- return xzm_malloc_zone_malloc_type_memalign_slow(zone, alignment, size,
- malloc_get_tsd_type_id());
-}
-
-static void *
-xzm_malloc_zone_malloc_type_malloc_with_options_slow(xzm_malloc_zone_t zone,
- size_t align, size_t size, malloc_zone_malloc_options_t options,
- malloc_type_id_t type_id)
-{
- void *ptr = NULL;
- if ((zone->xzz_flags & MALLOC_PURGEABLE) &&
- (size <= XZM_SMALL_BLOCK_SIZE_MAX)) {
- xzm_malloc_zone_t main = (xzm_malloc_zone_t)_xzm_malloc_zone_main(zone);
- ptr = xzm_malloc_zone_malloc_type_malloc_with_options(main, align, size,
- options, type_id);
- } else {
- ptr = xzm_malloc_zone_malloc_type_malloc_with_options(zone, align, size,
- options, type_id);
- }
- if (ptr && !(options & MALLOC_ZONE_MALLOC_OPTION_CLEAR) &&
- (zone->xzz_flags & MALLOC_DO_SCRIBBLE)) {
- memset(ptr, SCRIBBLE_BYTE, size);
- }
- return ptr;
-}
-
-static void *
-xzm_malloc_zone_malloc_with_options_slow(xzm_malloc_zone_t zone, size_t align,
- size_t size, malloc_zone_malloc_options_t options)
-{
- return xzm_malloc_zone_malloc_type_malloc_with_options_slow(zone, align,
- size, options, malloc_get_tsd_type_id());
-}
-
-#pragma mark libmalloc zone introspection
-size_t
-xzm_good_size(xzm_malloc_zone_t zone, size_t size)
-{
- if (size <= XZM_SMALL_BLOCK_SIZE_MAX) {
- xzm_main_malloc_zone_t main = _xzm_malloc_zone_main(zone);
- uint8_t bin = _xzm_bin(size);
- return main->xzmz_xzone_bin_sizes[bin];
- } else {
- // MAX() handles overflow in roundup()
- return MAX(roundup(size, XZM_SEGMENT_SLICE_SIZE), size);
- }
-}
-
-boolean_t
-xzm_check(xzm_malloc_zone_t zone)
-{
- // TODO: zone self-check
- //
- // Nano doesn't implement this - worth the trouble?
- (void)zone;
- return true;
-}
-
-void
-xzm_log(xzm_malloc_zone_t zone, void *log_address)
-{
- // Not implemented
- //
- // Doesn't seem that useful? Nano doesn't implement this either.
-}
-
-OS_ENUM(xzm_lock_action, int,
- XZM_LOCK_LOCK,
- XZM_LOCK_UNLOCK,
- XZM_LOCK_INIT,
-);
-
-static void
-_xzm_do_lock_action(_malloc_lock_s *lock, xzm_lock_action_t action)
-{
- switch (action) {
- case XZM_LOCK_LOCK:
- _malloc_lock_lock(lock);
- break;
- case XZM_LOCK_UNLOCK:
- _malloc_lock_unlock(lock);
- break;
- case XZM_LOCK_INIT:
- _malloc_lock_init(lock);
- break;
- default:
- xzm_debug_abort("invalid xzm lock action");
- break;
- }
-}
-
-static void
-_xzm_freelist_xzone_do_lock_action(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_lock_action_t action)
-{
- if (action == XZM_LOCK_LOCK) {
- for (xzm_allocation_index_t i = 0; i < zone->xzz_slot_count; i++) {
- _xzm_chunk_list_fork_lock(&_xzm_xzone_chunk_list_for_index(zone,
- xz, zone->xzz_partial_lists, i)->xcl_list);
- }
-
- _xzm_chunk_batch_list_fork_lock(&xz->xz_batch_list);
- _xzm_chunk_list_fork_lock(&xz->xz_empty_list);
- _xzm_chunk_list_fork_lock(&xz->xz_preallocated_list);
- _xzm_chunk_list_fork_lock(&xz->xz_all_list);
- }
-
- xzm_chunk_list_head_u head = {
- .xzch_value = os_atomic_load(&xz->xz_all_list.xzch_value, dependency),
- };
-
- xzm_chunk_t chunk = (xzm_chunk_t)head.xzch_ptr;
- while (chunk) {
- xzm_chunk_atomic_meta_u old_meta;
- if (action == XZM_LOCK_LOCK) {
- (void)_xzm_xzone_freelist_chunk_lock(chunk, &old_meta);
- } else {
- old_meta.xca_value = os_atomic_load_wide(
- &chunk->xzc_atomic_meta.xca_value, dependency);
- if (old_meta.xca_alloc_head != XZM_FREE_MADVISING &&
- old_meta.xca_alloc_head != XZM_FREE_MADVISED) {
- _xzm_xzone_freelist_chunk_unlock(chunk, old_meta);
- }
- }
- chunk = chunk->xzc_linkages[XZM_CHUNK_LINKAGE_ALL];
- }
-
- if (action != XZM_LOCK_LOCK) {
- _xzm_chunk_list_fork_unlock(&xz->xz_all_list);
- _xzm_chunk_batch_list_fork_unlock(&xz->xz_batch_list);
- _xzm_chunk_list_fork_unlock(&xz->xz_preallocated_list);
- _xzm_chunk_list_fork_unlock(&xz->xz_empty_list);
-
- for (xzm_allocation_index_t i = 0; i < zone->xzz_slot_count; i++) {
- _xzm_chunk_list_fork_unlock(&_xzm_xzone_chunk_list_for_index(zone,
- xz, zone->xzz_partial_lists, i)->xcl_list);
- }
-
- }
-}
-
-static void
-_xzm_small_xzone_lock_all(xzm_malloc_zone_t zone, xzm_xzone_t xz)
-{
- xzm_debug_assert(xz->xz_block_size > XZM_TINY_BLOCK_SIZE_MAX);
-
- while (true) {
- _malloc_lock_lock(&xz->xz_lock);
-
- xzm_chunk_t chunk = xz->xz_chunkq_batch;
- while (chunk) {
- _malloc_lock_lock(&chunk->xzc_lock);
- chunk = *_xzm_segment_slice_meta_batch_next(zone, chunk);
- xzm_debug_assert(_xzm_slice_meta_is_batch_pointer(zone, chunk));
- }
-
- LIST_FOREACH(chunk, &xz->xz_chunkq_partial, xzc_entry) {
- _malloc_lock_lock(&chunk->xzc_lock);
- }
-
- LIST_FOREACH(chunk, &xz->xz_chunkq_full, xzc_entry) {
- bool gotlock = _malloc_lock_trylock(&chunk->xzc_lock);
- if (!gotlock) {
- // Potential lock inversion - need to unwind and retry.
- xzm_chunk_t chunk2 = xz->xz_chunkq_batch;
-
- while (chunk2) {
- _malloc_lock_unlock(&chunk2->xzc_lock);
- chunk2 = *_xzm_segment_slice_meta_batch_next(zone, chunk2);
- xzm_debug_assert(_xzm_slice_meta_is_batch_pointer(zone,
- chunk2));
- }
-
- LIST_FOREACH(chunk2, &xz->xz_chunkq_partial, xzc_entry) {
- _malloc_lock_unlock(&chunk2->xzc_lock);
- }
-
- LIST_FOREACH(chunk2, &xz->xz_chunkq_full, xzc_entry) {
- if (chunk2 == chunk) {
- break;
- }
-
- _malloc_lock_unlock(&chunk2->xzc_lock);
- }
- break;
- }
- }
-
- if (chunk) {
- // Encountered a trylock failure, so we need to unlock the xzone to
- // let the holder of the chunk lock we need make progress.
- //
- // TODO: There's a priority inversion here because we're not doing
- // anything to push on the holder of the chunk lock we need. We
- // know exactly who we need to wait for, so we could give them an
- // artificial push by doing a timed ulock wait on their thread ID
- // after we release the lock - but before doing that we'd probably
- // just replace this whole approach instead.
- _malloc_lock_unlock(&xz->xz_lock);
- yield();
- } else {
- // Made it to the end of the full list without any trylock failures,
- // so we're done.
- break;
- }
- }
-}
-
-static void
-_xzm_allocation_slots_do_lock_action(xzm_malloc_zone_t zone,
- xzm_lock_action_t action)
-{
- for (uint8_t i = XZM_XZONE_INDEX_FIRST; i < zone->xzz_xzone_count; i++) {
- xzm_xzone_t xz = &zone->xzz_xzones[i];
-
- __unused xzm_slice_kind_t kind;
- if (xz->xz_block_size <= XZM_TINY_BLOCK_SIZE_MAX) {
- kind = XZM_SLICE_KIND_TINY_CHUNK;
- } else if (zone->xzz_small_freelist_enabled) {
- kind = XZM_SLICE_KIND_SMALL_FREELIST_CHUNK;
- } else {
- kind = XZM_SLICE_KIND_SMALL_CHUNK;
- }
-
- for (uint8_t j = 0; j < zone->xzz_slot_count; j++) {
- size_t slot_idx = (zone->xzz_xzone_count * j) + i;
- xzm_xzone_allocation_slot_t xas =
- &zone->xzz_xzone_allocation_slots[slot_idx];
-#if !CONFIG_TINY_ALLOCATION_SLOT_LOCK
- if (kind == XZM_SLICE_KIND_TINY_CHUNK ||
- kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK) {
- if (action == XZM_LOCK_LOCK) {
- _xzm_xzone_allocation_slot_fork_lock(xas);
- } else {
- _xzm_xzone_allocation_slot_fork_unlock(xas);
- }
- } else
-#endif // !CONFIG_TINY_ALLOCATION_SLOT_LOCK
- {
- _xzm_do_lock_action(&xas->xas_lock, action);
- }
- }
- }
-}
-
-static void
-_xzm_foreach_lock(xzm_malloc_zone_t zone, xzm_lock_action_t action)
-{
- // mzone-private locks
- //
- // There are 3 levels of locks:
- // - the xzone slot locks
- // - the xzone lock
- // - the chunk locks
- //
- // The chunk locks are the most difficult to handle for fork, as there are
- // so many of them. The simplest strategy that works is to just find and
- // acquire them all.
- //
- // The listed order is the general locking order, but when full chunks
- // transition back to partial their chunk lock is held while acquiring the
- // xzone lock. It's therefore not safe to hold the xzone lock while trying
- // to acquire the chunk lock of a full chunk. To deal with that, use
- // trylock to acquire the locks of full chunks, and on failure back off,
- // release the xzone lock and retry.
- //
- // It's not necessary to acquire the chunk locks of chunks that are empty
- // and in the transition from the partial list to the isolation list,
- // because they won't be allocated from or freed to on the child side of
- // fork - they'll effectively leak when the thread responsible for them
- // disappears as part of fork.
- //
- // TODO: this is not efficient, particularly in processes that have lots of
- // chunks at the time of fork. Before we pursue general userspace
- // enablement on macOS, where 3p code needs fork() to be reasonably fast, we
- // should do something better, but this is good enough for the situations
- // where it just needs to be correct.
-
- // The fork-lock bits send waiters to the fork lock, so we need to ensure
- // that we hold the fork lock whenever any of the fork-lock bits are set.
- // That means we need to acquire it before setting any of the fork-lock bits
- // and release it only after clearing all of them.
- //
- // The fork lock also provides mutual exclusion for mutation of the
- // fork-lock bits. This is needed to prevent collisions between concurrent
- // executions of foreach-lock: one thread un-setting the fork-lock bits must
- // not trample another one making a pass setting them.
- if (action == XZM_LOCK_LOCK) {
- _xzm_do_lock_action(&zone->xzz_fork_lock, action);
-
- // The zone lock (xzz_lock) is used for tiny chunk walk lock
- // synchronization (among other things). It is acquired under the
- // allocation slot lock when allocating from a partial chunk, and needs
- // to be held before performing any of the fork walk-locking, so it
- // needs to be taken here after the allocation slots are locked and
- // before any of the tiny chunks are locked.
- _xzm_do_lock_action(&zone->xzz_lock, action);
-
- // The freelist allocation slots must be locked before the all-chunk
- // lists are walked and unlocked after, so that no new chunks can be
- // allocated and added in the middle. That's important because we need
- // the set of chunks we walk during the lock phase to be the same as
- // during the unlock phase.
- _xzm_allocation_slots_do_lock_action(zone, action);
- }
-
- for (uint8_t i = XZM_XZONE_INDEX_FIRST; i < zone->xzz_xzone_count; i++) {
- xzm_xzone_t xz = &zone->xzz_xzones[i];
-
- xzm_slice_kind_t kind;
- if (xz->xz_block_size <= XZM_TINY_BLOCK_SIZE_MAX) {
- kind = XZM_SLICE_KIND_TINY_CHUNK;
- } else if (zone->xzz_small_freelist_enabled) {
- kind = XZM_SLICE_KIND_SMALL_FREELIST_CHUNK;
- } else {
- kind = XZM_SLICE_KIND_SMALL_CHUNK;
- }
-
- if (kind == XZM_SLICE_KIND_TINY_CHUNK ||
- kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK) {
- _xzm_freelist_xzone_do_lock_action(zone, xz, action);
- } else if (kind == XZM_SLICE_KIND_SMALL_CHUNK) {
- if (action == XZM_LOCK_LOCK) {
- _xzm_small_xzone_lock_all(zone, xz);
- } else {
- xzm_chunk_t chunk = xz->xz_chunkq_batch;
- while (chunk) {
- _xzm_do_lock_action(&chunk->xzc_lock, action);
- chunk = *_xzm_segment_slice_meta_batch_next(zone, chunk);
- xzm_debug_assert(_xzm_slice_meta_is_batch_pointer(zone,
- chunk));
- }
- LIST_FOREACH(chunk, &xz->xz_chunkq_partial, xzc_entry) {
- _xzm_do_lock_action(&chunk->xzc_lock, action);
- }
- LIST_FOREACH(chunk, &xz->xz_chunkq_full, xzc_entry) {
- _xzm_do_lock_action(&chunk->xzc_lock, action);
- }
-
- _xzm_do_lock_action(&xz->xz_lock, action);
- }
- }
- }
-
- if (action != XZM_LOCK_LOCK) {
- _xzm_allocation_slots_do_lock_action(zone, action);
- _xzm_do_lock_action(&zone->xzz_lock, action);
- _xzm_do_lock_action(&zone->xzz_fork_lock, action);
- }
-}
-
-void
-xzm_force_lock(xzm_malloc_zone_t zone)
-{
- _xzm_foreach_lock(zone, XZM_LOCK_LOCK);
-}
-
-void
-xzm_force_unlock(xzm_malloc_zone_t zone)
-{
- _xzm_foreach_lock(zone, XZM_LOCK_UNLOCK);
-}
-
-void
-xzm_reinit_lock(xzm_malloc_zone_t zone)
-{
- _xzm_foreach_lock(zone, XZM_LOCK_INIT);
-}
-
-boolean_t
-xzm_locked(xzm_malloc_zone_t zone)
-{
- // This mechanism is dead:
- // - nanov2 doesn't implement it despite having locks that could lead to the
- // deadlock this is meant to prevent
- // - there's an obvious copy-paste bug in szone_locked() for medium
- //
- // There are no references to malloc_gdb_po_unsafe() anywhere. It should
- // all be removed.
- xzm_abort("xzm_locked not implemented");
- return true;
-}
-
-static void
-_xzm_global_state_lock(xzm_main_malloc_zone_t main, xzm_lock_action_t action)
-{
- // main zone shared locks
- for (uint8_t i = XZM_XZONE_INDEX_FIRST; i < main->xzmz_base.xzz_xzone_count;
- i++) {
- xzm_isolation_zone_t iz = &main->xzmz_isolation_zones[i];
- _xzm_do_lock_action(&iz->xziz_lock, action);
- }
-
- for (uint8_t i = 0; i < main->xzmz_segment_group_count; i++) {
- xzm_segment_group_t sg = &main->xzmz_segment_groups[i];
- _xzm_do_lock_action(&sg->xzsg_alloc_lock, action);
- _xzm_do_lock_action(&sg->xzsg_lock, action);
-#if CONFIG_XZM_DEFERRED_RECLAIM
- if (sg->xzsg_id == XZM_SEGMENT_GROUP_DATA_LARGE) {
- _xzm_do_lock_action(&sg->xzsg_cache.xzsc_lock, action);
- }
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
- }
-
- for (uint8_t i = 0; i < main->xzmz_range_group_count; i++) {
- xzm_range_group_t rg = &main->xzmz_range_groups[i];
- _xzm_do_lock_action(&rg->xzrg_lock, action);
- }
-
- // The extended segment table lock is held while allocating from the segment
- // table metapool, so we need to acquire this lock before the metapool locks
- _xzm_do_lock_action(&main->xzmz_extended_segment_table_lock, action);
-
- for (int i = 0; i < main->xzmz_metapool_count; i++) {
- xzm_metapool_t mp = &main->xzmz_metapools[i];
- _xzm_do_lock_action(&mp->xzmp_lock, action);
- }
-
- _xzm_do_lock_action(&main->xzmz_mzones_lock, action);
- _xzm_do_lock_action(&main->xzmz_thread_cache_list_lock, action);
-
-#if CONFIG_XZM_DEFERRED_RECLAIM
- if (main->xzmz_reclaim_buffer) {
- _xzm_do_lock_action(&main->xzmz_reclaim_buffer->xrb_lock, action);
- }
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
-}
-
-void
-xzm_force_lock_global_state(malloc_zone_t *main_zone)
-{
- xzm_debug_assert(_xzm_malloc_zone_is_xzm(main_zone));
- xzm_debug_assert(_xzm_malloc_zone_is_main((xzm_malloc_zone_t)main_zone));
- xzm_main_malloc_zone_t main_ref = (xzm_main_malloc_zone_t)main_zone;
- _xzm_global_state_lock(main_ref, XZM_LOCK_LOCK);
-}
-
-void
-xzm_force_unlock_global_state(malloc_zone_t *main_zone)
-{
- xzm_debug_assert(_xzm_malloc_zone_is_xzm(main_zone));
- xzm_debug_assert(_xzm_malloc_zone_is_main((xzm_malloc_zone_t)main_zone));
- xzm_main_malloc_zone_t main_ref = (xzm_main_malloc_zone_t)main_zone;
- _xzm_global_state_lock(main_ref, XZM_LOCK_UNLOCK);
-}
-
-void
-xzm_force_reinit_lock_global_state(malloc_zone_t *main_zone)
-{
- xzm_debug_assert(_xzm_malloc_zone_is_xzm(main_zone));
- xzm_debug_assert(_xzm_malloc_zone_is_main((xzm_malloc_zone_t)main_zone));
- xzm_main_malloc_zone_t main_ref = (xzm_main_malloc_zone_t)main_zone;
- _xzm_global_state_lock(main_ref, XZM_LOCK_INIT);
-}
-
-static void
-xzm_malloc_zone_destroy(xzm_malloc_zone_t zone)
-{
- xzm_main_malloc_zone_t main_ref = zone->xzz_main_ref;
- // It is not sane to permit destroy on the main mzone in the general case:
- // the libsystem initializer makes allocations that need to stay live. A
- // sufficiently early interposing allocator (e.g. ASan) might plausibly be
- // able to do it early enough, but it doesn't do any harm to keep the main
- // zone around empty in that case.
- if (_xzm_malloc_zone_is_main(zone)) {
- return;
- }
-
- // Hold the zone lock while modifying the zone's chunk lists to guard
- // against possible fork() problems (e.g. a tiny chunk's walk_locked bit
- // being set)
- _malloc_lock_lock(&zone->xzz_lock);
-
- // During zone destroy, all the chunks belonging to this zone should exist
- // on zone local lists:
- // - tiny chunks should be on the xzone's ALL list
- // - small chunks will either be in a slot or on the partial or full list
- // - large and huge chunks will be on the zone's chunkq large
- // We require that clients not be allocating from or freeing to zones that
- // are mid destroy, so we don't need to acquire any locks while pulling
- // chunks off of these lists
- LIST_HEAD(, xzm_slice_s) chunks_to_free;
- LIST_INIT(&chunks_to_free);
-
- for (int i = XZM_XZONE_INDEX_FIRST; i < zone->xzz_xzone_count; i++) {
- xzm_xzone_t xz = &zone->xzz_xzones[i];
- if (xz->xz_block_size > XZM_TINY_BLOCK_SIZE_MAX &&
- !zone->xzz_small_freelist_enabled) {
- // small xzone, pull chunks off of the batch, partial, and full list
- xzm_slice_t chunk = xz->xz_chunkq_batch;
- xzm_slice_t temp_chunk = NULL;
- while (chunk) {
- xzm_debug_assert(!chunk->xzc_bits.xzcb_preallocated);
- LIST_INSERT_HEAD(&chunks_to_free, chunk, xzc_entry);
- temp_chunk = *_xzm_segment_slice_meta_batch_next(zone, chunk);
- xzm_debug_assert(_xzm_slice_meta_is_batch_pointer(zone,
- temp_chunk));
- xzm_debug_assert(main_ref->xzmz_batch_size);
-#if CONFIG_XZM_DEFERRED_RECLAIM
- // Reset its reclaim id so it does not appear to have been deferred
- *_xzm_slice_meta_reclaim_id(zone, chunk) = VM_RECLAIM_ID_NULL;
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
- chunk = temp_chunk;
- }
- LIST_FOREACH_SAFE(chunk, &xz->xz_chunkq_partial, xzc_entry,
- temp_chunk) {
- xzm_debug_assert(!chunk->xzc_bits.xzcb_preallocated);
- LIST_REMOVE(chunk, xzc_entry);
- LIST_INSERT_HEAD(&chunks_to_free, chunk, xzc_entry);
- }
- LIST_FOREACH_SAFE(chunk, &xz->xz_chunkq_full, xzc_entry,
- temp_chunk) {
- xzm_debug_assert(!chunk->xzc_bits.xzcb_preallocated);
- LIST_REMOVE(chunk, xzc_entry);
- LIST_INSERT_HEAD(&chunks_to_free, chunk, xzc_entry);
- }
- // Slots
- for (xzm_allocation_index_t j = 0; j < zone->xzz_slot_count; j++) {
- xzm_xzone_allocation_slot_t xas =
- _xzm_xzone_allocation_slot_for_index(zone, xz, j);
- chunk = xas->xas_chunk;
- if (chunk) {
- xzm_debug_assert(!chunk->xzc_bits.xzcb_preallocated);
- xzm_debug_assert(chunk->xzc_alloc_idx == j+1);
- xzm_debug_assert(!chunk->xzc_bits.xzcb_on_partial_list);
- xas->xas_chunk = NULL;
- LIST_INSERT_HEAD(&chunks_to_free, chunk, xzc_entry);
- }
- }
- // Preallocated chunks
- LIST_FOREACH_SAFE(chunk, &xz->xz_chunkq_preallocated, xzc_entry,
- temp_chunk)
- {
- LIST_REMOVE(chunk, xzc_entry);
- // This chunk won't have an mzone set, but otherwise will be
- // initialized enough to be freed
- _xzm_xzone_fresh_chunk_init(xz, chunk,
- XZM_SLICE_KIND_SMALL_CHUNK);
-#if CONFIG_XZM_DEFERRED_RECLAIM
- xzm_debug_assert(*_xzm_slice_meta_reclaim_id(zone, chunk) ==
- VM_RECLAIM_ID_NULL);
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
-
- LIST_INSERT_HEAD(&chunks_to_free, chunk, xzc_entry);
- }
- } else {
- // freelist xzone, empty the batch list, which will reset reclaim ids
- xzm_slice_t chunk = NULL;
- while ((chunk = _xzm_chunk_list_pop(zone, &xz->xz_batch_list,
- XZM_CHUNK_LINKAGE_BATCH, NULL))) {
- xzm_debug_assert(main_ref->xzmz_batch_size);
- }
- // enumerate all chunks via the ALL list
- while ((chunk = _xzm_chunk_list_pop(zone, &xz->xz_all_list,
- XZM_CHUNK_LINKAGE_ALL, NULL))) {
- xzm_assert(!chunk->xzc_bits.xzcb_preallocated);
- chunk->xzc_atomic_meta.xca_on_partial_list = false;
- chunk->xzc_atomic_meta.xca_on_empty_list = false;
- chunk->xzc_atomic_meta.xca_alloc_idx = XZM_SLOT_INDEX_EMPTY;
-
- LIST_INSERT_HEAD(&chunks_to_free, chunk, xzc_entry);
- }
- xzm_slice_kind_t kind;
- if (xz->xz_block_size > XZM_TINY_BLOCK_SIZE_MAX) {
- kind = XZM_SLICE_KIND_TINY_CHUNK;
- } else {
- kind = XZM_SLICE_KIND_SMALL_FREELIST_CHUNK;
- }
-
- // Preallocated freelist chunks
- while ((chunk = _xzm_chunk_list_pop(zone, &xz->xz_preallocated_list,
- XZM_CHUNK_LINKAGE_MAIN, NULL))) {
- // chunk should be madvised and reset by the free path below
- _xzm_xzone_fresh_chunk_init(xz, chunk, kind);
- xzm_debug_assert(chunk->xzc_atomic_meta.xca_alloc_head !=
- XZM_FREE_MADVISED);
-#if CONFIG_XZM_DEFERRED_RECLAIM
- xzm_debug_assert(*_xzm_slice_meta_reclaim_id(zone, chunk) ==
- VM_RECLAIM_ID_NULL);
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
-
- LIST_INSERT_HEAD(&chunks_to_free, chunk, xzc_entry);
- }
- }
- }
-
- _malloc_lock_unlock(&zone->xzz_lock);
-
- xzm_slice_t span = NULL;
- xzm_slice_t temp_span = NULL;
- LIST_FOREACH_SAFE(span, &chunks_to_free, xzc_entry, temp_span) {
- xzm_xzone_t xz = &zone->xzz_xzones[span->xzc_xzone_idx];
-#if CONFIG_XZM_DEFERRED_RECLAIM
- // If this chunk is non-sequestered and has a reclaim index, we need to
- // get it back from the reclaim buffer so it can be returned to the span
- // queue in xzm_chunk_free
- if (!xz->xz_sequestered && _xzm_chunk_should_defer_reclamation(main_ref, span)) {
- int retries = 0;
- while (_xzm_segment_slice_is_deferred(_xzm_segment_for_slice(zone,
- span), span)) {
- if (xzm_chunk_mark_used(zone, span, NULL)) {
- xzm_debug_assert(span->xzc_bits.xzcb_kind ==
- XZM_SLICE_KIND_TINY_CHUNK ||
- span->xzc_bits.xzcb_kind ==
- XZM_SLICE_KIND_SMALL_FREELIST_CHUNK);
- // We don't know if the chunk was successfully reclaimed, so
- // reset the allocation head so that it is synchronously
- // madvised
- span->xzc_atomic_meta.xca_alloc_head = XZM_FREE_NULL;
- break;
- }
- xzm_reclaim_force_sync(main_ref->xzmz_reclaim_buffer);
- xzm_assert(retries < 10);
- retries++;
- }
- }
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
- if (span->xzc_bits.xzcb_kind == XZM_SLICE_KIND_TINY_CHUNK ||
- span->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK) {
- // If the chunk is already madvised, it should already be cleared if
- // it's below the zero on free threshold
- if (span->xzc_atomic_meta.xca_alloc_head != XZM_FREE_MADVISED) {
- size_t size = 0;
- void *chunk_start = _xzm_chunk_start_ptr(zone, span, &size);
- xzm_debug_assert(size == XZM_TINY_CHUNK_SIZE ||
- size == XZM_SMALL_FREELIST_CHUNK_SIZE);
- if (xz->xz_block_size <= XZM_ZERO_ON_FREE_THRESHOLD) {
-#if CONFIG_MTE
- if (xz->xz_tagged) {
- memtag_disable_checking();
- bzero(chunk_start, size);
- memtag_enable_checking();
- } else {
-#endif
- bzero(chunk_start, size);
-#if CONFIG_MTE
- }
-#endif
- }
-
- xzm_segment_group_t sg = _xzm_segment_group_for_slice(zone,
- span);
- xzm_segment_group_segment_madvise_chunk(sg, span);
- span->xzc_atomic_meta.xca_free_count = 0;
- span->xzc_atomic_meta.xca_alloc_head = XZM_FREE_MADVISED;
- }
- span->xzc_bits.xzcb_on_partial_list = false;
- _xzm_xzone_chunk_free(zone, &zone->xzz_xzones[span->xzc_xzone_idx],
- span, false);
- } else if (span->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_CHUNK) {
- // Small chunks are typically, in the non-MTE configuration,
- // madvised when their individual blocks are freed. Since we're
- // freeing the whole chunk, we cautiously madvise the whole chunk
- bool madvise_needed = true;
- span->xzc_bits.xzcb_on_partial_list = false;
- _xzm_xzone_chunk_free(zone, &zone->xzz_xzones[span->xzc_xzone_idx],
- span, madvise_needed);
- } else {
- xzm_abort_with_reason("Unexpected chunk kind",
- span->xzc_bits.xzcb_kind);
- }
- }
-
- LIST_FOREACH_SAFE(span, &zone->xzz_chunkq_large, xzc_entry, temp_span) {
- // NB: _xzm_free_large_huge removes the chunk from chunkq_large, which
- // is why we need to use LIST_FOREACH_SAFE here
- _xzm_free_large_huge(zone, span);
- }
-
- xzm_metapool_t mp = &main_ref->xzmz_metapools[XZM_METAPOOL_MZONE_IDX];
- xzm_reused_mzone_index_t reused = xzm_metapool_alloc(mp);
- reused->xrmi_mzone_idx = zone->xzz_mzone_idx;
- _malloc_lock_lock(&main_ref->xzmz_mzones_lock);
- SLIST_INSERT_HEAD(&main_ref->xzmz_reusable_mzidxq, reused, xrmi_mzone_entry);
- _malloc_lock_unlock(&main_ref->xzmz_mzones_lock);
-
- mvm_deallocate_plat(zone, zone->xzz_total_size, 0, NULL);
-}
-
-#pragma mark Test helpers
-
-bool
-xzm_ptr_lookup_4test(xzm_malloc_zone_t zone, void *ptr,
- xzm_slice_kind_t *kind_out, xzm_segment_group_id_t *sgid_out,
- xzm_xzone_bucket_t *bucket_out)
-{
- xzm_xzone_t xz = NULL;
- xzm_chunk_t chunk = _xzm_ptr_lookup(zone, ptr, &xz, NULL, NULL);
- if (!chunk) {
- return false;
- }
-
- xzm_slice_kind_t kind = chunk->xzc_bits.xzcb_kind;
- *kind_out = kind;
-
- xzm_segment_t segment = _xzm_segment_for_slice(zone, chunk);
- *sgid_out = segment->xzs_segment_group->xzsg_id;
-
- if (_xzm_slice_kind_uses_xzones(kind)) {
- xzm_assert(xz);
- *bucket_out = xz->xz_bucket;
- }
-
- return true;
-}
-
-uint8_t
-xzm_type_choose_ptr_bucket_4test(const xzm_bucketing_keys_t *const keys,
- uint8_t ptr_bucket_count, malloc_type_descriptor_t type_desc)
-{
- return _xzm_type_choose_ptr_bucket(keys, ptr_bucket_count, type_desc);
-}
-
-#pragma mark Zone creation and initialization
-
-struct xzm_process_config_s {
- xzm_slot_config_t xzpc_slot_config;
- bool xzpc_madvise_workaround;
- bool xzpc_disable_vm_user_ranges;
-};
-
-typedef const struct xzm_process_config_s *xzm_process_config_t;
-
-#if CONFIG_MALLOC_PROCESS_IDENTITY
-
-static const struct xzm_process_config_s _xzm_launchd_process_config = {
- // almost everything in launchd happens on the event queue
- .xzpc_slot_config = XZM_SLOT_SINGLE,
-};
-
-static const struct xzm_process_config_s _xzm_notifyd_process_config = {
- // almost everything in notifyd happens on its global workloop
- .xzpc_slot_config = XZM_SLOT_SINGLE,
-};
-
-static const xzm_process_config_t _xzm_process_configs[MALLOC_PROCESS_COUNT] = {
- [MALLOC_PROCESS_LAUNCHD] = &_xzm_launchd_process_config,
- [MALLOC_PROCESS_NOTIFYD] = &_xzm_notifyd_process_config,
-};
-
-#endif // CONFIG_MALLOC_PROCESS_IDENTITY
-
-#define XZM_FRONT_RANDOM_SIZE 32
-
-static xzm_front_index_t
-_xzm_random_front_index(uint8_t front_random[static XZM_FRONT_RANDOM_SIZE],
- size_t allocation_front_count, xzm_xzone_index_t xz_idx)
-{
- if (allocation_front_count == 1) {
- return XZM_FRONT_INDEX_DEFAULT;
- }
-
- xzm_assert(allocation_front_count == 2);
-
- size_t byte = xz_idx / CHAR_BIT;
- size_t bit = xz_idx % CHAR_BIT;
-
- xzm_assert(byte < XZM_FRONT_RANDOM_SIZE);
- return (bool)(front_random[byte] & (1 << bit));
-}
-
-// mimalloc: MI_PAGE_QUEUES_EMPTY
-static const size_t _xzm_bin_sizes[] = {
- 16,
- 32,
- 48,
- 64,
- 80,
- 96,
- 112,
- 128,
- 160,
- 192,
- 224,
- 256,
- 320,
- 384,
- 448,
- 512,
- 640,
- 768,
- 896,
- 1024,
- 1280,
- 1536,
- 1792,
- 2048,
- 2560,
- 3072,
- 3584,
- 4096,
- 5120,
- 6144,
- 7168,
- 8192,
- 10240,
- 12288,
- 14336,
- 16384,
- 20480,
- 24576,
- 28672,
- 32768,
-};
-
-// mimalloc: MI_SEGMENT_SPAN_QUEUES_EMPTY
-static const size_t _xzm_span_queue_slice_counts[] = {
- 1,
- 2,
- 3,
- 4,
- 5,
- 6,
- 7,
- 10,
- 12,
- 14,
- 16,
- 20,
- 24,
- 28,
- 32,
- 40,
- 48,
- 56,
- 64,
- 80,
- 96,
- 112,
- 128,
- 160,
- 192,
- 224,
- 256,
-};
-
-MALLOC_STATIC_ASSERT(
- countof_unsafe(_xzm_span_queue_slice_counts) == XZM_SPAN_QUEUE_COUNT,
- "all span queues have slice counts");
-
-static const char ptr_buckets_boot_arg[] = "xzone_ptr_buckets";
-static const char xzone_slot_config_boot_arg[] = "malloc_xzone_slot_config";
-static const char xzone_guard_pages_boot_arg[] = "xzone_guard_pages";
-
-static void
-_xzm_initialize_const_zone_data(xzm_malloc_zone_t zone,
- size_t size, xzm_mzone_index_t mzone_idx, size_t xzone_count,
- size_t slot_count, xzm_xzone_t xzones,
- xzm_xzone_allocation_slot_t slots, xzm_main_malloc_zone_t main_ref,
- xzm_slot_config_t initial_slot_config, uint32_t initial_slot_threshold,
- xzm_slot_config_t max_slot_config,
- uint32_t list_upgrade_threshold_single,
- uint32_t list_upgrade_threshold_cluster, uint32_t list_upgrade_period,
- uint32_t slot_upgrade_threshold_single,
- uint32_t slot_upgrade_threshold_cluster, uint32_t slot_upgrade_period,
- uint64_t tiny_thrash_threshold, uint64_t small_thrash_threshold,
- uint64_t small_thrash_limit_size, uint64_t debug_flags,
- bool small_freelist, xzm_slot_config_t max_list_config,
- xzm_chunk_list_t partial_lists)
-{
- // We're making some assumptions about layout here, but I think that's fine
- xzm_debug_assert((uintptr_t)zone + size >= (uintptr_t)slots +
- sizeof(struct xzm_xzone_allocation_slot_s) * xzone_count * slot_count);
- xzm_debug_assert((uintptr_t)slots >= (uintptr_t)xzones +
- sizeof(struct xzm_xzone_s) * xzone_count);
-
- *zone = (struct xzm_malloc_zone_s){
- .xzz_basic_zone = {
- .version = 16,
- .size = (void *)xzm_malloc_zone_size,
- .malloc = (void *)xzm_malloc_zone_malloc,
- .calloc = (void *)xzm_malloc_zone_calloc,
- .valloc = (void *)xzm_malloc_zone_valloc,
- .free = (void *)xzm_malloc_zone_free,
- .realloc = (void *)xzm_malloc_zone_realloc,
- .destroy = (void *)xzm_malloc_zone_destroy,
- // Note: batch_malloc and batch_free introduce way too much
- // complexity and maintenance burden to be worth whatever
- // marginal performance benefit they may offer, so we don't
- // really want to implement them.
- //
- // Technically they're supposed to be optional, so ideally we'd
- // just leave them NULL. However, we've never had a default
- // zone that didn't have them, so we open ourselves up to
- // compatibility risks with code that either isn't prepared to
- // deal with getting 0 or isn't prepared to wrap a zone that
- // doesn't have an implementation. Even within the project
- // we depend on the default zone having an implementation: the
- // virtual default zone calls through without checking for NULL. So,
- // rather than leaving them NULL, we keep trivial implementations
- // that just wrap plain malloc and free.
- .batch_malloc = malloc_zone_batch_malloc_fallback,
- .batch_free = malloc_zone_batch_free_fallback,
- .introspect = (struct malloc_introspection_t
- *)&xzm_malloc_zone_introspect,
- .memalign = (void *)xzm_malloc_zone_memalign,
- .free_definite_size = (void *)xzm_malloc_zone_free_definite_size,
- .pressure_relief = (void *)malloc_zone_pressure_relief_fallback,
- .claimed_address = (void *)xzm_malloc_zone_claimed_address,
- .try_free_default = (void *)xzm_malloc_zone_try_free_default,
- .malloc_with_options = (void *)xzm_malloc_zone_malloc_with_options,
-
- .malloc_type_malloc = (void *)xzm_malloc_zone_malloc_type_malloc,
- .malloc_type_calloc = (void *)xzm_malloc_zone_malloc_type_calloc,
- .malloc_type_realloc = (void *)xzm_malloc_zone_malloc_type_realloc,
- .malloc_type_memalign = (void *)xzm_malloc_zone_malloc_type_memalign,
- .malloc_type_malloc_with_options =
- (void *)xzm_malloc_zone_malloc_type_malloc_with_options,
- },
- .xzz_total_size = size,
- .xzz_mzone_idx = mzone_idx,
- .xzz_xzone_count = xzone_count,
- .xzz_slot_count = slot_count,
- .xzz_xzones = xzones,
- .xzz_xzone_allocation_slots = slots,
- .xzz_partial_lists = partial_lists,
- .xzz_main_ref = main_ref,
- .xzz_max_list_config = max_list_config,
- .xzz_initial_slot_config = initial_slot_config,
- .xzz_max_slot_config = max_slot_config,
- .xzz_small_freelist_enabled = small_freelist,
- .xzz_list_upgrade_threshold = {
- list_upgrade_threshold_single,
- list_upgrade_threshold_cluster,
- },
- .xzz_list_upgrade_period = list_upgrade_period,
- .xzz_slot_initial_threshold = initial_slot_threshold,
- .xzz_slot_upgrade_threshold = {
- slot_upgrade_threshold_single,
- slot_upgrade_threshold_cluster,
- },
- .xzz_slot_upgrade_period = slot_upgrade_period,
- .xzz_tiny_thrash_threshold = tiny_thrash_threshold,
- .xzz_small_thrash_threshold = small_thrash_threshold,
- .xzz_small_thrash_limit_size = small_thrash_limit_size,
- .xzz_lock = _MALLOC_LOCK_INIT,
- .xzz_fork_lock = _MALLOC_LOCK_INIT,
- .xzz_flags = debug_flags,
- };
-
- bool use_slowpath_zone_functions = false;
- if (debug_flags & MALLOC_DO_SCRIBBLE ||
- debug_flags & MALLOC_PURGEABLE) {
- use_slowpath_zone_functions = true;
- }
-
- if (use_slowpath_zone_functions) {
- zone->xzz_basic_zone.malloc = (void *)xzm_malloc_zone_malloc_slow;
- zone->xzz_basic_zone.calloc = (void *)xzm_malloc_zone_calloc_slow;
- zone->xzz_basic_zone.valloc = (void *)xzm_malloc_zone_valloc_slow;
- zone->xzz_basic_zone.free = (void *)xzm_malloc_zone_free_slow;
- zone->xzz_basic_zone.realloc = (void *)xzm_malloc_zone_realloc_slow;
- zone->xzz_basic_zone.memalign = (void *)xzm_malloc_zone_memalign_slow;
- zone->xzz_basic_zone.free_definite_size =
- (void *)xzm_malloc_zone_free_definite_size_slow;
- zone->xzz_basic_zone.try_free_default =
- (void *)xzm_malloc_zone_try_free_default_slow;
- zone->xzz_basic_zone.malloc_with_options =
- (void *)xzm_malloc_zone_malloc_with_options_slow;
- zone->xzz_basic_zone.malloc_type_malloc =
- (void *)xzm_malloc_zone_malloc_type_malloc_slow;
- zone->xzz_basic_zone.malloc_type_calloc =
- (void *)xzm_malloc_zone_malloc_type_calloc_slow;
- zone->xzz_basic_zone.malloc_type_realloc =
- (void *)xzm_malloc_zone_malloc_type_realloc_slow;
- zone->xzz_basic_zone.malloc_type_memalign =
- (void *)xzm_malloc_zone_malloc_type_memalign_slow;
- zone->xzz_basic_zone.malloc_type_malloc_with_options =
- (void *)xzm_malloc_zone_malloc_type_malloc_with_options_slow;
- }
-}
-
-static void
-_xzm_initialize_xzone_data(xzm_malloc_zone_t zone,
- xzm_slot_config_t list_config, xzm_guard_page_config_t guard_config,
- uint8_t front_random[XZM_FRONT_RANDOM_SIZE], bool all_data)
-{
- xzm_main_malloc_zone_t main_ref = _xzm_malloc_zone_main(zone);
- bool is_main = _xzm_malloc_zone_is_main(zone);
-
- xzm_debug_assert((is_main && front_random) || (!is_main && !front_random));
-
- // NOTE: although this is technically a layering violation with Libc and
- // CoreCrypto, we and they believe it to be safe
- //
- // TODO: allow overriding with a fixed value for testing
- uint64_t freelist_cookie = 0;
- arc4random_buf(&freelist_cookie, sizeof(freelist_cookie));
- if (!freelist_cookie) {
- freelist_cookie = 0xdeaddeaddeaddeadull;
- }
-#if CONFIG_MTE
- if (zone->xzz_memtag_config.enabled) {
- // Clear the tag bits from the freelist cookie to ensure they can always be
- // used to store a block's tag
- freelist_cookie = (uint64_t)memtag_strip_address(
- (uint8_t *)freelist_cookie);
- }
-#endif
- zone->xzz_freelist_cookie = freelist_cookie;
-
- xzm_xzone_index_t xzidx = XZM_XZONE_INDEX_FIRST;
- for (size_t i = 0; i < countof(_xzm_bin_sizes); i++) {
- if (is_main) {
- main_ref->xzmz_xzone_bin_offsets[i] = xzidx;
- }
- size_t bin_buckets = main_ref->xzmz_xzone_bin_bucket_counts[i];
- for (int bucket = 0; bucket < bin_buckets; xzidx++, bucket++) {
- xzm_xzone_t xz = &zone->xzz_xzones[xzidx];
- size_t block_size = main_ref->xzmz_xzone_bin_sizes[i];
- size_t chunk_size;
- if (block_size <= XZM_TINY_BLOCK_SIZE_MAX) {
- chunk_size = XZM_TINY_CHUNK_SIZE;
- } else if (zone->xzz_small_freelist_enabled) {
- chunk_size = XZM_SMALL_FREELIST_CHUNK_SIZE;
- } else {
- chunk_size = XZM_SMALL_CHUNK_SIZE;
- }
-
- size_t chunk_capacity = chunk_size / block_size;
-
- // The early budget is only in the main zone xzones, all other
- // zones decrement those
- uint16_t early_budget = 0;
- if (is_main && main_ref->xzmz_mfm_address) {
- if (block_size <= 256) {
- early_budget = 2048 / block_size;
- } else if (block_size <= 512) {
- early_budget = 4096 / block_size;
- } else if (block_size <= 2048) {
- early_budget = 8192 / block_size;
- } else if (block_size <= 8192) {
- early_budget = 1;
- }
- }
-
- xzm_segment_group_id_t sgid;
- bool sequestered;
- if ((bucket == XZM_XZONE_BUCKET_DATA || all_data) &&
-#if XZM_NARROW_BUCKETING
- !main_ref->xzmz_narrow_bucketing &&
-#endif
- !XZM_BUCKET_POINTER_ONLY) {
- sgid = XZM_SEGMENT_GROUP_DATA;
- sequestered = false;
- } else {
- sgid = XZM_SEGMENT_GROUP_POINTER_XZONES;
- sequestered = true;
- }
-
- // Enable sequestering for small chunks to avoid coalescing with
- // adjacent chunks that may not already be marked as free
- if (block_size > XZM_TINY_BLOCK_SIZE_MAX &&
- main_ref->xzmz_defer_small) {
- sequestered = true;
- }
-
- xzm_front_index_t front;
- if (is_main) {
- if (sgid == XZM_SEGMENT_GROUP_POINTER_XZONES) {
- front = _xzm_random_front_index(front_random,
- main_ref->xzmz_allocation_front_count, xzidx);
- } else {
- xzm_debug_assert(sgid == XZM_SEGMENT_GROUP_DATA);
- front = XZM_FRONT_INDEX_DEFAULT;
- }
- } else {
- xzm_xzone_t main_xz = &main_ref->xzmz_base.xzz_xzones[xzidx];
- front = main_xz->xz_front;
- }
-
- uint8_t run_length = 0;
- uint8_t guard_density = 0;
- if (guard_config->xgpc_enabled) {
- if (guard_config->xgpc_enabled_for_data) {
- // To avoid unbounded VA growth caused by unusable free
- // spans in the data segments, all xzones are sequestered
- // when guards are enabled
- sequestered = true;
- }
-
- if (sequestered) {
- if (block_size <= XZM_TINY_BLOCK_SIZE_MAX) {
- run_length = guard_config->xgpc_max_run_tiny;
- guard_density = guard_config->xgpc_tiny_guard_density;
- } else {
- run_length = guard_config->xgpc_max_run_small;
- guard_density = guard_config->xgpc_small_guard_density;
- }
- }
- }
-
- *xz = (struct xzm_xzone_s){
- .xz_segment_group_id = sgid,
- .xz_front = front,
- .xz_block_size = block_size,
- .xz_quo_magic = XZM_MAGIC_QUO(block_size),
- .xz_align_magic = XZM_MAGIC_ALIGNED(block_size),
- .xz_chunk_capacity = (uint32_t)chunk_capacity,
- .xz_lock = _MALLOC_LOCK_INIT,
- .xz_early_budget = early_budget,
- .xz_idx = xzidx,
- .xz_mzone_idx = zone->xzz_mzone_idx,
- .xz_bucket = (xzm_xzone_bucket_t)bucket,
- .xz_sequestered = sequestered,
- .xz_slot_config = XZM_SLOT_SINGLE,
- .xz_guard_config = {
- .xxgc_max_run_length = run_length,
- .xxgc_density = guard_density,
- }
- };
-
- // Initialize all possible slots with the initial slot config
- xzm_debug_assert(zone->xzz_slot_count ==
- _xzm_get_limit_allocation_index(zone->xzz_max_slot_config));
- xzm_allocation_index_t limit_list =
- _xzm_get_limit_allocation_index(zone->xzz_max_list_config);
- for (xzm_allocation_index_t i = 0; i < zone->xzz_slot_count;
- ++i) {
- xzm_xzone_allocation_slot_t xas =
- _xzm_xzone_allocation_slot_for_index(zone, xz, i);
- xas->xas_counters.xsc_slot_config = xz->xz_slot_config;
-#if CONFIG_TINY_ALLOCATION_SLOT_LOCK
- _malloc_lock_init(&xas->xas_lock);
-#endif // CONFIG_TINY_ALLOCATION_SLOT_LOCK
-
- if (i >= limit_list) {
- continue;
- }
-
- xzm_chunk_list_t xcl =
- _xzm_xzone_chunk_list_for_index(zone, xz,
- zone->xzz_partial_lists, i);
- xcl->xcl_counters.xsc_slot_config = xz->xz_list_config;
- }
-
-#if CONFIG_MTE
- xz->xz_tagged = _xzm_zone_memtag_block(zone, block_size,
- /*data=*/(bucket == XZM_XZONE_BUCKET_DATA));
-
- // Disable the early allocator for zones for which we don't
- // enable tagging. This is because MFM will always provide
- // tagged memory, and this might cause the client to violate
- // MTE mapping policies that are expected to hold for
- // data-only allocations, which are normally not tagged in
- // the default configuration.
- // `zone->xzz_memtag_config.enabled` is used as proxy for
- // `mfm_memtag_enabled`.
- if (zone->xzz_memtag_config.enabled && !xz->xz_tagged) {
- xz->xz_early_budget = 0;
- }
-#endif // CONFIG_MTE
- }
- }
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static const char *
-_xzm_development_only_getenv(const char **envp, const char *key)
-{
- return malloc_internal_security_policy ? _simple_getenv(envp, key) : NULL;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static const char *
-_xzm_production_getenv(const char **envp, const char *key)
-{
- return _simple_getenv(envp, key);
-}
-
-#ifdef _simple_getenv
-#undef _simple_getenv
-#endif
-
-#define _simple_getenv(...) \
- static_assert(0, "Use _xzm_(development_only|release)_getenv instead")
-
-
-malloc_zone_t *
-xzm_main_malloc_zone_create(unsigned debug_flags, const char **envp,
- const char **apple, const char *bootargs)
-{
- bool security_critical = false, security_critical_max_perf = false;
-#if CONFIG_MALLOC_PROCESS_IDENTITY
- security_critical = malloc_process_is_security_critical(
- malloc_process_identity);
- security_critical_max_perf = malloc_process_is_security_critical_max_perf(
- malloc_process_identity);
-#endif // CONFIG_MALLOC_PROCESS_IDENTITY
-
- uint8_t front_random[XZM_FRONT_RANDOM_SIZE];
-
- xzm_bucketing_keys_t bucketing_keys;
- // executable_boothash is a salted hash of the concatenation of the current
- // boot session UUID and cdhash of the main executable
- const char *boothash = _xzm_production_getenv(apple, "executable_boothash");
- if (!boothash) {
- // executable_boothash isn't populated when the executable isn't
- // codesigned (rdar://118451590). This should be very rare (only when
- // the system is running with reduced security), so to guard against
- // accidentally breaking the boothash, crash in known security-critical
- // processes
- if (security_critical) {
- xzm_abort("couldn't find executable_boothash");
- }
-
- arc4random_buf(&bucketing_keys, sizeof(bucketing_keys));
- } else {
- size_t boothash_len = strlen(boothash);
- if (boothash_len < 32) {
- xzm_abort_with_reason("invalid executable_boothash length",
- boothash_len);
- }
-
- const size_t part_len = 16;
- char boothash_part[part_len + 1];
- unsigned long long value;
- const size_t keys_cnt = sizeof(bucketing_keys.xbk_key_data) /
- sizeof(bucketing_keys.xbk_key_data[0]);
-
- for (size_t i = 0; i < keys_cnt; i++) {
- memcpy(boothash_part, &boothash[i * part_len], part_len);
- boothash_part[part_len] = '\0';
-
- value = strtoull(boothash_part, NULL, 16);
- if ((value == 0 && errno == EINVAL) ||
- (value == ULLONG_MAX && errno == ERANGE)) {
- xzm_abort("invalid executable_boothash string");
- }
- bucketing_keys.xbk_key_data[i] = value;
- }
- }
-
- // XXX We need a bunch of additional per-boot entropy for allocation front
- // assignments. It may make sense to increase the amount of per-boot
- // entropy supplied by the kernel, but for now we make do by deriving
- // from what we've already got.
- //
- // TODO: If we do continue to derive, we should probably be seeding a PRNG
- // instead
- static_assert(sizeof(front_random) == CCSHA256_OUTPUT_SIZE,
- "front_random size");
-
- const struct ccdigest_info *di = ccsha256_di();
- ccdigest_di_decl(di, dc);
- ccdigest_init(di, dc);
- char diversifier[] = "xzone malloc front random";
- ccdigest_update(di, dc, sizeof(diversifier), diversifier);
- ccdigest_update(di, dc, sizeof(bucketing_keys),
- bucketing_keys.xbk_key_data);
- ccdigest_final(di, dc, front_random);
- ccdigest_di_clear(di, dc);
-
-
- // TODO: scrub executable_boothash from the apple array like we do for
- // malloc_entropy?
-
- xzm_process_config_t process_config = NULL;
-#if CONFIG_MALLOC_PROCESS_IDENTITY
- if (malloc_process_identity != MALLOC_PROCESS_NONE) {
- process_config = _xzm_process_configs[malloc_process_identity];
- }
-#endif // CONFIG_MALLOC_PROCESS_IDENTITY
-
- bool use_slowpath_zone_functions = false;
- if (debug_flags & MALLOC_DO_SCRIBBLE) {
- use_slowpath_zone_functions = true;
- }
-
- xzm_slot_config_t max_slot_config = XZM_SLOT_CPU;
-
- size_t override_ptr_bucket_count = 0;
-#if XZM_NARROW_BUCKETING
- // Platforms that support narrow bucketing default to using it
- bool narrow_bucketing = true;
-#endif
-
- char value_buf[256];
- const char *flag = malloc_common_value_for_key_copy(bootargs,
- ptr_buckets_boot_arg, value_buf, sizeof(value_buf));
- if (flag) {
- const char *endp;
- long value = malloc_common_convert_to_long(flag, &endp);
- if (!*endp && value > 0 && value <= XZM_POINTER_BUCKETS_MAX) {
- override_ptr_bucket_count = (size_t)value;
- } else {
- malloc_report(ASL_LEVEL_ERR,
- "Invalid xzone_ptr_buckets value %ld - ignored.\n", value);
- }
- }
-
-#if MALLOC_TARGET_IOS_ONLY || TARGET_OS_WATCH
- if (!security_critical) {
-#if MALLOC_TARGET_IOS_ONLY
- override_ptr_bucket_count = 2;
-#elif TARGET_OS_WATCH
- override_ptr_bucket_count = 1;
-#endif
- }
-#endif
-
- // There are a few ways MallocMaxMagazines might have been set:
- // - JetsamProperties sets it for a number of daemons on iOS
- // - A handful of projects found the envvar on their own and set it
- // themselves in their launchd plists
- //
- // Pending an audit and rebalancing of bucketing by security sensitivity,
- // give processes that have this set and that we don't otherwise have
- // explicit policy for a reduced bucket configuration.
- //
- // If set to 1, we should also give them a reduced slot config.
- bool allow_malloc_max_magazines = true;
- // Note: this is load-bearing for wifip2pd
- if (security_critical) {
- allow_malloc_max_magazines = false;
-#if TARGET_OS_VISION
- // On visionOS, don't clamp the security-critical-max-perf processes to
- // per-cluster scaling
- if (!security_critical_max_perf) {
- max_slot_config = XZM_SLOT_CLUSTER;
- }
-#else
- // On the other platforms, since we're already clamping the
- // security-critical-max-perf processes to per-cluster scaling without
- // issue, clamp in all cases
- max_slot_config = XZM_SLOT_CLUSTER;
-#endif
- }
- if (allow_malloc_max_magazines) {
- flag = _xzm_production_getenv(envp, "MallocMaxMagazines");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 1) {
- max_slot_config = XZM_SLOT_SINGLE;
- } else if (value == 2 || value == UINT16_MAX) {
- max_slot_config = XZM_SLOT_CLUSTER;
- }
-
-#if MALLOC_TARGET_IOS_ONLY || MALLOC_TARGET_DK_IOS || \
- TARGET_OS_OSX || MALLOC_TARGET_DK_OSX
- if (value == 1 || value == 2 || value == UINT16_MAX) {
- override_ptr_bucket_count = 1;
- }
-#endif
- }
- }
-
- flag = _xzm_development_only_getenv(envp, "MallocXzonePtrBucketCount");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value <= XZM_POINTER_BUCKETS_MAX) {
- override_ptr_bucket_count = (size_t)value;
-#if XZM_NARROW_BUCKETING
- narrow_bucketing = false;
-#endif
- }
- }
-
- size_t ptr_bucket_count = override_ptr_bucket_count ?:
- XZM_XZONE_DEFAULT_POINTER_BUCKET_COUNT;
-
- size_t bucket_base = XZM_XZONE_BUCKET_POINTER_BASE;
-#if XZM_NARROW_BUCKETING
- if (narrow_bucketing) {
- bucket_base = 0;
- }
-#endif
-
- bool use_early_allocator = true;
-#if XZM_NARROW_BUCKETING
- if (narrow_bucketing && ptr_bucket_count == 1) {
- use_early_allocator = false;
- }
-#endif
-
- flag = _xzm_production_getenv(envp, "MallocXzoneEarlyAlloc");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 0 || value == 1) {
- use_early_allocator = (bool)value;
- } else {
- malloc_report(ASL_LEVEL_ERR,
- "MallocXzoneEarlyAlloc must be 0 or 1.\n");
- }
- }
-
- if (use_early_allocator) {
- mfm_initialize();
- }
-
- size_t buckets_per_bin = bucket_base + ptr_bucket_count;
-
- size_t bin_count = countof(_xzm_bin_sizes);
- size_t xzone_count = 1 + bin_count * buckets_per_bin;
- xzm_assert(xzone_count <= UINT8_MAX);
-
- bool madvise_workaround = false;
- if (process_config && process_config->xzpc_madvise_workaround) {
- madvise_workaround = true;
- }
-
- flag = _xzm_development_only_getenv(envp, "MallocXzoneMadviseWorkaround");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 0 || value == 1) {
- madvise_workaround = (bool)value;
- }
- }
-
-#if CONFIG_MTE
- struct xzm_memtag_config_s memtag_config = {0};
-
- // If the process has been spawned by setting has_sec_transition=1,
- // load a default configuration for MTE support in xzone.
- // Note that we still allow overriding the configuration through
- // environment variables.
- if (malloc_has_sec_transition) {
- memtag_config.enabled = true;
- memtag_config.tag_data = false;
- memtag_config.max_block_size = XZM_SMALL_BLOCK_SIZE_MAX;
- if (malloc_sec_transition_policy & /*TASK_SEC_POLICY_USER_DATA*/0x02) {
- memtag_config.tag_data = true;
- }
- }
-
- uint64_t max_supported_memtag_block_size = XZM_SMALL_BLOCK_SIZE_MAX;
-
- // MTE debug mode is available on public builds
- flag = _xzm_production_getenv(envp, "MallocTagAll");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 1) {
- memtag_config.tag_data = true;
- memtag_config.max_block_size = max_supported_memtag_block_size;
-
- if (!malloc_has_sec_transition) {
- malloc_report(MALLOC_REPORT_CRASH,
- "Malloc MTE debug mode (MallocTagAll=1) requires the "
- "process to be started with MTE enabled.\n");
- }
- }
- }
-
- flag = _xzm_development_only_getenv(envp, "MallocTagAllInternal");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 1) {
- memtag_config.tag_data = true;
- memtag_config.max_block_size = max_supported_memtag_block_size;
- }
- // Skip sanity check for general MTE enablement.
- }
-
- flag = _xzm_development_only_getenv(envp, "MallocXzoneMemtagEnable");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 0 || value == 1) {
- memtag_config.enabled = (bool)value;
- }
- }
-
- flag = _xzm_development_only_getenv(envp, "MallocXzoneMemtagTagData");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 0 || value == 1) {
- memtag_config.tag_data = (bool)value;
- }
- }
-
- flag = _xzm_development_only_getenv(envp, "MallocXzoneMemtagMaxBlockSize");
- if (flag) {
- unsigned long long value = strtoull(flag, NULL, 10);
- // TODO: allow all sizes?
- if (value <= XZM_SMALL_BLOCK_SIZE_MAX && !(value & 0xf)) {
- memtag_config.max_block_size = value;
- }
- }
-#endif // CONFIG_MTE
-
- bool has_vm_user_ranges = true;
- if (process_config && process_config->xzpc_disable_vm_user_ranges) {
- has_vm_user_ranges = false;
- }
-
- flag = _xzm_development_only_getenv(envp, "MallocXzoneHasRanges");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 0 || value == 1) {
- has_vm_user_ranges = (bool)value;
- }
- }
-
-#if XZM_NARROW_BUCKETING
- // Don't waste a PTE trying to separate data at the range level if it's
- // mixed into the same xzones anyway
- if (narrow_bucketing && !security_critical) {
- has_vm_user_ranges = false;
- }
-#endif
-
- bool thread_caching = false;
-
-#if CONFIG_NANOZONE
- bool nano_config = (_malloc_engaged_nano == NANO_V2);
-
-#if MALLOC_TARGET_DK_OSX
- // TODO: clean up macOS DriverKit nano enablement config
- nano_config = false;
-#endif
-
- bool security_critical_allows_nano_config = false;
-#if CONFIG_MALLOC_PROCESS_IDENTITY
- if (malloc_process_identity == MALLOC_PROCESS_HARDENED_HEAP_CONFIG) {
- security_critical_allows_nano_config = true;
- }
-#endif
-
- if (security_critical && !security_critical_allows_nano_config) {
- // Load-bearing for MTLCompilerService
- nano_config = false;
- }
-
- if (nano_config) {
- max_slot_config = XZM_SLOT_CPU;
-
- // Not dealing with thread caching in the simulator yet
-#if !TARGET_OS_SIMULATOR
-#if CONFIG_FEATUREFLAGS_SIMPLE
- thread_caching = os_feature_enabled_simple(libmalloc,
- SecureAllocator_ThreadCaching, false);
-#else
- // Not for DriverKit yet
-#endif // CONFIG_FEATUREFLAGS_SIMPLE
-#endif // !TARGET_OS_SIMULATOR
- }
-
- flag = _xzm_production_getenv(envp, "MallocXzoneThreadCaching");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 0 || value == 1) {
- thread_caching = (value == 1);
- } else {
- malloc_report(ASL_LEVEL_ERR,
- "MallocXzoneThreadCaching must be one of 0,1 - got %ld\n",
- value);
- }
- }
-#endif // CONFIG_NANOZONE
-
- if (process_config && process_config->xzpc_slot_config != XZM_SLOT_LAST) {
- max_slot_config = process_config->xzpc_slot_config;
- }
-
- flag = malloc_common_value_for_key_copy(bootargs,
- xzone_slot_config_boot_arg, value_buf, sizeof(value_buf));
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value < XZM_SLOT_LAST) {
- max_slot_config = (xzm_slot_config_t)value;
- }
- }
-
- flag = _xzm_development_only_getenv(envp, "MallocXzoneSlotConfig");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value < XZM_SLOT_LAST) {
- max_slot_config = (xzm_slot_config_t)value;
- }
- }
-
- xzm_slot_config_t slot_config = XZM_SLOT_SINGLE;
- uint32_t slot_threshold = 128;
- uint8_t chunk_threshold = 1;
-#if TARGET_OS_OSX
- // On macOS, not known processes get initial per-cluster
- if (!(security_critical && !security_critical_max_perf) &&
- !malloc_space_efficient_enabled) {
- slot_config = XZM_SLOT_CLUSTER;
- }
-#endif // TARGET_OS_OSX
- flag = _xzm_development_only_getenv(envp, "MallocXzoneInitialSlotConfig");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value < XZM_SLOT_LAST) {
- slot_config = (xzm_slot_config_t)value;
- }
- }
- flag = _xzm_development_only_getenv(envp, "MallocXzoneInitialSlotThreshold");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value < UINT32_MAX) {
- slot_threshold = (uint32_t)value;
- }
- }
- flag = _xzm_development_only_getenv(envp, "MallocXzoneInitialChunkThreshold");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value > 0 && value < UINT8_MAX) {
- chunk_threshold = (uint8_t)value;
- }
- }
-
- uint32_t list_upgrade_threshold_single = 32;
- uint32_t list_upgrade_threshold_cluster = 128;
- uint32_t slot_upgrade_threshold_single = 64;
- uint32_t slot_upgrade_threshold_cluster = 256;
- flag = _xzm_development_only_getenv(envp,
- "MallocXzoneListUpgradeThreshold");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value < UINT32_MAX) {
- list_upgrade_threshold_single = (uint32_t)value;
- list_upgrade_threshold_cluster = (uint32_t)value;
- }
- }
-
- flag = _xzm_development_only_getenv(envp,
- "MallocXzoneListUpgradeThresholdSingle");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value < UINT32_MAX) {
- list_upgrade_threshold_single = (uint32_t)value;
- }
- }
-
- flag = _xzm_development_only_getenv(envp,
- "MallocXzoneListUpgradeThresholdCluster");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value < UINT32_MAX) {
- list_upgrade_threshold_cluster = (uint32_t)value;
- }
- }
-
- flag = _xzm_development_only_getenv(envp,
- "MallocXzoneSlotUpgradeThreshold");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value < UINT32_MAX) {
- slot_upgrade_threshold_single = (uint32_t)value;
- slot_upgrade_threshold_cluster = (uint32_t)value;
- }
- }
-
- flag = _xzm_development_only_getenv(envp,
- "MallocXzoneSlotUpgradeThresholdSingle");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value < UINT32_MAX) {
- slot_upgrade_threshold_single = (uint32_t)value;
- }
- }
-
- flag = _xzm_development_only_getenv(envp,
- "MallocXzoneSlotUpgradeThresholdCluster");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value < UINT32_MAX) {
- slot_upgrade_threshold_cluster = (uint32_t)value;
- }
- }
-
- // Reset contention counters every 1K allocations to prevent long-lived
- // processes from slowly accruing additional slots
- uint32_t list_upgrade_period = 512;
- uint32_t slot_upgrade_period = 1024;
- flag = _xzm_development_only_getenv(envp, "MallocXzoneListUpgradePeriod");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value < UINT32_MAX) {
- list_upgrade_period = (uint32_t)value;
- }
- }
-
- flag = _xzm_development_only_getenv(envp, "MallocXzoneSlotUpgradePeriod");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value < UINT32_MAX) {
- slot_upgrade_period = (uint32_t)value;
- }
- }
-
- uint8_t slot_count = 0;
- switch (max_slot_config) {
- case XZM_SLOT_SINGLE:
- slot_count = 1;
- break;
- case XZM_SLOT_CLUSTER:
-#if !CONFIG_XZM_CLUSTER_AWARE
- slot_count = MIN(2, logical_ncpus);
- break;
-#else // !CONFIG_XZM_CLUSTER_AWARE
- if (ncpuclusters > 1) {
- slot_count = (uint8_t)ncpuclusters;
- break;
- }
- MALLOC_FALLTHROUGH;
-#endif // !CONFIG_XZM_CLUSTER_AWARE
- case XZM_SLOT_CPU:
- max_slot_config = XZM_SLOT_CPU; // handle fallthrough
- slot_count = (uint8_t)logical_ncpus;
- break;
- default:
- xzm_abort("Invalid xzone slot config");
- }
- if (slot_config > max_slot_config) {
- slot_config = max_slot_config;
- }
-
- const uint64_t nsec_per_msec = 1000000ull;
- mach_timebase_info_data_t tb_info;
- kern_return_t kr = mach_timebase_info(&tb_info);
- if (kr) {
- xzm_abort_with_reason("mach_timebase_info failed", kr);
- }
-
- uint64_t small_thrash_threshold = 0;
- uint64_t small_thrash_limit_size = 0;
-
- uint64_t tiny_thrash_threshold_ns = 1 * nsec_per_msec;
- flag = _xzm_development_only_getenv(envp, "MallocXzoneTinyThrashThreshold");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value < UINT32_MAX) {
- tiny_thrash_threshold_ns = (uint64_t)value * nsec_per_msec;
- }
- }
- uint64_t tiny_thrash_threshold =
- (tiny_thrash_threshold_ns * tb_info.denom / tb_info.numer);
-
- uint64_t small_thrash_threshold_ns = 0;
- uint64_t small_thrash_threshold_default_ns = 1 * nsec_per_msec;
-
- // Cap to the size limit for szone SMALL
- small_thrash_limit_size = KiB(16);
-
-#if CONFIG_NANOZONE
- if (nano_config) {
- small_thrash_threshold_ns = small_thrash_threshold_default_ns;
- }
-#else
- (void)small_thrash_threshold_default_ns;
-#endif
-
-#if CONFIG_MTE
- if (memtag_config.enabled) {
- small_thrash_threshold_ns = small_thrash_threshold_default_ns;
- }
-#endif
-
- flag = _xzm_development_only_getenv(envp,
- "MallocXzoneSmallThrashThreshold");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value < UINT32_MAX) {
- small_thrash_threshold_ns = (uint64_t)value * nsec_per_msec;
- }
- }
-
- small_thrash_threshold =
- (small_thrash_threshold_ns * tb_info.denom / tb_info.numer);
-
- flag = _xzm_development_only_getenv(envp,
- "MallocXzoneSmallThrashLimitSize");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value <= XZM_SMALL_BLOCK_SIZE_MAX) {
- small_thrash_limit_size = (uint64_t)value;
- }
- }
-
- uint32_t thread_cache_activation_period = 16384;
- uint32_t thread_cache_activation_contentions = 256;
- uint64_t thread_cache_activation_time = 0;
-#if CONFIG_XZM_THREAD_CACHE
- flag = _xzm_production_getenv(envp, "MallocXzoneThreadCacheActivationPeriod");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value < UINT32_MAX) {
- thread_cache_activation_period = (uint32_t)value;
- }
- }
-
- flag = _xzm_production_getenv(envp,
- "MallocXzoneThreadCacheActivationContentions");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value < UINT32_MAX) {
- thread_cache_activation_contentions = (uint32_t)value;
- }
- }
-
- uint64_t thread_cache_activation_time_ns = 1000 * nsec_per_msec;
- flag = _xzm_production_getenv(envp, "MallocXzoneThreadCacheActivationTime");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value < UINT64_MAX) {
- thread_cache_activation_time_ns = (uint64_t)value * nsec_per_msec;
- }
- }
-
- thread_cache_activation_time =
- (thread_cache_activation_time_ns * tb_info.denom / tb_info.numer);
-#endif // CONFIG_XZM_THREAD_CACHE
-
-#if TARGET_OS_VISION
- bool vision_max_perf = !aggressive_madvise_enabled;
-#endif
-
- uint32_t huge_cache_max_entry_bytes = 0;
-
-#if CONFIG_XZM_DEFERRED_RECLAIM
- mach_vm_reclaim_count_t reclaim_buffer_count =
- XZM_RECLAIM_BUFFER_COUNT_DEFAULT;
- mach_vm_reclaim_count_t max_reclaim_buffer_count =
- XZM_RECLAIM_BUFFER_MAX_COUNT_DEFAULT;
- uint16_t huge_cache_size = XZM_HUGE_CACHE_SIZE_DEFAULT; // number of entries
- huge_cache_max_entry_bytes = XZM_HUGE_CACHE_MAX_ENTRY_BYTES_DEFAULT;
- bool defer_tiny = XZM_DEFERRED_RECLAIM_ENABLED_DEFAULT;
- bool defer_small = XZM_DEFERRED_RECLAIM_ENABLED_DEFAULT;
- bool defer_large = XZM_DEFERRED_RECLAIM_ENABLED_DEFAULT;
-
-#if TARGET_OS_VISION
- if (vision_max_perf) {
- defer_tiny = true;
- defer_small = true;
- defer_large = true;
- huge_cache_size = XZM_HUGE_CACHE_SIZE_ENABLED;
- }
-#endif
-
- flag = _xzm_development_only_getenv(envp, "MallocDeferredReclaim");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 0 || value == 1) {
- defer_tiny = (value == 1);
- defer_small = (value == 1);
- defer_large = (value == 1);
- huge_cache_size = (value == 1) ? XZM_HUGE_CACHE_SIZE_ENABLED : 0;
- } else {
- malloc_report(ASL_LEVEL_ERR, "MallocDeferredReclaim must be one of 0,1 - got %ld\n", value);
- }
- }
-
- flag = _xzm_development_only_getenv(envp, "MallocDeferredReclaimBufferCount");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value <= UINT32_MAX) {
- reclaim_buffer_count = (mach_vm_reclaim_count_t)value;
- }
- }
- flag = _xzm_development_only_getenv(envp, "MallocDeferredReclaimBufferMaxCount");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value <= UINT32_MAX) {
- max_reclaim_buffer_count = (mach_vm_reclaim_count_t)value;
- }
- }
-
- // Round capacities up to page-alignment, or down to the maximum
- reclaim_buffer_count = mach_vm_reclaim_round_capacity(reclaim_buffer_count);
- max_reclaim_buffer_count = mach_vm_reclaim_round_capacity(max_reclaim_buffer_count);
-
- // MallocLargeCache enables both the huge cache and deferred reclamation
- // for all large allocations
- flag = _xzm_production_getenv(envp, "MallocLargeCache");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 0 || value == 1) {
- huge_cache_size = (value == 1) ? XZM_HUGE_CACHE_SIZE_ENABLED : 0;
- defer_large = (value == 1);
- defer_tiny = (value == 1);
- defer_small = (value == 1);
- } else {
- malloc_report(ASL_LEVEL_ERR, "MallocLargeCache must be 0 or 1.\n");
- }
- }
-
-#if CONFIG_NANOZONE && !TARGET_OS_OSX
- // MallocLargeCache is only supported for processes with Nano
- if (!nano_config) {
- defer_large = false;
- huge_cache_size = 0;
-#if MALLOC_TARGET_IOS_ONLY
- defer_tiny = false;
- defer_small = false;
-#endif // MALLOC_TARGET_IOS_ONLY
- }
-#endif // CONFIG_NANOZONE && !TARGET_OS_OSX
-
- flag = _xzm_development_only_getenv(envp, "MallocXzoneHugeCacheSize");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value <= UINT16_MAX) {
- huge_cache_size = (uint16_t)value;
- } else {
- malloc_report(ASL_LEVEL_ERR,
- "xzm: unsupported value for MallocXzoneHugeCacheSize (%ld)",
- value);
- }
- }
-
- flag = _xzm_development_only_getenv(envp, "MallocXzoneHugeCacheMaxEntryBytes");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value <= UINT32_MAX) {
- huge_cache_max_entry_bytes = (uint32_t)value;
- } else {
- malloc_report(ASL_LEVEL_ERR,
- "xzm: unsupported value for MallocXzoneHugeCacheMaxEntryBytes (%ld)",
- value);
- }
- }
-
- flag = _xzm_development_only_getenv(envp, "MallocXzoneDeferTiny");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 0 || value == 1) {
- defer_tiny = (value == 1);
- } else {
- malloc_report(ASL_LEVEL_ERR, "MallocXzoneDeferTiny must be one of 0,1 - got %ld\n", value);
- }
- }
- flag = _xzm_development_only_getenv(envp, "MallocXzoneDeferSmall");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 0 || value == 1) {
- defer_small = (value == 1);
- } else {
- malloc_report(ASL_LEVEL_ERR, "MallocXzoneDeferSmall must be one of 0,1 - got %ld\n", value);
- }
- }
- flag = _xzm_development_only_getenv(envp, "MallocXzoneDeferLarge");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 0 || value == 1) {
- defer_large = (value == 1);
- } else {
- malloc_report(ASL_LEVEL_ERR, "MallocXzoneDeferLarge must be one of "
- "0,1 - got %ld\n", value);
- }
- }
-
- // Known processes do not get deferred reclamation
- if ((security_critical && !security_critical_max_perf) ||
- malloc_space_efficient_enabled) {
- defer_tiny = false;
- defer_small = false;
- defer_large = false;
- huge_cache_size = 0;
- }
-
- if (huge_cache_size && !defer_large) {
- // Deferral of xzones is only supported in conjunction with large/huge
- malloc_report(ASL_LEVEL_ERR, "Huge cache requires deferred reclamation "
- "for large.\n");
- defer_large = true;
- }
-
- if ((defer_tiny || defer_small) && !defer_large) {
- // Deferral of xzones is only supported in conjunction with large/huge
- malloc_report(ASL_LEVEL_ERR, "Deferred reclamation cannot be used for "
- "xzones without large\n");
- defer_large = true;
- }
-#else // CONFIG_XZM_DEFERRED_RECLAIM
- const uint16_t huge_cache_size = 0;
- const bool defer_tiny = false;
- const bool defer_small = false;
- const bool defer_large = false;
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
-
- bool small_freelist = false;
- bool guards_enabled = false;
- bool guards_enabled_data = false;
- uint8_t tiny_run_size = 0;
- uint8_t tiny_guard_density = 0;
- uint8_t small_run_size = 0;
- uint8_t small_guard_density = 0;
- uint8_t batch_size = 0;
-#if TARGET_OS_OSX
- batch_size = !malloc_space_efficient_enabled ? 10 : 0;
- small_freelist = (!security_critical || security_critical_max_perf)
- && !malloc_space_efficient_enabled;
-#elif TARGET_OS_VISION
- batch_size = vision_max_perf ? 10 : 0;
-#endif
-
- // Default config:
- // - Guards enabled and batching disabled for security critical (read,
- // known) processes
- // - Data guarded
- // - Tiny max run size = 8
- // - Tiny density = 64 (20% guard density)
- // - Small max run size = 3
- // - Small density = 32 (11% on 4-page small chunk config)
-
- if (security_critical) {
- guards_enabled = true;
- }
-
- if (security_critical && !security_critical_max_perf) {
- batch_size = 0;
- }
-
- flag = _xzm_development_only_getenv(envp, "MallocXzoneGuarded");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 0 || value == 1) {
- guards_enabled = (bool)value;
- }
- }
-
- flag = malloc_common_value_for_key_copy(bootargs,
- xzone_guard_pages_boot_arg, value_buf, sizeof(value_buf));
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 0 || value == 1) {
- guards_enabled = (bool)value;
- } else {
- malloc_report(ASL_LEVEL_ERR, "%s must be 0 or 1.\n",
- xzone_guard_pages_boot_arg);
- }
- }
-
- if (guards_enabled) {
- guards_enabled_data = true;
- flag = _xzm_development_only_getenv(envp, "MallocXzoneGuardedData");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 0 || value == 1) {
- guards_enabled_data = (bool)value;
- }
- }
-
- tiny_run_size = 8;
- flag = _xzm_development_only_getenv(envp, "MallocXzoneGuardTinyRun");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value > 0) {
- tiny_run_size = value;
- }
- }
-
- tiny_guard_density = 64;
- flag = _xzm_development_only_getenv(envp,
- "MallocXzoneGuardTinyDensity");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value > 0) {
- tiny_guard_density = value;
- }
- }
-
- small_run_size = 3;
- flag = _xzm_development_only_getenv(envp, "MallocXzoneGuardSmallRun");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value > 0) {
- small_run_size = value;
- }
- }
-
- small_guard_density = 32;
- flag = _xzm_development_only_getenv(envp,
- "MallocXzoneGuardSmallDensity");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value > 0) {
- small_guard_density = value;
- }
- }
- }
-
- flag = _xzm_development_only_getenv(envp, "MallocXzoneBatchSize");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value < (1u << XZM_BATCH_SIZE_BITS)) {
- batch_size = value;
- }
- }
-
- flag = _xzm_development_only_getenv(envp, "MallocSmallFreelist");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 0 || value == 1) {
- small_freelist = (bool)value;
- }
- }
-
-#if CONFIG_XZM_CLUSTER_AWARE
- bool per_cluster_segment_groups = false;
-#endif // CONFIG_XZM_CLUSTER_AWARE
-#if TARGET_OS_OSX
- // On macOS, use per-cluster segment groups by default when not space
- // efficient
- per_cluster_segment_groups = !malloc_space_efficient_enabled;
-#elif TARGET_OS_VISION && CONFIG_XZM_CLUSTER_AWARE
- per_cluster_segment_groups = vision_max_perf;
-#endif
- size_t segment_group_ids_count = XZM_SEGMENT_GROUP_IDS_COUNT;
- flag = _xzm_development_only_getenv(envp, "MallocXzoneDataOnly");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 1) {
- segment_group_ids_count = XZM_SEGMENT_GROUP_IDS_COUNT_DATA_ONLY;
- } else if (value != 0) {
- malloc_report(ASL_LEVEL_ERR,
- "MallocXzoneDataOnly must be 0 or 1.\n");
- }
- }
-
-# if CONFIG_XZM_CLUSTER_AWARE
- flag = _xzm_development_only_getenv(envp, "MallocXzonePerClusterSegmentGroups");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 0 || value == 1) {
- per_cluster_segment_groups = value;
- } else {
- malloc_report(ASL_LEVEL_ERR,
- "MallocXzonePerClusterSegmentGroups must be 0 or 1.\n");
- }
- }
-# endif // CONFIG_XZM_CLUSTER_AWARE
-
- // By default, have two allocation fronts
- size_t allocation_front_count = 2;
-
- flag = _xzm_development_only_getenv(envp, "MallocXzoneAllocationFronts");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 1 || value == 2) {
- allocation_front_count = (size_t)value;
- } else {
- malloc_report(ASL_LEVEL_ERR,
- "Unsupported MallocXzoneAllocationFronts\n");
- }
- }
-
- xzm_slot_config_t max_list_config = XZM_SLOT_SINGLE;
-#if TARGET_OS_OSX
- // On macOS, upgrade the max list config to the max slot config unless it
- // is a known process
- if ((!security_critical || security_critical_max_perf) &&
- !malloc_space_efficient_enabled) {
- max_list_config = max_slot_config;
- }
-#endif // TARGET_OS_OSX
- xzm_slot_config_t list_config = XZM_SLOT_SINGLE;
- flag = _xzm_development_only_getenv(envp, "MallocXzoneListConfig");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value < XZM_SLOT_LAST) {
- list_config = (xzm_slot_config_t)value;
- }
- }
-
- flag = _xzm_development_only_getenv(envp, "MallocXzoneMaxListConfig");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value >= 0 && value < XZM_SLOT_LAST && value < max_slot_config) {
- max_list_config = (xzm_slot_config_t)value;
- }
- }
-
- bool segment_deallocate = true;
-#if TARGET_OS_OSX
- // On macOS, skip deallocation unless it is a known process
- if ((!security_critical || security_critical_max_perf) &&
- !malloc_space_efficient_enabled) {
- segment_deallocate = false;
- }
-#endif // TARGET_OS_OSX
- flag = _xzm_development_only_getenv(envp, "MallocXzoneSegmentDeallocate");
- if (flag) {
- long value = strtol(flag, NULL, 10);
- if (value == 0 || value == 1) {
- segment_deallocate = (bool)value;
- }
- }
-
-#if CONFIG_XZM_CLUSTER_AWARE
- // Known processes do not get per-cluster segment groups
- if (security_critical && !security_critical_max_perf) {
- per_cluster_segment_groups = false;
- }
-#endif // CONFIG_XZM_CLUSTER_AWARE
-
- // There is one PTR range group per allocation front, and one PTR_LARGE
- // (when applicable) and DATA each globally
- size_t range_group_count = allocation_front_count +
- (XZM_RANGE_GROUP_COUNT - 1);
-#if !CONFIG_XZM_CLUSTER_AWARE
- size_t segment_group_cluster_count = 1;
-#else
- size_t segment_group_cluster_count = per_cluster_segment_groups ? ncpuclusters : 1;
-#endif // !CONFIG_XZM_CLUSTER_AWARE
-
- size_t segment_group_front_count = segment_group_ids_count;
- if (segment_group_ids_count > XZM_SEGMENT_GROUP_IDS_COUNT_DATA_ONLY &&
- allocation_front_count > 1) {
- // If POINTER_XZONES is active it has N fronts
- segment_group_front_count += allocation_front_count - 1;
- }
-
- size_t segment_group_count = segment_group_front_count *
- segment_group_cluster_count;
-
- size_t metapool_count = XZM_METAPOOL_COUNT;
-
- size_t tail_allocation_offset = sizeof(struct xzm_main_malloc_zone_s);
-
- size_t main_xzones_offset = tail_allocation_offset;
- tail_allocation_offset += sizeof(struct xzm_xzone_s) * xzone_count;
-
- size_t main_xzone_slots_offset = tail_allocation_offset;
- size_t total_slot_count = xzone_count * slot_count;
- tail_allocation_offset +=
- sizeof(struct xzm_xzone_allocation_slot_s) * total_slot_count;
-
- size_t main_xzone_partial_offset = tail_allocation_offset;
- tail_allocation_offset += sizeof(struct xzm_chunk_list_s) * total_slot_count;
-
- size_t bin_sizes_offset = tail_allocation_offset;
- tail_allocation_offset += sizeof(uint64_t) * bin_count;
-
- size_t bin_bucket_counts_offset = tail_allocation_offset;
- tail_allocation_offset += sizeof(uint8_t) * bin_count;
-
- size_t xzone_bin_offsets_offset = tail_allocation_offset;
- tail_allocation_offset += sizeof(uint8_t) * bin_count;
-
- size_t isolation_zones_offset = tail_allocation_offset;
- tail_allocation_offset += sizeof(struct xzm_isolation_zone_s) * xzone_count;
-
- size_t range_groups_offset = tail_allocation_offset;
- tail_allocation_offset +=
- sizeof(struct xzm_range_group_s) * range_group_count;
-
- size_t segment_groups_offset = tail_allocation_offset;
- tail_allocation_offset +=
- sizeof(struct xzm_segment_group_s) * segment_group_count;
-
- size_t metapools_offset = tail_allocation_offset;
- tail_allocation_offset += sizeof(struct xzm_metapool_s) * metapool_count;
-
- size_t segment_table_offset = tail_allocation_offset;
- tail_allocation_offset +=
- sizeof(xzm_segment_table_entry_s) * XZM_SEGMENT_TABLE_ENTRIES;
-
-#if CONFIG_EXTERNAL_METADATA_LARGE
- size_t extended_segment_table_offset = tail_allocation_offset;
- tail_allocation_offset += sizeof(xzm_extended_segment_table_entry_s) *
- XZM_EXTENDED_SEGMENT_TABLE_ENTRIES;
-#endif // CONFIG_EXTERNAL_METADATA_LARGE
-
- size_t total_size = tail_allocation_offset;
-
- int flags = VM_FLAGS_ANYWHERE;
- int tag = 0;
- plat_map_t *plat_map_ptr = NULL;
- tag = VM_MEMORY_MALLOC;
- mach_vm_address_t vm_addr = (mach_vm_address_t)mvm_allocate_plat(0,
- total_size, 0, flags, MALLOC_GUARDED_METADATA, tag, plat_map_ptr);
- if (!vm_addr) {
- xzm_abort("Failed to allocate xzm zone");
- }
-
- xzm_main_malloc_zone_t main = (xzm_main_malloc_zone_t)vm_addr;
- xzm_xzone_t main_xzones_ptr = (xzm_xzone_t)((uintptr_t)main + main_xzones_offset);
- xzm_xzone_allocation_slot_t main_xzone_slots_ptr = (xzm_xzone_allocation_slot_t)
- ((uint8_t*)main + main_xzone_slots_offset);
-
- xzm_chunk_list_t partial_lists = (xzm_chunk_list_t)((uint8_t *)main +
- main_xzone_partial_offset);
-
- xzm_mzone_index_t mzone_idx = XZM_MZONE_INDEX_MAIN;
-
- *main = (struct xzm_main_malloc_zone_s){
- .xzmz_total_size = total_size,
- .xzmz_bucketing_keys = bucketing_keys,
-#if XZM_NARROW_BUCKETING
- .xzmz_narrow_bucketing = narrow_bucketing,
-#endif
- .xzmz_madvise_workaround = madvise_workaround,
- .xzmz_defer_tiny = defer_tiny,
- .xzmz_defer_small = defer_small,
- .xzmz_defer_large = defer_large,
- .xzmz_deallocate_segment = segment_deallocate,
- .xzmz_range_group_count = (uint8_t)range_group_count,
- .xzmz_segment_group_ids_count = segment_group_ids_count,
- .xzmz_segment_group_front_count = segment_group_front_count,
- .xzmz_segment_group_count = segment_group_count,
- .xzmz_metapool_count = (uint8_t)metapool_count,
- .xzmz_allocation_front_count = (uint8_t)allocation_front_count,
- .xzmz_mfm_address = mfm_zone_address(),
- .xzmz_batch_size = batch_size,
- .xzmz_bin_count = (uint8_t)bin_count,
- .xzmz_ptr_bucket_count = (uint8_t)ptr_bucket_count,
- .xzmz_xzone_chunk_threshold = (uint8_t)chunk_threshold,
- .xzmz_xzone_bin_sizes = (uint64_t *)
- ((uintptr_t)main + bin_sizes_offset),
- .xzmz_xzone_bin_bucket_counts = (uint8_t *)
- ((uintptr_t)main + bin_bucket_counts_offset),
- .xzmz_xzone_bin_offsets = (uint8_t *)
- ((uintptr_t)main + xzone_bin_offsets_offset),
- .xzmz_isolation_zones = (struct xzm_isolation_zone_s *)
- ((uintptr_t)main + isolation_zones_offset),
- .xzmz_range_groups = (struct xzm_range_group_s *)
- ((uintptr_t)main + range_groups_offset),
- .xzmz_segment_groups = (struct xzm_segment_group_s *)
- ((uintptr_t)main + segment_groups_offset),
- .xzmz_metapools = (struct xzm_metapool_s *)(
- (uintptr_t)main + metapools_offset),
- .xzmz_segment_table = (xzm_segment_table_entry_s *)
- ((uintptr_t)main + segment_table_offset),
-#if CONFIG_EXTERNAL_METADATA_LARGE
- .xzmz_extended_segment_table_entries =
- XZM_EXTENDED_SEGMENT_TABLE_ENTRIES,
- .xzmz_extended_segment_table = (xzm_extended_segment_table_entry_s *)
- ((uintptr_t)main + extended_segment_table_offset),
-#endif // CONFIG_EXTERNAL_METADATA_LARGE
- .xzmz_max_mzone_idx = mzone_idx,
- .xzmz_mzones_lock = _MALLOC_LOCK_INIT,
- .xzmz_guard_config = {
- .xgpc_enabled = guards_enabled,
- .xgpc_enabled_for_data = guards_enabled_data,
- .xgpc_max_run_tiny = tiny_run_size,
- .xgpc_tiny_guard_density = tiny_guard_density,
- .xgpc_max_run_small = small_run_size,
- .xgpc_small_guard_density = small_guard_density,
- },
- .xzmz_thread_cache_list =
- LIST_HEAD_INITIALIZER(main->xzmz_thread_cache_list),
- };
- _xzm_initialize_const_zone_data(&main->xzmz_base, total_size, mzone_idx,
- xzone_count, slot_count, main_xzones_ptr, main_xzone_slots_ptr,
- NULL, slot_config, slot_threshold, max_slot_config,
- list_upgrade_threshold_single, list_upgrade_threshold_cluster,
- list_upgrade_period, slot_upgrade_threshold_single,
- slot_upgrade_threshold_cluster, slot_upgrade_period,
- tiny_thrash_threshold, small_thrash_threshold,
- small_thrash_limit_size, debug_flags, small_freelist,
- max_list_config, partial_lists);
-#if CONFIG_MTE
- main->xzmz_base.xzz_memtag_config = memtag_config;
-#endif // CONFIG_MTE
-
-#if CONFIG_XZM_DEFERRED_RECLAIM
- if (defer_tiny || defer_small || defer_large || huge_cache_size) {
- bool success = xzm_reclaim_init(main, reclaim_buffer_count,
- max_reclaim_buffer_count);
- if (!success) {
- huge_cache_size = 0;
- main->xzmz_defer_tiny = defer_tiny = false;
- main->xzmz_defer_small = defer_small = false;
- main->xzmz_defer_large = defer_large = false;
- }
- }
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
-
- main->xzmz_base.xzz_thread_cache_enabled = thread_caching;
- main->xzmz_base.xzz_thread_cache_xzone_activation_period =
- thread_cache_activation_period;
- main->xzmz_base.xzz_thread_cache_xzone_activation_contentions =
- thread_cache_activation_contentions;
- main->xzmz_base.xzz_thread_cache_xzone_activation_time =
- thread_cache_activation_time;
-
-#if CONFIG_XZM_THREAD_CACHE
- if (thread_caching) {
- int rc = pthread_key_init_np(__TSD_MALLOC_XZONE_THREAD_CACHE,
- _xzm_xzone_thread_cache_destructor);
- if (os_unlikely(rc)) {
- xzm_abort_with_reason("pthread_key_init_np failed", rc);
- }
- }
-#endif // CONFIG_XZM_THREAD_CACHE
-
- // Initialize the main mzone structures
- for (size_t i = 0; i < bin_count; i++) {
- main->xzmz_xzone_bin_sizes[i] = _xzm_bin_sizes[i];
- }
-
- for (size_t i = 0; i < bin_count; i++) {
- main->xzmz_xzone_bin_bucket_counts[i] = buckets_per_bin;
- }
-
- for (size_t i = 0; i < xzone_count; i++) {
- xzm_isolation_zone_t iz = &main->xzmz_isolation_zones[i];
- _malloc_lock_init(&iz->xziz_lock);
- }
-
- size_t rg_idx = 0;
- for (size_t i = 0; i < XZM_RANGE_GROUP_COUNT; i++) {
- xzm_range_group_id_t rgid = (xzm_range_group_id_t)i;
- size_t rg_fronts = (rgid == XZM_RANGE_GROUP_PTR) ?
- allocation_front_count : 1;
- for (size_t j = 0; j < rg_fronts; j++) {
- xzm_range_group_t rg = &main->xzmz_range_groups[rg_idx];
- rg->xzrg_id = rgid;
- rg->xzrg_front = (xzm_front_index_t)j;
- rg->xzrg_main_ref = main;
- _malloc_lock_init(&rg->xzrg_lock);
-
- rg_idx++;
- }
- }
-
- if (has_vm_user_ranges) {
- xzm_main_malloc_zone_init_range_groups(main);
- }
-
- if (!main->xzmz_use_ranges) {
- // If we don't actually have ranges we can't do allocation fronts
- allocation_front_count = 1;
- main->xzmz_allocation_front_count = 1;
- }
-
- for (size_t i = 0; i < segment_group_count; i++) {
- xzm_segment_group_t sg = &main->xzmz_segment_groups[i];
- size_t sg_front_idx = i % segment_group_front_count;
- sg->xzsg_id = sg_front_idx < XZM_SEGMENT_GROUP_POINTER_XZONES ?
- (xzm_segment_group_id_t)sg_front_idx :
- XZM_SEGMENT_GROUP_POINTER_XZONES;
-
- _malloc_lock_init(&sg->xzsg_lock);
- _malloc_lock_init(&sg->xzsg_alloc_lock);
-
- if (_xzm_segment_group_id_is_data(sg->xzsg_id)) {
- sg->xzsg_range_group =
- &main->xzmz_range_groups[XZM_RANGE_GROUP_DATA];
- } else if (sg->xzsg_id == XZM_SEGMENT_GROUP_POINTER_LARGE) {
-
- if (!sg->xzsg_range_group) {
- xzm_front_index_t front = _xzm_random_front_index(front_random,
- allocation_front_count, XZM_XZONE_INDEX_INVALID);
- size_t rg_idx = XZM_RANGE_GROUP_PTR + front;
- sg->xzsg_range_group = &main->xzmz_range_groups[rg_idx];
- }
- } else {
- xzm_assert(sg->xzsg_id == XZM_SEGMENT_GROUP_POINTER_XZONES);
- size_t ptr_front_idx =
- sg_front_idx - XZM_SEGMENT_GROUP_POINTER_XZONES;
- size_t rg_idx = XZM_RANGE_GROUP_PTR + ptr_front_idx;
- xzm_debug_assert(rg_idx < main->xzmz_range_group_count);
- sg->xzsg_range_group = &main->xzmz_range_groups[rg_idx];
- }
-
- sg->xzsg_front = sg->xzsg_range_group->xzrg_front;
- sg->xzsg_main_ref = main;
-
- if (i == XZM_SEGMENT_GROUP_DATA_LARGE) {
- // Huge allocations are all routed to the same segment group,
- // so only initialize the huge cache for that one segment group
- sg->xzsg_cache = (struct xzm_segment_cache_s){
- .xzsc_max_count = huge_cache_size,
- .xzsc_count = 0,
- // TODO: consider limiting the maximum entry size based on the
- // reclaim threshold
- .xzsc_max_entry_slices =
- (huge_cache_max_entry_bytes / XZM_SEGMENT_SLICE_SIZE),
- };
- TAILQ_INIT(&sg->xzsg_cache.xzsc_head);
- _malloc_lock_init(&sg->xzsg_cache.xzsc_lock);
- }
-
- for (size_t j = 0; j < XZM_SPAN_QUEUE_COUNT; j++) {
- sg->xzsg_spans[j].xzsq_slice_count =
- (xzm_slice_count_t)_xzm_span_queue_slice_counts[j];
- }
- }
-
- // NOTE: The order of these metapool allocators matters for the fork lock,
- // we have to grab the metadata metapool last, since the other metapools
- // acquire its lock after taking their own
- xzm_metapool_t metadata_mp = &main->xzmz_metapools[XZM_METAPOOL_METADATA];
- uint32_t mp_metadata_size = MAX(sizeof(struct xzm_metapool_block_s),
- sizeof(struct xzm_metapool_slab_s));
- uint32_t mp_metadata_slab_size = KiB(16); // arbitrary slab size
- xzm_metapool_init(metadata_mp, XZM_METAPOOL_METADATA, VM_MEMORY_MALLOC,
- mp_metadata_slab_size, mp_metadata_size, mp_metadata_size, NULL);
-
- xzm_metapool_t segment_mp = &main->xzmz_metapools[XZM_METAPOOL_SEGMENT];
- xzm_metapool_init(segment_mp, XZM_METAPOOL_SEGMENT, VM_MEMORY_MALLOC,
- XZM_METAPOOL_SEGMENT_SLAB_SIZE, XZM_METAPOOL_SEGMENT_ALIGN,
- XZM_METAPOOL_SEGMENT_BLOCK_SIZE, metadata_mp);
-
- xzm_metapool_t leaf_table_mp = &main->xzmz_metapools[XZM_METAPOOL_SEGMENT_TABLE];
- xzm_metapool_init(leaf_table_mp, XZM_METAPOOL_SEGMENT_TABLE,
- VM_MEMORY_MALLOC, XZM_METAPOOL_SEGMENT_TABLE_SLAB_SIZE,
- XZM_SEGMENT_TABLE_ALIGN, XZM_SEGMENT_TABLE_SIZE, metadata_mp);
-
- xzm_metapool_t mzone_idx_mp = &main->xzmz_metapools[XZM_METAPOOL_MZONE_IDX];
- xzm_metapool_init(mzone_idx_mp, XZM_METAPOOL_MZONE_IDX, VM_MEMORY_MALLOC,
- XZM_METAPOOL_MZIDX_SLAB_SIZE, XZM_METAPOOL_MZIDX_BLOCK_ALIGN,
- XZM_METAPOOL_MZIDX_BLOCK_SIZE, NULL);
-
- xzm_metapool_t tc_mp = &main->xzmz_metapools[XZM_METAPOOL_THREAD_CACHE];
-
- // XXX Needs to be re-worked if we allow per-bin bucket counts
- xzm_debug_assert(_xzm_bin_sizes[XZM_THREAD_CACHE_BINS - 1] ==
- XZM_THREAD_CACHE_THRESHOLD);
- size_t thread_cache_xzone_count =
- 1 + (XZM_THREAD_CACHE_BINS * buckets_per_bin);
-
- main->xzmz_base.xzz_thread_cache_xzone_count =
- (uint8_t)thread_cache_xzone_count;
-
- size_t thread_cache_block_size = sizeof(struct xzm_thread_cache_s) +
- (thread_cache_xzone_count * sizeof(xzm_xzone_thread_cache_u));
- xzm_metapool_init(tc_mp, XZM_METAPOOL_THREAD_CACHE,
- VM_MEMORY_MALLOC_NANO,
- XZM_METAPOOL_THREAD_CACHE_SLAB_SIZE, 0,
- (uint32_t)thread_cache_block_size, metadata_mp);
-
- // Initialize the per-mzone structures
- bool data_only = (main->xzmz_segment_group_ids_count ==
- XZM_SEGMENT_GROUP_IDS_COUNT_DATA_ONLY);
- _xzm_initialize_xzone_data(&main->xzmz_base, list_config, &main->xzmz_guard_config, front_random, data_only);
-
- flag = _xzm_production_getenv(envp, "MallocReportConfig");
- if (flag) {
- // Report our config to stderr for debugging purposes
-#if CONFIG_XZM_DEFERRED_RECLAIM
- int vm_reclaim_enabled;
- size_t vm_reclaim_enabled_size = sizeof(vm_reclaim_enabled);
- int kr = sysctlbyname("vm.reclaim.enabled",
- &vm_reclaim_enabled, &vm_reclaim_enabled_size, 0, 0);
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
- malloc_report(ASL_LEVEL_INFO,
- "XZM Config:\n"
- "\tData Only: %d\n"
- "\tAllocation Fronts: %d\n"
-#if XZM_NARROW_BUCKETING
- "\tNarrow Bucketing: %d\n"
-#endif
- "\tGuards Enabled: %d\n"
- "\tScribble: %d\n"
- "\tTiny/Small Batch Max: %d\n"
-#if CONFIG_XZM_DEFERRED_RECLAIM
- "\tDefer Tiny: %d\n"
- "\tDefer Small: %d\n"
- "\tDefer Large: %d\n"
- "\tHuge Cache Size: %d\n"
- "\tHuge Cache Max Entry Bytes: %u\n"
- "\tReclaim Buffer Count: %u/%u (%s)\n"
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
- "\tSmall Freelist: %u\n"
-#if CONFIG_VM_USER_RANGES || CONFIG_MACOS_RANGES
- "\tData Range: 0x%llx/%lu\n"
- "\tPointer Range 1: 0x%llx/%lu\n"
- "\tPointer Range 2: 0x%llx/%lu\n"
-#endif // CONFIG_VM_USER_RANGES || CONFIG_MACOS_RANGES
- "\tEarly Allocator: %s\n"
- "\tSegment Deallocate: %u\n"
-#if CONFIG_MTE
- "\tMTE (enabled/data/max size): %d/%d/%llu\n"
-#endif
- "\tInitial Slot Config: %s/%s (Chunk, Size Thresholds: %u, %u)\n"
- "\tInitial List Config: %s/%s\n"
- "\tList Upgrade Thresholds: %d/%d, %d/%d\n"
- "\tSlot Upgrade Thresholds: %d/%d, %d/%d\n"
- "\tTiny Thrash Threshold: %llu ms\n"
- "\tSmall Thrash Threshold: %llu ms, %llu bytes\n"
-#if CONFIG_XZM_THREAD_CACHE
- "\tThread Caching: %s (%u allocs, %u contentions, %llu ms)\n"
-#endif
- "\tPointer Bucket Count: %lu\n",
- data_only,
- (int)allocation_front_count,
-#if XZM_NARROW_BUCKETING
- main->xzmz_narrow_bucketing,
-#endif
- guards_enabled, !!(debug_flags & MALLOC_DO_SCRIBBLE),
- batch_size,
-#if CONFIG_XZM_DEFERRED_RECLAIM
- defer_tiny, defer_small, defer_large,
- huge_cache_size, huge_cache_max_entry_bytes,
- reclaim_buffer_count, max_reclaim_buffer_count,
- (kr || !vm_reclaim_enabled) ? "DISABLED": "ENABLED",
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
- small_freelist,
-#if CONFIG_VM_USER_RANGES || CONFIG_MACOS_RANGES
- !malloc_internal_security_policy ? 0 :
- main->xzmz_range_groups[XZM_RANGE_GROUP_DATA].xzrg_base,
- main->xzmz_range_groups[XZM_RANGE_GROUP_DATA].xzrg_size,
- !malloc_internal_security_policy ? 0 :
- main->xzmz_range_groups[XZM_RANGE_GROUP_PTR + 0].xzrg_base,
- main->xzmz_range_groups[XZM_RANGE_GROUP_PTR + 0].xzrg_size,
- !malloc_internal_security_policy ||
- main->xzmz_allocation_front_count < 2 ? 0 :
- main->xzmz_range_groups[XZM_RANGE_GROUP_PTR + 1].xzrg_base,
- main->xzmz_allocation_front_count < 2 ? 0 :
- main->xzmz_range_groups[XZM_RANGE_GROUP_PTR + 1].xzrg_size,
-#endif // CONFIG_VM_USER_RANGES || CONFIG_MACOS_RANGES
- main->xzmz_mfm_address ? "enabled" : "disabled",
- segment_deallocate,
-#if CONFIG_MTE
- memtag_config.enabled, memtag_config.tag_data,
- memtag_config.max_block_size,
-#endif
- _xzm_slot_config_to_string(slot_config),
- _xzm_slot_config_to_string(max_slot_config),
- chunk_threshold, slot_threshold,
- _xzm_slot_config_to_string(list_config),
- _xzm_slot_config_to_string(max_list_config),
- list_upgrade_threshold_single,
- list_upgrade_period, list_upgrade_threshold_cluster,
- list_upgrade_period, slot_upgrade_threshold_single,
- slot_upgrade_period, slot_upgrade_threshold_cluster,
- slot_upgrade_period,
- tiny_thrash_threshold_ns / nsec_per_msec,
- small_thrash_threshold_ns / nsec_per_msec,
- small_thrash_limit_size,
-#if CONFIG_XZM_THREAD_CACHE
- thread_caching ? "enabled" : "disabled",
- thread_cache_activation_period,
- thread_cache_activation_contentions,
- thread_cache_activation_time_ns / nsec_per_msec,
-#endif // CONFIG_XZM_THREAD_CACHE
- (unsigned long)ptr_bucket_count);
- }
-
- return &main->xzmz_base.xzz_basic_zone;
-}
-
-MALLOC_NOEXPORT
-malloc_zone_t *
-xzm_malloc_zone_create(unsigned debug_flags, xzm_main_malloc_zone_t main_ref)
-{
- size_t tail_allocation_offset = sizeof(struct xzm_malloc_zone_s);
- size_t xzones_offset = tail_allocation_offset;
-
- uint8_t xzone_count = main_ref->xzmz_base.xzz_xzone_count;
- uint8_t slot_count = main_ref->xzmz_base.xzz_slot_count;
- tail_allocation_offset += sizeof(struct xzm_xzone_s) * xzone_count;
- size_t slots_offset = tail_allocation_offset;
-
- size_t total_slot_count = xzone_count * slot_count;
- tail_allocation_offset += sizeof(struct xzm_xzone_allocation_slot_s) *
- total_slot_count;
-
- size_t xzone_partial_offset = tail_allocation_offset;
- tail_allocation_offset += sizeof(struct xzm_chunk_list_s) * total_slot_count;
-
- // Try to pop a previously destroyed mzone index from the reuse list,
- // otherwise use the next unused value
- _malloc_lock_lock(&main_ref->xzmz_mzones_lock);
- xzm_mzone_index_t mzone_idx;
- xzm_reused_mzone_index_t reuse = SLIST_FIRST(&main_ref->xzmz_reusable_mzidxq);
- if (reuse) {
- SLIST_REMOVE_HEAD(&main_ref->xzmz_reusable_mzidxq, xrmi_mzone_entry);
- mzone_idx = reuse->xrmi_mzone_idx;
- xzm_metapool_t mp = &main_ref->xzmz_metapools[XZM_METAPOOL_MZONE_IDX];
- xzm_metapool_free(mp, reuse);
- } else if (main_ref->xzmz_max_mzone_idx == XZM_MZONE_INDEX_MAX) {
- mzone_idx = XZM_MZONE_INDEX_INVALID;
- } else {
- main_ref->xzmz_max_mzone_idx += 1;
- mzone_idx = main_ref->xzmz_max_mzone_idx;
- }
- _malloc_lock_unlock(&main_ref->xzmz_mzones_lock);
-
- if (os_unlikely(mzone_idx == XZM_MZONE_INDEX_INVALID)) {
- return NULL;
- }
-
- // allocate a new mzone
- int flags = VM_FLAGS_ANYWHERE;
- int tag = 0;
- plat_map_t *plat_map_ptr = NULL;
- tag = VM_MEMORY_MALLOC;
-
- mach_vm_address_t vm_addr = (mach_vm_address_t)mvm_allocate_plat(0,
- tail_allocation_offset, 0, flags, MALLOC_GUARDED_METADATA, tag,
- plat_map_ptr);
- if (!vm_addr) {
- return NULL;
- }
- xzm_malloc_zone_t new_zone = (xzm_malloc_zone_t)vm_addr;
-
- xzm_chunk_list_t partial_lists = (xzm_chunk_list_t)((uint8_t *)new_zone +
- xzone_partial_offset);
-
- // Currently we copy the slot upgrade parameters from the main zone into
- // newly created zones. To limit fragmentation, we could cap the new
- // zones at single/per cluster. Pre-xzm, new malloc zones were all
- // scalable zones (you could only have one nano zone), so capping
- // scalability of non-default zones has prior art
- _xzm_initialize_const_zone_data(new_zone, tail_allocation_offset, mzone_idx,
- xzone_count, slot_count,
- (xzm_xzone_t)((uintptr_t)new_zone + xzones_offset),
- (xzm_xzone_allocation_slot_t)((uintptr_t)new_zone + slots_offset),
- main_ref, main_ref->xzmz_base.xzz_initial_slot_config,
- main_ref->xzmz_base.xzz_slot_initial_threshold,
- main_ref->xzmz_base.xzz_max_slot_config,
- main_ref->xzmz_base.xzz_list_upgrade_threshold[0],
- main_ref->xzmz_base.xzz_list_upgrade_threshold[1],
- main_ref->xzmz_base.xzz_list_upgrade_period,
- main_ref->xzmz_base.xzz_slot_upgrade_threshold[0],
- main_ref->xzmz_base.xzz_slot_upgrade_threshold[1],
- main_ref->xzmz_base.xzz_slot_upgrade_period,
- main_ref->xzmz_base.xzz_tiny_thrash_threshold,
- /* small_thrash_threshold */ 0, /* small_thrash_limit_size */ 0,
- debug_flags, main_ref->xzmz_base.xzz_small_freelist_enabled,
- main_ref->xzmz_base.xzz_max_list_config, partial_lists);
-#if CONFIG_MTE
- // TODO: Do we want a different MTE config in new zones?
- new_zone->xzz_memtag_config = main_ref->xzmz_base.xzz_memtag_config;
-#endif // CONFIG_MTE
-
- xzm_debug_assert(new_zone != NULL);
-
- bool data_only = (main_ref->xzmz_segment_group_ids_count ==
- XZM_SEGMENT_GROUP_IDS_COUNT_DATA_ONLY);
- // Currently new zones are never nano, so start all new xzone slot configs
- // at single.
- // TODO: if the envvar is set (MallocXzoneInitialSlotConfig), use
- // that slot config in non-default zones
- _xzm_initialize_xzone_data(new_zone, XZM_SLOT_SINGLE,
- &main_ref->xzmz_guard_config, NULL, data_only);
- LIST_INIT(&new_zone->xzz_chunkq_large);
-
- return &new_zone->xzz_basic_zone;
-}
-
-#endif // CONFIG_XZONE_MALLOC