Loading...
--- libmalloc/libmalloc-792.41.1/src/xzone_malloc/xzone_inline_internal.h
+++ /dev/null
@@ -1,1350 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2022, Microsoft Research, Daan Leijen
-Copyright © 2025 Apple Inc.
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" in the same directory as this file.
------------------------------------------------------------------------------*/
-
-#if CONFIG_XZONE_MALLOC
-
-#ifndef __XZONE_INLINE_INTERNAL_H__
-#define __XZONE_INLINE_INTERNAL_H__
-
-#if !__has_feature(bounds_safety)
-
-#define xzm_abort(msg) ({ \
- _os_set_crash_log_cause_and_message(0, "BUG IN LIBMALLOC: " msg); \
- __builtin_trap(); \
-})
-#define xzm_abort_with_reason(msg, reason) ({ \
- _os_set_crash_log_cause_and_message((reason), "BUG IN LIBMALLOC: " msg); \
- __builtin_trap(); \
-})
-#define xzm_client_abort(msg) ({ \
- _os_set_crash_log_cause_and_message(0, "BUG IN CLIENT OF LIBMALLOC: " msg); \
- __builtin_trap(); \
-})
-#define xzm_client_abort_with_reason(msg, reason) ({ \
- _os_set_crash_log_cause_and_message((reason), "BUG IN CLIENT OF LIBMALLOC: " msg); \
- __builtin_trap(); \
-})
-
-#define _xzm_assert_stringify(x) #x
-#define xzm_assert_stringify(x) _xzm_assert_stringify(x)
-
-// mimalloc: mi_assert
-#define xzm_assert(pred) \
- if (os_unlikely(!(pred))) { \
- xzm_abort("malloc assertion \"" #pred "\" failed " \
- "(" __FILE__ ":" xzm_assert_stringify(__LINE__) ")"); \
- }
-
-// mimalloc: mi_assert_internal
-#ifdef DEBUG
-#define xzm_debug_assert xzm_assert
-#define xzm_debug_abort xzm_abort
-#define xzm_debug_abort_with_reason xzm_abort_with_reason
-#ifndef __assert_only
-#define __assert_only
-#endif
-#else // DEBUG
-#define xzm_debug_assert(...)
-#define xzm_debug_abort(...)
-#define xzm_debug_abort_with_reason(...)
-#ifndef __assert_only
-#define __assert_only __unused
-#endif
-#endif // DEBUG
-
-MALLOC_INLINE
-static void
-_xzm_corruption_detected(void *corrupt_block)
-{
- // TODO: load the corrupt value into a register so it also appears in crash
- // reports
- xzm_client_abort_with_reason("memory corruption of free block",
- corrupt_block);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_main_malloc_zone_t
-_xzm_malloc_zone_main(xzm_malloc_zone_t zone)
-{
- return zone->xzz_main_ref ?: (xzm_main_malloc_zone_t)zone;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_malloc_zone_is_main(xzm_malloc_zone_t zone)
-{
- return !zone->xzz_main_ref;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_malloc_zone_is_xzm(malloc_zone_t *zone)
-{
- return zone->version >= 14 &&
- zone->introspect->zone_type == MALLOC_ZONE_TYPE_XZONE;
-}
-
-#pragma mark magic math
-
-// see https://lemire.me/blog/2019/02/20/more-fun-with-fast-remainders-when-the-divisor-is-a-constant/
-//
-// Implementation copied from zalloc
-#define XZM_MAGIC_QUO(s) (((1ull << 32) - 1) / (uint64_t)(s) + 1)
-#define XZM_MAGIC_ALIGNED(s) (~0u / (uint32_t)(s) + 1)
-
-// Returns (offs / size) if offs is small enough and magic = XZM_MAGIC_QUO(size)
-static inline uint32_t
-XZM_FAST_QUO(uint64_t offs, uint64_t __unused size, uint64_t magic)
-{
- uint32_t quo = (offs * magic) >> 32;
- xzm_debug_assert(offs / size == quo);
- return quo;
-}
-
-// Returns (offs % size) if offs is small enough and magic ==
-// XZM_MAGIC_QUO(size)
-static inline uint32_t
-XZM_FAST_MOD(uint64_t offs, uint64_t magic, uint64_t size)
-{
- uint32_t lowbits = (uint32_t)(offs * magic);
-
- uint32_t mod = (lowbits * size) >> 32;
- xzm_debug_assert(offs % size == mod);
- return mod;
-}
-
-// Returns whether (offs % size) == 0 if offs is small enough and magic ==
-// XZM_MAGIC_ALIGNED(size)
-static inline bool
-XZM_FAST_ALIGNED(uint64_t offs, uint64_t __unused size, uint32_t magic)
-{
- bool aligned = (uint32_t)(offs * magic) < magic;
- xzm_debug_assert(aligned == ((offs % size) == 0));
- return aligned;
-}
-
-#pragma mark metadata helpers
-
-// mimalloc: mi_segment_map_index_of
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static size_t
-_xzm_segment_table_index_of(const void *segment_body, size_t *extended_idx)
-{
- uintptr_t segment_bits = (uintptr_t)segment_body;
- if (segment_bits >= XZM_LIMIT_ADDRESS) {
- *extended_idx = 0;
- return XZM_SEGMENT_TABLE_ENTRIES;
- }
-
- uintptr_t segindex = segment_bits / XZM_SEGMENT_SIZE;
-#if CONFIG_EXTERNAL_METADATA_LARGE
- // The segment map index we return in the large address space is the index
- // into the bottom level table, the extended_idx is the index into top table
- // (setting extended_idx = 0 implies that the body is in the low 64GB of VA)
- *extended_idx = segindex / XZM_SEGMENT_TABLE_ENTRIES;
- xzm_debug_assert(*extended_idx < XZM_EXTENDED_SEGMENT_TABLE_ENTRIES);
- return segindex % XZM_SEGMENT_TABLE_ENTRIES;
-#else
- *extended_idx = 0;
- xzm_debug_assert(segindex < XZM_SEGMENT_TABLE_ENTRIES);
- return segindex;
-#endif // CONFIG_EXTERNAL_METADATA_LARGE
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_segment_t
-_xzm_segment_table_entry_to_segment(xzm_segment_table_entry_s entry)
-{
- return (xzm_segment_t)
- ((uintptr_t)entry.xste_val << XZM_METAPOOL_SEGMENT_BLOCK_SHIFT);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_segment_table_entry_s
-_xzm_segment_to_segment_table_entry(xzm_segment_t segment, bool normal)
-{
- xzm_debug_assert((uintptr_t)segment % XZM_METAPOOL_SEGMENT_ALIGN == 0);
- // TODO: On MacOS (47 bit address space), the upper portion of the address
- // space won't fit in this encoding
- xzm_assert(((uintptr_t)segment >> XZM_METAPOOL_SEGMENT_BLOCK_SHIFT) <
- XZM_SEGMENT_TABLE_LIMIT_ENTRY);
- return (xzm_segment_table_entry_s) {
- .xste_val = (uint32_t)
- ((uintptr_t)segment >> XZM_METAPOOL_SEGMENT_BLOCK_SHIFT),
- .xste_normal = normal,
- };
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_segment_table_entry_s *
-_xzm_ptr_to_table_entry(const void *segment_body,
- xzm_main_malloc_zone_t main)
-{
- size_t ext_idx = 0;
- size_t index = _xzm_segment_table_index_of(segment_body, &ext_idx);
-#if CONFIG_EXTERNAL_METADATA_LARGE
- if (ext_idx == 0) {
- if (os_unlikely(index >= XZM_SEGMENT_TABLE_ENTRIES)) {
- // Pointer out of bounds, greater than XZM_LIMIT_ADDRESS
- return NULL;
- }
- // This pointer is in the first 64GB of VA, so it comes directly from
- // the segment table in the main zone
- xzm_debug_assert((uintptr_t)segment_body < XZM_SEGMENT_TABLE_COVERAGE);
- return &main->xzmz_segment_table[index];
- } else if (ext_idx >= XZM_EXTENDED_SEGMENT_TABLE_ENTRIES) {
- return NULL;
- } else {
- xzm_segment_table_entry_s *leaf_table;
- xzm_extended_segment_table_entry_s *map;
- map = main->xzmz_extended_segment_table;
- xzm_debug_assert(map != 0);
- leaf_table = (void *)((uintptr_t)(map[ext_idx].xeste_val) *
- XZM_SEGMENT_TABLE_ALIGN);
- if (leaf_table == NULL) {
- // there are no segments in the given 64GB span
- return NULL;
- }
- return &leaf_table[index];
- }
-#else
- if (index >= XZM_SEGMENT_TABLE_ENTRIES) {
- return NULL;
- }
- return &main->xzmz_segment_table[index];
-#endif // CONFIG_EXTERNAL_METADATA_LARGE
-
-}
-
-// mimalloc: _mi_segment_of
-// Note: This will allow inner pointers, or any pointer inside the (4MB) segment
-// granule of an allocated segment
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_segment_t
-xzm_segment_table_query(xzm_main_malloc_zone_t main, const void *ptr)
-{
- xzm_segment_table_entry_s *leaf_entry;
- leaf_entry = _xzm_ptr_to_table_entry(ptr, main);
- if (leaf_entry == NULL) {
- return NULL;
- }
- return _xzm_segment_table_entry_to_segment(*leaf_entry);
-}
-
-// mimalloc: _mi_page_segment
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_segment_t
-_xzm_segment_for_slice(xzm_malloc_zone_t zone, xzm_slice_t slice)
-{
- xzm_segment_t segment = (xzm_segment_t)
- ((uintptr_t)slice & ~(XZM_METAPOOL_SEGMENT_BLOCK_SIZE - 1));
- xzm_debug_assert(!segment || (slice >= segment->xzs_slices &&
- slice < (segment->xzs_slices + segment->xzs_slice_entry_count)));
- return segment;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_segment_group_t
-_xzm_segment_group_for_slice(xzm_malloc_zone_t zone, xzm_slice_t slice)
-{
- xzm_segment_t segment = _xzm_segment_for_slice(zone, slice);
- return segment->xzs_segment_group;
-}
-
-// mimalloc: mi_slice_index
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_slice_count_t
-_xzm_slice_index(xzm_segment_t segment, xzm_slice_t slice)
-{
- xzm_debug_assert((uintptr_t)slice >= (uintptr_t)segment->xzs_slices);
- ptrdiff_t index = slice - segment->xzs_slices;
- xzm_debug_assert(index < (ptrdiff_t)segment->xzs_slice_entry_count);
- return (xzm_slice_count_t)index;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static const char *
-_xzm_range_group_id_to_string(xzm_range_group_id_t id)
-{
- switch(id) {
- case XZM_RANGE_GROUP_DATA:
- return "data";
- case XZM_RANGE_GROUP_PTR:
- return "pointer";
- default:
- return "unknown";
- }
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static const char *
-_xzm_segment_group_id_to_string(xzm_segment_group_id_t id)
-{
- switch(id) {
- case XZM_SEGMENT_GROUP_DATA:
- return "data";
- case XZM_SEGMENT_GROUP_DATA_LARGE:
- return "data_large";
- case XZM_SEGMENT_GROUP_POINTER_XZONES:
- return "pointer_xzones";
- case XZM_SEGMENT_GROUP_POINTER_LARGE:
- return "pointer_large";
- default:
- return "unknown";
- }
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_segment_group_id_is_data(xzm_segment_group_id_t id)
-{
- switch(id) {
- case XZM_SEGMENT_GROUP_DATA:
- case XZM_SEGMENT_GROUP_DATA_LARGE:
- return true;
- case XZM_SEGMENT_GROUP_POINTER_XZONES:
- case XZM_SEGMENT_GROUP_POINTER_LARGE:
- return false;
- default:
- xzm_abort_with_reason("unknown segment group id", id);
- }
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static uint8_t *
-_xzm_segment_slice_index_start(xzm_segment_t segment, xzm_slice_count_t idx)
-{
- return (uint8_t *)((uintptr_t)segment->xzs_segment_body +
- (idx * XZM_SEGMENT_SLICE_SIZE));
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static uint8_t *
-_xzm_segment_slice_start(xzm_segment_t segment, xzm_slice_t slice)
-{
- return _xzm_segment_slice_index_start(segment,
- _xzm_slice_index(segment, slice));
- // TODO: mimalloc offset optimization for small block sizes?
-}
-
-// FIXME: `zone` parameter isn't used in _xzm_segment_for_slice(),
-// _xzm_slice_start(), _xzm_chunk_start(), _xzm_chunk_start_ptr() and can be
-// removed.
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static uint8_t *
-_xzm_slice_start(xzm_malloc_zone_t zone, xzm_slice_t slice)
-{
- return _xzm_segment_slice_start(_xzm_segment_for_slice(zone, slice), slice);
-}
-
-// mimalloc: _mi_page_start
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static uintptr_t
-_xzm_chunk_start(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
- size_t *chunk_size_out)
-{
- if (chunk_size_out) {
- switch (chunk->xzc_bits.xzcb_kind) {
- case XZM_SLICE_KIND_TINY_CHUNK:
- *chunk_size_out = XZM_TINY_CHUNK_SIZE;
- break;
- case XZM_SLICE_KIND_SMALL_CHUNK:
- *chunk_size_out = XZM_SMALL_CHUNK_SIZE;
- break;
- case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
- *chunk_size_out = XZM_SMALL_FREELIST_CHUNK_SIZE;
- break;
- case XZM_SLICE_KIND_LARGE_CHUNK:
- case XZM_SLICE_KIND_HUGE_CHUNK:
- *chunk_size_out = ((size_t)chunk->xzcs_slice_count) <<
- XZM_SEGMENT_SLICE_SHIFT;
- break;
- default:
- xzm_abort_with_reason("asking for start of chunk with invalid kind",
- (unsigned)chunk->xzc_bits.xzcb_kind);
- }
- }
-
- return (uintptr_t)_xzm_slice_start(zone, (xzm_slice_t)chunk);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static uint8_t *
-_xzm_chunk_start_ptr(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
- size_t *chunk_size_out)
-{
- xzm_debug_assert(chunk_size_out);
- uintptr_t ptr = _xzm_chunk_start(zone, chunk, chunk_size_out);
- return (uint8_t *)ptr;
-}
-
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_slice_count_t
-_xzm_segment_slice_count(xzm_segment_t segment)
-{
- return segment->xzs_slice_count;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static size_t
-_xzm_segment_size(xzm_segment_t segment)
-{
- xzm_slice_count_t body_slice_count = _xzm_segment_slice_count(segment);
- return (size_t)body_slice_count << XZM_SEGMENT_SLICE_SHIFT;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static uint8_t *
-_xzm_segment_start(xzm_segment_t segment)
-{
- return _xzm_segment_slice_index_start(segment, 0);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static size_t
-_xzm_segment_slice_offset_of(xzm_segment_t segment, uintptr_t ptr)
-{
- const ptrdiff_t diff = ptr - (uintptr_t)_xzm_segment_start(segment);
-#ifdef DEBUG
- // Huge segments aren't always a multiple of the segment size, so it's
- // possible for malloc_size() to be passed a pointer that is within a
- // segment granule, but isn't within the segment that owns that granule. We
- // need to not crash in the debug dylib when that happens
- size_t rounded_size = roundup(_xzm_segment_size(segment), XZM_SEGMENT_SIZE);
- xzm_debug_assert(diff >= 0 && diff < (ptrdiff_t)rounded_size);
-#endif // DEBUG
- return (size_t)diff;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_slice_count_t
-_xzm_segment_slice_offset_index(xzm_segment_t segment, size_t offset)
-{
- return (xzm_slice_count_t)(offset >> XZM_SEGMENT_SLICE_SHIFT);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_slice_count_t
-_xzm_segment_slice_index_of(xzm_segment_t segment, uintptr_t ptr)
-{
- size_t offset = _xzm_segment_slice_offset_of(segment, ptr);
- return _xzm_segment_slice_offset_index(segment, offset);
-}
-
-// mimalloc: _mi_segment_page_of
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_slice_t
-_xzm_segment_slice_of(xzm_segment_t segment, uintptr_t ptr)
-{
- xzm_slice_count_t idx = _xzm_segment_slice_index_of(segment, ptr);
- if (os_likely(idx < segment->xzs_slice_entry_count)) {
- return &segment->xzs_slices[idx];
- } else {
- return NULL;
- }
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static uint8_t *
-_xzm_segment_slice_index_end(xzm_segment_t segment, xzm_slice_count_t idx)
-{
- return _xzm_segment_slice_index_start(segment, idx + 1);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static uint8_t *
-_xzm_segment_slice_end_of(xzm_segment_t segment, uintptr_t ptr)
-{
- xzm_slice_count_t idx = _xzm_segment_slice_index_of(segment, ptr);
- return _xzm_segment_slice_index_end(segment, idx);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static uint8_t *
-_xzm_segment_end(xzm_segment_t segment)
-{
- return _xzm_segment_slice_index_end(segment, segment->xzs_slice_count - 1);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static uint8_t *
-_xzm_segment_slice_start_of(xzm_segment_t segment, uintptr_t ptr)
-{
- xzm_slice_count_t idx = _xzm_segment_slice_index_of(segment, ptr);
- return _xzm_segment_slice_index_start(segment, idx);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_slice_t
-_xzm_segment_slices_begin(xzm_segment_t segment)
-{
- return &segment->xzs_slices[0];
-}
-
-// mimalloc: mi_segment_slices_end
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_slice_t
-_xzm_segment_slices_end(xzm_segment_t segment)
-{
- // Return a one-past-the-end pointer without immediately trapping
- return &segment->xzs_slices[segment->xzs_slice_entry_count];
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static const char *
-_xzm_segment_kind_to_string(xzm_segment_kind_t kind)
-{
- switch(kind) {
- case XZM_SEGMENT_KIND_NORMAL:
- return "normal_segment";
- case XZM_SEGMENT_KIND_HUGE:
- return "huge_segment";
- default:
- return "unknown";
- }
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static const char *
-_xzm_metapool_id_to_string(xzm_metapool_id_t id)
-{
- switch(id) {
- case XZM_METAPOOL_SEGMENT:
- return "segment metadata slab";
- case XZM_METAPOOL_SEGMENT_TABLE:
- return "segment table slab";
- case XZM_METAPOOL_MZONE_IDX:
- return "mzone index slab";
- case XZM_METAPOOL_THREAD_CACHE:
- return "thread cache slab";
- case XZM_METAPOOL_METADATA:
- return "metapool metadata slab";
- default:
- return "unknown slab";
- }
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_span_contains_slice(xzm_slice_t span, xzm_slice_t slice)
-{
- switch (span->xzc_bits.xzcb_kind) {
- case XZM_SLICE_KIND_TINY_CHUNK:
- case XZM_SLICE_KIND_SINGLE_FREE:
- return (span == slice);
- case XZM_SLICE_KIND_SMALL_CHUNK:
- case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
- case XZM_SLICE_KIND_LARGE_CHUNK:
- case XZM_SLICE_KIND_HUGE_CHUNK:
- case XZM_SLICE_KIND_MULTI_FREE:
- case XZM_SLICE_KIND_GUARD:
- xzm_debug_assert(slice >= span);
- return (slice < span + span->xzcs_slice_count);
- default:
- return false;
- }
-}
-
-// mimalloc: mi_slice_first
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_slice_t
-_xzm_span_slice_first(xzm_slice_t slice)
-{
- // "likely" in the sense that the most common case for lookups will be tiny
- // chunks
- if (os_likely(slice->xzc_bits.xzcb_kind != XZM_SLICE_KIND_MULTI_BODY)) {
- return slice;
- }
-
- xzm_slice_t out_slice = (xzm_slice_t)
- ((uintptr_t)slice - slice->xzsl_slice_offset_bytes);
-
- xzm_debug_assert(out_slice >= ((xzm_segment_t)((uintptr_t)slice &
- ~(XZM_METAPOOL_SEGMENT_BLOCK_SIZE - 1)))->xzs_slices);
- if (os_likely(_xzm_span_contains_slice(out_slice, slice))) {
- return out_slice;
- }
- // not contained in the span we refer to - leave it up to the caller to
- // handle this
- return slice;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_slice_kind_is_chunk(xzm_slice_kind_t kind)
-{
- switch (kind) {
- case XZM_SLICE_KIND_INVALID:
- case XZM_SLICE_KIND_SINGLE_FREE:
- case XZM_SLICE_KIND_MULTI_FREE:
- case XZM_SLICE_KIND_MULTI_BODY:
- return false;
- case XZM_SLICE_KIND_TINY_CHUNK:
- case XZM_SLICE_KIND_SMALL_CHUNK:
- case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
- case XZM_SLICE_KIND_LARGE_CHUNK:
- case XZM_SLICE_KIND_HUGE_CHUNK:
- return true;
- default:
- xzm_abort_with_reason("bad chunk kind", (unsigned)kind);
- }
-}
-
-// Like _xzm_slice_kind_is_chunk, but doesn't abort on totally bogus kinds
-// (useful during enumeration when we have no guarantees about what we're
-// looking at)
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_slice_kind_is_chunk_safe(xzm_slice_kind_t kind)
-{
- switch (kind) {
- case XZM_SLICE_KIND_TINY_CHUNK:
- case XZM_SLICE_KIND_SMALL_CHUNK:
- case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
- case XZM_SLICE_KIND_LARGE_CHUNK:
- case XZM_SLICE_KIND_HUGE_CHUNK:
- return true;
- default:
- return false;
- }
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_slice_kind_uses_xzones(xzm_slice_kind_t kind)
-{
- switch (kind) {
- case XZM_SLICE_KIND_TINY_CHUNK:
- case XZM_SLICE_KIND_SMALL_CHUNK:
- case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
- return true;
- default:
- return false;
- }
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_chunk_should_defer_reclamation(xzm_main_malloc_zone_t main,
- xzm_chunk_t chunk)
-{
- xzm_debug_assert(_xzm_slice_kind_is_chunk(chunk->xzc_bits.xzcb_kind));
- switch (chunk->xzc_bits.xzcb_kind) {
- case XZM_SLICE_KIND_TINY_CHUNK:
- return main->xzmz_defer_tiny;
- case XZM_SLICE_KIND_SMALL_CHUNK:
- case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
- return main->xzmz_defer_small;
- case XZM_SLICE_KIND_LARGE_CHUNK:
- case XZM_SLICE_KIND_HUGE_CHUNK:
- return main->xzmz_defer_large;
- default:
- xzm_abort("Attempt to check for deferred reclamation on "
- "non-chunk slice");
- }
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static const char *
-_xzm_slice_kind_to_string(xzm_slice_kind_t kind)
-{
- switch (kind) {
- case XZM_SLICE_KIND_INVALID:
- return "invalid";
- case XZM_SLICE_KIND_SINGLE_FREE:
- return "single_free";
- case XZM_SLICE_KIND_MULTI_FREE:
- return "multi_free";
- case XZM_SLICE_KIND_MULTI_BODY:
- return "multi_body";
- case XZM_SLICE_KIND_TINY_CHUNK:
- return "tiny_chunk";
- case XZM_SLICE_KIND_SMALL_CHUNK:
- return "small_chunk";
- case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
- return "small_freelist_chunk";
- case XZM_SLICE_KIND_LARGE_CHUNK:
- return "large_chunk";
- case XZM_SLICE_KIND_HUGE_CHUNK:
- return "huge_chunk";
- case XZM_SLICE_KIND_GUARD:
- return "guard_page";
- default:
- return "unknown";
- }
-}
-
-// mimalloc: _mi_segment_page_of
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_chunk_t
-_xzm_segment_chunk_of(xzm_segment_t segment, uintptr_t ptr)
-{
- xzm_slice_t slice = _xzm_segment_slice_of(segment, ptr);
- if (!slice) {
- return NULL;
- }
-
- xzm_slice_t first = _xzm_span_slice_first(slice);
- return _xzm_slice_kind_is_chunk(first->xzc_bits.xzcb_kind) ? first : NULL;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static size_t
-_xzm_segment_offset(xzm_segment_t segment, xzm_slice_count_t chunk_idx,
- xzm_block_index_t block_idx, uint64_t block_size)
-{
- return chunk_idx * XZM_SEGMENT_SLICE_SIZE + block_idx * block_size;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_block_index_t
-_xzm_segment_offset_chunk_block_index_of(xzm_segment_t segment,
- xzm_slice_count_t chunk_idx, uint64_t block_size, size_t offset)
-{
- xzm_debug_assert(offset >= chunk_idx * XZM_SEGMENT_SLICE_SIZE);
- return (xzm_block_index_t)
- (offset - chunk_idx * XZM_SEGMENT_SLICE_SIZE) / block_size;
-}
-
-// mimalloc: mi_page_block_size
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static uint64_t
-_xzm_chunk_block_size(xzm_malloc_zone_t zone, xzm_chunk_t chunk)
-{
- switch (chunk->xzc_bits.xzcb_kind) {
- case XZM_SLICE_KIND_TINY_CHUNK:
- case XZM_SLICE_KIND_SMALL_CHUNK:
- case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
- // TODO: depending on the size class scheme, it may be better to
- // directly compute the block size from the xzone index using the
- // inverse of the bin function
- return zone->xzz_xzones[chunk->xzc_xzone_idx].xz_block_size;
- case XZM_SLICE_KIND_LARGE_CHUNK:
- case XZM_SLICE_KIND_HUGE_CHUNK:
- return ((uint64_t)chunk->xzcs_slice_count) << XZM_SEGMENT_SLICE_SHIFT;
- default:
- xzm_abort_with_reason("asking for size of chunk with invalid kind",
- (unsigned)chunk->xzc_bits.xzcb_kind);
- }
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_slice_count_t
-_xzm_chunk_slice_count(xzm_chunk_t chunk)
-{
- return (chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_TINY_CHUNK) ? 1 :
- chunk->xzcs_slice_count;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_slice_t
-_xzm_chunk_slices_of(xzm_chunk_t chunk, size_t num_slices)
-{
- return (xzm_slice_t)chunk;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_chunk_is_empty(xzm_malloc_zone_t zone, xzm_xzone_t xz, xzm_chunk_t chunk)
-{
- switch (chunk->xzc_bits.xzcb_kind) {
- case XZM_SLICE_KIND_SMALL_CHUNK:
- return chunk->xzc_used == 0;
- case XZM_SLICE_KIND_TINY_CHUNK:
- case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
- return chunk->xzc_atomic_meta.xca_free_count == xz->xz_chunk_capacity ||
- chunk->xzc_atomic_meta.xca_alloc_head == XZM_FREE_MADVISING ||
- chunk->xzc_atomic_meta.xca_alloc_head == XZM_FREE_MADVISED;
- default:
- xzm_abort_with_reason("bad chunk kind",
- (unsigned int)chunk->xzc_bits.xzcb_kind);
- }
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_chunk_is_full(xzm_malloc_zone_t zone, xzm_xzone_t xz, xzm_chunk_t chunk)
-{
- switch (chunk->xzc_bits.xzcb_kind) {
- case XZM_SLICE_KIND_SMALL_CHUNK:
- return (chunk->xzc_used == xz->xz_chunk_capacity);
- case XZM_SLICE_KIND_TINY_CHUNK:
- case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
- return chunk->xzc_atomic_meta.xca_free_count == 0 &&
- chunk->xzc_atomic_meta.xca_alloc_head != XZM_FREE_MADVISING &&
- chunk->xzc_atomic_meta.xca_alloc_head != XZM_FREE_MADVISED;
- default:
- xzm_abort_with_reason("bad chunk kind",
- (unsigned int)chunk->xzc_bits.xzcb_kind);
- }
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static uint32_t
-_xzm_xzone_free_mask(xzm_xzone_t xz, size_t chunk_capacity)
-{
- xzm_debug_assert(!xz || xz->xz_chunk_capacity == chunk_capacity);
- return (uint32_t)((1ull << chunk_capacity) - 1);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static void
-_xzm_chunk_reset_free(xzm_xzone_t xz, xzm_chunk_t chunk, bool reusable)
-{
- if (!reusable) {
- chunk->xzc_xzone_idx = XZM_XZONE_INDEX_INVALID;
- }
-
- switch (chunk->xzc_bits.xzcb_kind) {
- case XZM_SLICE_KIND_TINY_CHUNK:
- case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
- // Tiny chunks should only be freed when a zone is destroyed, at which
- // point we don't need to support concurrent access to those chunks. As
- // such, we can access the atomic state non-atomically in this path
- xzm_debug_assert(chunk->xzc_atomic_meta.xca_alloc_head ==
- XZM_FREE_MADVISED);
- xzm_debug_assert(chunk->xzc_atomic_meta.xca_free_count == 0);
-
- if (!reusable) {
- // Reset everything to 0 to allow reuse of this slice for any
- // purpose
- chunk->xzc_atomic_meta.xca_value = 0;
- chunk->xzc_freelist_block_size = 0;
- chunk->xzc_freelist_chunk_capacity = 0;
-#if CONFIG_MTE
- chunk->xzc_tagged = false;
-#endif
- }
- break;
- case XZM_SLICE_KIND_SMALL_CHUNK:
- chunk->xzc_used = 0;
- chunk->xzc_alloc_idx = XZM_SLOT_INDEX_EMPTY;
- if (reusable) {
- chunk->xzc_free |= _xzm_xzone_free_mask(xz, xz->xz_chunk_capacity);
- } else {
- chunk->xzc_free = 0;
- }
- break;
- default:
- xzm_abort_with_reason("bad chunk kind",
- (unsigned int)chunk->xzc_bits.xzcb_kind);
- }
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_slice_count_t
-_xzm_free_span_slice_count(xzm_free_span_t span)
-{
- return (span->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SINGLE_FREE) ? 1 :
- span->xzcs_slice_count;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_slice_count_t
-_xzm_free_span_size(xzm_free_span_t span)
-{
- return _xzm_free_span_slice_count(span) << XZM_SEGMENT_SLICE_SHIFT;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_block_offset_t
-_xzm_chunk_offset_of_ptr(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
- uintptr_t ptr)
-{
- uintptr_t start = _xzm_chunk_start(zone, chunk, NULL);
-#if CONFIG_MTE
- // Remove tag bits for pointer arithmetic
- ptr = (uintptr_t)memtag_strip_address((uint8_t *)ptr);
-#endif
- xzm_block_offset_t offset = (xzm_block_offset_t)(ptr - start);
- return offset;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_block_offset_t
-_xzm_chunk_block_offset(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
- xzm_block_t block)
-{
- return _xzm_chunk_offset_of_ptr(zone, chunk, (uintptr_t)block);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static uint8_t *
-_xzm_chunk_block_start_of(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
- uintptr_t ptr)
-{
- xzm_debug_assert(chunk);
- xzm_debug_assert(ptr);
-
- xzm_block_offset_t offset = _xzm_chunk_offset_of_ptr(zone, chunk, ptr);
- size_t adjust = offset % _xzm_chunk_block_size(zone, chunk);
- return (uint8_t *)(ptr - adjust);
-}
-
-// mimalloc: _mi_page_ptr_unalign
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_block_t
-_xzm_chunk_block_of(xzm_malloc_zone_t zone, xzm_chunk_t chunk, uintptr_t ptr)
-{
- return (xzm_block_t)_xzm_chunk_block_start_of(zone, chunk, ptr);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_block_index_t
-_xzm_chunk_block_index(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
- xzm_block_t block)
-{
- return (xzm_block_index_t)(_xzm_chunk_block_offset(zone, chunk, block) /
- _xzm_chunk_block_size(zone, chunk));
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_block_index_t
-_xzm_chunk_block_index_of_ptr(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
- uintptr_t ptr)
-{
- return (xzm_block_index_t)(_xzm_chunk_offset_of_ptr(zone, chunk, ptr) /
- _xzm_chunk_block_size(zone, chunk));
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static uint8_t *
-_xzm_chunk_block_index_start(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
- xzm_block_index_t idx)
-{
- xzm_debug_assert(idx <
- zone->xzz_xzones[chunk->xzc_xzone_idx].xz_chunk_capacity);
- return (uint8_t *)(_xzm_chunk_start(zone, chunk, NULL) +
- (idx * _xzm_chunk_block_size(zone, chunk)));
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_small_chunk_block_index_is_free(xzm_chunk_t chunk,
- xzm_block_index_t block_index)
-{
- // Only applicable to chunks which use a bitmap freelist implementation
- xzm_debug_assert(chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_CHUNK);
- return (bool)(chunk->xzc_free & (1u << block_index));
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_chunk_block_index_range_is_free(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
- xzm_block_index_t start, xzm_block_index_t end)
-{
- // Only applicable to chunks which use a bitmap freelist implementation
- xzm_debug_assert(chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_CHUNK);
- xzm_debug_assert(end >= start);
-
- // Check the inclusive span from start to end
- uint32_t span = (end - start) + 1;
- xzm_debug_assert(span <= 32);
-
- uint32_t mask = (uint32_t)(((1ull << span) - 1) << start);
- return (chunk->xzc_free & mask) == mask;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static void
-_xzm_chunk_block_free_slices_on_allocate(const xzm_chunk_t chunk,
- xzm_slice_count_t chunk_idx, uint32_t chunk_capacity,
- xzm_block_index_t block_idx, uint64_t block_size,
- xzm_slice_count_t *slice_idx, xzm_slice_count_t *num_slices)
-{
- xzm_debug_assert(chunk &&
- chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_CHUNK);
-
- // Offset of this block relative to the body of the segment
- const size_t block = _xzm_segment_offset(NULL, chunk_idx, block_idx,
- block_size);
- const size_t block_end = block + block_size - 1;
-
- // Find the beginning of the first slice touched by the block
- const xzm_slice_count_t first_slice =
- _xzm_segment_slice_offset_index(NULL, block);
- // Find the beginning of the slice after the last touched by the block
- const xzm_slice_count_t limit_slice =
- _xzm_segment_slice_offset_index(NULL, block_end) + 1;
-
- // Find the blocks corresponding to these slices
- const xzm_block_index_t first_block =
- _xzm_segment_offset_chunk_block_index_of(NULL, chunk_idx, block_size,
- first_slice * XZM_SEGMENT_SLICE_SIZE);
- const xzm_block_index_t end_block =
- _xzm_segment_offset_chunk_block_index_of(NULL, chunk_idx, block_size,
- limit_slice * XZM_SEGMENT_SLICE_SIZE - 1);
-
- // Compute the offset of the first slice to populate, by checking if
- // any of the affected blocks are already in use (not all free).
- // If so, then we start with the end of the slice corresponding to the
- // beginning of our current block, instead of the beginning of that
- // slice
- const xzm_slice_count_t left = !_xzm_chunk_block_index_range_is_free(NULL,
- chunk, first_block, block_idx) ?
- _xzm_segment_slice_offset_index(NULL, block) + 1 : first_slice;
-
- // Compute the offset of the last slice to populate, by checking if
- // any of the affected blocks are already in use, as above.
- // If so, then we end with the beginning of the slice corresponding to
- // the end of our current block, instead of the end of that slice.
- // Exclude the end block if it is partial, because it is never free
- const xzm_block_index_t last_block =
- (end_block != chunk_capacity ? end_block : end_block - 1);
- xzm_debug_assert(block_idx <= last_block && last_block < chunk_capacity);
- const xzm_slice_count_t right = !_xzm_chunk_block_index_range_is_free(NULL,
- chunk, block_idx, last_block) ?
- _xzm_segment_slice_offset_index(NULL, block_end) : limit_slice;
-
- *slice_idx = left;
- *num_slices = (left <= right ? right - left : 0);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static void
-_xzm_chunk_block_free_slices_on_deallocate(const xzm_chunk_t chunk,
- xzm_slice_count_t chunk_idx, uint32_t chunk_capacity,
- xzm_block_index_t block_idx, uint64_t block_size,
- xzm_slice_count_t *slice_idx, xzm_slice_count_t *num_slices)
-{
- xzm_debug_assert(chunk &&
- chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_CHUNK);
-
- // Find the contiguous span from left (inclusive) to right (exclusive)
- xzm_slice_count_t left, right;
-
- // Offset of this block relative to the body of the segment
- const size_t block = _xzm_segment_offset(NULL, chunk_idx, block_idx,
- block_size);
- const size_t block_end = block + block_size - 1;
-
- // Determine the start of the first slice touched by this block
- const xzm_slice_count_t first_slice =
- _xzm_segment_slice_offset_index(NULL, block);
- // Determine end of last slice touched by this block
- const xzm_slice_count_t limit_slice =
- _xzm_segment_slice_offset_index(NULL, block_end) + 1;
-
- // Round the slice boundaries down to their corresponding blocks
- const xzm_block_index_t first_block =
- _xzm_segment_offset_chunk_block_index_of(NULL, chunk_idx, block_size,
- first_slice * XZM_SEGMENT_SLICE_SIZE);
- xzm_block_index_t last_block =
- _xzm_segment_offset_chunk_block_index_of(NULL, chunk_idx, block_size,
- limit_slice * XZM_SEGMENT_SLICE_SIZE - 1);
-
- // If the slices of this chunk are not perfectly divisible by the
- // block-size, the "partial" chunk at the end of the slice will never
- // be free because we cannot allocate from it
- if (last_block == chunk_capacity) {
- last_block -= 1;
- }
-
- // Determine if LHS blocks are free, and include/exclude them from the
- // madvisable range accordingly
- if (_xzm_chunk_block_index_range_is_free(NULL, chunk, first_block, block_idx)) {
- left = first_slice;
- } else {
- left = _xzm_segment_slice_offset_index(NULL, block) + 1;
- }
- xzm_debug_assert(left >= chunk_idx);
-
- // Determine if RHS blocks are free, and include/exclude them from the
- // madvisable range accordingly
- if (_xzm_chunk_block_index_range_is_free(NULL, chunk, block_idx, last_block)) {
- right = limit_slice;
- } else {
- right = _xzm_segment_slice_offset_index(NULL, block_end);
- }
- xzm_debug_assert(right <= chunk_idx + chunk->xzcs_slice_count);
-
- *slice_idx = left;
- *num_slices = (left <= right ? right - left : 0);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_chunk_t *
-_xzm_segment_slice_meta_batch_next(xzm_malloc_zone_t zone, xzm_slice_t slice)
-{
- xzm_segment_t segment = _xzm_segment_for_slice(zone, slice);
- xzm_xzone_slice_metadata_u *metadata =
- &segment->xzs_slice_metadata[_xzm_slice_index(segment, slice)];
- return &metadata->xzsm_batch_next;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_slice_meta_is_batch_pointer(xzm_malloc_zone_t zone, xzm_slice_t slice)
-{
-#if CONFIG_XZM_DEFERRED_RECLAIM
- if ((mach_vm_reclaim_id_t)slice == VM_RECLAIM_ID_NULL) {
- return false;
- }
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
- const uintptr_t slice_addr = (uintptr_t)slice;
- xzm_segment_t segment = _xzm_segment_for_slice(zone, slice);
- return !slice || (slice_addr >= (uintptr_t)(segment->xzs_slices) &&
- slice_addr < (uintptr_t)(segment->xzs_slices + segment->xzs_slice_entry_count));
-}
-
-#if CONFIG_XZM_DEFERRED_RECLAIM
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static mach_vm_reclaim_id_t *
-_xzm_segment_slice_meta_reclaim_id(xzm_segment_t segment,
- xzm_slice_t slice)
-{
- xzm_xzone_slice_metadata_u *metadata =
- &segment->xzs_slice_metadata[_xzm_slice_index(segment, slice)];
- return &metadata->xzsm_reclaim_id;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static mach_vm_reclaim_id_t *
-_xzm_slice_meta_reclaim_id(xzm_malloc_zone_t zone, xzm_slice_t slice)
-{
- xzm_segment_t segment = _xzm_segment_for_slice(zone, slice);
- return _xzm_segment_slice_meta_reclaim_id(segment, slice);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_segment_slice_is_deferred(xzm_segment_t segment, xzm_slice_t slice)
-{
- mach_vm_reclaim_id_t *reclaim_index = _xzm_segment_slice_meta_reclaim_id(
- segment, slice);
- return (*reclaim_index != VM_RECLAIM_ID_NULL);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_slice_is_deferred(xzm_malloc_zone_t zone, xzm_slice_t slice)
-{
- xzm_segment_t segment = _xzm_segment_for_slice(zone, slice);
- return _xzm_segment_slice_is_deferred(segment, slice);
-}
-
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_slice_kind_is_free_span(xzm_slice_kind_t kind)
-{
- return (kind == XZM_SLICE_KIND_SINGLE_FREE ||
- kind == XZM_SLICE_KIND_MULTI_FREE);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static uint32_t
-_xzm_xzone_slice_free_mask(xzm_malloc_zone_t zone, xzm_slice_t slice)
-{
- xzm_chunk_t chunk = _xzm_span_slice_first(slice);
- xzm_debug_assert(chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_CHUNK);
- xzm_xzone_t xz = &zone->xzz_xzones[chunk->xzc_xzone_idx];
-
- uintptr_t slice_start = (uintptr_t)_xzm_slice_start(zone, slice);
- uintptr_t slice_end = slice_start + XZM_SEGMENT_SLICE_SIZE - 1;
-
- // If the slices of this chunk are not perfectly divisible by the
- // block-size, there will be a "partial" block at the end of the chunk
- // needing special consideration
- xzm_block_index_t first = _xzm_chunk_block_index_of_ptr(zone, chunk,
- slice_start);
- if (first == xz->xz_chunk_capacity) {
- // This slice corresponds to the partial block, nothing can be allocated from it
- return 0u;
- }
- xzm_block_index_t last = _xzm_chunk_block_index_of_ptr(zone, chunk,
- slice_end);
- if (last == xz->xz_chunk_capacity) {
- // The partial chunk resides at the end of this slice, exclude it from
- // the mask
- last--;
- }
-
- xzm_block_index_t span = (last - first) + 1;
- xzm_debug_assert(first <= last);
-
- return (uint32_t)(((1ull << span) - 1) << first);
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static uintptr_t
-_xzm_introspect_rebase(uintptr_t orig_base, void *new_base, size_t size,
- void *ptr, size_t ptr_size)
-{
- if ((uintptr_t)ptr < orig_base) {
- return 0;
- }
-
- uintptr_t offset = (uintptr_t)ptr - orig_base;
- uintptr_t offset_end;
- if (os_add_overflow(offset, ptr_size, &offset_end)) {
- return 0;
- }
- if (offset_end > size) {
- return 0;
- }
-
- return (uintptr_t)new_base + offset;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_xzone_allocation_slot_t
-_xzm_xzone_allocation_slot_for_index(xzm_malloc_zone_t zone, xzm_xzone_t xz,
- xzm_allocation_index_t alloc_idx)
-{
- xzm_xzone_index_t xz_idx = xz->xz_idx;
- size_t alloc_base_idx = alloc_idx * zone->xzz_xzone_count;
- xzm_debug_assert(alloc_base_idx + xz_idx <
- zone->xzz_slot_count * zone->xzz_xzone_count);
- return &zone->xzz_xzone_allocation_slots[alloc_base_idx + xz_idx];
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static const char *
-_xzm_slot_config_to_string(xzm_slot_config_t slot_config)
-{
- switch(slot_config) {
- case XZM_SLOT_SINGLE:
- return "SINGLE";
- case XZM_SLOT_CLUSTER:
- return "CLUSTER";
- case XZM_SLOT_CPU:
- return "CPU";
- case XZM_SLOT_LAST:
- default:
- xzm_debug_abort("unexpected slot config");
- return "UNKNOWN";
- }
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_segment_group_has_madvise_workaround(xzm_segment_group_t sg)
-{
- return sg->xzsg_main_ref->xzmz_madvise_workaround;
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_segment_group_uses_deferred_reclamation(xzm_segment_group_t sg)
-{
-#if CONFIG_XZM_DEFERRED_RECLAIM
- switch(sg->xzsg_id) {
- case XZM_SEGMENT_GROUP_DATA:
- case XZM_SEGMENT_GROUP_POINTER_XZONES:
- // XXX: There is an implicit assumption that tiny chunks are
- // always sequestered. If tiny chunks every support recirculation,
- // they'll be subject to deferred reclaim alongside their small
- // counterparts once freed back to the segment group
- return sg->xzsg_main_ref->xzmz_defer_small;
- case XZM_SEGMENT_GROUP_POINTER_LARGE:
- case XZM_SEGMENT_GROUP_DATA_LARGE:
- return sg->xzsg_main_ref->xzmz_defer_large;
- default:
- xzm_abort_with_reason("unknown segment group id", sg->xzsg_id);
- }
-#else // CONFIG_XZM_DEFERRED_RECLAIM
- return false;
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
-}
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static size_t
-_xzm_segment_group_min_block_size(xzm_segment_group_t sg)
-{
- // Note: large alignments can force small allocations into segment groups for
- // bigger allocations, so this query is not always precise (which is
- // acceptable for our purposes).
- const size_t small_block_size_min = 16;
- const size_t large_block_size_min = XZM_SMALL_BLOCK_SIZE_MAX + 1;
-
- switch (sg->xzsg_id) {
- case XZM_SEGMENT_GROUP_POINTER_XZONES:
- case XZM_SEGMENT_GROUP_DATA:
- return small_block_size_min;
- case XZM_SEGMENT_GROUP_POINTER_LARGE:
- case XZM_SEGMENT_GROUP_DATA_LARGE:
- return large_block_size_min;
- default:
- xzm_abort_with_reason("unknown segment group id", sg->xzsg_id);
- }
-}
-
-#if CONFIG_MTE
-
-// Return whether this zone might contain tagged allocations. This is used as
-// the fast path check that avoids touching the xzone (xzm_xzone_t). Note that
-// unlike `xz->xz_tagged` this function does not take the <size,data>
-// characteristics of the allocation into account, so it should only be used
-// when this precision is not required.
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_zone_memtag_enabled(xzm_malloc_zone_t zone)
-{
- return zone->xzz_memtag_config.enabled;
-}
-
-// Return whether we tag allocations with these <size,data> characteristics
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_zone_memtag_block(xzm_malloc_zone_t zone, size_t block_size, bool data)
-{
- struct xzm_memtag_config_s *cfg = &(zone->xzz_memtag_config);
- return cfg->enabled &&
- (block_size <= cfg->max_block_size) &&
- (!data || cfg->tag_data);
-}
-
-// Return whether we tag allocations of this size in this segment group
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_segment_group_memtag_block(xzm_segment_group_t sg, size_t block_size)
-{
- xzm_malloc_zone_t zone = &(sg->xzsg_main_ref->xzmz_base);
- bool data = _xzm_segment_group_id_is_data(sg->xzsg_id);
- // Note: the block_size may be less than the min_block_size for the segment
- // group in the case where alignment is forcing the allocation of a
- // smaller-than-normal block from the segment group
- size_t min_block_size = _xzm_segment_group_min_block_size(sg);
- return block_size >= min_block_size &&
- _xzm_zone_memtag_block(zone, block_size, data);
-}
-
-// Return whether this segment group might contain tagged allocations
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static bool
-_xzm_segment_group_memtag_enabled(xzm_segment_group_t sg)
-{
- xzm_malloc_zone_t zone = &(sg->xzsg_main_ref->xzmz_base);
- bool data = _xzm_segment_group_id_is_data(sg->xzsg_id);
- size_t min_block_size = _xzm_segment_group_min_block_size(sg);
- return _xzm_zone_memtag_block(zone, min_block_size, data);
-}
-
-#endif // CONFIG_MTE
-
-#if CONFIG_XZM_THREAD_CACHE
-
-MALLOC_ALWAYS_INLINE MALLOC_INLINE
-static xzm_thread_cache_t
-_xzm_get_thread_cache(void)
-{
- return _pthread_getspecific_direct(__TSD_MALLOC_XZONE_THREAD_CACHE);
-}
-
-#endif // CONFIG_XZM_THREAD_CACHE
-
-#endif // __has_feature(bounds_safety)
-
-#endif // __XZONE_INLINE_INTERNAL_H__
-
-#endif // CONFIG_XZONE_MALLOC