/* ----------------------------------------------------------------------------
Copyright (c) 2018-2022, Microsoft Research, Daan Leijen
Copyright © 2025 Apple Inc.
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" in the same directory as this file.
-----------------------------------------------------------------------------*/

#if CONFIG_XZONE_MALLOC

#ifndef __XZONE_INLINE_INTERNAL_H__
#define __XZONE_INLINE_INTERNAL_H__

#if !__has_feature(bounds_safety)

#define xzm_abort(msg)  ({ \
	_os_set_crash_log_cause_and_message(0, "BUG IN LIBMALLOC: " msg); \
	__builtin_trap(); \
})
#define xzm_abort_with_reason(msg, reason)  ({ \
	_os_set_crash_log_cause_and_message((reason), "BUG IN LIBMALLOC: " msg); \
	__builtin_trap(); \
})
#define xzm_client_abort(msg)  ({ \
	_os_set_crash_log_cause_and_message(0, "BUG IN CLIENT OF LIBMALLOC: " msg); \
	__builtin_trap(); \
})
#define xzm_client_abort_with_reason(msg, reason)  ({ \
	_os_set_crash_log_cause_and_message((reason), "BUG IN CLIENT OF LIBMALLOC: " msg); \
	__builtin_trap(); \
})

#define _xzm_assert_stringify(x) #x
#define xzm_assert_stringify(x) _xzm_assert_stringify(x)

// mimalloc: mi_assert
#define xzm_assert(pred) \
	if (os_unlikely(!(pred))) { \
		xzm_abort("malloc assertion \"" #pred "\" failed " \
				"(" __FILE__ ":" xzm_assert_stringify(__LINE__) ")"); \
	}

// mimalloc: mi_assert_internal
#ifdef DEBUG
#define xzm_debug_assert xzm_assert
#define xzm_debug_abort xzm_abort
#define xzm_debug_abort_with_reason xzm_abort_with_reason
#ifndef __assert_only
#define __assert_only
#endif
#else // DEBUG
#define xzm_debug_assert(...)
#define xzm_debug_abort(...)
#define xzm_debug_abort_with_reason(...)
#ifndef __assert_only
#define __assert_only __unused
#endif
#endif // DEBUG

MALLOC_INLINE
static void
_xzm_corruption_detected(void *corrupt_block)
{
	// TODO: load the corrupt value into a register so it also appears in crash
	// reports
	xzm_client_abort_with_reason("memory corruption of free block",
			corrupt_block);
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_main_malloc_zone_t
_xzm_malloc_zone_main(xzm_malloc_zone_t zone)
{
	return zone->xzz_main_ref ?: (xzm_main_malloc_zone_t)zone;
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_malloc_zone_is_main(xzm_malloc_zone_t zone)
{
	return !zone->xzz_main_ref;
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_malloc_zone_is_xzm(malloc_zone_t *zone)
{
	return zone->version >= 14 &&
			zone->introspect->zone_type == MALLOC_ZONE_TYPE_XZONE;
}

#pragma mark magic math

// see https://lemire.me/blog/2019/02/20/more-fun-with-fast-remainders-when-the-divisor-is-a-constant/
//
// Implementation copied from zalloc
#define XZM_MAGIC_QUO(s)      (((1ull << 32) - 1) / (uint64_t)(s) + 1)
#define XZM_MAGIC_ALIGNED(s)  (~0u / (uint32_t)(s) + 1)

// Returns (offs / size) if offs is small enough and magic = XZM_MAGIC_QUO(size)
static inline uint32_t
XZM_FAST_QUO(uint64_t offs, uint64_t __unused size, uint64_t magic)
{
	uint32_t quo = (offs * magic) >> 32;
	xzm_debug_assert(offs / size == quo);
	return quo;
}

// Returns (offs % size) if offs is small enough and magic ==
// XZM_MAGIC_QUO(size)
static inline uint32_t
XZM_FAST_MOD(uint64_t offs, uint64_t magic, uint64_t size)
{
	uint32_t lowbits = (uint32_t)(offs * magic);

	uint32_t mod = (lowbits * size) >> 32;
	xzm_debug_assert(offs % size == mod);
	return mod;
}

// Returns whether (offs % size) == 0 if offs is small enough and magic ==
// XZM_MAGIC_ALIGNED(size)
static inline bool
XZM_FAST_ALIGNED(uint64_t offs, uint64_t __unused size, uint32_t magic)
{
	bool aligned = (uint32_t)(offs * magic) < magic;
	xzm_debug_assert(aligned == ((offs % size) == 0));
	return aligned;
}

#pragma mark metadata helpers

// mimalloc: mi_segment_map_index_of
MALLOC_ALWAYS_INLINE MALLOC_INLINE
static size_t
_xzm_segment_table_index_of(const void *segment_body, size_t *extended_idx)
{
	uintptr_t segment_bits = (uintptr_t)segment_body;
	if (segment_bits >= XZM_LIMIT_ADDRESS) {
		*extended_idx = 0;
		return XZM_SEGMENT_TABLE_ENTRIES;
	}

	uintptr_t segindex = segment_bits / XZM_SEGMENT_SIZE;
#if CONFIG_EXTERNAL_METADATA_LARGE
	// The segment map index we return in the large address space is the index
	// into the bottom level table, the extended_idx is the index into top table
	// (setting extended_idx = 0 implies that the body is in the low 64GB of VA)
	*extended_idx = segindex / XZM_SEGMENT_TABLE_ENTRIES;
	xzm_debug_assert(*extended_idx < XZM_EXTENDED_SEGMENT_TABLE_ENTRIES);
	return segindex % XZM_SEGMENT_TABLE_ENTRIES;
#else
	*extended_idx = 0;
	xzm_debug_assert(segindex < XZM_SEGMENT_TABLE_ENTRIES);
	return segindex;
#endif // CONFIG_EXTERNAL_METADATA_LARGE
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_segment_t
_xzm_segment_table_entry_to_segment(xzm_segment_table_entry_s entry)
{
	return (xzm_segment_t)
			((uintptr_t)entry.xste_val << XZM_METAPOOL_SEGMENT_BLOCK_SHIFT);
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_segment_table_entry_s
_xzm_segment_to_segment_table_entry(xzm_segment_t segment, bool normal)
{
	xzm_debug_assert((uintptr_t)segment % XZM_METAPOOL_SEGMENT_ALIGN == 0);
	// TODO: On MacOS (47 bit address space), the upper portion of the address
	// space won't fit in this encoding
	xzm_assert(((uintptr_t)segment >> XZM_METAPOOL_SEGMENT_BLOCK_SHIFT) <
			XZM_SEGMENT_TABLE_LIMIT_ENTRY);
	return (xzm_segment_table_entry_s) {
		.xste_val = (uint32_t)
				((uintptr_t)segment >> XZM_METAPOOL_SEGMENT_BLOCK_SHIFT),
		.xste_normal = normal,
	};
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_segment_table_entry_s *
_xzm_ptr_to_table_entry(const void *segment_body,
		xzm_main_malloc_zone_t main)
{
	size_t ext_idx = 0;
	size_t index = _xzm_segment_table_index_of(segment_body, &ext_idx);
#if CONFIG_EXTERNAL_METADATA_LARGE
	if (ext_idx == 0) {
		if (os_unlikely(index >= XZM_SEGMENT_TABLE_ENTRIES)) {
			// Pointer out of bounds, greater than XZM_LIMIT_ADDRESS
			return NULL;
		}
		// This pointer is in the first 64GB of VA, so it comes directly from
		// the segment table in the main zone
		xzm_debug_assert((uintptr_t)segment_body < XZM_SEGMENT_TABLE_COVERAGE);
		return &main->xzmz_segment_table[index];
	} else if (ext_idx >= XZM_EXTENDED_SEGMENT_TABLE_ENTRIES) {
		return NULL;
	} else {
		xzm_segment_table_entry_s *leaf_table;
		xzm_extended_segment_table_entry_s *map;
		map = main->xzmz_extended_segment_table;
		xzm_debug_assert(map != 0);
		leaf_table = (void *)((uintptr_t)(map[ext_idx].xeste_val) *
				XZM_SEGMENT_TABLE_ALIGN);
		if (leaf_table == NULL) {
			// there are no segments in the given 64GB span
			return NULL;
		}
		return &leaf_table[index];
	}
#else
	if (index >= XZM_SEGMENT_TABLE_ENTRIES) {
		return NULL;
	}
	return &main->xzmz_segment_table[index];
#endif // CONFIG_EXTERNAL_METADATA_LARGE

}

// mimalloc: _mi_segment_of
// Note: This will allow inner pointers, or any pointer inside the (4MB) segment
// granule of an allocated segment
MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_segment_t
xzm_segment_table_query(xzm_main_malloc_zone_t main, const void *ptr)
{
	xzm_segment_table_entry_s *leaf_entry;
	leaf_entry = _xzm_ptr_to_table_entry(ptr, main);
	if (leaf_entry == NULL) {
		return NULL;
	}
	return _xzm_segment_table_entry_to_segment(*leaf_entry);
}

// mimalloc: _mi_page_segment
MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_segment_t
_xzm_segment_for_slice(xzm_malloc_zone_t zone, xzm_slice_t slice)
{
	xzm_segment_t segment = (xzm_segment_t)
			((uintptr_t)slice & ~(XZM_METAPOOL_SEGMENT_BLOCK_SIZE - 1));
	xzm_debug_assert(!segment || (slice >= segment->xzs_slices &&
			slice < (segment->xzs_slices + segment->xzs_slice_entry_count)));
	return segment;
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_segment_group_t
_xzm_segment_group_for_slice(xzm_malloc_zone_t zone, xzm_slice_t slice)
{
	xzm_segment_t segment = _xzm_segment_for_slice(zone, slice);
	return segment->xzs_segment_group;
}

// mimalloc: mi_slice_index
MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_slice_count_t
_xzm_slice_index(xzm_segment_t segment, xzm_slice_t slice)
{
	xzm_debug_assert((uintptr_t)slice >= (uintptr_t)segment->xzs_slices);
	ptrdiff_t index = slice - segment->xzs_slices;
	xzm_debug_assert(index < (ptrdiff_t)segment->xzs_slice_entry_count);
	return (xzm_slice_count_t)index;
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static const char *
_xzm_range_group_id_to_string(xzm_range_group_id_t id)
{
	switch(id) {
	case XZM_RANGE_GROUP_DATA:
		return "data";
	case XZM_RANGE_GROUP_PTR:
		return "pointer";
	default:
		return "unknown";
	}
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static const char *
_xzm_segment_group_id_to_string(xzm_segment_group_id_t id)
{
	switch(id) {
	case XZM_SEGMENT_GROUP_DATA:
		return "data";
	case XZM_SEGMENT_GROUP_DATA_LARGE:
		return "data_large";
	case XZM_SEGMENT_GROUP_POINTER_XZONES:
		return "pointer_xzones";
	case XZM_SEGMENT_GROUP_POINTER_LARGE:
		return "pointer_large";
	default:
		return "unknown";
	}
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_segment_group_id_is_data(xzm_segment_group_id_t id)
{
	switch(id) {
	case XZM_SEGMENT_GROUP_DATA:
	case XZM_SEGMENT_GROUP_DATA_LARGE:
		return true;
	case XZM_SEGMENT_GROUP_POINTER_XZONES:
	case XZM_SEGMENT_GROUP_POINTER_LARGE:
		return false;
	default:
		xzm_abort_with_reason("unknown segment group id", id);
	}
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static uint8_t *
_xzm_segment_slice_index_start(xzm_segment_t segment, xzm_slice_count_t idx)
{
	return (uint8_t *)((uintptr_t)segment->xzs_segment_body +
			(idx * XZM_SEGMENT_SLICE_SIZE));
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static uint8_t *
_xzm_segment_slice_start(xzm_segment_t segment, xzm_slice_t slice)
{
	return _xzm_segment_slice_index_start(segment,
			_xzm_slice_index(segment, slice));
	// TODO: mimalloc offset optimization for small block sizes?
}

// FIXME: `zone` parameter isn't used in _xzm_segment_for_slice(),
// _xzm_slice_start(), _xzm_chunk_start(), _xzm_chunk_start_ptr() and can be
// removed.

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static uint8_t *
_xzm_slice_start(xzm_malloc_zone_t zone, xzm_slice_t slice)
{
	return _xzm_segment_slice_start(_xzm_segment_for_slice(zone, slice), slice);
}

// mimalloc: _mi_page_start
MALLOC_ALWAYS_INLINE MALLOC_INLINE
static uintptr_t
_xzm_chunk_start(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
		size_t *chunk_size_out)
{
	if (chunk_size_out) {
		switch (chunk->xzc_bits.xzcb_kind) {
		case XZM_SLICE_KIND_TINY_CHUNK:
			*chunk_size_out = XZM_TINY_CHUNK_SIZE;
			break;
		case XZM_SLICE_KIND_SMALL_CHUNK:
			*chunk_size_out = XZM_SMALL_CHUNK_SIZE;
			break;
		case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
			*chunk_size_out = XZM_SMALL_FREELIST_CHUNK_SIZE;
			break;
		case XZM_SLICE_KIND_LARGE_CHUNK:
		case XZM_SLICE_KIND_HUGE_CHUNK:
			*chunk_size_out = ((size_t)chunk->xzcs_slice_count) <<
					XZM_SEGMENT_SLICE_SHIFT;
			break;
		default:
			xzm_abort_with_reason("asking for start of chunk with invalid kind",
					(unsigned)chunk->xzc_bits.xzcb_kind);
		}
	}

	return (uintptr_t)_xzm_slice_start(zone, (xzm_slice_t)chunk);
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static uint8_t *
_xzm_chunk_start_ptr(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
		size_t *chunk_size_out)
{
	xzm_debug_assert(chunk_size_out);
	uintptr_t ptr = _xzm_chunk_start(zone, chunk, chunk_size_out);
	return (uint8_t *)ptr;
}


MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_slice_count_t
_xzm_segment_slice_count(xzm_segment_t segment)
{
	return segment->xzs_slice_count;
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static size_t
_xzm_segment_size(xzm_segment_t segment)
{
	xzm_slice_count_t body_slice_count = _xzm_segment_slice_count(segment);
	return (size_t)body_slice_count << XZM_SEGMENT_SLICE_SHIFT;
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static uint8_t *
_xzm_segment_start(xzm_segment_t segment)
{
	return _xzm_segment_slice_index_start(segment, 0);
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static size_t
_xzm_segment_slice_offset_of(xzm_segment_t segment, uintptr_t ptr)
{
	const ptrdiff_t diff = ptr - (uintptr_t)_xzm_segment_start(segment);
#ifdef DEBUG
	// Huge segments aren't always a multiple of the segment size, so it's
	// possible for malloc_size() to be passed a pointer that is within a
	// segment granule, but isn't within the segment that owns that granule. We
	// need to not crash in the debug dylib when that happens
	size_t rounded_size = roundup(_xzm_segment_size(segment), XZM_SEGMENT_SIZE);
	xzm_debug_assert(diff >= 0 && diff < (ptrdiff_t)rounded_size);
#endif // DEBUG
	return (size_t)diff;
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_slice_count_t
_xzm_segment_slice_offset_index(xzm_segment_t segment, size_t offset)
{
	return (xzm_slice_count_t)(offset >> XZM_SEGMENT_SLICE_SHIFT);
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_slice_count_t
_xzm_segment_slice_index_of(xzm_segment_t segment, uintptr_t ptr)
{
	size_t offset = _xzm_segment_slice_offset_of(segment, ptr);
	return _xzm_segment_slice_offset_index(segment, offset);
}

// mimalloc: _mi_segment_page_of
MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_slice_t
_xzm_segment_slice_of(xzm_segment_t segment, uintptr_t ptr)
{
	xzm_slice_count_t idx = _xzm_segment_slice_index_of(segment, ptr);
	if (os_likely(idx < segment->xzs_slice_entry_count)) {
		return &segment->xzs_slices[idx];
	} else {
		return NULL;
	}
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static uint8_t *
_xzm_segment_slice_index_end(xzm_segment_t segment, xzm_slice_count_t idx)
{
	return _xzm_segment_slice_index_start(segment, idx + 1);
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static uint8_t *
_xzm_segment_slice_end_of(xzm_segment_t segment, uintptr_t ptr)
{
	xzm_slice_count_t idx = _xzm_segment_slice_index_of(segment, ptr);
	return _xzm_segment_slice_index_end(segment, idx);
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static uint8_t *
_xzm_segment_end(xzm_segment_t segment)
{
	return _xzm_segment_slice_index_end(segment, segment->xzs_slice_count - 1);
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static uint8_t *
_xzm_segment_slice_start_of(xzm_segment_t segment, uintptr_t ptr)
{
	xzm_slice_count_t idx = _xzm_segment_slice_index_of(segment, ptr);
	return _xzm_segment_slice_index_start(segment, idx);
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_slice_t
_xzm_segment_slices_begin(xzm_segment_t segment)
{
	return &segment->xzs_slices[0];
}

// mimalloc: mi_segment_slices_end
MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_slice_t
_xzm_segment_slices_end(xzm_segment_t segment)
{
	// Return a one-past-the-end pointer without immediately trapping
	return &segment->xzs_slices[segment->xzs_slice_entry_count];
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static const char *
_xzm_segment_kind_to_string(xzm_segment_kind_t kind)
{
	switch(kind) {
	case XZM_SEGMENT_KIND_NORMAL:
		return "normal_segment";
	case XZM_SEGMENT_KIND_HUGE:
		return "huge_segment";
	default:
		return "unknown";
	}
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static const char *
_xzm_metapool_id_to_string(xzm_metapool_id_t id)
{
	switch(id) {
	case XZM_METAPOOL_SEGMENT:
		return "segment metadata slab";
	case XZM_METAPOOL_SEGMENT_TABLE:
		return "segment table slab";
	case XZM_METAPOOL_MZONE_IDX:
		return "mzone index slab";
	case XZM_METAPOOL_THREAD_CACHE:
		return "thread cache slab";
	case XZM_METAPOOL_METADATA:
		return "metapool metadata slab";
	default:
		return "unknown slab";
	}
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_span_contains_slice(xzm_slice_t span, xzm_slice_t slice)
{
	switch (span->xzc_bits.xzcb_kind) {
	case XZM_SLICE_KIND_TINY_CHUNK:
	case XZM_SLICE_KIND_SINGLE_FREE:
		return (span == slice);
	case XZM_SLICE_KIND_SMALL_CHUNK:
	case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
	case XZM_SLICE_KIND_LARGE_CHUNK:
	case XZM_SLICE_KIND_HUGE_CHUNK:
	case XZM_SLICE_KIND_MULTI_FREE:
	case XZM_SLICE_KIND_GUARD:
		xzm_debug_assert(slice >= span);
		return (slice < span + span->xzcs_slice_count);
	default:
		return false;
	}
}

// mimalloc: mi_slice_first
MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_slice_t
_xzm_span_slice_first(xzm_slice_t slice)
{
	// "likely" in the sense that the most common case for lookups will be tiny
	// chunks
	if (os_likely(slice->xzc_bits.xzcb_kind != XZM_SLICE_KIND_MULTI_BODY)) {
		return slice;
	}

	xzm_slice_t out_slice = (xzm_slice_t)
			((uintptr_t)slice - slice->xzsl_slice_offset_bytes);

	xzm_debug_assert(out_slice >= ((xzm_segment_t)((uintptr_t)slice &
			~(XZM_METAPOOL_SEGMENT_BLOCK_SIZE - 1)))->xzs_slices);
	if (os_likely(_xzm_span_contains_slice(out_slice, slice))) {
		return out_slice;
	}
	// not contained in the span we refer to - leave it up to the caller to
	// handle this
	return slice;
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_slice_kind_is_chunk(xzm_slice_kind_t kind)
{
	switch (kind) {
	case XZM_SLICE_KIND_INVALID:
	case XZM_SLICE_KIND_SINGLE_FREE:
	case XZM_SLICE_KIND_MULTI_FREE:
	case XZM_SLICE_KIND_MULTI_BODY:
		return false;
	case XZM_SLICE_KIND_TINY_CHUNK:
	case XZM_SLICE_KIND_SMALL_CHUNK:
	case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
	case XZM_SLICE_KIND_LARGE_CHUNK:
	case XZM_SLICE_KIND_HUGE_CHUNK:
		return true;
	default:
		xzm_abort_with_reason("bad chunk kind", (unsigned)kind);
	}
}

// Like _xzm_slice_kind_is_chunk, but doesn't abort on totally bogus kinds
// (useful during enumeration when we have no guarantees about what we're
// looking at)
MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_slice_kind_is_chunk_safe(xzm_slice_kind_t kind)
{
	switch (kind) {
	case XZM_SLICE_KIND_TINY_CHUNK:
	case XZM_SLICE_KIND_SMALL_CHUNK:
	case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
	case XZM_SLICE_KIND_LARGE_CHUNK:
	case XZM_SLICE_KIND_HUGE_CHUNK:
		return true;
	default:
		return false;
	}
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_slice_kind_uses_xzones(xzm_slice_kind_t kind)
{
	switch (kind) {
	case XZM_SLICE_KIND_TINY_CHUNK:
	case XZM_SLICE_KIND_SMALL_CHUNK:
	case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
		return true;
	default:
		return false;
	}
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_chunk_should_defer_reclamation(xzm_main_malloc_zone_t main,
		xzm_chunk_t chunk)
{
	xzm_debug_assert(_xzm_slice_kind_is_chunk(chunk->xzc_bits.xzcb_kind));
	switch (chunk->xzc_bits.xzcb_kind) {
	case XZM_SLICE_KIND_TINY_CHUNK:
		return main->xzmz_defer_tiny;
	case XZM_SLICE_KIND_SMALL_CHUNK:
	case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
		return main->xzmz_defer_small;
	case XZM_SLICE_KIND_LARGE_CHUNK:
	case XZM_SLICE_KIND_HUGE_CHUNK:
		return main->xzmz_defer_large;
	default:
		xzm_abort("Attempt to check for deferred reclamation on "
				"non-chunk slice");
	}
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static const char *
_xzm_slice_kind_to_string(xzm_slice_kind_t kind)
{
	switch (kind) {
	case XZM_SLICE_KIND_INVALID:
		return "invalid";
	case XZM_SLICE_KIND_SINGLE_FREE:
		return "single_free";
	case XZM_SLICE_KIND_MULTI_FREE:
		return "multi_free";
	case XZM_SLICE_KIND_MULTI_BODY:
		return "multi_body";
	case XZM_SLICE_KIND_TINY_CHUNK:
		return "tiny_chunk";
	case XZM_SLICE_KIND_SMALL_CHUNK:
		return "small_chunk";
	case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
		return "small_freelist_chunk";
	case XZM_SLICE_KIND_LARGE_CHUNK:
		return "large_chunk";
	case XZM_SLICE_KIND_HUGE_CHUNK:
		return "huge_chunk";
	case XZM_SLICE_KIND_GUARD:
		return "guard_page";
	default:
		return "unknown";
	}
}

// mimalloc: _mi_segment_page_of
MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_chunk_t
_xzm_segment_chunk_of(xzm_segment_t segment, uintptr_t ptr)
{
	xzm_slice_t slice = _xzm_segment_slice_of(segment, ptr);
	if (!slice) {
		return NULL;
	}

	xzm_slice_t first = _xzm_span_slice_first(slice);
	return _xzm_slice_kind_is_chunk(first->xzc_bits.xzcb_kind) ? first : NULL;
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static size_t
_xzm_segment_offset(xzm_segment_t segment, xzm_slice_count_t chunk_idx,
		xzm_block_index_t block_idx, uint64_t block_size)
{
	return chunk_idx * XZM_SEGMENT_SLICE_SIZE + block_idx * block_size;
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_block_index_t
_xzm_segment_offset_chunk_block_index_of(xzm_segment_t segment,
		xzm_slice_count_t chunk_idx, uint64_t block_size, size_t offset)
{
	xzm_debug_assert(offset >= chunk_idx * XZM_SEGMENT_SLICE_SIZE);
	return (xzm_block_index_t)
			(offset - chunk_idx * XZM_SEGMENT_SLICE_SIZE) / block_size;
}

// mimalloc: mi_page_block_size
MALLOC_ALWAYS_INLINE MALLOC_INLINE
static uint64_t
_xzm_chunk_block_size(xzm_malloc_zone_t zone, xzm_chunk_t chunk)
{
	switch (chunk->xzc_bits.xzcb_kind) {
	case XZM_SLICE_KIND_TINY_CHUNK:
	case XZM_SLICE_KIND_SMALL_CHUNK:
	case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
		// TODO: depending on the size class scheme, it may be better to
		// directly compute the block size from the xzone index using the
		// inverse of the bin function
		return zone->xzz_xzones[chunk->xzc_xzone_idx].xz_block_size;
	case XZM_SLICE_KIND_LARGE_CHUNK:
	case XZM_SLICE_KIND_HUGE_CHUNK:
		return ((uint64_t)chunk->xzcs_slice_count) << XZM_SEGMENT_SLICE_SHIFT;
	default:
		xzm_abort_with_reason("asking for size of chunk with invalid kind",
				(unsigned)chunk->xzc_bits.xzcb_kind);
	}
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_slice_count_t
_xzm_chunk_slice_count(xzm_chunk_t chunk)
{
	return (chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_TINY_CHUNK) ? 1 :
			chunk->xzcs_slice_count;
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_slice_t
_xzm_chunk_slices_of(xzm_chunk_t chunk, size_t num_slices)
{
	return (xzm_slice_t)chunk;
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_chunk_is_empty(xzm_malloc_zone_t zone, xzm_xzone_t xz, xzm_chunk_t chunk)
{
	switch (chunk->xzc_bits.xzcb_kind) {
	case XZM_SLICE_KIND_SMALL_CHUNK:
		return chunk->xzc_used == 0;
	case XZM_SLICE_KIND_TINY_CHUNK:
	case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
		return chunk->xzc_atomic_meta.xca_free_count == xz->xz_chunk_capacity ||
				chunk->xzc_atomic_meta.xca_alloc_head == XZM_FREE_MADVISING ||
				chunk->xzc_atomic_meta.xca_alloc_head == XZM_FREE_MADVISED;
	default:
		xzm_abort_with_reason("bad chunk kind",
				(unsigned int)chunk->xzc_bits.xzcb_kind);
	}
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_chunk_is_full(xzm_malloc_zone_t zone, xzm_xzone_t xz, xzm_chunk_t chunk)
{
	switch (chunk->xzc_bits.xzcb_kind) {
	case XZM_SLICE_KIND_SMALL_CHUNK:
		return (chunk->xzc_used == xz->xz_chunk_capacity);
	case XZM_SLICE_KIND_TINY_CHUNK:
	case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
		return chunk->xzc_atomic_meta.xca_free_count == 0 &&
				chunk->xzc_atomic_meta.xca_alloc_head != XZM_FREE_MADVISING &&
				chunk->xzc_atomic_meta.xca_alloc_head != XZM_FREE_MADVISED;
	default:
		xzm_abort_with_reason("bad chunk kind",
				(unsigned int)chunk->xzc_bits.xzcb_kind);
	}
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static uint32_t
_xzm_xzone_free_mask(xzm_xzone_t xz, size_t chunk_capacity)
{
	xzm_debug_assert(!xz || xz->xz_chunk_capacity == chunk_capacity);
	return (uint32_t)((1ull << chunk_capacity) - 1);
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static void
_xzm_chunk_reset_free(xzm_xzone_t xz, xzm_chunk_t chunk, bool reusable)
{
	if (!reusable) {
		chunk->xzc_xzone_idx = XZM_XZONE_INDEX_INVALID;
	}

	switch (chunk->xzc_bits.xzcb_kind) {
	case XZM_SLICE_KIND_TINY_CHUNK:
	case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
		// Tiny chunks should only be freed when a zone is destroyed, at which
		// point we don't need to support concurrent access to those chunks. As
		// such, we can access the atomic state non-atomically in this path
		xzm_debug_assert(chunk->xzc_atomic_meta.xca_alloc_head ==
				XZM_FREE_MADVISED);
		xzm_debug_assert(chunk->xzc_atomic_meta.xca_free_count == 0);

		if (!reusable) {
			// Reset everything to 0 to allow reuse of this slice for any
			// purpose
			chunk->xzc_atomic_meta.xca_value = 0;
			chunk->xzc_freelist_block_size = 0;
			chunk->xzc_freelist_chunk_capacity = 0;
#if CONFIG_MTE
			chunk->xzc_tagged = false;
#endif
		}
		break;
	case XZM_SLICE_KIND_SMALL_CHUNK:
		chunk->xzc_used = 0;
		chunk->xzc_alloc_idx = XZM_SLOT_INDEX_EMPTY;
		if (reusable) {
			chunk->xzc_free |= _xzm_xzone_free_mask(xz, xz->xz_chunk_capacity);
		} else {
			chunk->xzc_free = 0;
		}
		break;
	default:
		xzm_abort_with_reason("bad chunk kind",
				(unsigned int)chunk->xzc_bits.xzcb_kind);
	}
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_slice_count_t
_xzm_free_span_slice_count(xzm_free_span_t span)
{
	return (span->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SINGLE_FREE) ? 1 :
			span->xzcs_slice_count;
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_slice_count_t
_xzm_free_span_size(xzm_free_span_t span)
{
	return _xzm_free_span_slice_count(span) << XZM_SEGMENT_SLICE_SHIFT;
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_block_offset_t
_xzm_chunk_offset_of_ptr(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
		uintptr_t ptr)
{
	uintptr_t start = _xzm_chunk_start(zone, chunk, NULL);
#if CONFIG_MTE
	// Remove tag bits for pointer arithmetic
	ptr = (uintptr_t)memtag_strip_address((uint8_t *)ptr);
#endif
	xzm_block_offset_t offset = (xzm_block_offset_t)(ptr - start);
	return offset;
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_block_offset_t
_xzm_chunk_block_offset(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
		xzm_block_t block)
{
	return _xzm_chunk_offset_of_ptr(zone, chunk, (uintptr_t)block);
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static uint8_t *
_xzm_chunk_block_start_of(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
		uintptr_t ptr)
{
	xzm_debug_assert(chunk);
	xzm_debug_assert(ptr);

	xzm_block_offset_t offset = _xzm_chunk_offset_of_ptr(zone, chunk, ptr);
	size_t adjust = offset % _xzm_chunk_block_size(zone, chunk);
	return (uint8_t *)(ptr - adjust);
}

// mimalloc: _mi_page_ptr_unalign
MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_block_t
_xzm_chunk_block_of(xzm_malloc_zone_t zone, xzm_chunk_t chunk, uintptr_t ptr)
{
	return (xzm_block_t)_xzm_chunk_block_start_of(zone, chunk, ptr);
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_block_index_t
_xzm_chunk_block_index(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
		xzm_block_t block)
{
	return (xzm_block_index_t)(_xzm_chunk_block_offset(zone, chunk, block) /
			_xzm_chunk_block_size(zone, chunk));
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_block_index_t
_xzm_chunk_block_index_of_ptr(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
		uintptr_t ptr)
{
	return (xzm_block_index_t)(_xzm_chunk_offset_of_ptr(zone, chunk, ptr) /
			_xzm_chunk_block_size(zone, chunk));
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static uint8_t *
_xzm_chunk_block_index_start(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
		xzm_block_index_t idx)
{
	xzm_debug_assert(idx <
			zone->xzz_xzones[chunk->xzc_xzone_idx].xz_chunk_capacity);
	return (uint8_t *)(_xzm_chunk_start(zone, chunk, NULL) +
			(idx * _xzm_chunk_block_size(zone, chunk)));
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_small_chunk_block_index_is_free(xzm_chunk_t chunk,
		xzm_block_index_t block_index)
{
	// Only applicable to chunks which use a bitmap freelist implementation
	xzm_debug_assert(chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_CHUNK);
	return (bool)(chunk->xzc_free & (1u << block_index));
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_chunk_block_index_range_is_free(xzm_malloc_zone_t zone, xzm_chunk_t chunk,
		xzm_block_index_t start, xzm_block_index_t end)
{
	// Only applicable to chunks which use a bitmap freelist implementation
	xzm_debug_assert(chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_CHUNK);
	xzm_debug_assert(end >= start);

	// Check the inclusive span from start to end
	uint32_t span = (end - start) + 1;
	xzm_debug_assert(span <= 32);

	uint32_t mask = (uint32_t)(((1ull << span) - 1) << start);
	return (chunk->xzc_free & mask) == mask;
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static void
_xzm_chunk_block_free_slices_on_allocate(const xzm_chunk_t chunk,
		xzm_slice_count_t chunk_idx, uint32_t chunk_capacity,
		xzm_block_index_t block_idx, uint64_t block_size,
		xzm_slice_count_t *slice_idx, xzm_slice_count_t *num_slices)
{
	xzm_debug_assert(chunk &&
			chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_CHUNK);

	// Offset of this block relative to the body of the segment
	const size_t block = _xzm_segment_offset(NULL, chunk_idx, block_idx,
			block_size);
	const size_t block_end = block + block_size - 1;

	// Find the beginning of the first slice touched by the block
	const xzm_slice_count_t first_slice =
			_xzm_segment_slice_offset_index(NULL, block);
	// Find the beginning of the slice after the last touched by the block
	const xzm_slice_count_t limit_slice =
			_xzm_segment_slice_offset_index(NULL, block_end) + 1;

	// Find the blocks corresponding to these slices
	const xzm_block_index_t first_block =
			_xzm_segment_offset_chunk_block_index_of(NULL, chunk_idx, block_size,
			first_slice * XZM_SEGMENT_SLICE_SIZE);
	const xzm_block_index_t end_block =
			_xzm_segment_offset_chunk_block_index_of(NULL, chunk_idx, block_size,
			limit_slice * XZM_SEGMENT_SLICE_SIZE - 1);

	// Compute the offset of the first slice to populate, by checking if
	// any of the affected blocks are already in use (not all free).
	// If so, then we start with the end of the slice corresponding to the
	// beginning of our current block, instead of the beginning of that
	// slice
	const xzm_slice_count_t left = !_xzm_chunk_block_index_range_is_free(NULL,
			chunk, first_block, block_idx) ?
			_xzm_segment_slice_offset_index(NULL, block) + 1 : first_slice;

	// Compute the offset of the last slice to populate, by checking if
	// any of the affected blocks are already in use, as above.
	// If so, then we end with the beginning of the slice corresponding to
	// the end of our current block, instead of the end of that slice.
	// Exclude the end block if it is partial, because it is never free
	const xzm_block_index_t last_block =
			(end_block != chunk_capacity ? end_block : end_block - 1);
	xzm_debug_assert(block_idx <= last_block && last_block < chunk_capacity);
	const xzm_slice_count_t right = !_xzm_chunk_block_index_range_is_free(NULL,
			chunk, block_idx, last_block) ?
			_xzm_segment_slice_offset_index(NULL, block_end) : limit_slice;

	*slice_idx = left;
	*num_slices = (left <= right ? right - left : 0);
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static void
_xzm_chunk_block_free_slices_on_deallocate(const xzm_chunk_t chunk,
		xzm_slice_count_t chunk_idx, uint32_t chunk_capacity,
		xzm_block_index_t block_idx, uint64_t block_size,
		xzm_slice_count_t *slice_idx, xzm_slice_count_t *num_slices)
{
	xzm_debug_assert(chunk &&
			chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_CHUNK);

	// Find the contiguous span from left (inclusive) to right (exclusive)
	xzm_slice_count_t left, right;

	// Offset of this block relative to the body of the segment
	const size_t block = _xzm_segment_offset(NULL, chunk_idx, block_idx,
			block_size);
	const size_t block_end = block + block_size - 1;

	// Determine the start of the first slice touched by this block
	const xzm_slice_count_t first_slice =
			_xzm_segment_slice_offset_index(NULL, block);
	// Determine end of last slice touched by this block
	const xzm_slice_count_t limit_slice =
			_xzm_segment_slice_offset_index(NULL, block_end) + 1;

	// Round the slice boundaries down to their corresponding blocks
	const xzm_block_index_t first_block =
			_xzm_segment_offset_chunk_block_index_of(NULL, chunk_idx, block_size,
			first_slice * XZM_SEGMENT_SLICE_SIZE);
	xzm_block_index_t last_block =
			_xzm_segment_offset_chunk_block_index_of(NULL, chunk_idx, block_size,
			limit_slice * XZM_SEGMENT_SLICE_SIZE - 1);

	// If the slices of this chunk are not perfectly divisible by the
	// block-size, the "partial" chunk at the end of the slice will never
	// be free because we cannot allocate from it
	if (last_block == chunk_capacity) {
		last_block -= 1;
	}

	// Determine if LHS blocks are free, and include/exclude them from the
	// madvisable range accordingly
	if (_xzm_chunk_block_index_range_is_free(NULL, chunk, first_block, block_idx)) {
		left = first_slice;
	} else {
		left = _xzm_segment_slice_offset_index(NULL, block) + 1;
	}
	xzm_debug_assert(left >= chunk_idx);

	// Determine if RHS blocks are free, and include/exclude them from the
	// madvisable range accordingly
	if (_xzm_chunk_block_index_range_is_free(NULL, chunk, block_idx, last_block)) {
		right = limit_slice;
	} else {
		right = _xzm_segment_slice_offset_index(NULL, block_end);
	}
	xzm_debug_assert(right <= chunk_idx + chunk->xzcs_slice_count);

	*slice_idx = left;
	*num_slices = (left <= right ? right - left : 0);
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_chunk_t *
_xzm_segment_slice_meta_batch_next(xzm_malloc_zone_t zone, xzm_slice_t slice)
{
	xzm_segment_t segment = _xzm_segment_for_slice(zone, slice);
	xzm_xzone_slice_metadata_u *metadata =
			&segment->xzs_slice_metadata[_xzm_slice_index(segment, slice)];
	return &metadata->xzsm_batch_next;
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_slice_meta_is_batch_pointer(xzm_malloc_zone_t zone, xzm_slice_t slice)
{
#if CONFIG_XZM_DEFERRED_RECLAIM
	if ((mach_vm_reclaim_id_t)slice == VM_RECLAIM_ID_NULL) {
		return false;
	}
#endif // CONFIG_XZM_DEFERRED_RECLAIM
	const uintptr_t slice_addr = (uintptr_t)slice;
	xzm_segment_t segment = _xzm_segment_for_slice(zone, slice);
	return !slice || (slice_addr >= (uintptr_t)(segment->xzs_slices) &&
			slice_addr < (uintptr_t)(segment->xzs_slices + segment->xzs_slice_entry_count));
}

#if CONFIG_XZM_DEFERRED_RECLAIM

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static mach_vm_reclaim_id_t *
_xzm_segment_slice_meta_reclaim_id(xzm_segment_t segment,
		xzm_slice_t slice)
{
	xzm_xzone_slice_metadata_u *metadata =
			&segment->xzs_slice_metadata[_xzm_slice_index(segment, slice)];
	return &metadata->xzsm_reclaim_id;
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static mach_vm_reclaim_id_t *
_xzm_slice_meta_reclaim_id(xzm_malloc_zone_t zone, xzm_slice_t slice)
{
	xzm_segment_t segment = _xzm_segment_for_slice(zone, slice);
	return _xzm_segment_slice_meta_reclaim_id(segment, slice);
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_segment_slice_is_deferred(xzm_segment_t segment, xzm_slice_t slice)
{
	mach_vm_reclaim_id_t *reclaim_index = _xzm_segment_slice_meta_reclaim_id(
			segment, slice);
	return (*reclaim_index != VM_RECLAIM_ID_NULL);
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_slice_is_deferred(xzm_malloc_zone_t zone, xzm_slice_t slice)
{
	xzm_segment_t segment = _xzm_segment_for_slice(zone, slice);
	return _xzm_segment_slice_is_deferred(segment, slice);
}

#endif // CONFIG_XZM_DEFERRED_RECLAIM

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_slice_kind_is_free_span(xzm_slice_kind_t kind)
{
	return (kind == XZM_SLICE_KIND_SINGLE_FREE ||
			kind == XZM_SLICE_KIND_MULTI_FREE);
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static uint32_t
_xzm_xzone_slice_free_mask(xzm_malloc_zone_t zone, xzm_slice_t slice)
{
	xzm_chunk_t chunk = _xzm_span_slice_first(slice);
	xzm_debug_assert(chunk->xzc_bits.xzcb_kind == XZM_SLICE_KIND_SMALL_CHUNK);
	xzm_xzone_t xz = &zone->xzz_xzones[chunk->xzc_xzone_idx];

	uintptr_t slice_start = (uintptr_t)_xzm_slice_start(zone, slice);
	uintptr_t slice_end = slice_start + XZM_SEGMENT_SLICE_SIZE - 1;

	// If the slices of this chunk are not perfectly divisible by the
	// block-size, there will be a "partial" block at the end of the chunk
	// needing special consideration
	xzm_block_index_t first = _xzm_chunk_block_index_of_ptr(zone, chunk,
			slice_start);
	if (first == xz->xz_chunk_capacity) {
		// This slice corresponds to the partial block, nothing can be allocated from it
		return 0u;
	}
	xzm_block_index_t last = _xzm_chunk_block_index_of_ptr(zone, chunk,
			slice_end);
	if (last == xz->xz_chunk_capacity) {
		// The partial chunk resides at the end of this slice, exclude it from
		// the mask
		last--;
	}

	xzm_block_index_t span = (last - first) + 1;
	xzm_debug_assert(first <= last);

	return (uint32_t)(((1ull << span) - 1) << first);
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static uintptr_t
_xzm_introspect_rebase(uintptr_t orig_base, void *new_base, size_t size,
		void *ptr, size_t ptr_size)
{
	if ((uintptr_t)ptr < orig_base) {
		return 0;
	}

	uintptr_t offset = (uintptr_t)ptr - orig_base;
	uintptr_t offset_end;
	if (os_add_overflow(offset, ptr_size, &offset_end)) {
		return 0;
	}
	if (offset_end > size) {
		return 0;
	}

	return (uintptr_t)new_base + offset;
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_xzone_allocation_slot_t
_xzm_xzone_allocation_slot_for_index(xzm_malloc_zone_t zone, xzm_xzone_t xz,
		xzm_allocation_index_t alloc_idx)
{
	xzm_xzone_index_t xz_idx = xz->xz_idx;
	size_t alloc_base_idx = alloc_idx * zone->xzz_xzone_count;
	xzm_debug_assert(alloc_base_idx + xz_idx <
			zone->xzz_slot_count * zone->xzz_xzone_count);
	return &zone->xzz_xzone_allocation_slots[alloc_base_idx + xz_idx];
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static const char *
_xzm_slot_config_to_string(xzm_slot_config_t slot_config)
{
	switch(slot_config) {
	case XZM_SLOT_SINGLE:
		return "SINGLE";
	case XZM_SLOT_CLUSTER:
		return "CLUSTER";
	case XZM_SLOT_CPU:
		return "CPU";
	case XZM_SLOT_LAST:
	default:
		xzm_debug_abort("unexpected slot config");
		return "UNKNOWN";
	}
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_segment_group_has_madvise_workaround(xzm_segment_group_t sg)
{
	return sg->xzsg_main_ref->xzmz_madvise_workaround;
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_segment_group_uses_deferred_reclamation(xzm_segment_group_t sg)
{
#if CONFIG_XZM_DEFERRED_RECLAIM
	switch(sg->xzsg_id) {
	case XZM_SEGMENT_GROUP_DATA:
	case XZM_SEGMENT_GROUP_POINTER_XZONES:
		// XXX: There is an implicit assumption that tiny chunks are
		// always sequestered. If tiny chunks every support recirculation,
		// they'll be subject to deferred reclaim alongside their small
		// counterparts once freed back to the segment group
		return sg->xzsg_main_ref->xzmz_defer_small;
	case XZM_SEGMENT_GROUP_POINTER_LARGE:
	case XZM_SEGMENT_GROUP_DATA_LARGE:
		return sg->xzsg_main_ref->xzmz_defer_large;
	default:
		xzm_abort_with_reason("unknown segment group id", sg->xzsg_id);
	}
#else // CONFIG_XZM_DEFERRED_RECLAIM
	return false;
#endif // CONFIG_XZM_DEFERRED_RECLAIM
}

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static size_t
_xzm_segment_group_min_block_size(xzm_segment_group_t sg)
{
	// Note: large alignments can force small allocations into segment groups for
	// bigger allocations, so this query is not always precise (which is
	// acceptable for our purposes).
	const size_t small_block_size_min = 16;
	const size_t large_block_size_min = XZM_SMALL_BLOCK_SIZE_MAX + 1;

	switch (sg->xzsg_id) {
	case XZM_SEGMENT_GROUP_POINTER_XZONES:
	case XZM_SEGMENT_GROUP_DATA:
		return small_block_size_min;
	case XZM_SEGMENT_GROUP_POINTER_LARGE:
	case XZM_SEGMENT_GROUP_DATA_LARGE:
		return large_block_size_min;
	default:
		xzm_abort_with_reason("unknown segment group id", sg->xzsg_id);
	}
}

#if CONFIG_MTE

// Return whether this zone might contain tagged allocations.  This is used as
// the fast path check that avoids touching the xzone (xzm_xzone_t).  Note that
// unlike `xz->xz_tagged` this function does not take the <size,data>
// characteristics of the allocation into account, so it should only be used
// when this precision is not required.
MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_zone_memtag_enabled(xzm_malloc_zone_t zone)
{
	return zone->xzz_memtag_config.enabled;
}

// Return whether we tag allocations with these <size,data> characteristics
MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_zone_memtag_block(xzm_malloc_zone_t zone, size_t block_size, bool data)
{
	struct xzm_memtag_config_s *cfg = &(zone->xzz_memtag_config);
	return cfg->enabled &&
			(block_size <= cfg->max_block_size) &&
			(!data || cfg->tag_data);
}

// Return whether we tag allocations of this size in this segment group
MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_segment_group_memtag_block(xzm_segment_group_t sg, size_t block_size)
{
	xzm_malloc_zone_t zone = &(sg->xzsg_main_ref->xzmz_base);
	bool data = _xzm_segment_group_id_is_data(sg->xzsg_id);
	// Note: the block_size may be less than the min_block_size for the segment
	// group in the case where alignment is forcing the allocation of a
	// smaller-than-normal block from the segment group
	size_t min_block_size = _xzm_segment_group_min_block_size(sg);
	return block_size >= min_block_size &&
			_xzm_zone_memtag_block(zone, block_size, data);
}

// Return whether this segment group might contain tagged allocations
MALLOC_ALWAYS_INLINE MALLOC_INLINE
static bool
_xzm_segment_group_memtag_enabled(xzm_segment_group_t sg)
{
	xzm_malloc_zone_t zone = &(sg->xzsg_main_ref->xzmz_base);
	bool data = _xzm_segment_group_id_is_data(sg->xzsg_id);
	size_t min_block_size = _xzm_segment_group_min_block_size(sg);
	return _xzm_zone_memtag_block(zone, min_block_size, data);
}

#endif // CONFIG_MTE

#if CONFIG_XZM_THREAD_CACHE

MALLOC_ALWAYS_INLINE MALLOC_INLINE
static xzm_thread_cache_t
_xzm_get_thread_cache(void)
{
	return _pthread_getspecific_direct(__TSD_MALLOC_XZONE_THREAD_CACHE);
}

#endif // CONFIG_XZM_THREAD_CACHE

#endif // __has_feature(bounds_safety)

#endif // __XZONE_INLINE_INTERNAL_H__

#endif // CONFIG_XZONE_MALLOC