Loading...
--- /dev/null
+++ libmalloc/libmalloc-792.80.2/src/xzone_malloc/xzone_introspect.c
@@ -0,0 +1,1908 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2022, Microsoft Research, Daan Leijen
+Copyright © 2025 Apple Inc.
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" in the same directory as this file.
+-----------------------------------------------------------------------------*/
+
+#include "../internal.h"
+
+#if CONFIG_XZONE_MALLOC
+
+#define XZM_DEBUG_ENUMERATOR 0
+
+#pragma mark libmalloc segment introspection
+
+kern_return_t
+xzm_segment_group_segment_foreach_span(xzm_segment_t segment,
+ xzm_span_enumerator_t enumerator)
+{
+ xzm_slice_t end = _xzm_segment_slices_end(segment);
+ xzm_slice_t slice = _xzm_segment_slices_begin(segment);
+
+ if (segment->xzs_kind == XZM_SEGMENT_KIND_HUGE) {
+ return enumerator(slice, slice->xzcs_slice_count);
+ }
+
+ // Enumeration protocol: the kind bits of a chunk are updated last, after
+ // the rest of the chunk metadata is initialized, so if we see a chunk slice
+ // it should be valid. Anything else may be in an intermediate state and is
+ // not to be trusted, so rather than iterating span-by-span as we would
+ // under the lock we need to scan linearly from chunk to chunk.
+ kern_return_t kr;
+ while (slice < end) {
+ xzm_slice_kind_t kind = slice->xzc_bits.xzcb_kind;
+ if (_xzm_slice_kind_is_chunk_safe(kind) ||
+ // Guard pages aren't chunks, but should be enumerated like them
+ kind == XZM_SLICE_KIND_GUARD) {
+ xzm_slice_count_t slice_count;
+ if (kind == XZM_SLICE_KIND_TINY_CHUNK) {
+ slice_count = 1;
+ } else {
+ slice_count = slice->xzcs_slice_count;
+ }
+
+ kr = enumerator(slice, slice_count);
+ if (kr) {
+ return kr;
+ }
+
+ slice += slice_count;
+ } else {
+ // Scan forward to the next chunk.
+ xzm_slice_t first_slice = slice;
+ do {
+ slice++;
+ } while (!_xzm_slice_kind_is_chunk_safe(slice->xzc_bits.xzcb_kind) &&
+ slice->xzc_bits.xzcb_kind != XZM_SLICE_KIND_GUARD &&
+ slice < end);
+
+ // Report the free span.
+ kr = enumerator(first_slice,
+ (xzm_slice_count_t)(slice - first_slice));
+ if (kr) {
+ return kr;
+ }
+ }
+ }
+
+ return KERN_SUCCESS;
+}
+
+kern_return_t
+xzm_segment_table_foreach(xzm_segment_table_entry_s *segment_table,
+ size_t num_entries, xzm_segment_table_enumerator_t enumerator,
+ xzm_segment_t *last_segment_enumerated)
+{
+ xzm_segment_t last_segment = NULL;
+ if (last_segment_enumerated) {
+ last_segment = *last_segment_enumerated;
+ }
+ for (size_t i = 0; i < num_entries; i++) {
+ xzm_segment_t segment =
+ _xzm_segment_table_entry_to_segment(segment_table[i]);
+ if (!segment) {
+ continue;
+ }
+ // Huge segments can be in multiple adjacent entries in the segment map
+ // if the segment spans multiple segment granules. Only enumerate the
+ // first entry
+ if (segment == last_segment) {
+ continue;
+ } else {
+ last_segment = segment;
+ }
+
+ kern_return_t kr = enumerator((vm_address_t)segment);
+ if (kr) {
+ return kr;
+ }
+ }
+ if (last_segment_enumerated) {
+ *last_segment_enumerated = last_segment;
+ }
+ return KERN_SUCCESS;
+}
+
+#pragma mark libmalloc zone introspection
+
+#if CONFIG_XZM_THREAD_CACHE
+
+static kern_return_t
+_xzm_introspect_enumerate_thread_caches(task_t task, memory_reader_t reader,
+ xzm_main_malloc_zone_t main,
+ MALLOC_NOESCAPE xzm_thread_cache_enumerator_t thread_cache_enumerator)
+{
+ xzm_debug_assert(main->xzmz_base.xzz_thread_cache_enabled);
+
+ vm_address_t thread_cache_addr = (vm_address_t)LIST_FIRST(
+ &main->xzmz_thread_cache_list);
+ size_t thread_cache_size = sizeof(struct xzm_thread_cache_s) +
+ (main->xzmz_base.xzz_thread_cache_xzone_count *
+ sizeof(xzm_xzone_thread_cache_u));
+ while (thread_cache_addr != 0) {
+ xzm_thread_cache_t tc;
+ kern_return_t kr = reader(task, thread_cache_addr, thread_cache_size,
+ (void **)&tc);
+ if (kr) {
+ xzm_debug_abort("Failed to map thread cache");
+ return kr;
+ }
+
+ kr = thread_cache_enumerator(thread_cache_addr, tc);
+ if (kr) {
+ // Allow KERN_RETURN_MAX as a way to request early exit
+ if (kr != KERN_RETURN_MAX) {
+ xzm_debug_abort("Failed to enumerate thread cache");
+ }
+ return kr;
+ }
+
+ thread_cache_addr = (vm_address_t)LIST_NEXT(tc, xtc_linkage);
+ }
+
+ return KERN_SUCCESS;
+}
+
+#endif // CONFIG_XZM_THREAD_CACHE
+
+static kern_return_t
+_xzm_introspect_small_chunk_blocks(xzm_malloc_zone_t zone,
+ vm_address_t segment_addr, xzm_segment_t segment, xzm_chunk_t chunk,
+ xzm_slice_count_t slice_count, uintptr_t start, vm_address_t start_addr,
+ xzm_xzone_t xz, MALLOC_NOESCAPE xzm_chunk_enumerator_t chunk_enumerator)
+{
+ uint32_t block_size = (uint32_t)xz->xz_block_size;
+ size_t capacity = xz->xz_chunk_capacity;
+
+ union {
+ vm_range_t range;
+ bool free;
+ } blocks[XZM_CHUNK_MAX_BLOCK_COUNT] = { 0 };
+
+ size_t range_idx = 0;
+
+ for (xzm_block_index_t block_index = 0; block_index < capacity;
+ block_index++) {
+ if (!_xzm_small_chunk_block_index_is_free(chunk, block_index)) {
+ blocks[range_idx].range = (vm_range_t){
+ .address = start_addr + block_index * block_size,
+ .size = block_size,
+ };
+ range_idx++;
+ }
+ }
+
+ return chunk_enumerator(segment_addr, segment, chunk, slice_count,
+ start_addr, xz, (vm_range_t *)blocks, (unsigned)range_idx);
+}
+
+static kern_return_t
+_xzm_introspect_freelist_chunk_blocks(task_t task, memory_reader_t reader,
+ xzm_malloc_zone_t zone, vm_address_t segment_addr,
+ xzm_segment_t segment, xzm_chunk_t chunk, xzm_slice_count_t slice_count,
+ uintptr_t start, vm_address_t start_addr, xzm_xzone_t xz,
+ MALLOC_NOESCAPE xzm_chunk_enumerator_t chunk_enumerator)
+{
+ kern_return_t kr = KERN_FAILURE;
+
+ uint32_t block_size = (uint32_t)xz->xz_block_size;
+ size_t granule = block_size > XZM_TINY_BLOCK_SIZE_MAX ?
+ XZM_SMALL_GRANULE : XZM_GRANULE;
+ size_t capacity = xz->xz_chunk_capacity;
+
+ xzm_chunk_atomic_meta_u meta = chunk->xzc_atomic_meta;
+
+ if (meta.xca_alloc_head == XZM_FREE_MADVISING ||
+ meta.xca_alloc_head == XZM_FREE_MADVISED) {
+ xzm_debug_assert(meta.xca_free_count == 0);
+ // This chunk is madvised, so there can be no blocks in use
+ return chunk_enumerator(segment_addr, segment, chunk, slice_count,
+ start_addr, xz, NULL, 0);
+ }
+
+ xzm_debug_assert(meta.xca_free_count <= capacity);
+
+#if CONFIG_MTE
+ // This code can be invoked by memory tools that are not running
+ // under MTE, to introspect a zone mapped in from a process that
+ // is actually running under MTE. Therefore, we only ldg when we are running
+ // in a process spawned with has_sec_transition=1.
+ //
+ // This should only matter for the case of memory tools on non-MTE hardware
+ // introspecting processes running under MTE emulation, as we need to ensure
+ // we don't try to execute the unsupported MTE instructions. On real
+ // hardware, we expect memory tools to run with Allocation Tag Access
+ // disabled (SCTLR.ATA=0), so there should be no need to do anything to
+ // safely access the mapped memory of a remote process even if it is running
+ // under MTE.
+#endif
+
+ // To produce an array of all of the live blocks in a tiny chunk:
+ // - We walk the chunk freelist, marking everything on it as free
+ // - If the chunk is marked as installed to a thread cache:
+ // - We search the thread caches to find the specific one that the chunk
+ // is installed to
+ // - Once we find the cache containing the chunk, we walk the cache
+ // freelist, marking all of those blocks as free as well
+ // - We determine the bump offset as the difference between the free count
+ // and the number of blocks
+ // - Then we prepare the range array by initializing the range for each
+ // block below the bump offset that wasn't marked free on the first pass
+
+ // N.B. Nano handles inconsistent freelist state by assuming that whatever
+ // it saw before the inconsistency is what's on it. It might be better to
+ // return an error and let the caller know that the state is inconsistent,
+ // but for now we'll do as nano does for compatibility.
+
+ union {
+ vm_range_t range;
+ bool free;
+ } blocks[XZM_CHUNK_MAX_BLOCK_COUNT] = { 0 };
+
+ // First, walk the chunk freelist. We should only walk it to the length on
+ // the chunk. In the thread caching case, a detaching thread may be in the
+ // process of linking a local freelist to the end, but we'll pick those
+ // blocks up later and don't want to include them in this walk.
+ bool cached = (meta.xca_alloc_idx == XZM_SLOT_INDEX_THREAD_INSTALLED);
+
+ size_t block_granule_size = block_size / granule;
+ uint64_t block_offset = meta.xca_alloc_head;
+
+ size_t total_chunk_freelist_count = meta.xca_free_count;
+ size_t current_chunk_freelist_count = 0;
+ while (current_chunk_freelist_count < total_chunk_freelist_count &&
+ block_offset < XZM_FREE_LIMIT &&
+ block_offset % block_granule_size == 0) {
+ size_t block_index = block_offset / block_granule_size;
+ if (blocks[block_index].free) {
+ xzm_debug_abort("loop in freelist");
+ break;
+ }
+ current_chunk_freelist_count++;
+ blocks[block_index].free = true;
+
+ xzm_block_t block = (xzm_block_t)(
+ start + (block_offset * granule));
+#if CONFIG_MTE && !MALLOC_TARGET_EXCLAVES_INTROSPECTOR
+ if (malloc_has_sec_transition) {
+ block = (xzm_block_t)memtag_fixup_ptr((void *)block);
+ }
+#endif
+ block_offset = block->xzb_linkage.xzbl_next_offset;
+ }
+
+ size_t allocated_limit = capacity;
+ if (cached) {
+#if CONFIG_XZM_THREAD_CACHE
+ if (zone->xzz_main_ref) {
+ xzm_debug_abort("cached chunk on non-main zone");
+ goto fail;
+ }
+
+ // We should have walked the full reported length of the remote freelist
+ if (current_chunk_freelist_count != total_chunk_freelist_count) {
+ xzm_debug_abort("Cached chunk freelist walk incomplete");
+ // XXX By failing the enumeration here, we're being stricter than
+ // nano was about weirdness in the freelist. That seems worth the
+ // increased visibility into possible bugs this enumerator might
+ // have, so for this case fail hard rather than allowing it.
+ goto fail;
+ }
+
+ xzm_main_malloc_zone_t main = (xzm_main_malloc_zone_t)zone;
+
+ xzm_xzone_index_t xz_idx = xz->xz_idx;
+ if (xz_idx >= zone->xzz_thread_cache_xzone_count) {
+ xzm_debug_abort("out-of-bounds cached xzone index");
+ goto fail;
+ }
+
+ __block xzm_thread_cache_t tc = NULL;
+ __block xzm_xzone_thread_cache_t cache = NULL;
+
+ // This chunk is installed to a thread cache, so we need to go find the
+ // right one. The priority order is:
+ // - If a thread cache for a non-detaching thread cache holds the chunk,
+ // it owns it. There should be at most one such cache.
+ // - Otherwise, if one or more detaching caches holds it, the cache with
+ // the highest teardown generation owns it.
+ kern_return_t kr2 = _xzm_introspect_enumerate_thread_caches(task,
+ reader, main, ^(vm_address_t thread_cache_addr,
+ xzm_thread_cache_t curr_tc){
+ xzm_xzone_thread_cache_t curr_cache =
+ &curr_tc->xtc_xz_caches[xz_idx];
+ if ((curr_cache->xztc_state < XZM_FREE_LIMIT ||
+ curr_cache->xztc_state == XZM_FREE_NULL) &&
+ (vm_address_t)curr_cache->xztc_chunk_start == start_addr) {
+ // This is a match. Is is a better match?
+
+ // If the cache we're looking at isn't tearing down, it's a
+ // perfect match, and we can stop searching.
+ if (!curr_tc->xtc_teardown_gen) {
+ xzm_debug_assert(!tc || tc->xtc_teardown_gen);
+ tc = curr_tc;
+ cache = curr_cache;
+ return KERN_RETURN_MAX;
+ }
+
+ // Otherwise, if it is tearing down, if it's more recent than
+ // the last one we saw then it's the best match we've seen so
+ // far.
+ if (!tc || curr_tc->xtc_teardown_gen > tc->xtc_teardown_gen) {
+ tc = curr_tc;
+ cache = curr_cache;
+ }
+ }
+ return KERN_SUCCESS;
+ });
+
+ if (kr2 && kr2 != KERN_RETURN_MAX) {
+ xzm_debug_abort("Failure enumerating thread caches");
+ kr = kr2;
+ goto fail;
+ }
+
+ if (!tc) {
+ xzm_debug_abort("Failed to find cache for cached chunk");
+ goto fail;
+ }
+
+ xzm_debug_assert(tc && cache);
+
+ // Walk the local freelist to add its free blocks to the set of known
+ // free blocks.
+ size_t total_local_freelist_count = cache->xztc_free_count;
+ uint64_t block_offset = cache->xztc_head;
+ size_t current_local_freelist_count = 0;
+ while (current_local_freelist_count < total_local_freelist_count &&
+ block_offset < XZM_FREE_LIMIT &&
+ block_offset % block_granule_size == 0) {
+ size_t block_index = block_offset / block_granule_size;
+ if (blocks[block_index].free) {
+ xzm_debug_abort("loop in local freelist");
+ break;
+ }
+ current_local_freelist_count++;
+ blocks[block_index].free = true;
+
+ xzm_block_t block = (xzm_block_t)(
+ start + (block_offset * granule));
+#if CONFIG_MTE && !MALLOC_TARGET_EXCLAVES_INTROSPECTOR
+ if (malloc_has_sec_transition) {
+ block = (xzm_block_t)memtag_fixup_ptr((void *)block);
+ }
+#endif
+ block_offset = block->xzb_linkage.xzbl_next_offset;
+ }
+
+ xzm_debug_assert(block_offset == XZM_FREE_NULL);
+
+ // Account for the bump on the local freelist
+ if (current_local_freelist_count < total_local_freelist_count &&
+ total_local_freelist_count <= capacity) {
+ allocated_limit = capacity -
+ (total_local_freelist_count - current_local_freelist_count);
+ }
+#else // CONFIG_XZM_THREAD_CACHE
+ xzm_debug_abort("Unexpected cached chunk");
+ goto fail;
+#endif // CONFIG_XZM_THREAD_CACHE
+ } else {
+ xzm_debug_assert(block_offset == XZM_FREE_NULL);
+
+ // Account for the bump on the remote freelist
+ if (current_chunk_freelist_count < total_chunk_freelist_count &&
+ total_chunk_freelist_count <= capacity) {
+ allocated_limit = capacity -
+ (total_chunk_freelist_count - current_chunk_freelist_count);
+ }
+ }
+
+ size_t range_idx = 0;
+ for (size_t i = 0; i < allocated_limit; i++) {
+ if (!blocks[i].free) {
+ blocks[range_idx].range = (vm_range_t){
+ .address = start_addr + i * block_size,
+ .size = block_size,
+ };
+ range_idx++;
+ }
+ }
+
+ return chunk_enumerator(segment_addr, segment, chunk, slice_count,
+ start_addr, xz, (vm_range_t *)blocks, (unsigned)range_idx);
+
+fail:
+ xzm_debug_assert(kr);
+ return kr;
+}
+
+static kern_return_t
+_xzm_introspect_chunk_blocks(task_t task, memory_reader_t reader,
+ xzm_malloc_zone_t zone, vm_address_t segment_addr,
+ xzm_segment_t segment, xzm_chunk_t chunk, xzm_slice_count_t slice_count,
+ uintptr_t start, vm_address_t start_addr, xzm_xzone_t xz,
+ MALLOC_NOESCAPE xzm_chunk_enumerator_t chunk_enumerator)
+{
+ xzm_slice_kind_t kind = chunk->xzc_bits.xzcb_kind;
+ if (!_xzm_slice_kind_uses_xzones(kind)) {
+ // This is a large or huge chunk, which has exactly one block
+ vm_range_t range = {
+ .address = start_addr,
+ .size = slice_count * XZM_SEGMENT_SLICE_SIZE,
+ };
+
+ return chunk_enumerator(segment_addr, segment, chunk, slice_count,
+ start_addr, NULL, &range, 1);
+ }
+
+ uint32_t block_size = (uint32_t)xz->xz_block_size;
+ size_t capacity = xz->xz_chunk_capacity;
+ // Sanity check
+ if ((slice_count * XZM_SEGMENT_SLICE_SIZE) / block_size != capacity ||
+ capacity > XZM_CHUNK_MAX_BLOCK_COUNT) {
+ xzm_debug_abort("inconsistent xzone info");
+ return KERN_FAILURE;
+ }
+
+ if (kind == XZM_SLICE_KIND_SMALL_CHUNK) {
+ return _xzm_introspect_small_chunk_blocks(zone, segment_addr, segment,
+ chunk, slice_count, start, start_addr, xz, chunk_enumerator);
+ }
+
+ xzm_debug_assert(kind == XZM_SLICE_KIND_TINY_CHUNK ||
+ kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK);
+ return _xzm_introspect_freelist_chunk_blocks(task, reader, zone,
+ segment_addr, segment, chunk, slice_count, start, start_addr, xz,
+ chunk_enumerator);
+
+}
+
+static kern_return_t
+_xzm_introspect_enumerate(task_t task, memory_reader_t reader,
+ vm_address_t zone_address, xzm_malloc_zone_t zone,
+ vm_address_t main_address, xzm_main_malloc_zone_t main,
+ bool include_blocks,
+ MALLOC_NOESCAPE xzm_metapool_enumerator_t metapool_slab_enumerator,
+ MALLOC_NOESCAPE xzm_segment_enumerator_t segment_enumerator,
+ MALLOC_NOESCAPE xzm_chunk_enumerator_t chunk_enumerator,
+ MALLOC_NOESCAPE xzm_free_span_enumerator_t span_enumerator)
+{
+ bool zone_is_main = (zone_address == main_address);
+ xzm_debug_assert(!span_enumerator || zone_is_main);
+
+ size_t zone_size = zone_is_main ? main->xzmz_total_size :
+ zone->xzz_total_size;
+
+ if (zone_is_main) {
+ size_t metapools_size;
+ if (os_mul_overflow(main->xzmz_metapool_count,
+ sizeof(struct xzm_metapool_s), &metapools_size)) {
+ xzm_debug_abort("Failed to compute metapools size");
+ return KERN_FAILURE;
+ }
+ xzm_metapool_t metapools = (xzm_metapool_t)_xzm_introspect_rebase(
+ main_address, main, main->xzmz_total_size, main->xzmz_metapools,
+ metapools_size);
+ if (!metapools) {
+ xzm_debug_abort("Failed to rebase metapools");
+ return KERN_FAILURE;
+ }
+ for (int i = 0; i < main->xzmz_metapool_count; i++) {
+ xzm_metapool_t mp = &metapools[i];
+ vm_address_t slab_addr = (vm_address_t)SLIST_FIRST(&mp->xzmp_slabs);
+ while (slab_addr != 0) {
+ xzm_metapool_slab_t slab = NULL;
+
+ kern_return_t kr = reader(task, slab_addr,
+ sizeof(struct xzm_metapool_slab_s), (void **)&slab);
+ if (kr) {
+ xzm_debug_abort("Failed to map metapool slab");
+ return kr;
+ }
+
+ kr = metapool_slab_enumerator((vm_address_t)slab->xzmps_base,
+ mp->xzmp_slab_size, mp->xzmp_id);
+ if (kr) {
+ return kr;
+ }
+
+ slab_addr = (vm_address_t)SLIST_NEXT(slab, xzmps_entry);
+ }
+ }
+ }
+
+ size_t table_size;
+ if (os_mul_overflow(XZM_SEGMENT_TABLE_ENTRIES,
+ sizeof(xzm_segment_table_entry_s), &table_size)) {
+ xzm_debug_abort("failed to compute segment table size");
+ return KERN_FAILURE;
+ }
+ xzm_segment_table_entry_s *segment_table =
+ (xzm_segment_table_entry_s *)_xzm_introspect_rebase(main_address,
+ main, main->xzmz_total_size, main->xzmz_segment_table, table_size);
+ if (!segment_table) {
+ xzm_debug_abort("failed to rebase segment table");
+ return KERN_FAILURE;
+ }
+
+ xzm_segment_table_enumerator_t enumerator = ^(vm_address_t segment_addr){
+ xzm_segment_t segment;
+ // Even for huge segments, we don't need to map more than normal segment
+ // size because we don't need to see anything in the body of huge
+ // segments.
+ //
+ // XXX Note: the mapped segment is _not_ guaranteed to have the same
+ // alignment as the original segment, so many of the manipulation
+ // helpers can't be used with it.
+
+ // Map in the segment metadata to see how big it is.
+ kern_return_t kr = reader(task, segment_addr,
+ sizeof(struct xzm_segment_s), (void **)&segment);
+ if (kr) {
+ xzm_debug_abort("failed to map segment header");
+ return kr;
+ }
+
+ void *segment_body;
+ kr = reader(task, (vm_address_t)_xzm_segment_start(segment),
+ segment->xzs_slice_count * XZM_SEGMENT_SLICE_SIZE,
+ &segment_body);
+ if (kr) {
+ xzm_debug_abort("failed to map segment");
+ return kr;
+ }
+
+ kr = segment_enumerator(segment_addr, segment, " ");
+ if (kr) {
+ return kr;
+ }
+
+ return xzm_segment_group_segment_foreach_span(segment,
+ ^(xzm_slice_t span, xzm_slice_count_t slice_count){
+ ptrdiff_t idx = span - segment->xzs_slices;
+ size_t start_offset = idx * XZM_SEGMENT_SLICE_SIZE;
+ uintptr_t start = (uintptr_t)segment_body + start_offset;
+ uintptr_t orig_start = (uintptr_t)_xzm_segment_slice_index_start(
+ segment, (xzm_slice_count_t)idx);
+ vm_address_t start_addr = (vm_address_t)orig_start;
+
+ xzm_slice_kind_t kind = span->xzc_bits.xzcb_kind;
+ if (_xzm_slice_kind_is_chunk_safe(kind) &&
+ span->xzc_mzone_idx == zone->xzz_mzone_idx) {
+ // This is a chunk that belongs to this zone.
+ xzm_chunk_t chunk = span;
+
+ xzm_xzone_t xz = NULL;
+ if (_xzm_slice_kind_uses_xzones(kind)) {
+ xz = (xzm_xzone_t)_xzm_introspect_rebase(zone_address, zone,
+ zone_size, &zone->xzz_xzones[chunk->xzc_xzone_idx],
+ sizeof(struct xzm_xzone_s));
+ if (!xz) {
+ xzm_debug_abort("failed to rebase xzone");
+ return KERN_FAILURE;
+ }
+ }
+
+ if (include_blocks) {
+ return _xzm_introspect_chunk_blocks(task, reader, zone,
+ segment_addr, segment, chunk, slice_count, start,
+ start_addr, xz, chunk_enumerator);
+ } else {
+ return chunk_enumerator(segment_addr, segment, chunk,
+ slice_count, start_addr, xz, NULL, 0);
+ }
+ } else if (zone_is_main &&
+ span->xzc_mzone_idx == XZM_MZONE_INDEX_INVALID) {
+ // Include free spans and sequestered chunks when enumerating
+ // the main zone
+ //
+ // TODO: could include xzone for sequestered chunks that belong
+ // to one
+ return span_enumerator(segment_addr, segment, span,
+ slice_count, start_addr);
+ }
+
+ // Either a free span we don't care about or a Valid chunk that
+ // belongs to a different zone: skip, continue iteration
+ return KERN_SUCCESS;
+ });
+ };
+
+ xzm_segment_t last_segment_enumerated = NULL;
+ kern_return_t kr = xzm_segment_table_foreach(segment_table,
+ XZM_SEGMENT_TABLE_ENTRIES, enumerator, &last_segment_enumerated);
+ if (kr) {
+ return kr;
+ }
+
+ size_t ext_seg_table_size;
+ if (os_mul_overflow(main->xzmz_extended_segment_table_entries,
+ sizeof(xzm_extended_segment_table_entry_s), &ext_seg_table_size)) {
+ xzm_debug_abort("failed to compute extended segment table size");
+ return KERN_FAILURE;
+ }
+ xzm_extended_segment_table_entry_s *ext_segment_table =
+ (xzm_extended_segment_table_entry_s *) _xzm_introspect_rebase(
+ main_address, main, main->xzmz_total_size,
+ main->xzmz_extended_segment_table, ext_seg_table_size);
+ if (ext_segment_table) {
+ for (size_t i = 0; i < main->xzmz_extended_segment_table_entries; i++) {
+ // If leaf table pointer is non-null, map it in and enumerate over
+ // it
+ if (ext_segment_table[i].xeste_val != 0) {
+ // There is (or was) at least one segment in the 64GB span
+ // represented by this segment map entry
+ xzm_segment_table_entry_s *table;
+
+ vm_address_t table_addr = ((vm_address_t)ext_segment_table[i].xeste_val *
+ XZM_SEGMENT_TABLE_ALIGN);
+
+ kern_return_t kr = reader(task, table_addr,
+ XZM_SEGMENT_TABLE_SIZE, (void **)&table);
+ if (kr) {
+ xzm_debug_abort("Failed to map segment table");
+ return kr;
+ }
+
+ kr = xzm_segment_table_foreach(table, XZM_SEGMENT_TABLE_ENTRIES,
+ enumerator, &last_segment_enumerated);
+ if (kr) {
+ return kr;
+ }
+ }
+ }
+ }
+
+ return KERN_SUCCESS;
+}
+
+#if CONFIG_XZM_DEFERRED_RECLAIM
+
+static kern_return_t
+_xzm_introspect_map_reclaim_buffer(task_t task, memory_reader_t reader,
+ vm_address_t metadata_addr, xzm_reclaim_buffer_t *xzm_metadata_out,
+ mach_vm_reclaim_ring_t *buffer_out)
+{
+ xzm_reclaim_buffer_t xzm_buffer = NULL;
+ mach_vm_reclaim_ring_t buffer = NULL;
+ kern_return_t kr;
+
+ kr = reader(task, metadata_addr,
+ sizeof(struct xzm_reclaim_buffer_s), (void **)&xzm_buffer);
+ if (kr) {
+ xzm_debug_abort_with_reason("failed to map reclaim buffer metadata", kr);
+ goto out;
+ }
+
+ vm_address_t buffer_addr = (vm_address_t)xzm_buffer->xrb_ringbuffer;
+ size_t buffer_size = (xzm_buffer->xrb_len *
+ sizeof(struct mach_vm_reclaim_entry_s)) +
+ offsetof(struct mach_vm_reclaim_ring_s, entries);
+ if (buffer_addr != 0) {
+ xzm_debug_assert(buffer_size % vm_page_quanta_size == 0);
+ kr = reader(task, buffer_addr, buffer_size, (void **)&buffer);
+ if (kr) {
+ xzm_debug_abort_with_reason("failed to map reclaim buffer", kr);
+ goto out;
+ }
+ }
+
+out:
+ *xzm_metadata_out = xzm_buffer;
+ *buffer_out = buffer;
+ return kr;
+}
+
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+
+static kern_return_t
+_xzm_introspect_map_zone_and_main(task_t task, vm_address_t zone_address,
+ memory_reader_t reader, xzm_malloc_zone_t *zone_p_out,
+ xzm_main_malloc_zone_t *main_p_out, vm_address_t *main_address_out)
+{
+ xzm_malloc_zone_t zone = NULL;
+
+ // Map the base structure first to find its size, then map that full size
+ kern_return_t kr = reader(task, zone_address, sizeof(*zone),
+ (void **)&zone);
+ if (kr) {
+ xzm_debug_abort("failed to map zone");
+ return kr;
+ }
+
+ uint64_t zone_size = zone->xzz_total_size;
+ if (zone_size < sizeof(*zone)) {
+ xzm_debug_abort("inconsistent zone region info");
+ return KERN_FAILURE;
+ }
+
+ kr = reader(task, zone_address, zone_size, (void **)&zone);
+ if (kr) {
+ xzm_debug_abort("failed to map full zone");
+ return kr;
+ }
+
+ xzm_main_malloc_zone_t main = NULL;
+ uint64_t main_zone_size;
+ vm_address_t main_address = 0;
+ if (zone->xzz_main_ref) {
+ main_address = (vm_address_t)zone->xzz_main_ref;
+
+ kr = reader(task, main_address, sizeof(*main), (void **)&main);
+ if (kr) {
+ xzm_debug_abort("failed to map main zone");
+ return kr;
+ }
+
+ main_zone_size = main->xzmz_total_size;
+ if (main_zone_size < sizeof(*main)) {
+ xzm_debug_abort("inconsistent main zone info");
+ return KERN_FAILURE;
+ }
+
+ kr = reader(task, main_address, main_zone_size, (void **)&main);
+ if (kr) {
+ xzm_debug_abort("failed to map full main zone");
+ return kr;
+ }
+ } else {
+ main = (xzm_main_malloc_zone_t)zone;
+ if (main->xzmz_total_size != zone_size) {
+ xzm_debug_abort("inconsistent main zone size");
+ return KERN_FAILURE;
+ }
+
+ main_address = zone_address;
+ main_zone_size = zone_size;
+ }
+
+ if (main_zone_size < main->xzmz_total_size) {
+ xzm_debug_abort("inconsistent main region size");
+ return KERN_FAILURE;
+ }
+
+ xzm_assert(zone);
+ xzm_assert(main);
+ xzm_assert(main_address);
+ *zone_p_out = zone;
+ *main_p_out = main;
+ *main_address_out = main_address;
+
+ return KERN_SUCCESS;
+}
+
+static kern_return_t
+xzm_ptr_in_use_enumerator(task_t task, void *context, unsigned type_mask,
+ vm_address_t zone_address, memory_reader_t reader,
+ vm_range_recorder_t recorder)
+{
+ xzm_malloc_zone_t zone;
+ xzm_main_malloc_zone_t main;
+ vm_address_t main_address;
+ bool zone_is_main = false;
+
+ reader = reader_or_in_memory_fallback(reader, task);
+
+ bool record_admin = (type_mask & MALLOC_ADMIN_REGION_RANGE_TYPE);
+ bool record_ptr_region = (type_mask & MALLOC_PTR_REGION_RANGE_TYPE);
+ bool record_ptr_in_use = (type_mask & MALLOC_PTR_IN_USE_RANGE_TYPE);
+
+ kern_return_t kr = _xzm_introspect_map_zone_and_main(task, zone_address,
+ reader, &zone, &main, &main_address);
+ if (kr) {
+ return kr;
+ }
+
+ zone_is_main = (zone_address == main_address);
+
+ if (zone_is_main) {
+ vm_address_t mfm_addr = (vm_address_t)main->xzmz_mfm_address;
+ if (mfm_addr) {
+ kr = mfm_introspect.enumerator(task, context, type_mask, mfm_addr,
+ reader, recorder);
+ if (kr) {
+ return kr;
+ }
+ }
+ }
+
+ return _xzm_introspect_enumerate(task, reader, zone_address,
+ zone, main_address, main,
+ /* include_blocks */record_ptr_in_use,
+ ^(vm_address_t slab_addr, vm_size_t slab_size, xzm_metapool_id_t mp_id){
+ // Metapool slab enumerator
+ if (record_admin && zone_is_main) {
+ vm_range_t segment_meta_range = {
+ .address = slab_addr,
+ .size = slab_size,
+ };
+ recorder(task, context, MALLOC_ADMIN_REGION_RANGE_TYPE,
+ &segment_meta_range, 1);
+ }
+ return KERN_SUCCESS;
+ }, ^(vm_address_t segment_addr, xzm_segment_t segment, const char *indent){
+ // Segment enumerator
+ // Nothing to do, since segment metadata is recorded by the slab
+ // enumerator
+ return KERN_SUCCESS;
+ }, ^(vm_address_t segment_addr, xzm_segment_t segment, xzm_chunk_t chunk,
+ xzm_slice_count_t slice_count, vm_address_t start_addr,
+ xzm_xzone_t xz, vm_range_t *ranges, size_t count){
+ // Chunk enumerator
+ xzm_slice_kind_t kind = chunk->xzc_bits.xzcb_kind;
+ if (record_admin && kind == XZM_SLICE_KIND_HUGE_CHUNK) {
+ // Record the info slices of huge segments against the mzone they
+ // belong to
+ vm_range_t header_range = {
+ .address = segment_addr,
+ .size = XZM_METAPOOL_SEGMENT_BLOCK_SIZE,
+ };
+ recorder(task, context, MALLOC_ADMIN_REGION_RANGE_TYPE,
+ &header_range, 1);
+ }
+
+ if (!record_ptr_region && !record_ptr_in_use) {
+ return KERN_SUCCESS;
+ }
+
+ vm_range_t region_range = {
+ .address = start_addr,
+ .size = slice_count * XZM_SEGMENT_SLICE_SIZE,
+ };
+ if (_xzm_slice_kind_uses_xzones(kind)) {
+ if (record_ptr_region) {
+ recorder(task, context, MALLOC_PTR_REGION_RANGE_TYPE,
+ ®ion_range, 1);
+ }
+
+ if (record_ptr_in_use) {
+ recorder(task, context, MALLOC_PTR_IN_USE_RANGE_TYPE, ranges,
+ (unsigned)count);
+ }
+ } else {
+ recorder(task, context, MALLOC_PTR_IN_USE_RANGE_TYPE |
+ MALLOC_PTR_REGION_RANGE_TYPE, ®ion_range, 1);
+ }
+
+ return KERN_SUCCESS;
+ }, !zone_is_main ? NULL : ^(vm_address_t segment_addr,
+ xzm_segment_t segment, xzm_chunk_t span,
+ xzm_slice_count_t slice_count, vm_address_t start_addr){
+ // Main zone span enumerator
+
+ // Record all free spans and chunks with no mzone against the main zone,
+ // with the exception of huge chunks that could be in the reclaim buffer
+ if (record_ptr_region) {
+ bool should_record = true;
+
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ // Unfortunately, there's no way for us to reliably tell whether a
+ // given huge chunk is in the reclaim buffer, because when marking
+ // them free there's a window where we haven't yet stored the
+ // reclaim index in xzs_reclaim_id. So, we err on the side of
+ // caution and just never record them.
+ if (segment->xzs_kind == XZM_SEGMENT_KIND_HUGE &&
+ span->xzc_bits.xzcb_kind == XZM_SLICE_KIND_HUGE_CHUNK) {
+ should_record = false;
+ }
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+
+ if (should_record) {
+ vm_range_t region_range = {
+ .address = start_addr,
+ .size = slice_count * XZM_SEGMENT_SLICE_SIZE,
+ };
+ recorder(task, context, MALLOC_PTR_REGION_RANGE_TYPE,
+ ®ion_range, 1);
+ }
+ }
+
+ return KERN_SUCCESS;
+ });
+}
+
+#if XZM_DEBUG_ENUMERATOR
+
+struct xzm_debug_recorder_context_s {
+ vm_range_recorder_t *orig_recorder;
+ void *orig_context;
+};
+
+static void
+_xzm_debug_range_recorder(task_t task, void *context, unsigned type,
+ vm_range_t *ranges, unsigned count)
+{
+ const char *type_str = "(invalid?)";
+ switch (type) {
+ case MALLOC_PTR_IN_USE_RANGE_TYPE | MALLOC_PTR_REGION_RANGE_TYPE:
+ type_str = "PTR_IN_USE | PTR_REGION";
+ break;
+ case MALLOC_PTR_IN_USE_RANGE_TYPE:
+ type_str = "PTR_IN_USE";
+ break;
+ case MALLOC_PTR_REGION_RANGE_TYPE:
+ type_str = "PTR_REGION";
+ break;
+ case MALLOC_ADMIN_REGION_RANGE_TYPE:
+ type_str = "ADMIN_REGION";
+ break;
+ default:
+ break;
+ }
+
+ printf("XZM ENUMERATOR: %s (%x) - %p %u ranges\n", type_str, type, ranges,
+ count);
+ for (unsigned i = 0; i < count; i++) {
+ printf("XZM ENUMERATOR: %p[%u]: %p %llu\n", ranges, i,
+ (void *)ranges[i].address, (unsigned long long)ranges[i].size);
+ }
+
+ struct xzm_debug_recorder_context_s *ctx = context;
+ ctx->orig_recorder(task, ctx->orig_context, type, ranges, count);
+}
+
+static kern_return_t
+xzm_debug_ptr_in_use_enumerator(task_t task, void *context, unsigned type_mask,
+ vm_address_t zone_address, memory_reader_t reader,
+ vm_range_recorder_t recorder)
+{
+ struct xzm_debug_recorder_context_s ctx = {
+ .orig_recorder = recorder,
+ .orig_context = context,
+ };
+
+ return xzm_ptr_in_use_enumerator(task, &ctx, type_mask, zone_address,
+ reader, _xzm_debug_range_recorder);
+
+}
+
+#endif // XZM_DEBUG_ENUMERATOR
+
+static void
+xzm_print(task_t task, unsigned level, vm_address_t zone_address,
+ memory_reader_t reader, print_task_printer_t printer)
+{
+ xzm_malloc_zone_t zone;
+ xzm_main_malloc_zone_t main;
+ vm_address_t main_address;
+ bool zone_is_main = false;
+
+ kern_return_t kr = _xzm_introspect_map_zone_and_main(task, zone_address,
+ reader, &zone, &main, &main_address);
+ if (kr) {
+ return;
+ }
+
+ zone_is_main = (zone_address == main_address);
+
+ printer("Begin xzone malloc JSON:\n");
+ printer("{\n");
+ printer("\"desc\": \"xzone malloc\", \n");
+ printer("\"addr\": \"%p\", \n", zone_address);
+ printer("\"segment_size\": %zu, \n", XZM_SEGMENT_SIZE);
+ printer("\"slice_size\": %zu, \n", XZM_SEGMENT_SLICE_SIZE);
+ printer("\"mzone\": %d, \n", (int)zone->xzz_mzone_idx);
+ printer("\"is_main\": %d, \n", zone_is_main);
+ printer("\"max_list_config\": %d, \n", (int)zone->xzz_max_list_config);
+ printer("\"initial_slot_config\": %d, \n", (int)zone->xzz_initial_slot_config);
+ printer("\"slot_initial_threshold\": %u, \n", zone->xzz_slot_initial_threshold);
+ printer("\"max_slot_config\": %d, \n", (int)zone->xzz_max_slot_config);
+
+ // TODO: early allocator info
+
+ __block size_t dispositions_count = 0;
+ __block int *dispositions = NULL;
+ __block vm_address_t dispositions_start_addr = 0;
+ kern_return_t (^print_dispositions)(vm_address_t, vm_size_t, const char *);
+ print_dispositions = ^kern_return_t(vm_address_t addr, vm_size_t size, const char *indent) {
+ kern_return_t kr = KERN_SUCCESS;
+
+ // When operating on a core dump, no pages can be queried
+ if (task == TASK_NULL) {
+ return kr;
+ }
+
+ // If dispositions doesn't cover the full range of this request,
+ // (possibly) reallocate and re-query the VM
+ vm_address_t request_end = addr + size;
+ vm_address_t dispositions_end = dispositions_start_addr +
+ (dispositions_count * vm_page_size);
+ if ((dispositions_start_addr > addr) ||
+ (dispositions_end < request_end)) {
+ dispositions_start_addr = addr;
+
+ // This interface is usually used to query the disposition of a
+ // full segment, so to reduce the number of calls into the vm, request at least a segment
+ size_t request_pages = howmany(size, vm_page_size);
+ if (request_pages < (XZM_SEGMENT_SIZE / vm_page_size)) {
+ request_pages = XZM_SEGMENT_SIZE / vm_page_size;
+ }
+
+ // TODO: mixed page size difficulties
+ if (request_pages > dispositions_count) {
+ if (dispositions) {
+ mach_vm_deallocate(mach_task_self(),
+ (mach_vm_address_t)dispositions,
+ dispositions_count * sizeof(dispositions[0]));
+ dispositions = NULL;
+ }
+
+ dispositions_count = request_pages;
+ kr = mach_vm_allocate(mach_task_self(),
+ (mach_vm_address_t *)&dispositions,
+ dispositions_count * sizeof(dispositions[0]),
+ VM_FLAGS_ANYWHERE);
+ if (kr) {
+ xzm_debug_abort("failed to allocate memory for vm stats");
+ return kr;
+ }
+ }
+
+ mach_vm_size_t mvs_page_span = (mach_vm_size_t)request_pages;
+ kr = mach_vm_page_range_query(task,
+ (mach_vm_address_t)addr, MAX(size, XZM_SEGMENT_SIZE),
+ (mach_vm_address_t)dispositions, &mvs_page_span);
+ if (kr) {
+ xzm_debug_abort("Failed to query vm stats");
+ return kr;
+ }
+ }
+
+ printer("%s \"dispositions\": \"", indent);
+
+ size_t dirty_count = 0;
+ size_t swapped_count = 0;
+ size_t disposition_idx =
+ (addr - dispositions_start_addr) / vm_page_size;
+ for (size_t i = 0; i < (size / vm_page_size); i++) {
+ if (disposition_idx >= dispositions_count) {
+ xzm_debug_abort("inconsistent slice counts");
+ return KERN_FAILURE;
+ }
+
+ int disposition = dispositions[disposition_idx];
+ if (disposition & VM_PAGE_QUERY_PAGE_DIRTY) {
+ dirty_count++;
+ printer("d");
+ } else if (disposition & VM_PAGE_QUERY_PAGE_PAGED_OUT) {
+ swapped_count++;
+ printer("s");
+ } else {
+ printer("c");
+ }
+
+ disposition_idx++;
+ }
+
+ printer("\", \n"); // dispositions
+ printer("%s \"dirty_count\": %zu, \n", indent, dirty_count);
+ printer("%s \"swapped_count\": %zu, \n", indent, swapped_count);
+
+ return KERN_SUCCESS;
+ };
+
+ __block bool first_span = true;
+ __block bool print_segment_dispositions = true;
+ const xzm_segment_enumerator_t segment_enumerator =
+ ^(vm_address_t segment_addr, xzm_segment_t segment,
+ const char *indent) {
+ // Segment enumerator
+
+ if (!first_span) {
+ printer(", ");
+ }
+
+ printer("%s\"%p\": {\n", indent, (void *)segment_addr);
+ printer("%s \"addr\": \"%p\", \n", indent, (void *)segment_addr);
+ xzm_segment_group_id_t sg_id = segment->xzs_segment_group -
+ main->xzmz_segment_groups;
+ printer("%s \"segment_group\": \"%s\", \n", indent,
+ _xzm_segment_group_id_to_string(sg_id));
+ printer("%s \"body_addr\": \"%p\", \n", indent,
+ segment->xzs_segment_body);
+ printer("%s \"used\": %u, \n", indent, segment->xzs_used);
+ printer("%s \"kind\": \"%s\", \n", indent,
+ _xzm_segment_kind_to_string(segment->xzs_kind));
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ if (segment->xzs_reclaim_id == VM_RECLAIM_ID_NULL) {
+ printer("%s \"reclaim_id\": -1, \n", indent);
+ } else {
+ printer("%s \"reclaim_id\": %llu, \n", indent,
+ segment->xzs_reclaim_id);
+ }
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+ if (print_segment_dispositions) {
+ print_dispositions((vm_address_t)segment->xzs_segment_body,
+ segment->xzs_slice_count * XZM_SEGMENT_SLICE_SIZE, indent);
+ }
+
+ printer("%s \"slice_count\": %u, \n", indent,
+ segment->xzs_slice_count);
+ printer("%s \"slice_entry_count\": %u \n", indent,
+ segment->xzs_slice_entry_count);
+
+ printer("%s}\n", indent); // segment
+ first_span = false;
+
+ return KERN_SUCCESS;
+ };
+
+ if (zone_is_main) {
+ printer("\"bucketing_key\": \"%016llx%016llx\", \n",
+ main->xzmz_bucketing_keys.xbk_key_data[0],
+ main->xzmz_bucketing_keys.xbk_key_data[1]);
+ printer("\"guard_config\": {\n");
+ printer(" \"guards_enabled\": %d, \n",
+ main->xzmz_guard_config.xgpc_enabled);
+ printer(" \"data_guards_enabled\": %d, \n",
+ main->xzmz_guard_config.xgpc_enabled_for_data);
+ printer(" \"tiny_run_size\": %d, \n",
+ main->xzmz_guard_config.xgpc_max_run_tiny);
+ printer(" \"tiny_guard_density\": %d, \n",
+ main->xzmz_guard_config.xgpc_tiny_guard_density);
+ printer(" \"small_run_size\": %d, \n",
+ main->xzmz_guard_config.xgpc_max_run_small);
+ printer(" \"small_guard_density\": %d \n",
+ main->xzmz_guard_config.xgpc_small_guard_density);
+ printer("}, \n");
+ printer("\"chunk_threshold\": %u, \n", main->xzmz_xzone_chunk_threshold);
+ printer("\"ptr_bucket_count\": %d, \n", main->xzmz_ptr_bucket_count);
+ // guard_config
+
+#if CONFIG_MTE
+ printer("\"mte_config\": {\n");
+ printer(" \"enabled\": %d, \n",
+ (int)main->xzmz_base.xzz_memtag_config.enabled);
+ printer(" \"tag_data\": %d, \n",
+ (int)main->xzmz_base.xzz_memtag_config.tag_data);
+ printer(" \"max_block_size\": %d \n",
+ (int)main->xzmz_base.xzz_memtag_config.max_block_size);
+ printer("}, \n"); // mte_config
+#endif // CONFIG_MTE
+
+ printer("\"defer_tiny\": %s, \n", main->xzmz_defer_tiny ?
+ "true" : "false");
+ printer("\"defer_small\": %s, \n", main->xzmz_defer_small ?
+ "true" : "false");
+ printer("\"defer_large\": %s, \n", main->xzmz_defer_large ?
+ "true" : "false");
+ printer("\"deallocate_segment\": %s, \n", main->xzmz_deallocate_segment ?
+ "true" : "false");
+
+ printer("\"use_early_alloc\": %s, \n", main->xzmz_mfm_address ?
+ "true" : "false");
+
+ printer("\"batch_size\": %u, \n", main->xzmz_batch_size);
+
+#if CONFIG_XZM_DEFERRED_RECLAIM
+ if (main->xzmz_reclaim_buffer != NULL) {
+ vm_address_t xzm_buffer_addr = (vm_address_t)main->xzmz_reclaim_buffer;
+ xzm_reclaim_buffer_t xzm_reclaim_buffer;
+ mach_vm_reclaim_ring_t ringbuffer;
+ kr = _xzm_introspect_map_reclaim_buffer(task, reader,
+ xzm_buffer_addr, &xzm_reclaim_buffer, &ringbuffer);
+ if (kr) {
+ xzm_debug_abort("failed to map reclaim buffer");
+ return;
+ }
+ if (ringbuffer != NULL) {
+ printer("\"reclaim_buffer\": { \n");
+ printer(" \"buffer_len\": %llu, \n",
+ ringbuffer->len);
+ printer(" \"max_len\": %llu, \n",
+ ringbuffer->max_len);
+ printer(" \"sampling_period_abs\": %llu, \n", ringbuffer->sampling_period_abs);
+ printer(" \"last_sample_abs\": %llu, \n",
+ ringbuffer->last_sample_abs);
+ printer(" \"reclaimable_bytes\": %llu, \n",
+ os_atomic_load(&ringbuffer->reclaimable_bytes,
+ relaxed));
+ printer(" \"reclaimable_bytes_min\": %llu, \n",
+ os_atomic_load(&ringbuffer->reclaimable_bytes_min,
+ relaxed));
+
+ printer(" \"head\": %llu, \n",
+ os_atomic_load(&ringbuffer->head, relaxed));
+ printer(" \"busy\": %llu, \n",
+ os_atomic_load(&ringbuffer->busy, relaxed));
+ printer(" \"tail\": %llu, \n",
+ os_atomic_load(&ringbuffer->tail, relaxed));
+
+ printer(" \"entries\": [ \n");
+
+ for (mach_vm_reclaim_count_t i = 0; i < ringbuffer->len; i++) {
+ mach_vm_reclaim_entry_t entry = &ringbuffer->entries[i];
+ printer(" { \n");
+ printer(" \"id\": %u, \n", i);
+ printer(" \"address\": \"%p\", \n", entry->address);
+ printer(" \"size\": %u, \n", entry->size);
+ // TODO: add string decoder to libsyscall
+ printer(" \"behavior\": %u \n", entry->behavior);
+ printer(" }");
+ if (i < ringbuffer->len - 1) {
+ printer(",");
+ }
+ printer(" \n");
+ }
+ printer(" ] \n"); // entries
+ }
+ printer("}, \n"); // reclaim buffer
+ }
+
+#endif // CONFIG_XZM_DEFERRED_RECLAIM
+
+ printer("\"allocation_front_count\": %u, \n",
+ main->xzmz_allocation_front_count);
+ printer("\"range_group_count\": %u, \n", main->xzmz_range_group_count);
+ printer("\"range_groups\": {\n");
+
+ size_t range_group_size;
+ if (os_mul_overflow(main->xzmz_range_group_count,
+ sizeof(struct xzm_range_group_s), &range_group_size)) {
+ xzm_debug_abort("failed to compute range group size");
+ return;
+ }
+ struct xzm_range_group_s *mapped_range_groups =
+ (struct xzm_range_group_s *)_xzm_introspect_rebase(main_address,
+ main, main->xzmz_total_size, main->xzmz_range_groups,
+ range_group_size);
+ if (!mapped_range_groups) {
+ xzm_debug_abort("failed to map range_groups");
+ return;
+ }
+
+ for (uint8_t i = 0; i < main->xzmz_range_group_count; i++) {
+ printer(" ");
+ if (i) {
+ printer(", ");
+ }
+ xzm_range_group_t rg = &mapped_range_groups[i];
+ printer("\"%d\": {\n", (int)i);
+ printer(" \"id\": \"%s\", \n",
+ _xzm_range_group_id_to_string(rg->xzrg_id));
+ printer(" \"front\": %d, \n", (int)rg->xzrg_front);
+ printer(" \"lock\": %u, \n", *(uint32_t *)&rg->xzrg_lock);
+ printer(" \"base\": \"%p\", \n", (void *)rg->xzrg_base);
+ printer(" \"size\": %zu, \n", rg->xzrg_size);
+ printer(" \"skip_addr\": \"%p\", \n",
+ (void *)rg->xzrg_skip_addr);
+ printer(" \"skip_size\": %zu, \n", rg->xzrg_skip_size);
+ printer(" \"next\": \"%p\", \n", (void *)rg->xzrg_next);
+ printer(" \"remaining\": %zu, \n", rg->xzrg_remaining);
+ printer(" \"direction\": \"%s\"\n",
+ rg->xzrg_direction == XZM_FRONT_INCREASING ? "up" : "down");
+ printer(" }\n"); // range group
+ }
+
+ printer("}, \n"); // range_groups
+
+ printer("\"segment_group_ids_count\": %u, \n",
+ main->xzmz_segment_group_ids_count);
+ printer("\"segment_group_front_count\": %u, \n",
+ main->xzmz_segment_group_front_count);
+ printer("\"segment_group_count\": %u, \n",
+ main->xzmz_segment_group_count);
+ printer("\"segment_groups\": {\n");
+
+ size_t segment_group_size;
+ if (os_mul_overflow(main->xzmz_segment_group_count,
+ sizeof(struct xzm_segment_group_s), &segment_group_size)) {
+ xzm_debug_abort("failed to compute segment group size");
+ return;
+ }
+ struct xzm_segment_group_s *mapped_segment_groups =
+ (struct xzm_segment_group_s *)_xzm_introspect_rebase(main_address,
+ main, main->xzmz_total_size, main->xzmz_segment_groups,
+ segment_group_size);
+ if (!mapped_segment_groups) {
+ xzm_debug_abort("failed to map segment_groups");
+ return;
+ }
+
+ for (uint8_t i = 0; i < main->xzmz_segment_group_count; i++) {
+ printer(" ");
+ if (i) {
+ printer(", ");
+ }
+ xzm_segment_group_t sg = &mapped_segment_groups[i];
+ printer("\"%d\": {\n", (int)i);
+ printer(" \"id\": \"%s\", \n",
+ _xzm_segment_group_id_to_string(sg->xzsg_id));
+ printer(" \"front\": %d, \n", (int)sg->xzsg_front);
+ printer(" \"range_group\": \"%p\", \n",
+ sg->xzsg_range_group);
+ printer(" \"segment_cache\": { \n");
+ printer(" \"max_count\": %u, \n",
+ (unsigned)sg->xzsg_cache.xzsc_max_count);
+ printer(" \"count\": %u, \n",
+ (unsigned)sg->xzsg_cache.xzsc_count);
+ printer(" \"max_entry_slices\": %u, \n",
+ (unsigned)sg->xzsg_cache.xzsc_max_entry_slices);
+ printer(" \"segments\": { \n");
+ if (sg->xzsg_cache.xzsc_count) {
+ // Segments in the segment cache will not be present in the segment
+ // table, so they must be enumerated here
+ vm_address_t segment_addr = (vm_address_t)TAILQ_FIRST(
+ &sg->xzsg_cache.xzsc_head);
+ while (segment_addr != 0) {
+ xzm_segment_t segment;
+ kern_return_t kr = reader(task, segment_addr,
+ sizeof(struct xzm_segment_s), (void **)&segment);
+ if (kr) {
+ xzm_debug_abort("Failed to map cached segment");
+ return;
+ }
+
+ kr = segment_enumerator(segment_addr, segment,
+ " ");
+ if (kr) {
+ xzm_debug_abort("Failed to enumerate segment");
+ return;
+ }
+ segment_addr = (vm_address_t)TAILQ_NEXT(segment,
+ xzs_cache_entry);
+ }
+ }
+ printer(" } \n"); // segments
+ printer(" } \n"); // segment cache
+ printer(" }\n"); // segment group
+ }
+
+ printer("}, \n"); // segment_groups
+
+ printer("\"xzones\": {\n");
+
+ size_t xzone_size;
+ if (os_mul_overflow(main->xzmz_base.xzz_xzone_count,
+ sizeof(struct xzm_xzone_s), &xzone_size)) {
+ xzm_debug_abort("failed to compute xzone array size");
+ return;
+ }
+ uintptr_t rebased_xzones = _xzm_introspect_rebase(main_address, main,
+ main->xzmz_total_size, main->xzmz_base.xzz_xzones, xzone_size);
+ struct xzm_xzone_s *mapped_xzones =
+ (struct xzm_xzone_s *)rebased_xzones;
+ if (!mapped_xzones) {
+ xzm_debug_abort("failed to map main xzones");
+ return;
+ }
+
+ size_t slots_size;
+ if (os_mul3_overflow(main->xzmz_base.xzz_xzone_count,
+ main->xzmz_base.xzz_slot_count,
+ sizeof(struct xzm_xzone_allocation_slot_s), &slots_size)) {
+ xzm_debug_abort("failed to compute allocation slots size");
+ return;
+ }
+ uintptr_t rebased_slots = _xzm_introspect_rebase(main_address, main,
+ main->xzmz_total_size,
+ main->xzmz_base.xzz_xzone_allocation_slots, slots_size);
+ struct xzm_xzone_allocation_slot_s *mapped_slots =
+ (struct xzm_xzone_allocation_slot_s *)rebased_slots;
+ if (!mapped_slots) {
+ xzm_debug_abort("failed to map main allocation slots");
+ return;
+ }
+
+ for (uint8_t xzidx = XZM_XZONE_INDEX_FIRST;
+ xzidx < zone->xzz_xzone_count; xzidx++) {
+ xzm_xzone_t xz = &mapped_xzones[xzidx];
+ printer(" \"%d\": {\n", (int)xzidx);
+ printer(" \"early_budget\": %u, \n", xz->xz_early_budget);
+ printer(" \"id\": %d, \n", (int)xz->xz_idx);
+ printer(" \"bucket\": %d, \n", (int)xz->xz_bucket);
+ printer(" \"segment_group_id\": %d, \n",
+ xz->xz_segment_group_id);
+ printer(" \"front\": %d, \n", xz->xz_front);
+ printer(" \"batch_count\": %u, \n",
+ xz->xz_block_size <= XZM_TINY_BLOCK_SIZE_MAX ?
+ xz->xz_batch_list.xzch_batch_count :
+ xz->xz_chunkq_batch_count);
+ printer(" \"block_size\": %llu, \n", xz->xz_block_size);
+ printer(" \"chunk_count\": %llu, \n", xz->xz_chunk_count);
+ printer(" \"chunk_capacity\": %u, \n", xz->xz_chunk_capacity);
+ printer(" \"sequestered\": %d,\n", (int)xz->xz_sequestered);
+ printer(" \"list_config\": \"%s\",\n",
+ _xzm_slot_config_to_string(xz->xz_list_config));
+ printer(" \"slot_config\": \"%s\",\n",
+ _xzm_slot_config_to_string(xz->xz_slot_config));
+ printer(" \"allocation_slots\": [\n");
+
+ for (xzm_allocation_index_t slot_idx = 0;
+ slot_idx < zone->xzz_slot_count;
+ slot_idx++) {
+ xzm_xzone_allocation_slot_t xas = &mapped_slots[
+ (slot_idx * zone->xzz_xzone_count) + xzidx];
+ printer(" {\n");
+
+ if (xz->xz_block_size <= XZM_TINY_BLOCK_SIZE_MAX ||
+ zone->xzz_small_freelist_enabled) {
+ printer(" \"atomic_value\": \"0x%llx\",\n",
+ xas->xas_atomic.xasa_value);
+ printer(" \"xsg_locked\": \"0x%llx\",\n",
+ xas->xas_atomic.xasa_gate.xsg_locked);
+ printer(" \"xsg_waiters\": \"0x%llx\",\n",
+ xas->xas_atomic.xasa_gate.xsg_waiters);
+ printer(" \"xsc_ptr\": \"0x%llx\",\n",
+ xas->xas_atomic.xasa_chunk.xsc_ptr);
+
+ printer(" \"operations\": %lu,\n",
+ xas->xas_counters.xsc_ops);
+ printer(" \"contentions\": %lu,\n",
+ xas->xas_counters.xsc_contentions);
+
+ printer(" \"slot_config\": \"%s\",\n",
+ _xzm_slot_config_to_string(
+ xas->xas_counters.xsc_slot_config));
+ } else {
+ printer(" \"chunk\": \"%p\",\n",
+ (void *)xas->xas_chunk);
+ printer(" \"allocations\": %lu,\n",
+ xas->xas_allocs);
+ printer(" \"contentions\": %lu,\n",
+ xas->xas_contentions);
+ }
+
+ printer(" \"last_chunk_empty_ts\": %llu\n",
+ xas->xas_last_chunk_empty_ts);
+ printer(" }");
+ if (slot_idx < zone->xzz_slot_count - 1) {
+ printer(",");
+ }
+ printer("\n");
+ }
+
+ printer(" ]\n"); // allocation slots
+ printer(" }"); // xzone
+ if (xzidx < zone->xzz_xzone_count - 1) {
+ printer(",");
+ }
+ printer("\n");
+ }
+
+ printer("}, \n"); // xzones
+
+#if CONFIG_XZM_THREAD_CACHE
+ printer("\"thread_cache_enabled\": %s, \n",
+ zone->xzz_thread_cache_enabled ? "true" : "false");
+ printer("\"thread_cache_activation_period\": %lu, \n",
+ zone->xzz_thread_cache_xzone_activation_period);
+ printer("\"thread_cache_activation_contentions\": %lu, \n",
+ zone->xzz_thread_cache_xzone_activation_contentions);
+ printer("\"thread_cache_activation_time\": %llu, \n",
+ zone->xzz_thread_cache_xzone_activation_time);
+
+ if (zone->xzz_thread_cache_enabled) {
+ printer("\"thread_caches\": [ \n");
+ __block bool first_thread_cache = true;
+ kr = _xzm_introspect_enumerate_thread_caches(task, reader, main,
+ ^(vm_address_t thread_cache_addr, xzm_thread_cache_t tc){
+ printer(" ");
+ if (!first_thread_cache) {
+ printer(", ");
+ } else {
+ first_thread_cache = false;
+ }
+ printer("{\n");
+ printer(" \"thread\": \"%p\",\n", (void *)tc->xtc_thread);
+ printer(" \"xz_caches\": {\n", (void *)tc->xtc_thread);
+ for (uint8_t xzidx = XZM_XZONE_INDEX_FIRST;
+ xzidx < zone->xzz_thread_cache_xzone_count; xzidx++) {
+ xzm_xzone_thread_cache_t cache = &tc->xtc_xz_caches[xzidx];
+
+ printer(" \"%d\": {\n", (int)xzidx);
+ printer(" \"xz_idx\": %d, \n", (int)xzidx);
+
+ uint16_t head = cache->xztc_head;
+ if (head == XZM_XZONE_NOT_CACHED) {
+ printer(" \"head\": \"NOT_CACHED\", \n");
+ printer(" \"timestamp\": \"%llu\", \n",
+ cache->xztc_timestamp);
+ printer(" \"contentions\": \"%llu\", \n",
+ (uint64_t)cache->xztc_contentions);
+ printer(" \"allocs\": \"%llu\" \n",
+ (uint64_t)cache->xztc_allocs);
+ } else if (head == XZM_XZONE_CACHE_EMPTY) {
+ printer(" \"head\": \"EMPTY\" \n");
+ } else {
+ printer(" \"head\": \"0x%llx\", \n",
+ (uint64_t)head);
+ printer(" \"chunk\": \"%p\", \n",
+ cache->xztc_chunk);
+ printer(" \"chunk_start\": \"%p\", \n",
+ cache->xztc_chunk_start);
+ printer(" \"head_seqno\": \"0x%llx\", \n",
+ (uint64_t)cache->xztc_head_seqno);
+ printer(" \"free_count\": \"0x%llx\", \n",
+ (uint64_t)cache->xztc_free_count);
+ printer(" \"seqno\": \"0x%llx\" \n",
+ (uint64_t)cache->xztc_seqno);
+ }
+
+
+ printer(" }"); // xzone thread cache
+ if (xzidx < zone->xzz_thread_cache_xzone_count - 1) {
+ printer(",");
+ }
+ printer("\n");
+ }
+ printer(" } \n"); // xzone thread caches
+ printer(" } \n"); // thread cache
+
+ return KERN_SUCCESS;
+ });
+
+ printer("], \n"); // thread_caches
+ }
+#endif // CONFIG_XZM_THREAD_CACHE
+ }
+
+ printer("\"spans\": {\n");
+
+ first_span = true;
+ // un-cached segments will have their dispositions enumerated via the
+ // spans/chunks they contain
+ print_segment_dispositions = false;
+
+ kr = _xzm_introspect_enumerate(task, reader, zone_address,
+ zone, main_address, main,
+ /* include_blocks */false,
+ ^(vm_address_t slab_addr, vm_size_t slab_size, xzm_metapool_id_t
+ metapool_id) {
+ // Metapool slab enumerator
+
+ printer(" ");
+ if (!first_span) {
+ printer(", ");
+ }
+
+ printer("\"%p\": {\n", (void *)slab_addr);
+ printer(" \"addr\": \"%p\", \n", (void *)slab_addr);
+ printer(" \"kind\": \"%s\", \n",
+ _xzm_metapool_id_to_string(metapool_id));
+ print_dispositions(slab_addr, slab_size, " ");
+ printer(" \"size\": %u \n",
+ slab_size);
+
+ printer(" }\n");
+ first_span = false;
+
+ return KERN_SUCCESS;
+ },
+ segment_enumerator,
+ ^(vm_address_t segment_addr, xzm_segment_t segment, xzm_chunk_t chunk,
+ xzm_slice_count_t slice_count, vm_address_t start_addr,
+ xzm_xzone_t xz, vm_range_t *ranges, size_t count){
+ // Chunk enumerator
+
+ printer(" ");
+ if (!first_span) {
+ printer(", ");
+ }
+
+ printer("\"%p\": {\n", (void *)start_addr);
+ printer(" \"addr\": \"%p\", \n", (void *)start_addr);
+ printer(" \"metadata_addr\": \"%p\", \n", (void *)(segment_addr +
+ ((vm_address_t)chunk - (vm_address_t)segment)));
+ printer(" \"mzone\": %d, \n", (int)chunk->xzc_mzone_idx);
+ printer(" \"xzone\": %d, \n", (int)chunk->xzc_xzone_idx);
+ printer(" \"segment\": \"%p\", \n", (void *)segment_addr);
+ printer(" \"segment_group\": %zu, \n",
+ segment->xzs_segment_group - main->xzmz_segment_groups);
+
+ xzm_slice_kind_t kind = chunk->xzc_bits.xzcb_kind;
+ const char *kind_str = _xzm_slice_kind_to_string(kind);
+ printer(" \"kind\": \"%s\", \n", kind_str);
+ printer(" \"slice_count\": %u, \n", slice_count);
+ printer(" \"block_size\": %u, \n",
+ xz ? (unsigned)xz->xz_block_size : 0);
+ printer(" \"in_use\": 1, \n");
+
+ xzm_slice_count_t slice_index = (xzm_slice_count_t)
+ (chunk - segment->xzs_slices);
+ xzm_xzone_slice_metadata_u *metadata =
+ &segment->xzs_slice_metadata[slice_index];
+ printer(" \"slice_metadata\": \"%p\", \n",
+ metadata->xzsm_batch_next);
+
+ kern_return_t kr = print_dispositions(start_addr, slice_count *
+ XZM_SEGMENT_SLICE_SIZE, " ");
+ if (kr) {
+ return kr;
+ }
+
+ if (_xzm_slice_kind_uses_xzones(kind)) {
+ printer(" \"bucket\": %u,\n", (unsigned)xz->xz_bucket);
+ }
+
+ switch (kind) {
+ case XZM_SLICE_KIND_TINY_CHUNK:
+ case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
+ printer(" \"meta\": \"0x%llx\",\n",
+ chunk->xzc_atomic_meta.xca_value);
+ printer(" \"xca_alloc_head\": \"0x%llx\",\n",
+ chunk->xzc_atomic_meta.xca_alloc_head);
+ printer(" \"xca_free_count\": \"0x%llx\",\n",
+ chunk->xzc_atomic_meta.xca_free_count);
+ printer(" \"xca_alloc_idx\": \"0x%llx\",\n",
+ chunk->xzc_atomic_meta.xca_alloc_idx);
+ printer(" \"xca_on_partial_list\": \"0x%llx\",\n",
+ chunk->xzc_atomic_meta.xca_on_partial_list);
+ printer(" \"xca_on_empty_list\": \"0x%llx\",\n",
+ chunk->xzc_atomic_meta.xca_on_empty_list);
+ printer(" \"xca_walk_locked\": \"0x%llx\",\n",
+ chunk->xzc_atomic_meta.xca_walk_locked);
+ printer(" \"xca_head_seqno\": \"0x%llx\",\n",
+ chunk->xzc_atomic_meta.xca_head_seqno);
+ printer(" \"xca_seqno\": \"0x%llx\",\n",
+ chunk->xzc_atomic_meta.xca_seqno);
+ break;
+ case XZM_SLICE_KIND_SMALL_CHUNK:
+ printer(" \"free\": \"0x%x\",\n", chunk->xzc_free);
+ printer(" \"used\": %u,\n", (unsigned)chunk->xzc_used);
+ printer(" \"alloc_idx\": %u,\n",
+ (unsigned)chunk->xzc_alloc_idx);
+ break;
+ default:
+ break;
+ }
+
+ printer(" \"is_preallocated\": %d,\n",
+ (int)chunk->xzc_bits.xzcb_preallocated);
+ printer(" \"is_pristine\": %d\n",
+ (int)chunk->xzc_bits.xzcb_is_pristine);
+ printer(" }\n");
+ first_span = false;
+
+ return KERN_SUCCESS;
+ }, !zone_is_main ? NULL : ^(vm_address_t segment_addr,
+ xzm_segment_t segment, xzm_chunk_t span,
+ xzm_slice_count_t slice_count, vm_address_t start_addr){
+ // Main zone span enumerator
+
+ // TODO: better sharing with the chunk enumerator
+ printer(" ");
+ if (!first_span) {
+ printer(", ");
+ }
+
+ printer("\"%p\": {\n", (void *)start_addr);
+ printer(" \"addr\": \"%p\", \n", (void *)start_addr);
+ printer(" \"metadata_addr\": \"%p\", \n", (void *)(segment_addr +
+ ((vm_address_t)span - (vm_address_t)segment)));
+ printer(" \"mzone\": %d, \n", (int)span->xzc_mzone_idx);
+ printer(" \"xzone\": %d, \n", (int)span->xzc_xzone_idx);
+ printer(" \"segment\": \"%p\", \n", (void *)segment_addr);
+ printer(" \"segment_group\": %zu, \n",
+ segment->xzs_segment_group - main->xzmz_segment_groups);
+
+ xzm_slice_kind_t kind = span->xzc_bits.xzcb_kind;
+ const char *kind_str = _xzm_slice_kind_to_string(kind);
+ printer(" \"kind\": \"%s\", \n", kind_str);
+ printer(" \"slice_count\": %u, \n", slice_count);
+
+ xzm_slice_count_t slice_index = (xzm_slice_count_t)
+ (span - segment->xzs_slices);
+ xzm_xzone_slice_metadata_u *metadata =
+ &segment->xzs_slice_metadata[slice_index];
+ printer(" \"slice_metadata\": \"%p\", \n",
+ metadata->xzsm_batch_next);
+
+ kern_return_t kr = print_dispositions(start_addr, slice_count *
+ XZM_SEGMENT_SLICE_SIZE, " ");
+ if (kr) {
+ return kr;
+ }
+
+ printer(" \"is_preallocated\": %d,\n",
+ (int)span->xzc_bits.xzcb_preallocated);
+ printer(" \"in_use\": 0 \n");
+ printer(" }\n"); // span
+ first_span = false;
+
+ return KERN_SUCCESS;
+ });
+
+ if (dispositions) {
+ mach_vm_deallocate(mach_task_self(), (mach_vm_address_t)dispositions,
+ dispositions_count * sizeof(dispositions[0]));
+ }
+
+ if (kr) {
+ return;
+ }
+
+ printer("}\n"); // spans
+ printer("}\n"); // overall
+ printer("End xzone malloc JSON\n");
+}
+
+static void
+xzm_print_task(task_t task, unsigned level, vm_address_t zone_address,
+ memory_reader_t reader, print_task_printer_t printer)
+{
+ xzm_print(task, level, zone_address, reader, printer);
+}
+
+static void
+xzm_print_self(xzm_malloc_zone_t zone, boolean_t verbose)
+{
+ xzm_print(mach_task_self(), verbose ? MALLOC_VERBOSE_PRINT_LEVEL : 0,
+ (vm_address_t)zone, _malloc_default_reader, malloc_report_simple);
+}
+
+static kern_return_t
+xzm_statistics(task_t task, vm_address_t zone_address,
+ memory_reader_t reader, print_task_printer_t printer,
+ malloc_statistics_t *stats)
+{
+ // It's straightforward to compute blocks_in_use and size_in_use from an
+ // enumeration pass.
+ //
+ // size_allocated, which is the sum of the virtual sizes of all live
+ // non-metadata VM reservations, is also fairly easy.
+ //
+ // max_size_in_use, which is supposed to track the "high water mark of
+ // touched memory", is harder:
+ // - nano doesn't even try
+ // - szone malloc sort of does, but its accounting is wrong in several
+ // ways:
+ // - it doesn't account for madvise, so its calculation for "in-use"
+ // memory in regions is too pessimistic (not everything outside the
+ // pristine parts of a region was necessarily all in use at the same
+ // time)
+ // - regions that only became partially full before being deallocated
+ // are incorrectly assumed to have been fully used
+ // - it doesn't keep track of the high water mark in large, so the
+ // current size of large is taken as the max
+ //
+ // Keeping a running max of the sizes of live chunks in each segment group
+ // seems like it could be reasonable. We'd also have to somehow deal with
+ // sequestered empty chunks, though, since they look "live" to the range
+ // group as-is.
+ //
+ // Nano's policy of not even trying seems best, if we can get away with it,
+ // and the lack of complaints from default enablement of nano on macOS seems
+ // like strong evidence that we can, so that's what we should go with unless
+ // a compelling need to do otherwise arises.
+ (void)printer;
+ *stats = (malloc_statistics_t){ 0 };
+
+ reader = reader_or_in_memory_fallback(reader, task);
+
+ xzm_malloc_zone_t zone;
+ xzm_main_malloc_zone_t main;
+ vm_address_t main_address;
+ bool zone_is_main = false;
+
+ kern_return_t kr = _xzm_introspect_map_zone_and_main(task, zone_address,
+ reader, &zone, &main, &main_address);
+ if (kr) {
+ return kr;
+ }
+
+ zone_is_main = (zone_address == main_address);
+
+ if (zone_is_main) {
+ vm_address_t mfm_addr = (vm_address_t)main->xzmz_mfm_address;
+ if (mfm_addr) {
+ mfm_introspect.task_statistics(task, mfm_addr, reader, stats);
+
+ // We don't know how to report max_size_in_use, so don't confuse things
+ // by only including the max_size_in_use from mfm
+ stats->max_size_in_use = 0;
+ }
+ }
+
+ return _xzm_introspect_enumerate(task, reader, zone_address,
+ zone, main_address, main, /* include_blocks */ false,
+ ^(vm_address_t slab_addr, vm_size_t slab_size, xzm_metapool_id_t mp_id){
+ // Metapool slab enumerator
+ return KERN_SUCCESS;
+ }, ^(vm_address_t segment_addr, xzm_segment_t segment, const char *indent){
+ // Segment enumerator
+
+ // Nothing interesting for stats at the segment level
+ return KERN_SUCCESS;
+ }, ^(vm_address_t segment_addr, xzm_segment_t segment, xzm_chunk_t chunk,
+ xzm_slice_count_t slice_count, vm_address_t start_addr,
+ xzm_xzone_t xz, vm_range_t *ranges, size_t count){
+ // Chunk enumerator
+ size_t chunk_size = slice_count * XZM_SEGMENT_SLICE_SIZE;
+ size_t used = 0;
+
+ xzm_slice_kind_t kind = chunk->xzc_bits.xzcb_kind;
+ switch (kind) {
+ case XZM_SLICE_KIND_TINY_CHUNK:
+ case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:;
+ xzm_chunk_atomic_meta_u meta = chunk->xzc_atomic_meta;
+ uint32_t capacity = xz->xz_chunk_capacity;
+ if (meta.xca_alloc_head != XZM_FREE_MADVISING &&
+ meta.xca_alloc_head != XZM_FREE_MADVISED) {
+ used = (size_t)(capacity - meta.xca_free_count);
+ stats->blocks_in_use += used;
+ stats->size_in_use += used * xz->xz_block_size;
+ }
+ break;
+ case XZM_SLICE_KIND_SMALL_CHUNK:
+ used = chunk->xzc_used;
+ stats->blocks_in_use += used;
+ stats->size_in_use += used * xz->xz_block_size;
+ break;
+ default:
+ stats->blocks_in_use++;
+ stats->size_in_use += chunk_size;
+ break;
+ }
+ stats->size_allocated += chunk_size;
+
+ return KERN_SUCCESS;
+ }, !zone_is_main ? NULL : ^(vm_address_t segment_addr,
+ xzm_segment_t segment, xzm_chunk_t span,
+ xzm_slice_count_t slice_count, vm_address_t start_addr){
+ // Main zone span enumerator
+
+ // Record all free spans and chunks with no mzone against the main zone
+ stats->size_allocated += slice_count * XZM_SEGMENT_SLICE_SIZE;
+
+ return KERN_SUCCESS;
+ });
+}
+
+static void
+xzm_statistics_self(xzm_malloc_zone_t zone, malloc_statistics_t *stats)
+{
+ if (_xzm_malloc_zone_is_main(zone)) {
+ mfm_lock();
+ }
+
+ xzm_force_lock(zone);
+ xzm_statistics(mach_task_self(), (vm_address_t)zone, _malloc_default_reader,
+ malloc_report_simple, stats);
+ xzm_force_unlock(zone);
+ if (_xzm_malloc_zone_is_main(zone)) {
+ mfm_unlock();
+ }
+}
+
+static void
+xzm_statistics_task(task_t task, vm_address_t zone_address,
+ memory_reader_t reader, malloc_statistics_t *stats)
+{
+ xzm_statistics(task, zone_address, reader, NULL, stats);
+}
+
+
+const struct malloc_introspection_t xzm_malloc_zone_introspect = {
+#if XZM_DEBUG_ENUMERATOR
+ .enumerator = (void *)xzm_debug_ptr_in_use_enumerator,
+#else // XZM_DEBUG_ENUMERATOR
+ .enumerator = (void *)xzm_ptr_in_use_enumerator,
+#endif // XZM_DEBUG_ENUMERATOR
+ .print_task = (void *)xzm_print_task,
+ .good_size = (void *)xzm_good_size,
+ .check = (void *)xzm_check,
+ .print = (void *)xzm_print_self,
+ .statistics = (void *)xzm_statistics_self,
+ .task_statistics = (void*)xzm_statistics_task,
+ .log = (void *)xzm_log,
+ .zone_locked = (void *)xzm_locked,
+ .force_lock = (void *)xzm_force_lock,
+ .force_unlock = (void *)xzm_force_unlock,
+ .reinit_lock = (void *)xzm_reinit_lock,
+ // discharge checking is a vestigial interface relating to the historical
+ // ObjC gc - not to be implemented
+ .enable_discharge_checking = NULL,
+ .disable_discharge_checking = NULL,
+#ifdef __BLOCKS__
+ .enumerate_discharged_pointers = NULL,
+#else // __BLOCKS__
+ .enumerate_unavailable_without_blocks = NULL,
+#endif // __BLOCKS__
+ .zone_type = MALLOC_ZONE_TYPE_XZONE,
+};
+
+#endif // CONFIG_XZONE_MALLOC