Loading...
--- libmalloc/libmalloc-792.80.2/src/xzone_malloc/xzone_introspect.c
+++ /dev/null
@@ -1,1908 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2022, Microsoft Research, Daan Leijen
-Copyright © 2025 Apple Inc.
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" in the same directory as this file.
------------------------------------------------------------------------------*/
-
-#include "../internal.h"
-
-#if CONFIG_XZONE_MALLOC
-
-#define XZM_DEBUG_ENUMERATOR 0
-
-#pragma mark libmalloc segment introspection
-
-kern_return_t
-xzm_segment_group_segment_foreach_span(xzm_segment_t segment,
- xzm_span_enumerator_t enumerator)
-{
- xzm_slice_t end = _xzm_segment_slices_end(segment);
- xzm_slice_t slice = _xzm_segment_slices_begin(segment);
-
- if (segment->xzs_kind == XZM_SEGMENT_KIND_HUGE) {
- return enumerator(slice, slice->xzcs_slice_count);
- }
-
- // Enumeration protocol: the kind bits of a chunk are updated last, after
- // the rest of the chunk metadata is initialized, so if we see a chunk slice
- // it should be valid. Anything else may be in an intermediate state and is
- // not to be trusted, so rather than iterating span-by-span as we would
- // under the lock we need to scan linearly from chunk to chunk.
- kern_return_t kr;
- while (slice < end) {
- xzm_slice_kind_t kind = slice->xzc_bits.xzcb_kind;
- if (_xzm_slice_kind_is_chunk_safe(kind) ||
- // Guard pages aren't chunks, but should be enumerated like them
- kind == XZM_SLICE_KIND_GUARD) {
- xzm_slice_count_t slice_count;
- if (kind == XZM_SLICE_KIND_TINY_CHUNK) {
- slice_count = 1;
- } else {
- slice_count = slice->xzcs_slice_count;
- }
-
- kr = enumerator(slice, slice_count);
- if (kr) {
- return kr;
- }
-
- slice += slice_count;
- } else {
- // Scan forward to the next chunk.
- xzm_slice_t first_slice = slice;
- do {
- slice++;
- } while (!_xzm_slice_kind_is_chunk_safe(slice->xzc_bits.xzcb_kind) &&
- slice->xzc_bits.xzcb_kind != XZM_SLICE_KIND_GUARD &&
- slice < end);
-
- // Report the free span.
- kr = enumerator(first_slice,
- (xzm_slice_count_t)(slice - first_slice));
- if (kr) {
- return kr;
- }
- }
- }
-
- return KERN_SUCCESS;
-}
-
-kern_return_t
-xzm_segment_table_foreach(xzm_segment_table_entry_s *segment_table,
- size_t num_entries, xzm_segment_table_enumerator_t enumerator,
- xzm_segment_t *last_segment_enumerated)
-{
- xzm_segment_t last_segment = NULL;
- if (last_segment_enumerated) {
- last_segment = *last_segment_enumerated;
- }
- for (size_t i = 0; i < num_entries; i++) {
- xzm_segment_t segment =
- _xzm_segment_table_entry_to_segment(segment_table[i]);
- if (!segment) {
- continue;
- }
- // Huge segments can be in multiple adjacent entries in the segment map
- // if the segment spans multiple segment granules. Only enumerate the
- // first entry
- if (segment == last_segment) {
- continue;
- } else {
- last_segment = segment;
- }
-
- kern_return_t kr = enumerator((vm_address_t)segment);
- if (kr) {
- return kr;
- }
- }
- if (last_segment_enumerated) {
- *last_segment_enumerated = last_segment;
- }
- return KERN_SUCCESS;
-}
-
-#pragma mark libmalloc zone introspection
-
-#if CONFIG_XZM_THREAD_CACHE
-
-static kern_return_t
-_xzm_introspect_enumerate_thread_caches(task_t task, memory_reader_t reader,
- xzm_main_malloc_zone_t main,
- MALLOC_NOESCAPE xzm_thread_cache_enumerator_t thread_cache_enumerator)
-{
- xzm_debug_assert(main->xzmz_base.xzz_thread_cache_enabled);
-
- vm_address_t thread_cache_addr = (vm_address_t)LIST_FIRST(
- &main->xzmz_thread_cache_list);
- size_t thread_cache_size = sizeof(struct xzm_thread_cache_s) +
- (main->xzmz_base.xzz_thread_cache_xzone_count *
- sizeof(xzm_xzone_thread_cache_u));
- while (thread_cache_addr != 0) {
- xzm_thread_cache_t tc;
- kern_return_t kr = reader(task, thread_cache_addr, thread_cache_size,
- (void **)&tc);
- if (kr) {
- xzm_debug_abort("Failed to map thread cache");
- return kr;
- }
-
- kr = thread_cache_enumerator(thread_cache_addr, tc);
- if (kr) {
- // Allow KERN_RETURN_MAX as a way to request early exit
- if (kr != KERN_RETURN_MAX) {
- xzm_debug_abort("Failed to enumerate thread cache");
- }
- return kr;
- }
-
- thread_cache_addr = (vm_address_t)LIST_NEXT(tc, xtc_linkage);
- }
-
- return KERN_SUCCESS;
-}
-
-#endif // CONFIG_XZM_THREAD_CACHE
-
-static kern_return_t
-_xzm_introspect_small_chunk_blocks(xzm_malloc_zone_t zone,
- vm_address_t segment_addr, xzm_segment_t segment, xzm_chunk_t chunk,
- xzm_slice_count_t slice_count, uintptr_t start, vm_address_t start_addr,
- xzm_xzone_t xz, MALLOC_NOESCAPE xzm_chunk_enumerator_t chunk_enumerator)
-{
- uint32_t block_size = (uint32_t)xz->xz_block_size;
- size_t capacity = xz->xz_chunk_capacity;
-
- union {
- vm_range_t range;
- bool free;
- } blocks[XZM_CHUNK_MAX_BLOCK_COUNT] = { 0 };
-
- size_t range_idx = 0;
-
- for (xzm_block_index_t block_index = 0; block_index < capacity;
- block_index++) {
- if (!_xzm_small_chunk_block_index_is_free(chunk, block_index)) {
- blocks[range_idx].range = (vm_range_t){
- .address = start_addr + block_index * block_size,
- .size = block_size,
- };
- range_idx++;
- }
- }
-
- return chunk_enumerator(segment_addr, segment, chunk, slice_count,
- start_addr, xz, (vm_range_t *)blocks, (unsigned)range_idx);
-}
-
-static kern_return_t
-_xzm_introspect_freelist_chunk_blocks(task_t task, memory_reader_t reader,
- xzm_malloc_zone_t zone, vm_address_t segment_addr,
- xzm_segment_t segment, xzm_chunk_t chunk, xzm_slice_count_t slice_count,
- uintptr_t start, vm_address_t start_addr, xzm_xzone_t xz,
- MALLOC_NOESCAPE xzm_chunk_enumerator_t chunk_enumerator)
-{
- kern_return_t kr = KERN_FAILURE;
-
- uint32_t block_size = (uint32_t)xz->xz_block_size;
- size_t granule = block_size > XZM_TINY_BLOCK_SIZE_MAX ?
- XZM_SMALL_GRANULE : XZM_GRANULE;
- size_t capacity = xz->xz_chunk_capacity;
-
- xzm_chunk_atomic_meta_u meta = chunk->xzc_atomic_meta;
-
- if (meta.xca_alloc_head == XZM_FREE_MADVISING ||
- meta.xca_alloc_head == XZM_FREE_MADVISED) {
- xzm_debug_assert(meta.xca_free_count == 0);
- // This chunk is madvised, so there can be no blocks in use
- return chunk_enumerator(segment_addr, segment, chunk, slice_count,
- start_addr, xz, NULL, 0);
- }
-
- xzm_debug_assert(meta.xca_free_count <= capacity);
-
-#if CONFIG_MTE
- // This code can be invoked by memory tools that are not running
- // under MTE, to introspect a zone mapped in from a process that
- // is actually running under MTE. Therefore, we only ldg when we are running
- // in a process spawned with has_sec_transition=1.
- //
- // This should only matter for the case of memory tools on non-MTE hardware
- // introspecting processes running under MTE emulation, as we need to ensure
- // we don't try to execute the unsupported MTE instructions. On real
- // hardware, we expect memory tools to run with Allocation Tag Access
- // disabled (SCTLR.ATA=0), so there should be no need to do anything to
- // safely access the mapped memory of a remote process even if it is running
- // under MTE.
-#endif
-
- // To produce an array of all of the live blocks in a tiny chunk:
- // - We walk the chunk freelist, marking everything on it as free
- // - If the chunk is marked as installed to a thread cache:
- // - We search the thread caches to find the specific one that the chunk
- // is installed to
- // - Once we find the cache containing the chunk, we walk the cache
- // freelist, marking all of those blocks as free as well
- // - We determine the bump offset as the difference between the free count
- // and the number of blocks
- // - Then we prepare the range array by initializing the range for each
- // block below the bump offset that wasn't marked free on the first pass
-
- // N.B. Nano handles inconsistent freelist state by assuming that whatever
- // it saw before the inconsistency is what's on it. It might be better to
- // return an error and let the caller know that the state is inconsistent,
- // but for now we'll do as nano does for compatibility.
-
- union {
- vm_range_t range;
- bool free;
- } blocks[XZM_CHUNK_MAX_BLOCK_COUNT] = { 0 };
-
- // First, walk the chunk freelist. We should only walk it to the length on
- // the chunk. In the thread caching case, a detaching thread may be in the
- // process of linking a local freelist to the end, but we'll pick those
- // blocks up later and don't want to include them in this walk.
- bool cached = (meta.xca_alloc_idx == XZM_SLOT_INDEX_THREAD_INSTALLED);
-
- size_t block_granule_size = block_size / granule;
- uint64_t block_offset = meta.xca_alloc_head;
-
- size_t total_chunk_freelist_count = meta.xca_free_count;
- size_t current_chunk_freelist_count = 0;
- while (current_chunk_freelist_count < total_chunk_freelist_count &&
- block_offset < XZM_FREE_LIMIT &&
- block_offset % block_granule_size == 0) {
- size_t block_index = block_offset / block_granule_size;
- if (blocks[block_index].free) {
- xzm_debug_abort("loop in freelist");
- break;
- }
- current_chunk_freelist_count++;
- blocks[block_index].free = true;
-
- xzm_block_t block = (xzm_block_t)(
- start + (block_offset * granule));
-#if CONFIG_MTE && !MALLOC_TARGET_EXCLAVES_INTROSPECTOR
- if (malloc_has_sec_transition) {
- block = (xzm_block_t)memtag_fixup_ptr((void *)block);
- }
-#endif
- block_offset = block->xzb_linkage.xzbl_next_offset;
- }
-
- size_t allocated_limit = capacity;
- if (cached) {
-#if CONFIG_XZM_THREAD_CACHE
- if (zone->xzz_main_ref) {
- xzm_debug_abort("cached chunk on non-main zone");
- goto fail;
- }
-
- // We should have walked the full reported length of the remote freelist
- if (current_chunk_freelist_count != total_chunk_freelist_count) {
- xzm_debug_abort("Cached chunk freelist walk incomplete");
- // XXX By failing the enumeration here, we're being stricter than
- // nano was about weirdness in the freelist. That seems worth the
- // increased visibility into possible bugs this enumerator might
- // have, so for this case fail hard rather than allowing it.
- goto fail;
- }
-
- xzm_main_malloc_zone_t main = (xzm_main_malloc_zone_t)zone;
-
- xzm_xzone_index_t xz_idx = xz->xz_idx;
- if (xz_idx >= zone->xzz_thread_cache_xzone_count) {
- xzm_debug_abort("out-of-bounds cached xzone index");
- goto fail;
- }
-
- __block xzm_thread_cache_t tc = NULL;
- __block xzm_xzone_thread_cache_t cache = NULL;
-
- // This chunk is installed to a thread cache, so we need to go find the
- // right one. The priority order is:
- // - If a thread cache for a non-detaching thread cache holds the chunk,
- // it owns it. There should be at most one such cache.
- // - Otherwise, if one or more detaching caches holds it, the cache with
- // the highest teardown generation owns it.
- kern_return_t kr2 = _xzm_introspect_enumerate_thread_caches(task,
- reader, main, ^(vm_address_t thread_cache_addr,
- xzm_thread_cache_t curr_tc){
- xzm_xzone_thread_cache_t curr_cache =
- &curr_tc->xtc_xz_caches[xz_idx];
- if ((curr_cache->xztc_state < XZM_FREE_LIMIT ||
- curr_cache->xztc_state == XZM_FREE_NULL) &&
- (vm_address_t)curr_cache->xztc_chunk_start == start_addr) {
- // This is a match. Is is a better match?
-
- // If the cache we're looking at isn't tearing down, it's a
- // perfect match, and we can stop searching.
- if (!curr_tc->xtc_teardown_gen) {
- xzm_debug_assert(!tc || tc->xtc_teardown_gen);
- tc = curr_tc;
- cache = curr_cache;
- return KERN_RETURN_MAX;
- }
-
- // Otherwise, if it is tearing down, if it's more recent than
- // the last one we saw then it's the best match we've seen so
- // far.
- if (!tc || curr_tc->xtc_teardown_gen > tc->xtc_teardown_gen) {
- tc = curr_tc;
- cache = curr_cache;
- }
- }
- return KERN_SUCCESS;
- });
-
- if (kr2 && kr2 != KERN_RETURN_MAX) {
- xzm_debug_abort("Failure enumerating thread caches");
- kr = kr2;
- goto fail;
- }
-
- if (!tc) {
- xzm_debug_abort("Failed to find cache for cached chunk");
- goto fail;
- }
-
- xzm_debug_assert(tc && cache);
-
- // Walk the local freelist to add its free blocks to the set of known
- // free blocks.
- size_t total_local_freelist_count = cache->xztc_free_count;
- uint64_t block_offset = cache->xztc_head;
- size_t current_local_freelist_count = 0;
- while (current_local_freelist_count < total_local_freelist_count &&
- block_offset < XZM_FREE_LIMIT &&
- block_offset % block_granule_size == 0) {
- size_t block_index = block_offset / block_granule_size;
- if (blocks[block_index].free) {
- xzm_debug_abort("loop in local freelist");
- break;
- }
- current_local_freelist_count++;
- blocks[block_index].free = true;
-
- xzm_block_t block = (xzm_block_t)(
- start + (block_offset * granule));
-#if CONFIG_MTE && !MALLOC_TARGET_EXCLAVES_INTROSPECTOR
- if (malloc_has_sec_transition) {
- block = (xzm_block_t)memtag_fixup_ptr((void *)block);
- }
-#endif
- block_offset = block->xzb_linkage.xzbl_next_offset;
- }
-
- xzm_debug_assert(block_offset == XZM_FREE_NULL);
-
- // Account for the bump on the local freelist
- if (current_local_freelist_count < total_local_freelist_count &&
- total_local_freelist_count <= capacity) {
- allocated_limit = capacity -
- (total_local_freelist_count - current_local_freelist_count);
- }
-#else // CONFIG_XZM_THREAD_CACHE
- xzm_debug_abort("Unexpected cached chunk");
- goto fail;
-#endif // CONFIG_XZM_THREAD_CACHE
- } else {
- xzm_debug_assert(block_offset == XZM_FREE_NULL);
-
- // Account for the bump on the remote freelist
- if (current_chunk_freelist_count < total_chunk_freelist_count &&
- total_chunk_freelist_count <= capacity) {
- allocated_limit = capacity -
- (total_chunk_freelist_count - current_chunk_freelist_count);
- }
- }
-
- size_t range_idx = 0;
- for (size_t i = 0; i < allocated_limit; i++) {
- if (!blocks[i].free) {
- blocks[range_idx].range = (vm_range_t){
- .address = start_addr + i * block_size,
- .size = block_size,
- };
- range_idx++;
- }
- }
-
- return chunk_enumerator(segment_addr, segment, chunk, slice_count,
- start_addr, xz, (vm_range_t *)blocks, (unsigned)range_idx);
-
-fail:
- xzm_debug_assert(kr);
- return kr;
-}
-
-static kern_return_t
-_xzm_introspect_chunk_blocks(task_t task, memory_reader_t reader,
- xzm_malloc_zone_t zone, vm_address_t segment_addr,
- xzm_segment_t segment, xzm_chunk_t chunk, xzm_slice_count_t slice_count,
- uintptr_t start, vm_address_t start_addr, xzm_xzone_t xz,
- MALLOC_NOESCAPE xzm_chunk_enumerator_t chunk_enumerator)
-{
- xzm_slice_kind_t kind = chunk->xzc_bits.xzcb_kind;
- if (!_xzm_slice_kind_uses_xzones(kind)) {
- // This is a large or huge chunk, which has exactly one block
- vm_range_t range = {
- .address = start_addr,
- .size = slice_count * XZM_SEGMENT_SLICE_SIZE,
- };
-
- return chunk_enumerator(segment_addr, segment, chunk, slice_count,
- start_addr, NULL, &range, 1);
- }
-
- uint32_t block_size = (uint32_t)xz->xz_block_size;
- size_t capacity = xz->xz_chunk_capacity;
- // Sanity check
- if ((slice_count * XZM_SEGMENT_SLICE_SIZE) / block_size != capacity ||
- capacity > XZM_CHUNK_MAX_BLOCK_COUNT) {
- xzm_debug_abort("inconsistent xzone info");
- return KERN_FAILURE;
- }
-
- if (kind == XZM_SLICE_KIND_SMALL_CHUNK) {
- return _xzm_introspect_small_chunk_blocks(zone, segment_addr, segment,
- chunk, slice_count, start, start_addr, xz, chunk_enumerator);
- }
-
- xzm_debug_assert(kind == XZM_SLICE_KIND_TINY_CHUNK ||
- kind == XZM_SLICE_KIND_SMALL_FREELIST_CHUNK);
- return _xzm_introspect_freelist_chunk_blocks(task, reader, zone,
- segment_addr, segment, chunk, slice_count, start, start_addr, xz,
- chunk_enumerator);
-
-}
-
-static kern_return_t
-_xzm_introspect_enumerate(task_t task, memory_reader_t reader,
- vm_address_t zone_address, xzm_malloc_zone_t zone,
- vm_address_t main_address, xzm_main_malloc_zone_t main,
- bool include_blocks,
- MALLOC_NOESCAPE xzm_metapool_enumerator_t metapool_slab_enumerator,
- MALLOC_NOESCAPE xzm_segment_enumerator_t segment_enumerator,
- MALLOC_NOESCAPE xzm_chunk_enumerator_t chunk_enumerator,
- MALLOC_NOESCAPE xzm_free_span_enumerator_t span_enumerator)
-{
- bool zone_is_main = (zone_address == main_address);
- xzm_debug_assert(!span_enumerator || zone_is_main);
-
- size_t zone_size = zone_is_main ? main->xzmz_total_size :
- zone->xzz_total_size;
-
- if (zone_is_main) {
- size_t metapools_size;
- if (os_mul_overflow(main->xzmz_metapool_count,
- sizeof(struct xzm_metapool_s), &metapools_size)) {
- xzm_debug_abort("Failed to compute metapools size");
- return KERN_FAILURE;
- }
- xzm_metapool_t metapools = (xzm_metapool_t)_xzm_introspect_rebase(
- main_address, main, main->xzmz_total_size, main->xzmz_metapools,
- metapools_size);
- if (!metapools) {
- xzm_debug_abort("Failed to rebase metapools");
- return KERN_FAILURE;
- }
- for (int i = 0; i < main->xzmz_metapool_count; i++) {
- xzm_metapool_t mp = &metapools[i];
- vm_address_t slab_addr = (vm_address_t)SLIST_FIRST(&mp->xzmp_slabs);
- while (slab_addr != 0) {
- xzm_metapool_slab_t slab = NULL;
-
- kern_return_t kr = reader(task, slab_addr,
- sizeof(struct xzm_metapool_slab_s), (void **)&slab);
- if (kr) {
- xzm_debug_abort("Failed to map metapool slab");
- return kr;
- }
-
- kr = metapool_slab_enumerator((vm_address_t)slab->xzmps_base,
- mp->xzmp_slab_size, mp->xzmp_id);
- if (kr) {
- return kr;
- }
-
- slab_addr = (vm_address_t)SLIST_NEXT(slab, xzmps_entry);
- }
- }
- }
-
- size_t table_size;
- if (os_mul_overflow(XZM_SEGMENT_TABLE_ENTRIES,
- sizeof(xzm_segment_table_entry_s), &table_size)) {
- xzm_debug_abort("failed to compute segment table size");
- return KERN_FAILURE;
- }
- xzm_segment_table_entry_s *segment_table =
- (xzm_segment_table_entry_s *)_xzm_introspect_rebase(main_address,
- main, main->xzmz_total_size, main->xzmz_segment_table, table_size);
- if (!segment_table) {
- xzm_debug_abort("failed to rebase segment table");
- return KERN_FAILURE;
- }
-
- xzm_segment_table_enumerator_t enumerator = ^(vm_address_t segment_addr){
- xzm_segment_t segment;
- // Even for huge segments, we don't need to map more than normal segment
- // size because we don't need to see anything in the body of huge
- // segments.
- //
- // XXX Note: the mapped segment is _not_ guaranteed to have the same
- // alignment as the original segment, so many of the manipulation
- // helpers can't be used with it.
-
- // Map in the segment metadata to see how big it is.
- kern_return_t kr = reader(task, segment_addr,
- sizeof(struct xzm_segment_s), (void **)&segment);
- if (kr) {
- xzm_debug_abort("failed to map segment header");
- return kr;
- }
-
- void *segment_body;
- kr = reader(task, (vm_address_t)_xzm_segment_start(segment),
- segment->xzs_slice_count * XZM_SEGMENT_SLICE_SIZE,
- &segment_body);
- if (kr) {
- xzm_debug_abort("failed to map segment");
- return kr;
- }
-
- kr = segment_enumerator(segment_addr, segment, " ");
- if (kr) {
- return kr;
- }
-
- return xzm_segment_group_segment_foreach_span(segment,
- ^(xzm_slice_t span, xzm_slice_count_t slice_count){
- ptrdiff_t idx = span - segment->xzs_slices;
- size_t start_offset = idx * XZM_SEGMENT_SLICE_SIZE;
- uintptr_t start = (uintptr_t)segment_body + start_offset;
- uintptr_t orig_start = (uintptr_t)_xzm_segment_slice_index_start(
- segment, (xzm_slice_count_t)idx);
- vm_address_t start_addr = (vm_address_t)orig_start;
-
- xzm_slice_kind_t kind = span->xzc_bits.xzcb_kind;
- if (_xzm_slice_kind_is_chunk_safe(kind) &&
- span->xzc_mzone_idx == zone->xzz_mzone_idx) {
- // This is a chunk that belongs to this zone.
- xzm_chunk_t chunk = span;
-
- xzm_xzone_t xz = NULL;
- if (_xzm_slice_kind_uses_xzones(kind)) {
- xz = (xzm_xzone_t)_xzm_introspect_rebase(zone_address, zone,
- zone_size, &zone->xzz_xzones[chunk->xzc_xzone_idx],
- sizeof(struct xzm_xzone_s));
- if (!xz) {
- xzm_debug_abort("failed to rebase xzone");
- return KERN_FAILURE;
- }
- }
-
- if (include_blocks) {
- return _xzm_introspect_chunk_blocks(task, reader, zone,
- segment_addr, segment, chunk, slice_count, start,
- start_addr, xz, chunk_enumerator);
- } else {
- return chunk_enumerator(segment_addr, segment, chunk,
- slice_count, start_addr, xz, NULL, 0);
- }
- } else if (zone_is_main &&
- span->xzc_mzone_idx == XZM_MZONE_INDEX_INVALID) {
- // Include free spans and sequestered chunks when enumerating
- // the main zone
- //
- // TODO: could include xzone for sequestered chunks that belong
- // to one
- return span_enumerator(segment_addr, segment, span,
- slice_count, start_addr);
- }
-
- // Either a free span we don't care about or a Valid chunk that
- // belongs to a different zone: skip, continue iteration
- return KERN_SUCCESS;
- });
- };
-
- xzm_segment_t last_segment_enumerated = NULL;
- kern_return_t kr = xzm_segment_table_foreach(segment_table,
- XZM_SEGMENT_TABLE_ENTRIES, enumerator, &last_segment_enumerated);
- if (kr) {
- return kr;
- }
-
- size_t ext_seg_table_size;
- if (os_mul_overflow(main->xzmz_extended_segment_table_entries,
- sizeof(xzm_extended_segment_table_entry_s), &ext_seg_table_size)) {
- xzm_debug_abort("failed to compute extended segment table size");
- return KERN_FAILURE;
- }
- xzm_extended_segment_table_entry_s *ext_segment_table =
- (xzm_extended_segment_table_entry_s *) _xzm_introspect_rebase(
- main_address, main, main->xzmz_total_size,
- main->xzmz_extended_segment_table, ext_seg_table_size);
- if (ext_segment_table) {
- for (size_t i = 0; i < main->xzmz_extended_segment_table_entries; i++) {
- // If leaf table pointer is non-null, map it in and enumerate over
- // it
- if (ext_segment_table[i].xeste_val != 0) {
- // There is (or was) at least one segment in the 64GB span
- // represented by this segment map entry
- xzm_segment_table_entry_s *table;
-
- vm_address_t table_addr = ((vm_address_t)ext_segment_table[i].xeste_val *
- XZM_SEGMENT_TABLE_ALIGN);
-
- kern_return_t kr = reader(task, table_addr,
- XZM_SEGMENT_TABLE_SIZE, (void **)&table);
- if (kr) {
- xzm_debug_abort("Failed to map segment table");
- return kr;
- }
-
- kr = xzm_segment_table_foreach(table, XZM_SEGMENT_TABLE_ENTRIES,
- enumerator, &last_segment_enumerated);
- if (kr) {
- return kr;
- }
- }
- }
- }
-
- return KERN_SUCCESS;
-}
-
-#if CONFIG_XZM_DEFERRED_RECLAIM
-
-static kern_return_t
-_xzm_introspect_map_reclaim_buffer(task_t task, memory_reader_t reader,
- vm_address_t metadata_addr, xzm_reclaim_buffer_t *xzm_metadata_out,
- mach_vm_reclaim_ring_t *buffer_out)
-{
- xzm_reclaim_buffer_t xzm_buffer = NULL;
- mach_vm_reclaim_ring_t buffer = NULL;
- kern_return_t kr;
-
- kr = reader(task, metadata_addr,
- sizeof(struct xzm_reclaim_buffer_s), (void **)&xzm_buffer);
- if (kr) {
- xzm_debug_abort_with_reason("failed to map reclaim buffer metadata", kr);
- goto out;
- }
-
- vm_address_t buffer_addr = (vm_address_t)xzm_buffer->xrb_ringbuffer;
- size_t buffer_size = (xzm_buffer->xrb_len *
- sizeof(struct mach_vm_reclaim_entry_s)) +
- offsetof(struct mach_vm_reclaim_ring_s, entries);
- if (buffer_addr != 0) {
- xzm_debug_assert(buffer_size % vm_page_quanta_size == 0);
- kr = reader(task, buffer_addr, buffer_size, (void **)&buffer);
- if (kr) {
- xzm_debug_abort_with_reason("failed to map reclaim buffer", kr);
- goto out;
- }
- }
-
-out:
- *xzm_metadata_out = xzm_buffer;
- *buffer_out = buffer;
- return kr;
-}
-
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
-
-static kern_return_t
-_xzm_introspect_map_zone_and_main(task_t task, vm_address_t zone_address,
- memory_reader_t reader, xzm_malloc_zone_t *zone_p_out,
- xzm_main_malloc_zone_t *main_p_out, vm_address_t *main_address_out)
-{
- xzm_malloc_zone_t zone = NULL;
-
- // Map the base structure first to find its size, then map that full size
- kern_return_t kr = reader(task, zone_address, sizeof(*zone),
- (void **)&zone);
- if (kr) {
- xzm_debug_abort("failed to map zone");
- return kr;
- }
-
- uint64_t zone_size = zone->xzz_total_size;
- if (zone_size < sizeof(*zone)) {
- xzm_debug_abort("inconsistent zone region info");
- return KERN_FAILURE;
- }
-
- kr = reader(task, zone_address, zone_size, (void **)&zone);
- if (kr) {
- xzm_debug_abort("failed to map full zone");
- return kr;
- }
-
- xzm_main_malloc_zone_t main = NULL;
- uint64_t main_zone_size;
- vm_address_t main_address = 0;
- if (zone->xzz_main_ref) {
- main_address = (vm_address_t)zone->xzz_main_ref;
-
- kr = reader(task, main_address, sizeof(*main), (void **)&main);
- if (kr) {
- xzm_debug_abort("failed to map main zone");
- return kr;
- }
-
- main_zone_size = main->xzmz_total_size;
- if (main_zone_size < sizeof(*main)) {
- xzm_debug_abort("inconsistent main zone info");
- return KERN_FAILURE;
- }
-
- kr = reader(task, main_address, main_zone_size, (void **)&main);
- if (kr) {
- xzm_debug_abort("failed to map full main zone");
- return kr;
- }
- } else {
- main = (xzm_main_malloc_zone_t)zone;
- if (main->xzmz_total_size != zone_size) {
- xzm_debug_abort("inconsistent main zone size");
- return KERN_FAILURE;
- }
-
- main_address = zone_address;
- main_zone_size = zone_size;
- }
-
- if (main_zone_size < main->xzmz_total_size) {
- xzm_debug_abort("inconsistent main region size");
- return KERN_FAILURE;
- }
-
- xzm_assert(zone);
- xzm_assert(main);
- xzm_assert(main_address);
- *zone_p_out = zone;
- *main_p_out = main;
- *main_address_out = main_address;
-
- return KERN_SUCCESS;
-}
-
-static kern_return_t
-xzm_ptr_in_use_enumerator(task_t task, void *context, unsigned type_mask,
- vm_address_t zone_address, memory_reader_t reader,
- vm_range_recorder_t recorder)
-{
- xzm_malloc_zone_t zone;
- xzm_main_malloc_zone_t main;
- vm_address_t main_address;
- bool zone_is_main = false;
-
- reader = reader_or_in_memory_fallback(reader, task);
-
- bool record_admin = (type_mask & MALLOC_ADMIN_REGION_RANGE_TYPE);
- bool record_ptr_region = (type_mask & MALLOC_PTR_REGION_RANGE_TYPE);
- bool record_ptr_in_use = (type_mask & MALLOC_PTR_IN_USE_RANGE_TYPE);
-
- kern_return_t kr = _xzm_introspect_map_zone_and_main(task, zone_address,
- reader, &zone, &main, &main_address);
- if (kr) {
- return kr;
- }
-
- zone_is_main = (zone_address == main_address);
-
- if (zone_is_main) {
- vm_address_t mfm_addr = (vm_address_t)main->xzmz_mfm_address;
- if (mfm_addr) {
- kr = mfm_introspect.enumerator(task, context, type_mask, mfm_addr,
- reader, recorder);
- if (kr) {
- return kr;
- }
- }
- }
-
- return _xzm_introspect_enumerate(task, reader, zone_address,
- zone, main_address, main,
- /* include_blocks */record_ptr_in_use,
- ^(vm_address_t slab_addr, vm_size_t slab_size, xzm_metapool_id_t mp_id){
- // Metapool slab enumerator
- if (record_admin && zone_is_main) {
- vm_range_t segment_meta_range = {
- .address = slab_addr,
- .size = slab_size,
- };
- recorder(task, context, MALLOC_ADMIN_REGION_RANGE_TYPE,
- &segment_meta_range, 1);
- }
- return KERN_SUCCESS;
- }, ^(vm_address_t segment_addr, xzm_segment_t segment, const char *indent){
- // Segment enumerator
- // Nothing to do, since segment metadata is recorded by the slab
- // enumerator
- return KERN_SUCCESS;
- }, ^(vm_address_t segment_addr, xzm_segment_t segment, xzm_chunk_t chunk,
- xzm_slice_count_t slice_count, vm_address_t start_addr,
- xzm_xzone_t xz, vm_range_t *ranges, size_t count){
- // Chunk enumerator
- xzm_slice_kind_t kind = chunk->xzc_bits.xzcb_kind;
- if (record_admin && kind == XZM_SLICE_KIND_HUGE_CHUNK) {
- // Record the info slices of huge segments against the mzone they
- // belong to
- vm_range_t header_range = {
- .address = segment_addr,
- .size = XZM_METAPOOL_SEGMENT_BLOCK_SIZE,
- };
- recorder(task, context, MALLOC_ADMIN_REGION_RANGE_TYPE,
- &header_range, 1);
- }
-
- if (!record_ptr_region && !record_ptr_in_use) {
- return KERN_SUCCESS;
- }
-
- vm_range_t region_range = {
- .address = start_addr,
- .size = slice_count * XZM_SEGMENT_SLICE_SIZE,
- };
- if (_xzm_slice_kind_uses_xzones(kind)) {
- if (record_ptr_region) {
- recorder(task, context, MALLOC_PTR_REGION_RANGE_TYPE,
- ®ion_range, 1);
- }
-
- if (record_ptr_in_use) {
- recorder(task, context, MALLOC_PTR_IN_USE_RANGE_TYPE, ranges,
- (unsigned)count);
- }
- } else {
- recorder(task, context, MALLOC_PTR_IN_USE_RANGE_TYPE |
- MALLOC_PTR_REGION_RANGE_TYPE, ®ion_range, 1);
- }
-
- return KERN_SUCCESS;
- }, !zone_is_main ? NULL : ^(vm_address_t segment_addr,
- xzm_segment_t segment, xzm_chunk_t span,
- xzm_slice_count_t slice_count, vm_address_t start_addr){
- // Main zone span enumerator
-
- // Record all free spans and chunks with no mzone against the main zone,
- // with the exception of huge chunks that could be in the reclaim buffer
- if (record_ptr_region) {
- bool should_record = true;
-
-#if CONFIG_XZM_DEFERRED_RECLAIM
- // Unfortunately, there's no way for us to reliably tell whether a
- // given huge chunk is in the reclaim buffer, because when marking
- // them free there's a window where we haven't yet stored the
- // reclaim index in xzs_reclaim_id. So, we err on the side of
- // caution and just never record them.
- if (segment->xzs_kind == XZM_SEGMENT_KIND_HUGE &&
- span->xzc_bits.xzcb_kind == XZM_SLICE_KIND_HUGE_CHUNK) {
- should_record = false;
- }
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
-
- if (should_record) {
- vm_range_t region_range = {
- .address = start_addr,
- .size = slice_count * XZM_SEGMENT_SLICE_SIZE,
- };
- recorder(task, context, MALLOC_PTR_REGION_RANGE_TYPE,
- ®ion_range, 1);
- }
- }
-
- return KERN_SUCCESS;
- });
-}
-
-#if XZM_DEBUG_ENUMERATOR
-
-struct xzm_debug_recorder_context_s {
- vm_range_recorder_t *orig_recorder;
- void *orig_context;
-};
-
-static void
-_xzm_debug_range_recorder(task_t task, void *context, unsigned type,
- vm_range_t *ranges, unsigned count)
-{
- const char *type_str = "(invalid?)";
- switch (type) {
- case MALLOC_PTR_IN_USE_RANGE_TYPE | MALLOC_PTR_REGION_RANGE_TYPE:
- type_str = "PTR_IN_USE | PTR_REGION";
- break;
- case MALLOC_PTR_IN_USE_RANGE_TYPE:
- type_str = "PTR_IN_USE";
- break;
- case MALLOC_PTR_REGION_RANGE_TYPE:
- type_str = "PTR_REGION";
- break;
- case MALLOC_ADMIN_REGION_RANGE_TYPE:
- type_str = "ADMIN_REGION";
- break;
- default:
- break;
- }
-
- printf("XZM ENUMERATOR: %s (%x) - %p %u ranges\n", type_str, type, ranges,
- count);
- for (unsigned i = 0; i < count; i++) {
- printf("XZM ENUMERATOR: %p[%u]: %p %llu\n", ranges, i,
- (void *)ranges[i].address, (unsigned long long)ranges[i].size);
- }
-
- struct xzm_debug_recorder_context_s *ctx = context;
- ctx->orig_recorder(task, ctx->orig_context, type, ranges, count);
-}
-
-static kern_return_t
-xzm_debug_ptr_in_use_enumerator(task_t task, void *context, unsigned type_mask,
- vm_address_t zone_address, memory_reader_t reader,
- vm_range_recorder_t recorder)
-{
- struct xzm_debug_recorder_context_s ctx = {
- .orig_recorder = recorder,
- .orig_context = context,
- };
-
- return xzm_ptr_in_use_enumerator(task, &ctx, type_mask, zone_address,
- reader, _xzm_debug_range_recorder);
-
-}
-
-#endif // XZM_DEBUG_ENUMERATOR
-
-static void
-xzm_print(task_t task, unsigned level, vm_address_t zone_address,
- memory_reader_t reader, print_task_printer_t printer)
-{
- xzm_malloc_zone_t zone;
- xzm_main_malloc_zone_t main;
- vm_address_t main_address;
- bool zone_is_main = false;
-
- kern_return_t kr = _xzm_introspect_map_zone_and_main(task, zone_address,
- reader, &zone, &main, &main_address);
- if (kr) {
- return;
- }
-
- zone_is_main = (zone_address == main_address);
-
- printer("Begin xzone malloc JSON:\n");
- printer("{\n");
- printer("\"desc\": \"xzone malloc\", \n");
- printer("\"addr\": \"%p\", \n", zone_address);
- printer("\"segment_size\": %zu, \n", XZM_SEGMENT_SIZE);
- printer("\"slice_size\": %zu, \n", XZM_SEGMENT_SLICE_SIZE);
- printer("\"mzone\": %d, \n", (int)zone->xzz_mzone_idx);
- printer("\"is_main\": %d, \n", zone_is_main);
- printer("\"max_list_config\": %d, \n", (int)zone->xzz_max_list_config);
- printer("\"initial_slot_config\": %d, \n", (int)zone->xzz_initial_slot_config);
- printer("\"slot_initial_threshold\": %u, \n", zone->xzz_slot_initial_threshold);
- printer("\"max_slot_config\": %d, \n", (int)zone->xzz_max_slot_config);
-
- // TODO: early allocator info
-
- __block size_t dispositions_count = 0;
- __block int *dispositions = NULL;
- __block vm_address_t dispositions_start_addr = 0;
- kern_return_t (^print_dispositions)(vm_address_t, vm_size_t, const char *);
- print_dispositions = ^kern_return_t(vm_address_t addr, vm_size_t size, const char *indent) {
- kern_return_t kr = KERN_SUCCESS;
-
- // When operating on a core dump, no pages can be queried
- if (task == TASK_NULL) {
- return kr;
- }
-
- // If dispositions doesn't cover the full range of this request,
- // (possibly) reallocate and re-query the VM
- vm_address_t request_end = addr + size;
- vm_address_t dispositions_end = dispositions_start_addr +
- (dispositions_count * vm_page_size);
- if ((dispositions_start_addr > addr) ||
- (dispositions_end < request_end)) {
- dispositions_start_addr = addr;
-
- // This interface is usually used to query the disposition of a
- // full segment, so to reduce the number of calls into the vm, request at least a segment
- size_t request_pages = howmany(size, vm_page_size);
- if (request_pages < (XZM_SEGMENT_SIZE / vm_page_size)) {
- request_pages = XZM_SEGMENT_SIZE / vm_page_size;
- }
-
- // TODO: mixed page size difficulties
- if (request_pages > dispositions_count) {
- if (dispositions) {
- mach_vm_deallocate(mach_task_self(),
- (mach_vm_address_t)dispositions,
- dispositions_count * sizeof(dispositions[0]));
- dispositions = NULL;
- }
-
- dispositions_count = request_pages;
- kr = mach_vm_allocate(mach_task_self(),
- (mach_vm_address_t *)&dispositions,
- dispositions_count * sizeof(dispositions[0]),
- VM_FLAGS_ANYWHERE);
- if (kr) {
- xzm_debug_abort("failed to allocate memory for vm stats");
- return kr;
- }
- }
-
- mach_vm_size_t mvs_page_span = (mach_vm_size_t)request_pages;
- kr = mach_vm_page_range_query(task,
- (mach_vm_address_t)addr, MAX(size, XZM_SEGMENT_SIZE),
- (mach_vm_address_t)dispositions, &mvs_page_span);
- if (kr) {
- xzm_debug_abort("Failed to query vm stats");
- return kr;
- }
- }
-
- printer("%s \"dispositions\": \"", indent);
-
- size_t dirty_count = 0;
- size_t swapped_count = 0;
- size_t disposition_idx =
- (addr - dispositions_start_addr) / vm_page_size;
- for (size_t i = 0; i < (size / vm_page_size); i++) {
- if (disposition_idx >= dispositions_count) {
- xzm_debug_abort("inconsistent slice counts");
- return KERN_FAILURE;
- }
-
- int disposition = dispositions[disposition_idx];
- if (disposition & VM_PAGE_QUERY_PAGE_DIRTY) {
- dirty_count++;
- printer("d");
- } else if (disposition & VM_PAGE_QUERY_PAGE_PAGED_OUT) {
- swapped_count++;
- printer("s");
- } else {
- printer("c");
- }
-
- disposition_idx++;
- }
-
- printer("\", \n"); // dispositions
- printer("%s \"dirty_count\": %zu, \n", indent, dirty_count);
- printer("%s \"swapped_count\": %zu, \n", indent, swapped_count);
-
- return KERN_SUCCESS;
- };
-
- __block bool first_span = true;
- __block bool print_segment_dispositions = true;
- const xzm_segment_enumerator_t segment_enumerator =
- ^(vm_address_t segment_addr, xzm_segment_t segment,
- const char *indent) {
- // Segment enumerator
-
- if (!first_span) {
- printer(", ");
- }
-
- printer("%s\"%p\": {\n", indent, (void *)segment_addr);
- printer("%s \"addr\": \"%p\", \n", indent, (void *)segment_addr);
- xzm_segment_group_id_t sg_id = segment->xzs_segment_group -
- main->xzmz_segment_groups;
- printer("%s \"segment_group\": \"%s\", \n", indent,
- _xzm_segment_group_id_to_string(sg_id));
- printer("%s \"body_addr\": \"%p\", \n", indent,
- segment->xzs_segment_body);
- printer("%s \"used\": %u, \n", indent, segment->xzs_used);
- printer("%s \"kind\": \"%s\", \n", indent,
- _xzm_segment_kind_to_string(segment->xzs_kind));
-#if CONFIG_XZM_DEFERRED_RECLAIM
- if (segment->xzs_reclaim_id == VM_RECLAIM_ID_NULL) {
- printer("%s \"reclaim_id\": -1, \n", indent);
- } else {
- printer("%s \"reclaim_id\": %llu, \n", indent,
- segment->xzs_reclaim_id);
- }
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
- if (print_segment_dispositions) {
- print_dispositions((vm_address_t)segment->xzs_segment_body,
- segment->xzs_slice_count * XZM_SEGMENT_SLICE_SIZE, indent);
- }
-
- printer("%s \"slice_count\": %u, \n", indent,
- segment->xzs_slice_count);
- printer("%s \"slice_entry_count\": %u \n", indent,
- segment->xzs_slice_entry_count);
-
- printer("%s}\n", indent); // segment
- first_span = false;
-
- return KERN_SUCCESS;
- };
-
- if (zone_is_main) {
- printer("\"bucketing_key\": \"%016llx%016llx\", \n",
- main->xzmz_bucketing_keys.xbk_key_data[0],
- main->xzmz_bucketing_keys.xbk_key_data[1]);
- printer("\"guard_config\": {\n");
- printer(" \"guards_enabled\": %d, \n",
- main->xzmz_guard_config.xgpc_enabled);
- printer(" \"data_guards_enabled\": %d, \n",
- main->xzmz_guard_config.xgpc_enabled_for_data);
- printer(" \"tiny_run_size\": %d, \n",
- main->xzmz_guard_config.xgpc_max_run_tiny);
- printer(" \"tiny_guard_density\": %d, \n",
- main->xzmz_guard_config.xgpc_tiny_guard_density);
- printer(" \"small_run_size\": %d, \n",
- main->xzmz_guard_config.xgpc_max_run_small);
- printer(" \"small_guard_density\": %d \n",
- main->xzmz_guard_config.xgpc_small_guard_density);
- printer("}, \n");
- printer("\"chunk_threshold\": %u, \n", main->xzmz_xzone_chunk_threshold);
- printer("\"ptr_bucket_count\": %d, \n", main->xzmz_ptr_bucket_count);
- // guard_config
-
-#if CONFIG_MTE
- printer("\"mte_config\": {\n");
- printer(" \"enabled\": %d, \n",
- (int)main->xzmz_base.xzz_memtag_config.enabled);
- printer(" \"tag_data\": %d, \n",
- (int)main->xzmz_base.xzz_memtag_config.tag_data);
- printer(" \"max_block_size\": %d \n",
- (int)main->xzmz_base.xzz_memtag_config.max_block_size);
- printer("}, \n"); // mte_config
-#endif // CONFIG_MTE
-
- printer("\"defer_tiny\": %s, \n", main->xzmz_defer_tiny ?
- "true" : "false");
- printer("\"defer_small\": %s, \n", main->xzmz_defer_small ?
- "true" : "false");
- printer("\"defer_large\": %s, \n", main->xzmz_defer_large ?
- "true" : "false");
- printer("\"deallocate_segment\": %s, \n", main->xzmz_deallocate_segment ?
- "true" : "false");
-
- printer("\"use_early_alloc\": %s, \n", main->xzmz_mfm_address ?
- "true" : "false");
-
- printer("\"batch_size\": %u, \n", main->xzmz_batch_size);
-
-#if CONFIG_XZM_DEFERRED_RECLAIM
- if (main->xzmz_reclaim_buffer != NULL) {
- vm_address_t xzm_buffer_addr = (vm_address_t)main->xzmz_reclaim_buffer;
- xzm_reclaim_buffer_t xzm_reclaim_buffer;
- mach_vm_reclaim_ring_t ringbuffer;
- kr = _xzm_introspect_map_reclaim_buffer(task, reader,
- xzm_buffer_addr, &xzm_reclaim_buffer, &ringbuffer);
- if (kr) {
- xzm_debug_abort("failed to map reclaim buffer");
- return;
- }
- if (ringbuffer != NULL) {
- printer("\"reclaim_buffer\": { \n");
- printer(" \"buffer_len\": %llu, \n",
- ringbuffer->len);
- printer(" \"max_len\": %llu, \n",
- ringbuffer->max_len);
- printer(" \"sampling_period_abs\": %llu, \n", ringbuffer->sampling_period_abs);
- printer(" \"last_sample_abs\": %llu, \n",
- ringbuffer->last_sample_abs);
- printer(" \"reclaimable_bytes\": %llu, \n",
- os_atomic_load(&ringbuffer->reclaimable_bytes,
- relaxed));
- printer(" \"reclaimable_bytes_min\": %llu, \n",
- os_atomic_load(&ringbuffer->reclaimable_bytes_min,
- relaxed));
-
- printer(" \"head\": %llu, \n",
- os_atomic_load(&ringbuffer->head, relaxed));
- printer(" \"busy\": %llu, \n",
- os_atomic_load(&ringbuffer->busy, relaxed));
- printer(" \"tail\": %llu, \n",
- os_atomic_load(&ringbuffer->tail, relaxed));
-
- printer(" \"entries\": [ \n");
-
- for (mach_vm_reclaim_count_t i = 0; i < ringbuffer->len; i++) {
- mach_vm_reclaim_entry_t entry = &ringbuffer->entries[i];
- printer(" { \n");
- printer(" \"id\": %u, \n", i);
- printer(" \"address\": \"%p\", \n", entry->address);
- printer(" \"size\": %u, \n", entry->size);
- // TODO: add string decoder to libsyscall
- printer(" \"behavior\": %u \n", entry->behavior);
- printer(" }");
- if (i < ringbuffer->len - 1) {
- printer(",");
- }
- printer(" \n");
- }
- printer(" ] \n"); // entries
- }
- printer("}, \n"); // reclaim buffer
- }
-
-#endif // CONFIG_XZM_DEFERRED_RECLAIM
-
- printer("\"allocation_front_count\": %u, \n",
- main->xzmz_allocation_front_count);
- printer("\"range_group_count\": %u, \n", main->xzmz_range_group_count);
- printer("\"range_groups\": {\n");
-
- size_t range_group_size;
- if (os_mul_overflow(main->xzmz_range_group_count,
- sizeof(struct xzm_range_group_s), &range_group_size)) {
- xzm_debug_abort("failed to compute range group size");
- return;
- }
- struct xzm_range_group_s *mapped_range_groups =
- (struct xzm_range_group_s *)_xzm_introspect_rebase(main_address,
- main, main->xzmz_total_size, main->xzmz_range_groups,
- range_group_size);
- if (!mapped_range_groups) {
- xzm_debug_abort("failed to map range_groups");
- return;
- }
-
- for (uint8_t i = 0; i < main->xzmz_range_group_count; i++) {
- printer(" ");
- if (i) {
- printer(", ");
- }
- xzm_range_group_t rg = &mapped_range_groups[i];
- printer("\"%d\": {\n", (int)i);
- printer(" \"id\": \"%s\", \n",
- _xzm_range_group_id_to_string(rg->xzrg_id));
- printer(" \"front\": %d, \n", (int)rg->xzrg_front);
- printer(" \"lock\": %u, \n", *(uint32_t *)&rg->xzrg_lock);
- printer(" \"base\": \"%p\", \n", (void *)rg->xzrg_base);
- printer(" \"size\": %zu, \n", rg->xzrg_size);
- printer(" \"skip_addr\": \"%p\", \n",
- (void *)rg->xzrg_skip_addr);
- printer(" \"skip_size\": %zu, \n", rg->xzrg_skip_size);
- printer(" \"next\": \"%p\", \n", (void *)rg->xzrg_next);
- printer(" \"remaining\": %zu, \n", rg->xzrg_remaining);
- printer(" \"direction\": \"%s\"\n",
- rg->xzrg_direction == XZM_FRONT_INCREASING ? "up" : "down");
- printer(" }\n"); // range group
- }
-
- printer("}, \n"); // range_groups
-
- printer("\"segment_group_ids_count\": %u, \n",
- main->xzmz_segment_group_ids_count);
- printer("\"segment_group_front_count\": %u, \n",
- main->xzmz_segment_group_front_count);
- printer("\"segment_group_count\": %u, \n",
- main->xzmz_segment_group_count);
- printer("\"segment_groups\": {\n");
-
- size_t segment_group_size;
- if (os_mul_overflow(main->xzmz_segment_group_count,
- sizeof(struct xzm_segment_group_s), &segment_group_size)) {
- xzm_debug_abort("failed to compute segment group size");
- return;
- }
- struct xzm_segment_group_s *mapped_segment_groups =
- (struct xzm_segment_group_s *)_xzm_introspect_rebase(main_address,
- main, main->xzmz_total_size, main->xzmz_segment_groups,
- segment_group_size);
- if (!mapped_segment_groups) {
- xzm_debug_abort("failed to map segment_groups");
- return;
- }
-
- for (uint8_t i = 0; i < main->xzmz_segment_group_count; i++) {
- printer(" ");
- if (i) {
- printer(", ");
- }
- xzm_segment_group_t sg = &mapped_segment_groups[i];
- printer("\"%d\": {\n", (int)i);
- printer(" \"id\": \"%s\", \n",
- _xzm_segment_group_id_to_string(sg->xzsg_id));
- printer(" \"front\": %d, \n", (int)sg->xzsg_front);
- printer(" \"range_group\": \"%p\", \n",
- sg->xzsg_range_group);
- printer(" \"segment_cache\": { \n");
- printer(" \"max_count\": %u, \n",
- (unsigned)sg->xzsg_cache.xzsc_max_count);
- printer(" \"count\": %u, \n",
- (unsigned)sg->xzsg_cache.xzsc_count);
- printer(" \"max_entry_slices\": %u, \n",
- (unsigned)sg->xzsg_cache.xzsc_max_entry_slices);
- printer(" \"segments\": { \n");
- if (sg->xzsg_cache.xzsc_count) {
- // Segments in the segment cache will not be present in the segment
- // table, so they must be enumerated here
- vm_address_t segment_addr = (vm_address_t)TAILQ_FIRST(
- &sg->xzsg_cache.xzsc_head);
- while (segment_addr != 0) {
- xzm_segment_t segment;
- kern_return_t kr = reader(task, segment_addr,
- sizeof(struct xzm_segment_s), (void **)&segment);
- if (kr) {
- xzm_debug_abort("Failed to map cached segment");
- return;
- }
-
- kr = segment_enumerator(segment_addr, segment,
- " ");
- if (kr) {
- xzm_debug_abort("Failed to enumerate segment");
- return;
- }
- segment_addr = (vm_address_t)TAILQ_NEXT(segment,
- xzs_cache_entry);
- }
- }
- printer(" } \n"); // segments
- printer(" } \n"); // segment cache
- printer(" }\n"); // segment group
- }
-
- printer("}, \n"); // segment_groups
-
- printer("\"xzones\": {\n");
-
- size_t xzone_size;
- if (os_mul_overflow(main->xzmz_base.xzz_xzone_count,
- sizeof(struct xzm_xzone_s), &xzone_size)) {
- xzm_debug_abort("failed to compute xzone array size");
- return;
- }
- uintptr_t rebased_xzones = _xzm_introspect_rebase(main_address, main,
- main->xzmz_total_size, main->xzmz_base.xzz_xzones, xzone_size);
- struct xzm_xzone_s *mapped_xzones =
- (struct xzm_xzone_s *)rebased_xzones;
- if (!mapped_xzones) {
- xzm_debug_abort("failed to map main xzones");
- return;
- }
-
- size_t slots_size;
- if (os_mul3_overflow(main->xzmz_base.xzz_xzone_count,
- main->xzmz_base.xzz_slot_count,
- sizeof(struct xzm_xzone_allocation_slot_s), &slots_size)) {
- xzm_debug_abort("failed to compute allocation slots size");
- return;
- }
- uintptr_t rebased_slots = _xzm_introspect_rebase(main_address, main,
- main->xzmz_total_size,
- main->xzmz_base.xzz_xzone_allocation_slots, slots_size);
- struct xzm_xzone_allocation_slot_s *mapped_slots =
- (struct xzm_xzone_allocation_slot_s *)rebased_slots;
- if (!mapped_slots) {
- xzm_debug_abort("failed to map main allocation slots");
- return;
- }
-
- for (uint8_t xzidx = XZM_XZONE_INDEX_FIRST;
- xzidx < zone->xzz_xzone_count; xzidx++) {
- xzm_xzone_t xz = &mapped_xzones[xzidx];
- printer(" \"%d\": {\n", (int)xzidx);
- printer(" \"early_budget\": %u, \n", xz->xz_early_budget);
- printer(" \"id\": %d, \n", (int)xz->xz_idx);
- printer(" \"bucket\": %d, \n", (int)xz->xz_bucket);
- printer(" \"segment_group_id\": %d, \n",
- xz->xz_segment_group_id);
- printer(" \"front\": %d, \n", xz->xz_front);
- printer(" \"batch_count\": %u, \n",
- xz->xz_block_size <= XZM_TINY_BLOCK_SIZE_MAX ?
- xz->xz_batch_list.xzch_batch_count :
- xz->xz_chunkq_batch_count);
- printer(" \"block_size\": %llu, \n", xz->xz_block_size);
- printer(" \"chunk_count\": %llu, \n", xz->xz_chunk_count);
- printer(" \"chunk_capacity\": %u, \n", xz->xz_chunk_capacity);
- printer(" \"sequestered\": %d,\n", (int)xz->xz_sequestered);
- printer(" \"list_config\": \"%s\",\n",
- _xzm_slot_config_to_string(xz->xz_list_config));
- printer(" \"slot_config\": \"%s\",\n",
- _xzm_slot_config_to_string(xz->xz_slot_config));
- printer(" \"allocation_slots\": [\n");
-
- for (xzm_allocation_index_t slot_idx = 0;
- slot_idx < zone->xzz_slot_count;
- slot_idx++) {
- xzm_xzone_allocation_slot_t xas = &mapped_slots[
- (slot_idx * zone->xzz_xzone_count) + xzidx];
- printer(" {\n");
-
- if (xz->xz_block_size <= XZM_TINY_BLOCK_SIZE_MAX ||
- zone->xzz_small_freelist_enabled) {
- printer(" \"atomic_value\": \"0x%llx\",\n",
- xas->xas_atomic.xasa_value);
- printer(" \"xsg_locked\": \"0x%llx\",\n",
- xas->xas_atomic.xasa_gate.xsg_locked);
- printer(" \"xsg_waiters\": \"0x%llx\",\n",
- xas->xas_atomic.xasa_gate.xsg_waiters);
- printer(" \"xsc_ptr\": \"0x%llx\",\n",
- xas->xas_atomic.xasa_chunk.xsc_ptr);
-
- printer(" \"operations\": %lu,\n",
- xas->xas_counters.xsc_ops);
- printer(" \"contentions\": %lu,\n",
- xas->xas_counters.xsc_contentions);
-
- printer(" \"slot_config\": \"%s\",\n",
- _xzm_slot_config_to_string(
- xas->xas_counters.xsc_slot_config));
- } else {
- printer(" \"chunk\": \"%p\",\n",
- (void *)xas->xas_chunk);
- printer(" \"allocations\": %lu,\n",
- xas->xas_allocs);
- printer(" \"contentions\": %lu,\n",
- xas->xas_contentions);
- }
-
- printer(" \"last_chunk_empty_ts\": %llu\n",
- xas->xas_last_chunk_empty_ts);
- printer(" }");
- if (slot_idx < zone->xzz_slot_count - 1) {
- printer(",");
- }
- printer("\n");
- }
-
- printer(" ]\n"); // allocation slots
- printer(" }"); // xzone
- if (xzidx < zone->xzz_xzone_count - 1) {
- printer(",");
- }
- printer("\n");
- }
-
- printer("}, \n"); // xzones
-
-#if CONFIG_XZM_THREAD_CACHE
- printer("\"thread_cache_enabled\": %s, \n",
- zone->xzz_thread_cache_enabled ? "true" : "false");
- printer("\"thread_cache_activation_period\": %lu, \n",
- zone->xzz_thread_cache_xzone_activation_period);
- printer("\"thread_cache_activation_contentions\": %lu, \n",
- zone->xzz_thread_cache_xzone_activation_contentions);
- printer("\"thread_cache_activation_time\": %llu, \n",
- zone->xzz_thread_cache_xzone_activation_time);
-
- if (zone->xzz_thread_cache_enabled) {
- printer("\"thread_caches\": [ \n");
- __block bool first_thread_cache = true;
- kr = _xzm_introspect_enumerate_thread_caches(task, reader, main,
- ^(vm_address_t thread_cache_addr, xzm_thread_cache_t tc){
- printer(" ");
- if (!first_thread_cache) {
- printer(", ");
- } else {
- first_thread_cache = false;
- }
- printer("{\n");
- printer(" \"thread\": \"%p\",\n", (void *)tc->xtc_thread);
- printer(" \"xz_caches\": {\n", (void *)tc->xtc_thread);
- for (uint8_t xzidx = XZM_XZONE_INDEX_FIRST;
- xzidx < zone->xzz_thread_cache_xzone_count; xzidx++) {
- xzm_xzone_thread_cache_t cache = &tc->xtc_xz_caches[xzidx];
-
- printer(" \"%d\": {\n", (int)xzidx);
- printer(" \"xz_idx\": %d, \n", (int)xzidx);
-
- uint16_t head = cache->xztc_head;
- if (head == XZM_XZONE_NOT_CACHED) {
- printer(" \"head\": \"NOT_CACHED\", \n");
- printer(" \"timestamp\": \"%llu\", \n",
- cache->xztc_timestamp);
- printer(" \"contentions\": \"%llu\", \n",
- (uint64_t)cache->xztc_contentions);
- printer(" \"allocs\": \"%llu\" \n",
- (uint64_t)cache->xztc_allocs);
- } else if (head == XZM_XZONE_CACHE_EMPTY) {
- printer(" \"head\": \"EMPTY\" \n");
- } else {
- printer(" \"head\": \"0x%llx\", \n",
- (uint64_t)head);
- printer(" \"chunk\": \"%p\", \n",
- cache->xztc_chunk);
- printer(" \"chunk_start\": \"%p\", \n",
- cache->xztc_chunk_start);
- printer(" \"head_seqno\": \"0x%llx\", \n",
- (uint64_t)cache->xztc_head_seqno);
- printer(" \"free_count\": \"0x%llx\", \n",
- (uint64_t)cache->xztc_free_count);
- printer(" \"seqno\": \"0x%llx\" \n",
- (uint64_t)cache->xztc_seqno);
- }
-
-
- printer(" }"); // xzone thread cache
- if (xzidx < zone->xzz_thread_cache_xzone_count - 1) {
- printer(",");
- }
- printer("\n");
- }
- printer(" } \n"); // xzone thread caches
- printer(" } \n"); // thread cache
-
- return KERN_SUCCESS;
- });
-
- printer("], \n"); // thread_caches
- }
-#endif // CONFIG_XZM_THREAD_CACHE
- }
-
- printer("\"spans\": {\n");
-
- first_span = true;
- // un-cached segments will have their dispositions enumerated via the
- // spans/chunks they contain
- print_segment_dispositions = false;
-
- kr = _xzm_introspect_enumerate(task, reader, zone_address,
- zone, main_address, main,
- /* include_blocks */false,
- ^(vm_address_t slab_addr, vm_size_t slab_size, xzm_metapool_id_t
- metapool_id) {
- // Metapool slab enumerator
-
- printer(" ");
- if (!first_span) {
- printer(", ");
- }
-
- printer("\"%p\": {\n", (void *)slab_addr);
- printer(" \"addr\": \"%p\", \n", (void *)slab_addr);
- printer(" \"kind\": \"%s\", \n",
- _xzm_metapool_id_to_string(metapool_id));
- print_dispositions(slab_addr, slab_size, " ");
- printer(" \"size\": %u \n",
- slab_size);
-
- printer(" }\n");
- first_span = false;
-
- return KERN_SUCCESS;
- },
- segment_enumerator,
- ^(vm_address_t segment_addr, xzm_segment_t segment, xzm_chunk_t chunk,
- xzm_slice_count_t slice_count, vm_address_t start_addr,
- xzm_xzone_t xz, vm_range_t *ranges, size_t count){
- // Chunk enumerator
-
- printer(" ");
- if (!first_span) {
- printer(", ");
- }
-
- printer("\"%p\": {\n", (void *)start_addr);
- printer(" \"addr\": \"%p\", \n", (void *)start_addr);
- printer(" \"metadata_addr\": \"%p\", \n", (void *)(segment_addr +
- ((vm_address_t)chunk - (vm_address_t)segment)));
- printer(" \"mzone\": %d, \n", (int)chunk->xzc_mzone_idx);
- printer(" \"xzone\": %d, \n", (int)chunk->xzc_xzone_idx);
- printer(" \"segment\": \"%p\", \n", (void *)segment_addr);
- printer(" \"segment_group\": %zu, \n",
- segment->xzs_segment_group - main->xzmz_segment_groups);
-
- xzm_slice_kind_t kind = chunk->xzc_bits.xzcb_kind;
- const char *kind_str = _xzm_slice_kind_to_string(kind);
- printer(" \"kind\": \"%s\", \n", kind_str);
- printer(" \"slice_count\": %u, \n", slice_count);
- printer(" \"block_size\": %u, \n",
- xz ? (unsigned)xz->xz_block_size : 0);
- printer(" \"in_use\": 1, \n");
-
- xzm_slice_count_t slice_index = (xzm_slice_count_t)
- (chunk - segment->xzs_slices);
- xzm_xzone_slice_metadata_u *metadata =
- &segment->xzs_slice_metadata[slice_index];
- printer(" \"slice_metadata\": \"%p\", \n",
- metadata->xzsm_batch_next);
-
- kern_return_t kr = print_dispositions(start_addr, slice_count *
- XZM_SEGMENT_SLICE_SIZE, " ");
- if (kr) {
- return kr;
- }
-
- if (_xzm_slice_kind_uses_xzones(kind)) {
- printer(" \"bucket\": %u,\n", (unsigned)xz->xz_bucket);
- }
-
- switch (kind) {
- case XZM_SLICE_KIND_TINY_CHUNK:
- case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:
- printer(" \"meta\": \"0x%llx\",\n",
- chunk->xzc_atomic_meta.xca_value);
- printer(" \"xca_alloc_head\": \"0x%llx\",\n",
- chunk->xzc_atomic_meta.xca_alloc_head);
- printer(" \"xca_free_count\": \"0x%llx\",\n",
- chunk->xzc_atomic_meta.xca_free_count);
- printer(" \"xca_alloc_idx\": \"0x%llx\",\n",
- chunk->xzc_atomic_meta.xca_alloc_idx);
- printer(" \"xca_on_partial_list\": \"0x%llx\",\n",
- chunk->xzc_atomic_meta.xca_on_partial_list);
- printer(" \"xca_on_empty_list\": \"0x%llx\",\n",
- chunk->xzc_atomic_meta.xca_on_empty_list);
- printer(" \"xca_walk_locked\": \"0x%llx\",\n",
- chunk->xzc_atomic_meta.xca_walk_locked);
- printer(" \"xca_head_seqno\": \"0x%llx\",\n",
- chunk->xzc_atomic_meta.xca_head_seqno);
- printer(" \"xca_seqno\": \"0x%llx\",\n",
- chunk->xzc_atomic_meta.xca_seqno);
- break;
- case XZM_SLICE_KIND_SMALL_CHUNK:
- printer(" \"free\": \"0x%x\",\n", chunk->xzc_free);
- printer(" \"used\": %u,\n", (unsigned)chunk->xzc_used);
- printer(" \"alloc_idx\": %u,\n",
- (unsigned)chunk->xzc_alloc_idx);
- break;
- default:
- break;
- }
-
- printer(" \"is_preallocated\": %d,\n",
- (int)chunk->xzc_bits.xzcb_preallocated);
- printer(" \"is_pristine\": %d\n",
- (int)chunk->xzc_bits.xzcb_is_pristine);
- printer(" }\n");
- first_span = false;
-
- return KERN_SUCCESS;
- }, !zone_is_main ? NULL : ^(vm_address_t segment_addr,
- xzm_segment_t segment, xzm_chunk_t span,
- xzm_slice_count_t slice_count, vm_address_t start_addr){
- // Main zone span enumerator
-
- // TODO: better sharing with the chunk enumerator
- printer(" ");
- if (!first_span) {
- printer(", ");
- }
-
- printer("\"%p\": {\n", (void *)start_addr);
- printer(" \"addr\": \"%p\", \n", (void *)start_addr);
- printer(" \"metadata_addr\": \"%p\", \n", (void *)(segment_addr +
- ((vm_address_t)span - (vm_address_t)segment)));
- printer(" \"mzone\": %d, \n", (int)span->xzc_mzone_idx);
- printer(" \"xzone\": %d, \n", (int)span->xzc_xzone_idx);
- printer(" \"segment\": \"%p\", \n", (void *)segment_addr);
- printer(" \"segment_group\": %zu, \n",
- segment->xzs_segment_group - main->xzmz_segment_groups);
-
- xzm_slice_kind_t kind = span->xzc_bits.xzcb_kind;
- const char *kind_str = _xzm_slice_kind_to_string(kind);
- printer(" \"kind\": \"%s\", \n", kind_str);
- printer(" \"slice_count\": %u, \n", slice_count);
-
- xzm_slice_count_t slice_index = (xzm_slice_count_t)
- (span - segment->xzs_slices);
- xzm_xzone_slice_metadata_u *metadata =
- &segment->xzs_slice_metadata[slice_index];
- printer(" \"slice_metadata\": \"%p\", \n",
- metadata->xzsm_batch_next);
-
- kern_return_t kr = print_dispositions(start_addr, slice_count *
- XZM_SEGMENT_SLICE_SIZE, " ");
- if (kr) {
- return kr;
- }
-
- printer(" \"is_preallocated\": %d,\n",
- (int)span->xzc_bits.xzcb_preallocated);
- printer(" \"in_use\": 0 \n");
- printer(" }\n"); // span
- first_span = false;
-
- return KERN_SUCCESS;
- });
-
- if (dispositions) {
- mach_vm_deallocate(mach_task_self(), (mach_vm_address_t)dispositions,
- dispositions_count * sizeof(dispositions[0]));
- }
-
- if (kr) {
- return;
- }
-
- printer("}\n"); // spans
- printer("}\n"); // overall
- printer("End xzone malloc JSON\n");
-}
-
-static void
-xzm_print_task(task_t task, unsigned level, vm_address_t zone_address,
- memory_reader_t reader, print_task_printer_t printer)
-{
- xzm_print(task, level, zone_address, reader, printer);
-}
-
-static void
-xzm_print_self(xzm_malloc_zone_t zone, boolean_t verbose)
-{
- xzm_print(mach_task_self(), verbose ? MALLOC_VERBOSE_PRINT_LEVEL : 0,
- (vm_address_t)zone, _malloc_default_reader, malloc_report_simple);
-}
-
-static kern_return_t
-xzm_statistics(task_t task, vm_address_t zone_address,
- memory_reader_t reader, print_task_printer_t printer,
- malloc_statistics_t *stats)
-{
- // It's straightforward to compute blocks_in_use and size_in_use from an
- // enumeration pass.
- //
- // size_allocated, which is the sum of the virtual sizes of all live
- // non-metadata VM reservations, is also fairly easy.
- //
- // max_size_in_use, which is supposed to track the "high water mark of
- // touched memory", is harder:
- // - nano doesn't even try
- // - szone malloc sort of does, but its accounting is wrong in several
- // ways:
- // - it doesn't account for madvise, so its calculation for "in-use"
- // memory in regions is too pessimistic (not everything outside the
- // pristine parts of a region was necessarily all in use at the same
- // time)
- // - regions that only became partially full before being deallocated
- // are incorrectly assumed to have been fully used
- // - it doesn't keep track of the high water mark in large, so the
- // current size of large is taken as the max
- //
- // Keeping a running max of the sizes of live chunks in each segment group
- // seems like it could be reasonable. We'd also have to somehow deal with
- // sequestered empty chunks, though, since they look "live" to the range
- // group as-is.
- //
- // Nano's policy of not even trying seems best, if we can get away with it,
- // and the lack of complaints from default enablement of nano on macOS seems
- // like strong evidence that we can, so that's what we should go with unless
- // a compelling need to do otherwise arises.
- (void)printer;
- *stats = (malloc_statistics_t){ 0 };
-
- reader = reader_or_in_memory_fallback(reader, task);
-
- xzm_malloc_zone_t zone;
- xzm_main_malloc_zone_t main;
- vm_address_t main_address;
- bool zone_is_main = false;
-
- kern_return_t kr = _xzm_introspect_map_zone_and_main(task, zone_address,
- reader, &zone, &main, &main_address);
- if (kr) {
- return kr;
- }
-
- zone_is_main = (zone_address == main_address);
-
- if (zone_is_main) {
- vm_address_t mfm_addr = (vm_address_t)main->xzmz_mfm_address;
- if (mfm_addr) {
- mfm_introspect.task_statistics(task, mfm_addr, reader, stats);
-
- // We don't know how to report max_size_in_use, so don't confuse things
- // by only including the max_size_in_use from mfm
- stats->max_size_in_use = 0;
- }
- }
-
- return _xzm_introspect_enumerate(task, reader, zone_address,
- zone, main_address, main, /* include_blocks */ false,
- ^(vm_address_t slab_addr, vm_size_t slab_size, xzm_metapool_id_t mp_id){
- // Metapool slab enumerator
- return KERN_SUCCESS;
- }, ^(vm_address_t segment_addr, xzm_segment_t segment, const char *indent){
- // Segment enumerator
-
- // Nothing interesting for stats at the segment level
- return KERN_SUCCESS;
- }, ^(vm_address_t segment_addr, xzm_segment_t segment, xzm_chunk_t chunk,
- xzm_slice_count_t slice_count, vm_address_t start_addr,
- xzm_xzone_t xz, vm_range_t *ranges, size_t count){
- // Chunk enumerator
- size_t chunk_size = slice_count * XZM_SEGMENT_SLICE_SIZE;
- size_t used = 0;
-
- xzm_slice_kind_t kind = chunk->xzc_bits.xzcb_kind;
- switch (kind) {
- case XZM_SLICE_KIND_TINY_CHUNK:
- case XZM_SLICE_KIND_SMALL_FREELIST_CHUNK:;
- xzm_chunk_atomic_meta_u meta = chunk->xzc_atomic_meta;
- uint32_t capacity = xz->xz_chunk_capacity;
- if (meta.xca_alloc_head != XZM_FREE_MADVISING &&
- meta.xca_alloc_head != XZM_FREE_MADVISED) {
- used = (size_t)(capacity - meta.xca_free_count);
- stats->blocks_in_use += used;
- stats->size_in_use += used * xz->xz_block_size;
- }
- break;
- case XZM_SLICE_KIND_SMALL_CHUNK:
- used = chunk->xzc_used;
- stats->blocks_in_use += used;
- stats->size_in_use += used * xz->xz_block_size;
- break;
- default:
- stats->blocks_in_use++;
- stats->size_in_use += chunk_size;
- break;
- }
- stats->size_allocated += chunk_size;
-
- return KERN_SUCCESS;
- }, !zone_is_main ? NULL : ^(vm_address_t segment_addr,
- xzm_segment_t segment, xzm_chunk_t span,
- xzm_slice_count_t slice_count, vm_address_t start_addr){
- // Main zone span enumerator
-
- // Record all free spans and chunks with no mzone against the main zone
- stats->size_allocated += slice_count * XZM_SEGMENT_SLICE_SIZE;
-
- return KERN_SUCCESS;
- });
-}
-
-static void
-xzm_statistics_self(xzm_malloc_zone_t zone, malloc_statistics_t *stats)
-{
- if (_xzm_malloc_zone_is_main(zone)) {
- mfm_lock();
- }
-
- xzm_force_lock(zone);
- xzm_statistics(mach_task_self(), (vm_address_t)zone, _malloc_default_reader,
- malloc_report_simple, stats);
- xzm_force_unlock(zone);
- if (_xzm_malloc_zone_is_main(zone)) {
- mfm_unlock();
- }
-}
-
-static void
-xzm_statistics_task(task_t task, vm_address_t zone_address,
- memory_reader_t reader, malloc_statistics_t *stats)
-{
- xzm_statistics(task, zone_address, reader, NULL, stats);
-}
-
-
-const struct malloc_introspection_t xzm_malloc_zone_introspect = {
-#if XZM_DEBUG_ENUMERATOR
- .enumerator = (void *)xzm_debug_ptr_in_use_enumerator,
-#else // XZM_DEBUG_ENUMERATOR
- .enumerator = (void *)xzm_ptr_in_use_enumerator,
-#endif // XZM_DEBUG_ENUMERATOR
- .print_task = (void *)xzm_print_task,
- .good_size = (void *)xzm_good_size,
- .check = (void *)xzm_check,
- .print = (void *)xzm_print_self,
- .statistics = (void *)xzm_statistics_self,
- .task_statistics = (void*)xzm_statistics_task,
- .log = (void *)xzm_log,
- .zone_locked = (void *)xzm_locked,
- .force_lock = (void *)xzm_force_lock,
- .force_unlock = (void *)xzm_force_unlock,
- .reinit_lock = (void *)xzm_reinit_lock,
- // discharge checking is a vestigial interface relating to the historical
- // ObjC gc - not to be implemented
- .enable_discharge_checking = NULL,
- .disable_discharge_checking = NULL,
-#ifdef __BLOCKS__
- .enumerate_discharged_pointers = NULL,
-#else // __BLOCKS__
- .enumerate_unavailable_without_blocks = NULL,
-#endif // __BLOCKS__
- .zone_type = MALLOC_ZONE_TYPE_XZONE,
-};
-
-#endif // CONFIG_XZONE_MALLOC