Loading...
--- xnu/xnu-12377.101.15/iokit/Kernel/IOKitDebug.cpp
+++ xnu/xnu-8020.140.41/iokit/Kernel/IOKitDebug.cpp
@@ -26,9 +26,10 @@
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
+
#include <sys/sysctl.h>
extern "C" {
-#include <vm/vm_kern_xnu.h>
+#include <vm/vm_kern.h>
#include <kern/task.h>
#include <kern/debug.h>
}
@@ -48,7 +49,7 @@
#include "IOKitKernelInternal.h"
TUNABLE_WRITEABLE(SInt64, gIOKitDebug, "io", DEBUG_INIT_VALUE);
-TUNABLE_DEV_WRITEABLE(SInt64, gIOKitTrace, "iotrace", 0);
+TUNABLE(SInt64, gIOKitTrace, "iotrace", 0);
#if DEVELOPMENT || DEBUG
#define IODEBUG_CTLFLAGS CTLFLAG_RW
@@ -56,7 +57,7 @@
#define IODEBUG_CTLFLAGS CTLFLAG_RD
#endif
-SYSCTL_QUAD(_debug, OID_AUTO, iotrace, IODEBUG_CTLFLAGS | CTLFLAG_LOCKED, &gIOKitTrace, "trace io");
+SYSCTL_QUAD(_debug, OID_AUTO, iotrace, CTLFLAG_RW | CTLFLAG_LOCKED, &gIOKitTrace, "trace io");
static int
sysctl_debug_iokit
@@ -73,8 +74,6 @@
SYSCTL_PROC(_debug, OID_AUTO, iokit,
CTLTYPE_QUAD | IODEBUG_CTLFLAGS | CTLFLAG_KERN | CTLFLAG_LOCKED,
&gIOKitDebug, 0, sysctl_debug_iokit, "Q", "boot_arg io");
-
-void (*gIOTrackingLeakScanCallback)(uint32_t notification) = NULL;
size_t debug_malloc_size;
size_t debug_iomalloc_size;
@@ -266,7 +265,7 @@
queue_chain_t link;
queue_head_t instances;
IOTrackingQueue * queue;
- IOTracking ** addresses;
+ IOTracking * addresses;
size_t size[2];
uint32_t crc;
uint32_t count;
@@ -616,7 +615,7 @@
}
queue_init(&site->instances);
- site->addresses = NULL;
+ site->addresses = (IOTracking *) &site->instances;
site->queue = queue;
site->crc = crc;
site->count = 0;
@@ -639,22 +638,10 @@
}
if (address) {
- IOTrackingAddress * memAddr = (typeof(memAddr))mem;
- uint32_t hashIdx;
-
- if (NULL == site->addresses) {
- site->addresses = kalloc_type(IOTracking *, queue->numSiteQs, Z_WAITOK_ZERO_NOFAIL);
- for (hashIdx = 0; hashIdx < queue->numSiteQs; hashIdx++) {
- site->addresses[hashIdx] = (IOTracking *) &site->instances;
- }
- }
- hashIdx = atop(memAddr->address) % queue->numSiteQs;
- if (queue_end(&site->instances, (queue_entry_t)site->addresses[hashIdx])) {
- queue_enter/*last*/ (&site->instances, mem, IOTracking *, link);
- } else {
- queue_insert_before(&site->instances, mem, site->addresses[hashIdx], IOTracking *, link);
- }
- site->addresses[hashIdx] = mem;
+ queue_enter/*last*/ (&site->instances, mem, IOTracking *, link);
+ if (queue_end(&site->instances, (queue_entry_t)site->addresses)) {
+ site->addresses = mem;
+ }
} else {
queue_enter_first(&site->instances, mem, IOTracking *, link);
}
@@ -668,12 +655,9 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-static void
-IOTrackingRemoveInternal(IOTrackingQueue * queue, IOTracking * mem, size_t size, uint32_t addressIdx)
-{
- IOTrackingCallSite * site;
- IOTrackingAddress * nextAddress;
-
+void
+IOTrackingRemove(IOTrackingQueue * queue, IOTracking * mem, size_t size)
+{
if (!mem->link.next) {
return;
}
@@ -681,57 +665,29 @@
IOTRecursiveLockLock(&queue->lock);
if (mem->link.next) {
assert(mem->site);
- site = mem->site;
-
- if ((-1U != addressIdx) && (mem == site->addresses[addressIdx])) {
- nextAddress = (IOTrackingAddress *) queue_next(&mem->link);
- if (!queue_end(&site->instances, &nextAddress->tracking.link)
- && (addressIdx != (atop(nextAddress->address) % queue->numSiteQs))) {
- nextAddress = (IOTrackingAddress *) &site->instances;
- }
- site->addresses[addressIdx] = &nextAddress->tracking;
- }
-
+
+ if (mem == mem->site->addresses) {
+ mem->site->addresses = (IOTracking *) queue_next(&mem->link);
+ }
remque(&mem->link);
- assert(site->count);
- site->count--;
- assert(site->size[0] >= size);
- site->size[0] -= size;
- if (!site->count) {
- assert(queue_empty(&site->instances));
- assert(!site->size[0]);
- assert(!site->size[1]);
-
- remque(&site->link);
+
+ assert(mem->site->count);
+ mem->site->count--;
+ assert(mem->site->size[0] >= size);
+ mem->site->size[0] -= size;
+ if (!mem->site->count) {
+ assert(queue_empty(&mem->site->instances));
+ assert(!mem->site->size[0]);
+ assert(!mem->site->size[1]);
+
+ remque(&mem->site->link);
assert(queue->siteCount);
queue->siteCount--;
- IOTrackingFreeCallSite(queue->type, &site);
+ IOTrackingFreeCallSite(queue->type, &mem->site);
}
mem->site = NULL;
}
IOTRecursiveLockUnlock(&queue->lock);
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-void
-IOTrackingRemove(IOTrackingQueue * queue, IOTracking * mem, size_t size)
-{
- return IOTrackingRemoveInternal(queue, mem, size, -1U);
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-void
-IOTrackingRemoveAddress(IOTrackingQueue * queue, IOTrackingAddress * mem, size_t size)
-{
- uint32_t addressIdx;
- uint64_t address;
-
- address = mem->address;
- addressIdx = atop(address) % queue->numSiteQs;
-
- return IOTrackingRemoveInternal(queue, &mem->tracking, size, addressIdx);
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -764,38 +720,24 @@
{
IOTrackingCallSite * site;
IOTrackingAddress * tracking;
- IOTrackingAddress * nextAddress;
- uint32_t idx, hashIdx;
+ uint32_t idx;
bool done;
address = ~address;
IOTRecursiveLockLock(&queue->lock);
-
- hashIdx = atop(address) % queue->numSiteQs;
-
done = false;
for (idx = 0; idx < queue->numSiteQs; idx++) {
queue_iterate(&queue->sites[idx], site, IOTrackingCallSite *, link)
{
- if (!site->addresses) {
- continue;
- }
- tracking = (IOTrackingAddress *) site->addresses[hashIdx];
+ tracking = (IOTrackingAddress *) site->addresses;
while (!queue_end(&site->instances, &tracking->tracking.link)) {
- nextAddress = (IOTrackingAddress *) queue_next(&tracking->tracking.link);
- if (!queue_end(&site->instances, &nextAddress->tracking.link)
- && (hashIdx != (atop(nextAddress->address) % queue->numSiteQs))) {
- nextAddress = (IOTrackingAddress *) &site->instances;
- }
if ((done = (address == tracking->address))) {
- if (tracking == (IOTrackingAddress *) site->addresses[hashIdx]) {
- site->addresses[hashIdx] = &nextAddress->tracking;
- }
- IOTrackingRemoveInternal(queue, &tracking->tracking, size, -1U);
+ IOTrackingRemove(queue, &tracking->tracking, size);
kfree_type(IOTrackingAddress, tracking);
break;
+ } else {
+ tracking = (IOTrackingAddress *) queue_next(&tracking->tracking.link);
}
- tracking = nextAddress;
}
if (done) {
break;
@@ -809,15 +751,11 @@
}
static void
-IOTrackingFreeCallSite(uint32_t type, IOTrackingCallSite ** pSite)
-{
- IOTrackingCallSite * site;
+IOTrackingFreeCallSite(uint32_t type, IOTrackingCallSite ** site)
+{
void ** ptr;
- site = *pSite;
- kfree_type(IOTracking *, site->queue->numSiteQs, site->addresses);
-
- ptr = reinterpret_cast<void **>(pSite);
+ ptr = reinterpret_cast<void **>(site);
if (kIOTrackingQueueTypeUser & type) {
kfree_type(IOTrackingCallSiteWithUser, *ptr);
} else {
@@ -849,7 +787,7 @@
IOTrackingUser * user;
IOTracking * tracking;
IOTrackingAddress * trackingAddress;
- uint32_t idx, hashIdx;
+ uint32_t idx;
bool addresses;
IOTRecursiveLockLock(&queue->lock);
@@ -863,12 +801,8 @@
addresses = false;
while (!queue_empty(&site->instances)) {
queue_remove_first(&site->instances, tracking, IOTracking *, link);
- if (site->addresses) {
- for (hashIdx = 0; !addresses && (hashIdx < queue->numSiteQs); hashIdx++) {
- if (tracking == site->addresses[hashIdx]) {
- addresses = true;
- }
- }
+ if (tracking == site->addresses) {
+ addresses = true;
}
if (addresses) {
trackingAddress = (typeof(trackingAddress))tracking;
@@ -997,21 +931,11 @@
count = ref->count;
size = origsize = ref->zoneSize;
- if (gIOTrackingLeakScanCallback) {
- gIOTrackingLeakScanCallback(kIOTrackingLeakScanStart);
- }
-
for (deadline = 0, vaddr = VM_MIN_KERNEL_AND_KEXT_ADDRESS;
;
vaddr += vincr) {
if ((mach_absolute_time() > deadline) || (vaddr >= VM_MAX_KERNEL_ADDRESS)) {
if (deadline) {
-#if SCHED_HYGIENE_DEBUG
- if (is) {
- // Reset the interrupt timeout to avoid panics
- ml_spin_debug_clear_self();
- }
-#endif /* SCHED_HYGIENE_DEBUG */
ml_set_interrupts_enabled(is);
IODelay(10);
}
@@ -1021,6 +945,7 @@
is = ml_set_interrupts_enabled(false);
clock_interval_to_deadline(10, kMillisecondScale, &deadline);
}
+
ppn = kernel_pmap_present_mapping(vaddr, &vincr, &vphysaddr);
// check noencrypt to avoid VM structs (map entries) with pointers
if (ppn && (!pmap_valid_page(ppn) || (!ref->zoneSize && pmap_is_noencrypt(ppn)))) {
@@ -1030,7 +955,6 @@
continue;
}
- vm_memtag_disable_checking();
for (ptrIdx = 0; ptrIdx < (page_size / sizeof(uintptr_t)); ptrIdx++) {
ptr = ((uintptr_t *)vphysaddr)[ptrIdx];
#if defined(HAS_APPLE_PAC)
@@ -1076,12 +1000,7 @@
// else move left
}
}
- vm_memtag_enable_checking();
ref->bytes += page_size;
- }
-
- if (gIOTrackingLeakScanCallback) {
- gIOTrackingLeakScanCallback(kIOTrackingLeakScanEnd);
}
}
@@ -1128,11 +1047,7 @@
uintptr_t inst;
uint32_t count, idx, numSites, dups, siteCount;
- /* BEGIN IGNORE CODESTYLE */
- __typed_allocators_ignore_push
instances = (typeof(instances))data->getBytesNoCopy();
- __typed_allocators_ignore_pop
- /* END IGNORE CODESTYLE */
count = (data->getLength() / sizeof(*instances));
qsort(instances, count, sizeof(*instances), &IOTrackingAddressCompare);
@@ -1149,11 +1064,7 @@
}
}
- /* BEGIN IGNORE CODESTYLE */
- __typed_allocators_ignore_push
leakData = OSData::withCapacity(128 * sizeof(IOTrackingCallSiteInfo));
- __typed_allocators_ignore_pop
- /* END IGNORE CODESTYLE */
for (numSites = 0, idx = 0; idx < count; idx++) {
inst = instances[idx];
@@ -1178,16 +1089,11 @@
instances[dups] = 0;
}
}
- // leak byte size is reported as:
- // (total bytes allocated by the callsite * number of leaked instances)
- // divided by (number of allocations by callsite)
siteInfo.count = siteCount;
- siteInfo.size[0] = (site->size[0] * siteCount) / site->count;
- siteInfo.size[1] = (site->size[1] * siteCount) / site->count;
+ siteInfo.size[0] = (site->size[0] * site->count) / siteCount;
+ siteInfo.size[1] = (site->size[1] * site->count) / siteCount;
CopyOutBacktraces(site, &siteInfo);
- __typed_allocators_ignore_push
leakData->appendBytes(&siteInfo, sizeof(siteInfo));
- __typed_allocators_ignore_pop
}
data->release();
@@ -1314,11 +1220,7 @@
}
if (!data) {
- /* BEGIN IGNORE CODESTYLE */
- __typed_allocators_ignore_push
data = OSData::withCapacity(1024 * sizeof(uintptr_t));
- __typed_allocators_ignore_pop
- /* END IGNORE CODESTYLE */
}
IOTRecursiveLockLock(&queue->lock);
@@ -1328,12 +1230,8 @@
addresses = false;
queue_iterate(&site->instances, instance, IOTracking *, link)
{
- if (site->addresses) {
- for (uint32_t hashIdx = 0; !addresses && (hashIdx < queue->numSiteQs); hashIdx++) {
- if (instance == site->addresses[hashIdx]) {
- addresses = true;
- }
- }
+ if (instance == site->addresses) {
+ addresses = true;
}
instFlags = (typeof(instFlags))instance;
if (addresses) {
@@ -1356,11 +1254,7 @@
}
if (!data) {
- /* BEGIN IGNORE CODESTYLE */
- __typed_allocators_ignore_push
data = OSData::withCapacity(128 * sizeof(IOTrackingCallSiteInfo));
- __typed_allocators_ignore_pop
- /* END IGNORE CODESTYLE */
}
IOTRecursiveLockLock(&queue->lock);
@@ -1388,12 +1282,8 @@
tsize[0] = tsize[1] = 0;
queue_iterate(&site->instances, instance, IOTracking *, link)
{
- if (site->addresses) {
- for (uint32_t hashIdx = 0; !addresses && (hashIdx < queue->numSiteQs); hashIdx++) {
- if (instance == site->addresses[hashIdx]) {
- addresses = true;
- }
- }
+ if (instance == site->addresses) {
+ addresses = true;
}
if (addresses) {
@@ -1434,9 +1324,7 @@
siteInfo.size[0] = tsize[0];
siteInfo.size[1] = tsize[1];
CopyOutBacktraces(site, &siteInfo);
- __typed_allocators_ignore_push
data->appendBytes(&siteInfo, sizeof(siteInfo));
- __typed_allocators_ignore_pop
}
}
assert(idx == num);
@@ -1494,9 +1382,7 @@
siteInfo.bt[1][j] = bt64[j];
}
}
- __typed_allocators_ignore_push
data->appendBytes(&siteInfo, sizeof(siteInfo));
- __typed_allocators_ignore_pop
}
}
assert(idx == num);
@@ -1533,11 +1419,7 @@
uint8_t sLen;
if (!data) {
- /* BEGIN IGNORE CODESTYLE */
- __typed_allocators_ignore_push
data = OSData::withCapacity(4096 * sizeof(uintptr_t));
- __typed_allocators_ignore_pop
- /* END IGNORE CODESTYLE */
}
// <len><name>...<len><name><0>
@@ -1556,9 +1438,8 @@
};
btref_decode_unslide(ref, siteInfo.bt[0]);
- __typed_allocators_ignore_push
+
data->appendBytes(&siteInfo, sizeof(siteInfo));
- __typed_allocators_ignore_pop
});
if (KERN_SUCCESS == kr) {
ret = kIOReturnSuccess;
@@ -1576,11 +1457,7 @@
case kIOTrackingGetMappings:
{
IOTrackingCallSiteInfo * siteInfos;
- /* BEGIN IGNORE CODESTYLE */
- __typed_allocators_ignore_push
siteInfos = (typeof(siteInfos))data->getBytesNoCopy();
- __typed_allocators_ignore_pop
- /* END IGNORE CODESTYLE */
num = (data->getLength() / sizeof(*siteInfos));
qsort(siteInfos, num, sizeof(*siteInfos), &IOTrackingCallSiteInfoCompare);
break;
@@ -1606,15 +1483,14 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#undef super
-#define super IOUserClient2022
-
-OSDefineMetaClassAndStructors(IOKitDiagnosticsClient, IOUserClient2022)
+#define super IOUserClient
+
+OSDefineMetaClassAndStructors(IOKitDiagnosticsClient, IOUserClient)
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
IOUserClient * IOKitDiagnosticsClient::withTask(task_t owningTask)
{
-#if IOTRACKING
IOKitDiagnosticsClient * inst;
inst = new IOKitDiagnosticsClient;
@@ -1623,16 +1499,7 @@
inst = NULL;
}
- inst->setProperty(kIOUserClientDefaultLockingKey, kOSBooleanTrue);
- inst->setProperty(kIOUserClientDefaultLockingSetPropertiesKey, kOSBooleanTrue);
- inst->setProperty(kIOUserClientDefaultLockingSingleThreadExternalMethodKey, kOSBooleanTrue);
-
- inst->setProperty(kIOUserClientEntitlementsKey, kOSBooleanFalse);
-
return inst;
-#else
- return NULL;
-#endif
}
IOReturn
@@ -1649,10 +1516,9 @@
return kr;
}
-
IOReturn
-IOTrackingMethodDispatched(OSObject * target, void * reference,
- IOExternalMethodArguments * args)
+IOKitDiagnosticsClient::externalMethod(uint32_t selector, IOExternalMethodArguments * args,
+ IOExternalMethodDispatch * dispatch, OSObject * target, void * reference)
{
IOReturn ret = kIOReturnBadArgument;
const IOKitDiagnosticsParameters * params;
@@ -1674,85 +1540,15 @@
names = (typeof(names))(params + 1);
}
- ret = IOTrackingDebug(args->selector, params->options, params->value, params->tag, params->zsize, names, namesLen, params->size, &result);
+ ret = IOTrackingDebug(selector, params->options, params->value, params->tag, params->zsize, names, namesLen, params->size, &result);
+
if ((kIOReturnSuccess == ret) && args->structureVariableOutputData) {
*args->structureVariableOutputData = result;
} else if (result) {
result->release();
}
+
return ret;
}
-IOReturn
-IOKitDiagnosticsClient::externalMethod(uint32_t selector, IOExternalMethodArgumentsOpaque * args)
-{
- static const IOExternalMethodDispatch2022 dispatchArray[] = {
- [kIOTrackingGetTracking] = {
- .function = &IOTrackingMethodDispatched,
- .checkScalarInputCount = 0,
- .checkStructureInputSize = kIOUCVariableStructureSize,
- .checkScalarOutputCount = 0,
- .checkStructureOutputSize = 0,
- .allowAsync = false,
- .checkEntitlement = NULL,
- },
- [kIOTrackingGetMappings] = {
- .function = &IOTrackingMethodDispatched,
- .checkScalarInputCount = 0,
- .checkStructureInputSize = kIOUCVariableStructureSize,
- .checkScalarOutputCount = 0,
- .checkStructureOutputSize = 0,
- .allowAsync = false,
- .checkEntitlement = NULL,
- },
- [kIOTrackingResetTracking] = {
- .function = &IOTrackingMethodDispatched,
- .checkScalarInputCount = 0,
- .checkStructureInputSize = kIOUCVariableStructureSize,
- .checkScalarOutputCount = 0,
- .checkStructureOutputSize = 0,
- .allowAsync = false,
- .checkEntitlement = NULL,
- },
- [kIOTrackingStartCapture] = {
- .function = &IOTrackingMethodDispatched,
- .checkScalarInputCount = 0,
- .checkStructureInputSize = kIOUCVariableStructureSize,
- .checkScalarOutputCount = 0,
- .checkStructureOutputSize = 0,
- .allowAsync = false,
- .checkEntitlement = NULL,
- },
- [kIOTrackingStopCapture] = {
- .function = &IOTrackingMethodDispatched,
- .checkScalarInputCount = 0,
- .checkStructureInputSize = kIOUCVariableStructureSize,
- .checkScalarOutputCount = 0,
- .checkStructureOutputSize = 0,
- .allowAsync = false,
- .checkEntitlement = NULL,
- },
- [kIOTrackingSetMinCaptureSize] = {
- .function = &IOTrackingMethodDispatched,
- .checkScalarInputCount = 0,
- .checkStructureInputSize = kIOUCVariableStructureSize,
- .checkScalarOutputCount = 0,
- .checkStructureOutputSize = 0,
- .allowAsync = false,
- .checkEntitlement = NULL,
- },
- [kIOTrackingLeaks] = {
- .function = &IOTrackingMethodDispatched,
- .checkScalarInputCount = 0,
- .checkStructureInputSize = kIOUCVariableStructureSize,
- .checkScalarOutputCount = 0,
- .checkStructureOutputSize = 0,
- .allowAsync = false,
- .checkEntitlement = NULL,
- },
- };
-
- return dispatchExternalMethod(selector, args, dispatchArray, sizeof(dispatchArray) / sizeof(dispatchArray[0]), this, NULL);
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */