Loading...
--- xnu/xnu-8020.140.41/iokit/Kernel/IOKitDebug.cpp
+++ xnu/xnu-8792.41.9/iokit/Kernel/IOKitDebug.cpp
@@ -26,7 +26,6 @@
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
-
#include <sys/sysctl.h>
extern "C" {
#include <vm/vm_kern.h>
@@ -49,7 +48,7 @@
#include "IOKitKernelInternal.h"
TUNABLE_WRITEABLE(SInt64, gIOKitDebug, "io", DEBUG_INIT_VALUE);
-TUNABLE(SInt64, gIOKitTrace, "iotrace", 0);
+TUNABLE_DEV_WRITEABLE(SInt64, gIOKitTrace, "iotrace", 0);
#if DEVELOPMENT || DEBUG
#define IODEBUG_CTLFLAGS CTLFLAG_RW
@@ -57,7 +56,7 @@
#define IODEBUG_CTLFLAGS CTLFLAG_RD
#endif
-SYSCTL_QUAD(_debug, OID_AUTO, iotrace, CTLFLAG_RW | CTLFLAG_LOCKED, &gIOKitTrace, "trace io");
+SYSCTL_QUAD(_debug, OID_AUTO, iotrace, IODEBUG_CTLFLAGS | CTLFLAG_LOCKED, &gIOKitTrace, "trace io");
static int
sysctl_debug_iokit
@@ -74,6 +73,8 @@
SYSCTL_PROC(_debug, OID_AUTO, iokit,
CTLTYPE_QUAD | IODEBUG_CTLFLAGS | CTLFLAG_KERN | CTLFLAG_LOCKED,
&gIOKitDebug, 0, sysctl_debug_iokit, "Q", "boot_arg io");
+
+void (*gIOTrackingLeakScanCallback)(uint32_t notification) = NULL;
size_t debug_malloc_size;
size_t debug_iomalloc_size;
@@ -265,7 +266,7 @@
queue_chain_t link;
queue_head_t instances;
IOTrackingQueue * queue;
- IOTracking * addresses;
+ IOTracking ** addresses;
size_t size[2];
uint32_t crc;
uint32_t count;
@@ -615,7 +616,7 @@
}
queue_init(&site->instances);
- site->addresses = (IOTracking *) &site->instances;
+ site->addresses = NULL;
site->queue = queue;
site->crc = crc;
site->count = 0;
@@ -638,10 +639,22 @@
}
if (address) {
- queue_enter/*last*/ (&site->instances, mem, IOTracking *, link);
- if (queue_end(&site->instances, (queue_entry_t)site->addresses)) {
- site->addresses = mem;
- }
+ IOTrackingAddress * memAddr = (typeof(memAddr))mem;
+ uint32_t hashIdx;
+
+ if (NULL == site->addresses) {
+ site->addresses = kalloc_type(IOTracking *, queue->numSiteQs, Z_WAITOK_ZERO_NOFAIL);
+ for (hashIdx = 0; hashIdx < queue->numSiteQs; hashIdx++) {
+ site->addresses[hashIdx] = (IOTracking *) &site->instances;
+ }
+ }
+ hashIdx = atop(memAddr->address) % queue->numSiteQs;
+ if (queue_end(&site->instances, (queue_entry_t)site->addresses[hashIdx])) {
+ queue_enter/*last*/ (&site->instances, mem, IOTracking *, link);
+ } else {
+ queue_insert_before(&site->instances, mem, site->addresses[hashIdx], IOTracking *, link);
+ }
+ site->addresses[hashIdx] = mem;
} else {
queue_enter_first(&site->instances, mem, IOTracking *, link);
}
@@ -655,9 +668,12 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-void
-IOTrackingRemove(IOTrackingQueue * queue, IOTracking * mem, size_t size)
-{
+static void
+IOTrackingRemoveInternal(IOTrackingQueue * queue, IOTracking * mem, size_t size, uint32_t addressIdx)
+{
+ IOTrackingCallSite * site;
+ IOTrackingAddress * nextAddress;
+
if (!mem->link.next) {
return;
}
@@ -665,29 +681,57 @@
IOTRecursiveLockLock(&queue->lock);
if (mem->link.next) {
assert(mem->site);
-
- if (mem == mem->site->addresses) {
- mem->site->addresses = (IOTracking *) queue_next(&mem->link);
- }
+ site = mem->site;
+
+ if ((-1U != addressIdx) && (mem == site->addresses[addressIdx])) {
+ nextAddress = (IOTrackingAddress *) queue_next(&mem->link);
+ if (!queue_end(&site->instances, &nextAddress->tracking.link)
+ && (addressIdx != (atop(nextAddress->address) % queue->numSiteQs))) {
+ nextAddress = (IOTrackingAddress *) &site->instances;
+ }
+ site->addresses[addressIdx] = &nextAddress->tracking;
+ }
+
remque(&mem->link);
-
- assert(mem->site->count);
- mem->site->count--;
- assert(mem->site->size[0] >= size);
- mem->site->size[0] -= size;
- if (!mem->site->count) {
- assert(queue_empty(&mem->site->instances));
- assert(!mem->site->size[0]);
- assert(!mem->site->size[1]);
-
- remque(&mem->site->link);
+ assert(site->count);
+ site->count--;
+ assert(site->size[0] >= size);
+ site->size[0] -= size;
+ if (!site->count) {
+ assert(queue_empty(&site->instances));
+ assert(!site->size[0]);
+ assert(!site->size[1]);
+
+ remque(&site->link);
assert(queue->siteCount);
queue->siteCount--;
- IOTrackingFreeCallSite(queue->type, &mem->site);
+ IOTrackingFreeCallSite(queue->type, &site);
}
mem->site = NULL;
}
IOTRecursiveLockUnlock(&queue->lock);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+void
+IOTrackingRemove(IOTrackingQueue * queue, IOTracking * mem, size_t size)
+{
+ return IOTrackingRemoveInternal(queue, mem, size, -1U);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+void
+IOTrackingRemoveAddress(IOTrackingQueue * queue, IOTrackingAddress * mem, size_t size)
+{
+ uint32_t addressIdx;
+ uint64_t address;
+
+ address = mem->address;
+ addressIdx = atop(address) % queue->numSiteQs;
+
+ return IOTrackingRemoveInternal(queue, &mem->tracking, size, addressIdx);
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -720,24 +764,38 @@
{
IOTrackingCallSite * site;
IOTrackingAddress * tracking;
- uint32_t idx;
+ IOTrackingAddress * nextAddress;
+ uint32_t idx, hashIdx;
bool done;
address = ~address;
IOTRecursiveLockLock(&queue->lock);
+
+ hashIdx = atop(address) % queue->numSiteQs;
+
done = false;
for (idx = 0; idx < queue->numSiteQs; idx++) {
queue_iterate(&queue->sites[idx], site, IOTrackingCallSite *, link)
{
- tracking = (IOTrackingAddress *) site->addresses;
+ if (!site->addresses) {
+ continue;
+ }
+ tracking = (IOTrackingAddress *) site->addresses[hashIdx];
while (!queue_end(&site->instances, &tracking->tracking.link)) {
+ nextAddress = (IOTrackingAddress *) queue_next(&tracking->tracking.link);
+ if (!queue_end(&site->instances, &nextAddress->tracking.link)
+ && (hashIdx != (atop(nextAddress->address) % queue->numSiteQs))) {
+ nextAddress = (IOTrackingAddress *) &site->instances;
+ }
if ((done = (address == tracking->address))) {
- IOTrackingRemove(queue, &tracking->tracking, size);
+ if (tracking == (IOTrackingAddress *) site->addresses[hashIdx]) {
+ site->addresses[hashIdx] = &nextAddress->tracking;
+ }
+ IOTrackingRemoveInternal(queue, &tracking->tracking, size, -1U);
kfree_type(IOTrackingAddress, tracking);
break;
- } else {
- tracking = (IOTrackingAddress *) queue_next(&tracking->tracking.link);
}
+ tracking = nextAddress;
}
if (done) {
break;
@@ -751,11 +809,15 @@
}
static void
-IOTrackingFreeCallSite(uint32_t type, IOTrackingCallSite ** site)
-{
+IOTrackingFreeCallSite(uint32_t type, IOTrackingCallSite ** pSite)
+{
+ IOTrackingCallSite * site;
void ** ptr;
- ptr = reinterpret_cast<void **>(site);
+ site = *pSite;
+ kfree_type(IOTracking *, site->queue->numSiteQs, site->addresses);
+
+ ptr = reinterpret_cast<void **>(pSite);
if (kIOTrackingQueueTypeUser & type) {
kfree_type(IOTrackingCallSiteWithUser, *ptr);
} else {
@@ -787,7 +849,7 @@
IOTrackingUser * user;
IOTracking * tracking;
IOTrackingAddress * trackingAddress;
- uint32_t idx;
+ uint32_t idx, hashIdx;
bool addresses;
IOTRecursiveLockLock(&queue->lock);
@@ -801,8 +863,12 @@
addresses = false;
while (!queue_empty(&site->instances)) {
queue_remove_first(&site->instances, tracking, IOTracking *, link);
- if (tracking == site->addresses) {
- addresses = true;
+ if (site->addresses) {
+ for (hashIdx = 0; !addresses && (hashIdx < queue->numSiteQs); hashIdx++) {
+ if (tracking == site->addresses[hashIdx]) {
+ addresses = true;
+ }
+ }
}
if (addresses) {
trackingAddress = (typeof(trackingAddress))tracking;
@@ -931,11 +997,21 @@
count = ref->count;
size = origsize = ref->zoneSize;
+ if (gIOTrackingLeakScanCallback) {
+ gIOTrackingLeakScanCallback(kIOTrackingLeakScanStart);
+ }
+
for (deadline = 0, vaddr = VM_MIN_KERNEL_AND_KEXT_ADDRESS;
;
vaddr += vincr) {
if ((mach_absolute_time() > deadline) || (vaddr >= VM_MAX_KERNEL_ADDRESS)) {
if (deadline) {
+#if SCHED_HYGIENE_DEBUG
+ if (is) {
+ // Reset the interrupt timeout to avoid panics
+ ml_spin_debug_clear_self();
+ }
+#endif /* SCHED_HYGIENE_DEBUG */
ml_set_interrupts_enabled(is);
IODelay(10);
}
@@ -1002,6 +1078,10 @@
}
ref->bytes += page_size;
}
+
+ if (gIOTrackingLeakScanCallback) {
+ gIOTrackingLeakScanCallback(kIOTrackingLeakScanEnd);
+ }
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -1230,8 +1310,12 @@
addresses = false;
queue_iterate(&site->instances, instance, IOTracking *, link)
{
- if (instance == site->addresses) {
- addresses = true;
+ if (site->addresses) {
+ for (uint32_t hashIdx = 0; !addresses && (hashIdx < queue->numSiteQs); hashIdx++) {
+ if (instance == site->addresses[hashIdx]) {
+ addresses = true;
+ }
+ }
}
instFlags = (typeof(instFlags))instance;
if (addresses) {
@@ -1282,8 +1366,12 @@
tsize[0] = tsize[1] = 0;
queue_iterate(&site->instances, instance, IOTracking *, link)
{
- if (instance == site->addresses) {
- addresses = true;
+ if (site->addresses) {
+ for (uint32_t hashIdx = 0; !addresses && (hashIdx < queue->numSiteQs); hashIdx++) {
+ if (instance == site->addresses[hashIdx]) {
+ addresses = true;
+ }
+ }
}
if (addresses) {
@@ -1483,14 +1571,15 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#undef super
-#define super IOUserClient
-
-OSDefineMetaClassAndStructors(IOKitDiagnosticsClient, IOUserClient)
+#define super IOUserClient2022
+
+OSDefineMetaClassAndStructors(IOKitDiagnosticsClient, IOUserClient2022)
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
IOUserClient * IOKitDiagnosticsClient::withTask(task_t owningTask)
{
+#if IOTRACKING
IOKitDiagnosticsClient * inst;
inst = new IOKitDiagnosticsClient;
@@ -1499,7 +1588,16 @@
inst = NULL;
}
+ inst->setProperty(kIOUserClientDefaultLockingKey, kOSBooleanTrue);
+ inst->setProperty(kIOUserClientDefaultLockingSetPropertiesKey, kOSBooleanTrue);
+ inst->setProperty(kIOUserClientDefaultLockingSingleThreadExternalMethodKey, kOSBooleanTrue);
+
+ inst->setProperty(kIOUserClientEntitlementsKey, kOSBooleanFalse);
+
return inst;
+#else
+ return NULL;
+#endif
}
IOReturn
@@ -1516,9 +1614,10 @@
return kr;
}
+
IOReturn
-IOKitDiagnosticsClient::externalMethod(uint32_t selector, IOExternalMethodArguments * args,
- IOExternalMethodDispatch * dispatch, OSObject * target, void * reference)
+IOTrackingMethodDispatched(OSObject * target, void * reference,
+ IOExternalMethodArguments * args)
{
IOReturn ret = kIOReturnBadArgument;
const IOKitDiagnosticsParameters * params;
@@ -1540,15 +1639,85 @@
names = (typeof(names))(params + 1);
}
- ret = IOTrackingDebug(selector, params->options, params->value, params->tag, params->zsize, names, namesLen, params->size, &result);
-
+ ret = IOTrackingDebug(args->selector, params->options, params->value, params->tag, params->zsize, names, namesLen, params->size, &result);
if ((kIOReturnSuccess == ret) && args->structureVariableOutputData) {
*args->structureVariableOutputData = result;
} else if (result) {
result->release();
}
-
return ret;
}
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+IOReturn
+IOKitDiagnosticsClient::externalMethod(uint32_t selector, IOExternalMethodArgumentsOpaque * args)
+{
+ static const IOExternalMethodDispatch2022 dispatchArray[] = {
+ [kIOTrackingGetTracking] = {
+ .function = &IOTrackingMethodDispatched,
+ .checkScalarInputCount = 0,
+ .checkStructureInputSize = kIOUCVariableStructureSize,
+ .checkScalarOutputCount = 0,
+ .checkStructureOutputSize = 0,
+ .allowAsync = false,
+ .checkEntitlement = NULL,
+ },
+ [kIOTrackingGetMappings] = {
+ .function = &IOTrackingMethodDispatched,
+ .checkScalarInputCount = 0,
+ .checkStructureInputSize = kIOUCVariableStructureSize,
+ .checkScalarOutputCount = 0,
+ .checkStructureOutputSize = 0,
+ .allowAsync = false,
+ .checkEntitlement = NULL,
+ },
+ [kIOTrackingResetTracking] = {
+ .function = &IOTrackingMethodDispatched,
+ .checkScalarInputCount = 0,
+ .checkStructureInputSize = kIOUCVariableStructureSize,
+ .checkScalarOutputCount = 0,
+ .checkStructureOutputSize = 0,
+ .allowAsync = false,
+ .checkEntitlement = NULL,
+ },
+ [kIOTrackingStartCapture] = {
+ .function = &IOTrackingMethodDispatched,
+ .checkScalarInputCount = 0,
+ .checkStructureInputSize = kIOUCVariableStructureSize,
+ .checkScalarOutputCount = 0,
+ .checkStructureOutputSize = 0,
+ .allowAsync = false,
+ .checkEntitlement = NULL,
+ },
+ [kIOTrackingStopCapture] = {
+ .function = &IOTrackingMethodDispatched,
+ .checkScalarInputCount = 0,
+ .checkStructureInputSize = kIOUCVariableStructureSize,
+ .checkScalarOutputCount = 0,
+ .checkStructureOutputSize = 0,
+ .allowAsync = false,
+ .checkEntitlement = NULL,
+ },
+ [kIOTrackingSetMinCaptureSize] = {
+ .function = &IOTrackingMethodDispatched,
+ .checkScalarInputCount = 0,
+ .checkStructureInputSize = kIOUCVariableStructureSize,
+ .checkScalarOutputCount = 0,
+ .checkStructureOutputSize = 0,
+ .allowAsync = false,
+ .checkEntitlement = NULL,
+ },
+ [kIOTrackingLeaks] = {
+ .function = &IOTrackingMethodDispatched,
+ .checkScalarInputCount = 0,
+ .checkStructureInputSize = kIOUCVariableStructureSize,
+ .checkScalarOutputCount = 0,
+ .checkStructureOutputSize = 0,
+ .allowAsync = false,
+ .checkEntitlement = NULL,
+ },
+ };
+
+ return dispatchExternalMethod(selector, args, dispatchArray, sizeof(dispatchArray) / sizeof(dispatchArray[0]), this, NULL);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */