Loading...
--- xnu/xnu-12377.121.6/iokit/Kernel/IOPerfControl.cpp
+++ xnu/xnu-7195.81.3/iokit/Kernel/IOPerfControl.cpp
@@ -7,9 +7,6 @@
#include <stdatomic.h>
#include <kern/thread_group.h>
-#include <kern/task.h>
-#include <sys/coalition.h>
-#include <kern/coalition.h>
#undef super
#define super OSObject
@@ -32,7 +29,7 @@
shared = atomic_load_explicit(&gIOPerfControlClientShared, memory_order_acquire);
if (shared == nullptr) {
IOPerfControlClient::IOPerfControlClientShared *expected = shared;
- shared = kalloc_type(IOPerfControlClientShared, Z_WAITOK);
+ shared = reinterpret_cast<IOPerfControlClient::IOPerfControlClientShared*>(kalloc(sizeof(IOPerfControlClientShared)));
if (!shared) {
return false;
}
@@ -40,7 +37,7 @@
atomic_init(&shared->maxDriverIndex, 0);
shared->interface = PerfControllerInterface{
- .version = PERFCONTROL_INTERFACE_VERSION_NONE,
+ .version = 0,
.registerDevice =
[](IOService *device) {
return kIOReturnSuccess;
@@ -62,9 +59,6 @@
.workEnd =
[](IOService *device, uint64_t token, PerfControllerInterface::WorkState *state, WorkEndArgs *args, bool done) {
},
- .workUpdate =
- [](IOService *device, uint64_t token, PerfControllerInterface::WorkState *state, WorkUpdateArgs *args) {
- },
};
shared->interfaceLock = IOLockAlloc();
@@ -81,12 +75,10 @@
memory_order_acquire)) {
IOLockFree(shared->interfaceLock);
shared->deviceRegistrationList->release();
- kfree_type(IOPerfControlClientShared, shared);
+ kfree(shared, sizeof(*shared));
shared = expected;
}
}
- workTable = NULL;
- workTableLock = NULL;
// Note: driverIndex is not guaranteed to be unique if maxDriverIndex wraps around. It is intended for debugging only.
driverIndex = atomic_fetch_add_explicit(&shared->maxDriverIndex, 1, memory_order_relaxed) + 1;
@@ -95,10 +87,11 @@
workTableLength = maxWorkCapacity + 1;
assertf(workTableLength <= kWorkTableMaxSize, "%zu exceeds max allowed capacity of %zu", workTableLength, kWorkTableMaxSize);
if (maxWorkCapacity > 0) {
- workTable = kalloc_type(WorkTableEntry, workTableLength, Z_WAITOK_ZERO);
+ workTable = reinterpret_cast<WorkTableEntry*>(kalloc(workTableLength * sizeof(WorkTableEntry)));
if (!workTable) {
goto error;
}
+ bzero(workTable, workTableLength * sizeof(WorkTableEntry));
workTableNextIndex = 1;
workTableLock = IOSimpleLockAlloc();
@@ -107,18 +100,14 @@
}
}
- bzero(&clientData, sizeof(clientData));
-
return true;
error:
if (workTable) {
- kfree_type(WorkTableEntry, workTableLength, workTable);
- workTable = NULL;
+ kfree(workTable, maxWorkCapacity * sizeof(WorkTableEntry));
}
if (workTableLock) {
IOSimpleLockFree(workTableLock);
- workTableLock = NULL;
}
return false;
shared_init_error:
@@ -129,54 +118,20 @@
if (shared->deviceRegistrationList) {
shared->deviceRegistrationList->release();
}
- kfree_type(IOPerfControlClientShared, shared);
+ kfree(shared, sizeof(*shared));
shared = nullptr;
}
return false;
}
-void
-IOPerfControlClient::free()
-{
- if (workTable) {
- kfree_type(WorkTableEntry, workTableLength, workTable);
- }
- if (workTableLock) {
- IOSimpleLockFree(workTableLock);
- }
- super::free();
-}
-
-void
-IOPerfControlClient::setDeviceType(IOPCDeviceType newDeviceType)
-{
- if (newDeviceType >= IOPCDeviceTypeMax) {
- panic("unknown device type %d", newDeviceType);
- }
-
- if (deviceType != IOPCDeviceTypeUnknown) {
- panic("deviceType already set to %d", deviceType);
- }
-
- deviceType = newDeviceType;
-}
-
-
IOPerfControlClient *
-IOPerfControlClient::copyClientForDeviceType(IOService *driver, uint64_t maxWorkCapacity, IOPCDeviceType deviceType)
+IOPerfControlClient::copyClient(IOService *driver, uint64_t maxWorkCapacity)
{
IOPerfControlClient *client = new IOPerfControlClient;
if (!client || !client->init(driver, maxWorkCapacity)) {
panic("could not create IOPerfControlClient");
}
- client->setDeviceType(deviceType);
return client;
-}
-
-IOPerfControlClient *
-IOPerfControlClient::copyClient(IOService *driver, uint64_t maxWorkCapacity)
-{
- return copyClientForDeviceType(driver, maxWorkCapacity, IOPCDeviceTypeUnknown);
}
/* Convert the per driver token into a globally unique token for the performance
@@ -192,26 +147,6 @@
return token | (static_cast<uint64_t>(driverIndex) << kWorkTableIndexBits);
}
-/* Accounting resources used in a work item to the containing coalition.
- * Contigent upon the PerfController signaling that it wants resource accounting
- * in the registerDevice()/registerDriverDevice calls. More device types can
- * be added here in the future.
- */
-void
-IOPerfControlClient::accountResources(coalition_t coal, PerfControllerInterface::PerfDeviceID device_type, PerfControllerInterface::ResourceAccounting *resources)
-{
- switch (device_type) {
- case PerfControllerInterface::PerfDeviceID::kANE:
- if (coal != nullptr) {
- coalition_update_ane_stats(coal, resources->mach_time_delta, resources->energy_nj_delta);
- }
- break;
-
- default:
- assertf(false, "Unexpected device type for IOPerfControlClient::accountResources: %llu", static_cast<uint64_t>(device_type));
- }
-}
-
/* With this implementation, tokens returned to the driver differ from tokens
* passed to the performance controller. This implementation has the nice
* property that tokens returns to the driver will aways be between 1 and
@@ -234,12 +169,6 @@
if (workTable[index].thread_group == nullptr) {
thread_group_retain(thread_group);
workTable[index].thread_group = thread_group;
- if (clientData.driverState.resource_accounting) {
- auto *coalition = task_get_coalition(current_task(), COALITION_TYPE_RESOURCE);
- assert(coalition != nullptr);
- coalition_retain(coalition);
- workTable[index].coal = coalition;
- }
token = index;
// next integer between 1 and workTableLength - 1
workTableNextIndex = (index % (workTableLength - 1)) + 1;
@@ -258,7 +187,7 @@
* this code will have to be modified to support dynamic table growth to support larger
* numbers of tokens.
*/
- panic("Tokens allocated for this device exceeded maximum of %zu.",
+ panic("Tokens allocated for this device exceeded maximum of %zu.\n",
workTableLength - 1); // - 1 since entry 0 is for kIOPerfControlClientWorkUntracked
}
#endif
@@ -279,7 +208,6 @@
auto &entry = workTable[token];
auto *thread_group = entry.thread_group;
- auto *coal = entry.coal;
bzero(&entry, sizeof(entry));
workTableNextIndex = token;
@@ -288,9 +216,6 @@
// This can call into the performance controller if the last reference is dropped here. Are we sure
// the driver isn't holding any locks? If not, we may want to async this to another context.
thread_group_release(thread_group);
- if (coal != nullptr) {
- coalition_release(coal);
- }
#endif
}
@@ -324,92 +249,36 @@
workTable[token].started = started;
}
-#if CONFIG_THREAD_GROUPS
-
-static struct thread_group *
-threadGroupForDextService(IOService *device)
-{
- assert(device);
-
- if (!device->hasUserServer()) {
- return NULL;
- }
-
- // Devices associated with a dext driver, must be called from dext
- // context to ensure that thread_group reference is valid.
- thread_t thread = current_thread();
- assert(get_threadtask(thread) != kernel_task);
- struct thread_group * thread_group = thread_group_get(thread);
- assert(thread_group != nullptr);
- return thread_group;
-}
-
-#endif /* CONFIG_THREAD_GROUPS */
-
IOReturn
-IOPerfControlClient::registerDevice(IOService *driver, IOService *device)
+IOPerfControlClient::registerDevice(__unused IOService *driver, IOService *device)
{
IOReturn ret = kIOReturnSuccess;
-#if CONFIG_THREAD_GROUPS
+
IOLockLock(shared->interfaceLock);
- clientData.device = device;
-
- if (device) {
- struct thread_group *dext_thread_group = threadGroupForDextService(device);
- if (dext_thread_group) {
- if (clientData.driverState.has_target_thread_group) {
- panic("driverState has already been initialized");
- }
- clientData.driverState.has_target_thread_group = true;
- clientData.driverState.target_thread_group_id = thread_group_get_id(dext_thread_group);
- clientData.driverState.target_thread_group_data = thread_group_get_machine_data(dext_thread_group);
-
- clientData.target_thread_group = dext_thread_group;
- thread_group_retain(dext_thread_group);
- }
- }
-
- if (shared->interface.version >= PERFCONTROL_INTERFACE_VERSION_3) {
- ret = shared->interface.registerDriverDevice(driver, device, &clientData.driverState);
- } else if (shared->interface.version >= PERFCONTROL_INTERFACE_VERSION_1) {
+ if (shared->interface.version > 0) {
ret = shared->interface.registerDevice(device);
} else {
- shared->deviceRegistrationList->setObject(this);
+ shared->deviceRegistrationList->setObject(device);
}
IOLockUnlock(shared->interfaceLock);
-#endif
+
return ret;
}
void
-IOPerfControlClient::unregisterDevice(IOService *driver, IOService *device)
-{
-#if CONFIG_THREAD_GROUPS
+IOPerfControlClient::unregisterDevice(__unused IOService *driver, IOService *device)
+{
IOLockLock(shared->interfaceLock);
- if (shared->interface.version >= PERFCONTROL_INTERFACE_VERSION_3) {
- shared->interface.unregisterDriverDevice(driver, device, &clientData.driverState);
- } else if (shared->interface.version >= PERFCONTROL_INTERFACE_VERSION_1) {
+ if (shared->interface.version > 0) {
shared->interface.unregisterDevice(device);
} else {
- shared->deviceRegistrationList->removeObject(this);
- }
-
- if (clientData.driverState.has_target_thread_group) {
- thread_group_release(clientData.target_thread_group);
- clientData.target_thread_group = nullptr;
-
- clientData.driverState.has_target_thread_group = false;
- clientData.driverState.target_thread_group_id = ~0ull;
- clientData.driverState.target_thread_group_data = nullptr;
- }
-
- clientData.device = nullptr;
+ shared->deviceRegistrationList->removeObject(device);
+ }
IOLockUnlock(shared->interfaceLock);
-#endif
}
uint64_t
@@ -427,7 +296,6 @@
.work_data = nullptr,
.work_data_size = 0,
.started = false,
- .driver_state = &clientData.driverState
};
if (!shared->interface.workCanSubmit(device, &state, args)) {
return kIOPerfControlClientWorkUntracked;
@@ -460,7 +328,6 @@
.work_data = nullptr,
.work_data_size = 0,
.started = false,
- .driver_state = &clientData.driverState
};
if (!shared->interface.workCanSubmit(device, &state, submitArgs)) {
return kIOPerfControlClientWorkUntracked;
@@ -499,7 +366,6 @@
.work_data = &entry->perfcontrol_data,
.work_data_size = sizeof(entry->perfcontrol_data),
.started = true,
- .driver_state = &clientData.driverState
};
shared->interface.workBegin(device, tokenToGlobalUniqueToken(token), &state, args);
markEntryStarted(token, true);
@@ -521,18 +387,8 @@
.work_data = &entry->perfcontrol_data,
.work_data_size = sizeof(entry->perfcontrol_data),
.started = entry->started,
- .driver_state = &clientData.driverState
- };
-
- if (shared->interface.version >= PERFCONTROL_INTERFACE_VERSION_4) {
- PerfControllerInterface::ResourceAccounting resources;
- shared->interface.workEndWithResources(device, tokenToGlobalUniqueToken(token), &state, args, &resources, done);
- if (clientData.driverState.resource_accounting) {
- accountResources(workTable[token].coal, clientData.driverState.device_type, &resources);
- }
- } else {
- shared->interface.workEnd(device, tokenToGlobalUniqueToken(token), &state, args, done);
- }
+ };
+ shared->interface.workEnd(device, tokenToGlobalUniqueToken(token), &state, args, done);
if (done) {
deallocateToken(token);
@@ -551,7 +407,6 @@
public:
uint64_t id;
struct thread_group *thread_group;
- coalition_t coal;
bool started;
uint8_t perfcontrol_data[32];
@@ -577,7 +432,6 @@
IOPerfControlWorkContext::reset()
{
thread_group = nullptr;
- coal = nullptr;
started = false;
bzero(perfcontrol_data, sizeof(perfcontrol_data));
}
@@ -586,7 +440,6 @@
IOPerfControlWorkContext::free()
{
assertf(thread_group == nullptr, "IOPerfControlWorkContext ID %llu being released without calling workEnd!\n", id);
- assertf(coal == nullptr, "IOPerfControlWorkContext ID %llu being released without calling workEnd!\n", id);
super::free();
}
@@ -604,7 +457,7 @@
return nullptr;
}
- return context;
+ return OSDynamicCast(OSObject, context);
}
bool
@@ -624,7 +477,6 @@
.work_data = &work_context->perfcontrol_data,
.work_data_size = sizeof(work_context->perfcontrol_data),
.started = true,
- .driver_state = &clientData.driverState
};
shared->interface.workBegin(device, work_context->id, &state, beginArgs);
@@ -659,19 +511,12 @@
.work_data = nullptr,
.work_data_size = 0,
.started = false,
- .driver_state = &clientData.driverState
};
if (!shared->interface.workCanSubmit(device, &state, args)) {
return false;
}
work_context->thread_group = thread_group_retain(thread_group);
- if (clientData.driverState.resource_accounting) {
- auto *coalition = task_get_coalition(current_task(), COALITION_TYPE_RESOURCE);
- assert(coalition != nullptr);
- work_context->coal = coalition;
- coalition_retain(coalition);
- }
state.work_data = &work_context->perfcontrol_data;
state.work_data_size = sizeof(work_context->perfcontrol_data);
@@ -681,32 +526,6 @@
return true;
#else
return false;
-#endif
-}
-
-void
-IOPerfControlClient::workUpdateWithContext(IOService *device, OSObject *context, WorkUpdateArgs *args)
-{
-#if CONFIG_THREAD_GROUPS
- IOPerfControlWorkContext *work_context = OSDynamicCast(IOPerfControlWorkContext, context);
-
- if (work_context == nullptr) {
- return;
- }
-
- if (work_context->thread_group == nullptr) {
- // This Work Context has not taken a refcount on a TG
- return;
- }
-
- PerfControllerInterface::WorkState state{
- .thread_group_id = thread_group_get_id(work_context->thread_group),
- .thread_group_data = thread_group_get_machine_data(work_context->thread_group),
- .work_data = &work_context->perfcontrol_data,
- .work_data_size = sizeof(work_context->perfcontrol_data),
- .driver_state = &clientData.driverState
- };
- shared->interface.workUpdate(device, work_context->id, &state, args);
#endif
}
@@ -733,7 +552,6 @@
.work_data = &work_context->perfcontrol_data,
.work_data_size = sizeof(work_context->perfcontrol_data),
.started = true,
- .driver_state = &clientData.driverState
};
shared->interface.workBegin(device, work_context->id, &state, args);
@@ -761,24 +579,12 @@
.work_data = &work_context->perfcontrol_data,
.work_data_size = sizeof(work_context->perfcontrol_data),
.started = work_context->started,
- .driver_state = &clientData.driverState
- };
-
- if (shared->interface.version >= PERFCONTROL_INTERFACE_VERSION_4) {
- PerfControllerInterface::ResourceAccounting resources;
- shared->interface.workEndWithResources(device, work_context->id, &state, args, &resources, done);
- if (clientData.driverState.resource_accounting) {
- accountResources(work_context->coal, clientData.driverState.device_type, &resources);
- }
- } else {
- shared->interface.workEnd(device, work_context->id, &state, args, done);
- }
+ };
+
+ shared->interface.workEnd(device, work_context->id, &state, args, done);
if (done) {
thread_group_release(work_context->thread_group);
- if (work_context->coal != nullptr) {
- coalition_release(work_context->coal);
- }
work_context->reset();
} else {
work_context->started = false;
@@ -791,81 +597,26 @@
}
IOReturn
-IOPerfControlClient::querySubmitterRole(IOService *device, task_t clientTask, uint32_t* role_out)
-{
- IOReturn result = kIOReturnNotFound;
-
- uint32_t role;
-
- switch (deviceType) {
- case IOPCDeviceTypeGPU:
- role = task_get_gpu_role(clientTask);
-
- KDBG(IMPORTANCE_CODE(IMP_QUERY_GPU_ROLE, 0), role);
-
- *role_out = role;
-
- result = kIOReturnSuccess;
- break;
- default:
- result = kIOReturnNotFound;
- }
-
- return result;
-}
-
-IOReturn
-IOPerfControlClient::registerPerformanceController(PerfControllerInterface *pci)
+IOPerfControlClient::registerPerformanceController(PerfControllerInterface pci)
{
IOReturn result = kIOReturnError;
IOLockLock(shared->interfaceLock);
- if (shared->interface.version == PERFCONTROL_INTERFACE_VERSION_NONE) {
- shared->interface.version = pci->version;
-
- if (pci->version >= PERFCONTROL_INTERFACE_VERSION_1) {
- assert(pci->registerDevice && pci->unregisterDevice && pci->workCanSubmit && pci->workSubmit && pci->workBegin && pci->workEnd);
- shared->interface.registerDevice = pci->registerDevice;
- shared->interface.unregisterDevice = pci->unregisterDevice;
- shared->interface.workCanSubmit = pci->workCanSubmit;
- shared->interface.workSubmit = pci->workSubmit;
- shared->interface.workBegin = pci->workBegin;
- shared->interface.workEnd = pci->workEnd;
- }
-
- if (pci->version >= PERFCONTROL_INTERFACE_VERSION_2) {
- if (pci->workUpdate != nullptr) {
- shared->interface.workUpdate = pci->workUpdate;
- }
- }
-
- if (pci->version >= PERFCONTROL_INTERFACE_VERSION_3) {
- assert(pci->registerDriverDevice && pci->unregisterDriverDevice);
- shared->interface.registerDriverDevice = pci->registerDriverDevice;
- shared->interface.unregisterDriverDevice = pci->unregisterDriverDevice;
- }
-
- if (pci->version >= PERFCONTROL_INTERFACE_VERSION_4) {
- assert(pci->workEndWithResources);
- shared->interface.workEndWithResources = pci->workEndWithResources;
- }
-
+ if (shared->interface.version == 0 && pci.version > 0) {
+ assert(pci.registerDevice && pci.unregisterDevice && pci.workCanSubmit && pci.workSubmit && pci.workBegin && pci.workEnd);
result = kIOReturnSuccess;
OSObject *obj;
while ((obj = shared->deviceRegistrationList->getAnyObject())) {
- IOPerfControlClient *client = OSDynamicCast(IOPerfControlClient, obj);
- IOPerfControlClientData *clientData = client->getClientData();
- if (clientData && clientData->device) {
- if (pci->version >= PERFCONTROL_INTERFACE_VERSION_3) {
- pci->registerDriverDevice(clientData->device->getProvider(), clientData->device, &(clientData->driverState));
- } else if (pci->version >= PERFCONTROL_INTERFACE_VERSION_1) {
- pci->registerDevice(clientData->device);
- }
+ IOService *device = OSDynamicCast(IOService, obj);
+ if (device) {
+ pci.registerDevice(device);
}
shared->deviceRegistrationList->removeObject(obj);
}
+
+ shared->interface = pci;
}
IOLockUnlock(shared->interfaceLock);