Loading...
--- xnu/xnu-12377.121.6/iokit/Kernel/IOLib.cpp
+++ xnu/xnu-8792.61.2/iokit/Kernel/IOLib.cpp
@@ -36,8 +36,7 @@
#include <IOKit/system.h>
#include <mach/sync_policy.h>
#include <machine/machine_routines.h>
-#include <vm/vm_kern_xnu.h>
-#include <vm/vm_map_xnu.h>
+#include <vm/vm_kern.h>
#include <libkern/c++/OSCPPDebug.h>
#include <IOKit/assert.h>
@@ -101,7 +100,6 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-lck_grp_t io_lck_grp;
lck_grp_t *IOLockGroup;
/*
@@ -127,11 +125,15 @@
static lck_mtx_t * gIOMallocContiguousEntriesLock;
#if __x86_64__
-enum { kIOPageableMaxAllocSize = 512ULL * 1024 * 1024 };
-enum { kIOPageableMapSize = 8ULL * kIOPageableMaxAllocSize };
+enum { kIOMaxPageableMaps = 8 };
+enum { kIOMaxFixedRanges = 4 };
+enum { kIOPageableMapSize = 512 * 1024 * 1024 };
+enum { kIOPageableMaxMapSize = 512 * 1024 * 1024 };
#else
-enum { kIOPageableMaxAllocSize = 96ULL * 1024 * 1024 };
-enum { kIOPageableMapSize = 16ULL * kIOPageableMaxAllocSize };
+enum { kIOMaxPageableMaps = 16 };
+enum { kIOMaxFixedRanges = 4 };
+enum { kIOPageableMapSize = 96 * 1024 * 1024 };
+enum { kIOPageableMaxMapSize = 96 * 1024 * 1024 };
#endif
typedef struct {
@@ -140,12 +142,15 @@
vm_offset_t end;
} IOMapData;
-#ifndef __BUILDING_XNU_LIBRARY__
-/* this makes clang emit a C and C++ symbol which confuses lldb rdar://135688747 */
-static
-#endif /* __BUILDING_XNU_LIBRARY__ */
-SECURITY_READ_ONLY_LATE(struct mach_vm_range) gIOKitPageableFixedRange;
-IOMapData gIOKitPageableMap;
+static SECURITY_READ_ONLY_LATE(struct mach_vm_range)
+gIOKitPageableFixedRanges[kIOMaxFixedRanges];
+
+static struct {
+ UInt32 count;
+ UInt32 hint;
+ IOMapData maps[kIOMaxPageableMaps];
+ lck_mtx_t * lock;
+} gIOKitPageableSpace;
#if defined(__x86_64__)
static iopa_t gIOPageablePageAllocator;
@@ -161,8 +166,14 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-KMEM_RANGE_REGISTER_STATIC(gIOKitPageableFixed,
- &gIOKitPageableFixedRange, kIOPageableMapSize);
+KMEM_RANGE_REGISTER_STATIC(gIOKitPageableFixed0,
+ &gIOKitPageableFixedRanges[0], kIOPageableMapSize);
+KMEM_RANGE_REGISTER_STATIC(gIOKitPageableFixed1,
+ &gIOKitPageableFixedRanges[1], kIOPageableMapSize);
+KMEM_RANGE_REGISTER_STATIC(gIOKitPageableFixed2,
+ &gIOKitPageableFixedRanges[2], kIOPageableMapSize);
+KMEM_RANGE_REGISTER_STATIC(gIOKitPageableFixed3,
+ &gIOKitPageableFixedRanges[3], kIOPageableMapSize);
void
IOLibInit(void)
{
@@ -172,8 +183,7 @@
return;
}
- lck_grp_init(&io_lck_grp, "IOKit", LCK_GRP_ATTR_NULL);
- IOLockGroup = &io_lck_grp;
+ IOLockGroup = lck_grp_alloc_init("IOKit", LCK_GRP_ATTR_NULL);
#if IOTRACKING
IOTrackingInit();
@@ -190,16 +200,19 @@
0);
#endif
- gIOKitPageableMap.map = kmem_suballoc(kernel_map,
- &gIOKitPageableFixedRange.min_address,
+ gIOKitPageableSpace.maps[0].map = kmem_suballoc(kernel_map,
+ &gIOKitPageableFixedRanges[0].min_address,
kIOPageableMapSize,
- VM_MAP_CREATE_DEFAULT,
- VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE,
- (kms_flags_t)(KMS_DATA | KMS_NOFAIL | KMS_NOSOFTLIMIT),
+ VM_MAP_CREATE_PAGEABLE,
+ VM_FLAGS_FIXED_RANGE_SUBALLOC,
+ (kms_flags_t)(KMS_PERMANENT | KMS_DATA | KMS_NOFAIL),
VM_KERN_MEMORY_IOKIT).kmr_submap;
- gIOKitPageableMap.address = gIOKitPageableFixedRange.min_address;
- gIOKitPageableMap.end = gIOKitPageableFixedRange.max_address;
+ gIOKitPageableSpace.maps[0].address = gIOKitPageableFixedRanges[0].min_address;
+ gIOKitPageableSpace.maps[0].end = gIOKitPageableFixedRanges[0].max_address;
+ gIOKitPageableSpace.lock = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL);
+ gIOKitPageableSpace.hint = 0;
+ gIOKitPageableSpace.count = 1;
gIOMallocContiguousEntriesLock = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL);
queue_init( &gIOMallocContiguousEntries );
@@ -364,7 +377,7 @@
void
IOFree(void * inAddress, vm_size_t size)
{
- IOFree_internal(KHEAP_DEFAULT, inAddress, size);
+ IOFree_internal(KHEAP_ANY, inAddress, size);
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -518,10 +531,8 @@
kma_flags = KMA_ZERO;
}
- if (kheap == KHEAP_DATA_PRIVATE) {
+ if (kheap == KHEAP_DATA_BUFFERS) {
kma_flags = (kma_flags_t) (kma_flags | KMA_DATA);
- } else if (kheap == KHEAP_DATA_SHARED) {
- kma_flags = (kma_flags_t) (kma_flags | KMA_DATA_SHARED);
}
alignment = (1UL << log2up((uint32_t) alignment));
@@ -643,7 +654,7 @@
IOMallocAligned_external(
vm_size_t size, vm_size_t alignment)
{
- return IOMallocAligned_internal(KHEAP_DATA_SHARED, size, alignment,
+ return IOMallocAligned_internal(KHEAP_DATA_BUFFERS, size, alignment,
Z_VM_TAG_BT_BIT);
}
@@ -652,7 +663,7 @@
void * address,
vm_size_t size)
{
- IOFreeAligned_internal(KHEAP_DATA_SHARED, address, size);
+ IOFreeAligned_internal(KHEAP_DATA_BUFFERS, address, size);
}
__typed_allocators_ignore_pop
@@ -710,8 +721,7 @@
mach_vm_size_t size,
mach_vm_address_t maxPhys,
mach_vm_size_t alignment,
- bool contiguous,
- bool noSoftLimit)
+ bool contiguous)
{
kern_return_t kr;
mach_vm_address_t address;
@@ -740,14 +750,8 @@
kma_flags_t options = KMA_ZERO;
vm_offset_t virt;
- if (kheap == KHEAP_DATA_PRIVATE) {
+ if (kheap == KHEAP_DATA_BUFFERS) {
options = (kma_flags_t) (options | KMA_DATA);
- } else if (kheap == KHEAP_DATA_SHARED) {
- options = (kma_flags_t) (options | KMA_DATA_SHARED);
- }
-
- if (noSoftLimit) {
- options = (kma_flags_t) (options | KMA_NOSOFTLIMIT);
}
adjustedSize = size;
@@ -786,21 +790,14 @@
address = 0;
}
} else {
- zalloc_flags_t zflags = Z_WAITOK;
-
- if (noSoftLimit) {
- zflags = (zalloc_flags_t)(zflags | Z_NOSOFTLIMIT);
- }
-
adjustedSize += alignMask;
if (adjustedSize < size) {
return 0;
}
-
/* BEGIN IGNORE CODESTYLE */
__typed_allocators_ignore_push // allocator implementation
allocationAddress = (mach_vm_address_t) kheap_alloc(kheap,
- adjustedSize, Z_VM_TAG_BT(zflags, VM_KERN_MEMORY_IOKIT));
+ adjustedSize, Z_VM_TAG_BT(Z_WAITOK, VM_KERN_MEMORY_IOKIT));
__typed_allocators_ignore_pop
/* END IGNORE CODESTYLE */
@@ -863,7 +860,7 @@
/* Do we want a physical address? */
if (!physicalAddress) {
address = IOKernelAllocateWithPhysicalRestrict(KHEAP_DEFAULT,
- size, 0 /*maxPhys*/, alignment, true, false /* noSoftLimit */);
+ size, 0 /*maxPhys*/, alignment, true);
} else {
do {
IOBufferMemoryDescriptor * bmd;
@@ -941,10 +938,84 @@
IOIteratePageableMaps(vm_size_t size,
IOIteratePageableMapsCallback callback, void * ref)
{
- if (size > kIOPageableMaxAllocSize) {
+ kern_return_t kr = kIOReturnNotReady;
+ kmem_return_t kmr;
+ vm_size_t segSize;
+ UInt32 attempts;
+ UInt32 index;
+ mach_vm_offset_t min;
+ int flags;
+
+ if (size > kIOPageableMaxMapSize) {
return kIOReturnBadArgument;
}
- return (*callback)(gIOKitPageableMap.map, ref);
+
+ do {
+ index = gIOKitPageableSpace.hint;
+ attempts = gIOKitPageableSpace.count;
+ while (attempts--) {
+ kr = (*callback)(gIOKitPageableSpace.maps[index].map, ref);
+ if (KERN_SUCCESS == kr) {
+ gIOKitPageableSpace.hint = index;
+ break;
+ }
+ if (index) {
+ index--;
+ } else {
+ index = gIOKitPageableSpace.count - 1;
+ }
+ }
+ if (KERN_NO_SPACE != kr) {
+ break;
+ }
+
+ lck_mtx_lock( gIOKitPageableSpace.lock );
+
+ index = gIOKitPageableSpace.count;
+ if (index >= (kIOMaxPageableMaps - 1)) {
+ lck_mtx_unlock( gIOKitPageableSpace.lock );
+ break;
+ }
+
+ if (size < kIOPageableMapSize) {
+ segSize = kIOPageableMapSize;
+ } else {
+ segSize = size;
+ }
+
+ /*
+ * Use the predefine ranges if available, else default to data
+ */
+ if (index < kIOMaxFixedRanges) {
+ min = gIOKitPageableFixedRanges[index].min_address;
+ flags = VM_FLAGS_FIXED_RANGE_SUBALLOC;
+ } else {
+ min = 0;
+ flags = VM_FLAGS_ANYWHERE;
+ }
+ kmr = kmem_suballoc(kernel_map,
+ &min,
+ segSize,
+ VM_MAP_CREATE_PAGEABLE,
+ flags,
+ (kms_flags_t)(KMS_PERMANENT | KMS_DATA),
+ VM_KERN_MEMORY_IOKIT);
+ if (kmr.kmr_return != KERN_SUCCESS) {
+ kr = kmr.kmr_return;
+ lck_mtx_unlock( gIOKitPageableSpace.lock );
+ break;
+ }
+
+ gIOKitPageableSpace.maps[index].map = kmr.kmr_submap;
+ gIOKitPageableSpace.maps[index].address = min;
+ gIOKitPageableSpace.maps[index].end = min + segSize;
+ gIOKitPageableSpace.hint = index;
+ gIOKitPageableSpace.count = index + 1;
+
+ lck_mtx_unlock( gIOKitPageableSpace.lock );
+ } while (true);
+
+ return kr;
}
struct IOMallocPageableRef {
@@ -957,7 +1028,7 @@
IOMallocPageableCallback(vm_map_t map, void * _ref)
{
struct IOMallocPageableRef * ref = (struct IOMallocPageableRef *) _ref;
- kma_flags_t flags = (kma_flags_t)(KMA_PAGEABLE | KMA_DATA_SHARED);
+ kma_flags_t flags = (kma_flags_t)(KMA_PAGEABLE | KMA_DATA);
return kmem_alloc( map, &ref->address, ref->size, flags, ref->tag );
}
@@ -971,7 +1042,7 @@
if (alignment > page_size) {
return NULL;
}
- if (size > kIOPageableMaxAllocSize) {
+ if (size > kIOPageableMaxMapSize) {
return NULL;
}
@@ -986,12 +1057,23 @@
}
vm_map_t
-IOPageableMapForAddress(uintptr_t address)
-{
- if (address < gIOKitPageableMap.address || address >= gIOKitPageableMap.end) {
- panic("IOPageableMapForAddress: address out of range");
- }
- return gIOKitPageableMap.map;
+IOPageableMapForAddress( uintptr_t address )
+{
+ vm_map_t map = NULL;
+ UInt32 index;
+
+ for (index = 0; index < gIOKitPageableSpace.count; index++) {
+ if ((address >= gIOKitPageableSpace.maps[index].address)
+ && (address < gIOKitPageableSpace.maps[index].end)) {
+ map = gIOKitPageableSpace.maps[index].map;
+ break;
+ }
+ }
+ if (!map) {
+ panic("IOPageableMapForAddress: null");
+ }
+
+ return map;
}
static void
@@ -1092,16 +1174,13 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-__typed_allocators_ignore_push
-
void *
IOMallocData_external(
vm_size_t size);
void *
IOMallocData_external(vm_size_t size)
{
- return IOMalloc_internal(KHEAP_DATA_PRIVATE, size, Z_VM_TAG_BT_BIT);
+ return IOMalloc_internal(KHEAP_DATA_BUFFERS, size, Z_VM_TAG_BT_BIT);
}
void *
@@ -1110,40 +1189,14 @@
void *
IOMallocZeroData_external(vm_size_t size)
{
- return IOMalloc_internal(KHEAP_DATA_PRIVATE, size, Z_ZERO_VM_TAG_BT_BIT);
-}
-
-void *
-IOMallocDataShareable_external(
- vm_size_t size);
-void *
-IOMallocDataShareable_external(vm_size_t size)
-{
- return IOMalloc_internal(KHEAP_DATA_SHARED, size, Z_VM_TAG_BT_BIT);
-}
-
-void *
-IOMallocZeroDataShareable_external(
- vm_size_t size);
-void *
-IOMallocZeroDataShareable_external(vm_size_t size)
-{
- return IOMalloc_internal(KHEAP_DATA_SHARED, size, Z_ZERO_VM_TAG_BT_BIT);
+ return IOMalloc_internal(KHEAP_DATA_BUFFERS, size, Z_ZERO_VM_TAG_BT_BIT);
}
void
IOFreeData(void * address, vm_size_t size)
{
- return IOFree_internal(KHEAP_DATA_PRIVATE, address, size);
-}
-
-void
-IOFreeDataShareable(void * address, vm_size_t size)
-{
- return IOFree_internal(KHEAP_DATA_SHARED, address, size);
-}
-
-__typed_allocators_ignore_pop
+ return IOFree_internal(KHEAP_DATA_BUFFERS, address, size);
+}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -1525,7 +1578,6 @@
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wformat-nonliteral"
-#pragma clang diagnostic ignored "-Wformat"
os_log_with_args(OS_LOG_DEFAULT, OS_LOG_TYPE_DEFAULT, format, ap, caller);
#pragma clang diagnostic pop