Loading...
--- xnu/xnu-3248.50.21/iokit/Kernel/IOBufferMemoryDescriptor.cpp
+++ xnu/xnu-2050.22.13/iokit/Kernel/IOBufferMemoryDescriptor.cpp
@@ -71,26 +71,208 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+#if 0
+#undef assert
+#define assert(ex) \
+ ((ex) ? (void)0 : Assert(__FILE__, __LINE__, # ex))
+#endif
+
+enum
+{
+ kIOPageAllocChunkBytes = (PAGE_SIZE / 64),
+ kIOPageAllocSignature = 'iopa'
+};
+
+struct io_pagealloc_t
+{
+ queue_chain_t link;
+ uint64_t avail;
+ uint32_t signature;
+};
+typedef struct io_pagealloc_t io_pagealloc_t;
+
+typedef char io_pagealloc_t_assert[(sizeof(io_pagealloc_t) <= kIOPageAllocChunkBytes) ? 1 : -1];
+
+IOSimpleLock * gIOPageAllocLock;
+queue_head_t gIOPageAllocList;
+vm_size_t gIOPageAllocCount;
+vm_size_t gIOPageAllocBytes;
+
+static io_pagealloc_t *
+iopa_allocpage(void)
+{
+ kern_return_t kr;
+ io_pagealloc_t * pa;
+ vm_address_t vmaddr = 0;
+
+ int options = 0; // KMA_LOMEM;
+ kr = kernel_memory_allocate(kernel_map, &vmaddr,
+ page_size, 0, options);
+ if (KERN_SUCCESS != kr) return (0);
+
+ bzero((void *) vmaddr, page_size);
+ pa = (typeof(pa)) (vmaddr + page_size - kIOPageAllocChunkBytes);
+
+ pa->signature = kIOPageAllocSignature;
+ pa->avail = -2ULL;
+
+ return (pa);
+}
+
+static void
+iopa_freepage(io_pagealloc_t * pa)
+{
+ kmem_free( kernel_map, trunc_page((uintptr_t) pa), page_size);
+}
+
+static uintptr_t
+iopa_allocinpage(io_pagealloc_t * pa, uint32_t count, uint64_t align)
+{
+ uint32_t n, s;
+ uint64_t avail = pa->avail;
+
+ assert(avail);
+
+ // find strings of count 1 bits in avail
+ for (n = count; n > 1; n -= s)
+ {
+ s = n >> 1;
+ avail = avail & (avail << s);
+ }
+ // and aligned
+ avail &= align;
+
+ if (avail)
+ {
+ n = __builtin_clzll(avail);
+ pa->avail &= ~((-1ULL << (64 - count)) >> n);
+ if (!pa->avail && pa->link.next)
+ {
+ remque(&pa->link);
+ pa->link.next = 0;
+ }
+ return (n * kIOPageAllocChunkBytes + trunc_page((uintptr_t) pa));
+ }
+
+ return (0);
+}
+
+static uint32_t
+log2up(uint32_t size)
+{
+ if (size <= 1) size = 0;
+ else size = 32 - __builtin_clz(size - 1);
+ return (size);
+}
+
+static uintptr_t
+iopa_alloc(vm_size_t bytes, uint32_t balign)
+{
+ static const uint64_t align_masks[] = {
+ 0xFFFFFFFFFFFFFFFF,
+ 0xAAAAAAAAAAAAAAAA,
+ 0x8888888888888888,
+ 0x8080808080808080,
+ 0x8000800080008000,
+ 0x8000000080000000,
+ 0x8000000000000000,
+ };
+ io_pagealloc_t * pa;
+ uintptr_t addr = 0;
+ uint32_t count;
+ uint64_t align;
+
+ if (!bytes) bytes = 1;
+ count = (bytes + kIOPageAllocChunkBytes - 1) / kIOPageAllocChunkBytes;
+ align = align_masks[log2up((balign + kIOPageAllocChunkBytes - 1) / kIOPageAllocChunkBytes)];
+
+ IOSimpleLockLock(gIOPageAllocLock);
+ pa = (typeof(pa)) queue_first(&gIOPageAllocList);
+ while (!queue_end(&gIOPageAllocList, &pa->link))
+ {
+ addr = iopa_allocinpage(pa, count, align);
+ if (addr)
+ {
+ gIOPageAllocBytes += bytes;
+ break;
+ }
+ pa = (typeof(pa)) queue_next(&pa->link);
+ }
+ IOSimpleLockUnlock(gIOPageAllocLock);
+ if (!addr)
+ {
+ pa = iopa_allocpage();
+ if (pa)
+ {
+ addr = iopa_allocinpage(pa, count, align);
+ IOSimpleLockLock(gIOPageAllocLock);
+ if (pa->avail) enqueue_head(&gIOPageAllocList, &pa->link);
+ gIOPageAllocCount++;
+ if (addr) gIOPageAllocBytes += bytes;
+ IOSimpleLockUnlock(gIOPageAllocLock);
+ }
+ }
+
+ if (addr)
+ {
+ assert((addr & ((1 << log2up(balign)) - 1)) == 0);
+ IOStatisticsAlloc(kIOStatisticsMallocAligned, bytes);
+#if IOALLOCDEBUG
+ debug_iomalloc_size += bytes;
+#endif
+ }
+
+ return (addr);
+}
+
+static void
+iopa_free(uintptr_t addr, vm_size_t bytes)
+{
+ io_pagealloc_t * pa;
+ uint32_t count;
+ uintptr_t chunk;
+
+ if (!bytes) bytes = 1;
+
+ chunk = (addr & page_mask);
+ assert(0 == (chunk & (kIOPageAllocChunkBytes - 1)));
+
+ pa = (typeof(pa)) (addr | (page_size - kIOPageAllocChunkBytes));
+ assert(kIOPageAllocSignature == pa->signature);
+
+ count = (bytes + kIOPageAllocChunkBytes - 1) / kIOPageAllocChunkBytes;
+ chunk /= kIOPageAllocChunkBytes;
+
+ IOSimpleLockLock(gIOPageAllocLock);
+ if (!pa->avail)
+ {
+ assert(!pa->link.next);
+ enqueue_tail(&gIOPageAllocList, &pa->link);
+ }
+ pa->avail |= ((-1ULL << (64 - count)) >> chunk);
+ if (pa->avail != -2ULL) pa = 0;
+ else
+ {
+ remque(&pa->link);
+ pa->link.next = 0;
+ pa->signature = 0;
+ gIOPageAllocCount--;
+ }
+ gIOPageAllocBytes -= bytes;
+ IOSimpleLockUnlock(gIOPageAllocLock);
+ if (pa) iopa_freepage(pa);
+
+#if IOALLOCDEBUG
+ debug_iomalloc_size -= bytes;
+#endif
+ IOStatisticsAlloc(kIOStatisticsFreeAligned, bytes);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
#define super IOGeneralMemoryDescriptor
OSDefineMetaClassAndStructors(IOBufferMemoryDescriptor,
IOGeneralMemoryDescriptor);
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-static uintptr_t IOBMDPageProc(iopa_t * a)
-{
- kern_return_t kr;
- vm_address_t vmaddr = 0;
- int options = 0; // KMA_LOMEM;
-
- kr = kernel_memory_allocate(kernel_map, &vmaddr,
- page_size, 0, options, VM_KERN_MEMORY_IOKIT);
-
- if (KERN_SUCCESS != kr) vmaddr = 0;
- else bzero((void *) vmaddr, page_size);
-
- return ((uintptr_t) vmaddr);
-}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -113,6 +295,7 @@
mach_vm_address_t alignment,
mach_vm_address_t physicalMask)
{
+ kern_return_t kr;
task_t mapTask = NULL;
vm_map_t vmmap = NULL;
mach_vm_address_t highestMask = 0;
@@ -121,7 +304,8 @@
bool mapped = false;
bool needZero;
- if (!capacity) return false;
+ if (!capacity)
+ return false;
_options = options;
_capacity = capacity;
@@ -145,7 +329,7 @@
IOMapper::checkForSystemMapper();
mapped = (0 != IOMapper::gSystem);
}
- needZero = (mapped || (0 != (kIOMemorySharingTypeMask & options)));
+ needZero = mapped;
if (physicalMask && (alignment <= 1))
{
@@ -167,8 +351,6 @@
_alignment = alignment;
- if ((capacity + alignment) < _capacity) return (false);
-
if ((inTask != kernel_task) && !(options & kIOMemoryPageable))
return false;
@@ -184,15 +366,53 @@
highestMask = 0;
}
- // set memory entry cache mode, pageable, purgeable
- iomdOptions |= ((options & kIOMapCacheMask) >> kIOMapCacheShift) << kIOMemoryBufferCacheShift;
+ // set flags for entry + object create
+ vm_prot_t memEntryCacheMode = VM_PROT_READ | VM_PROT_WRITE;
+
+ // set memory entry cache mode
+ switch (options & kIOMapCacheMask)
+ {
+ case kIOMapInhibitCache:
+ SET_MAP_MEM(MAP_MEM_IO, memEntryCacheMode);
+ break;
+
+ case kIOMapWriteThruCache:
+ SET_MAP_MEM(MAP_MEM_WTHRU, memEntryCacheMode);
+ break;
+
+ case kIOMapWriteCombineCache:
+ SET_MAP_MEM(MAP_MEM_WCOMB, memEntryCacheMode);
+ break;
+
+ case kIOMapCopybackCache:
+ SET_MAP_MEM(MAP_MEM_COPYBACK, memEntryCacheMode);
+ break;
+
+ case kIOMapCopybackInnerCache:
+ SET_MAP_MEM(MAP_MEM_INNERWBACK, memEntryCacheMode);
+ break;
+
+ case kIOMapDefaultCache:
+ default:
+ SET_MAP_MEM(MAP_MEM_NOOP, memEntryCacheMode);
+ break;
+ }
+
if (options & kIOMemoryPageable)
{
iomdOptions |= kIOMemoryBufferPageable;
- if (options & kIOMemoryPurgeable) iomdOptions |= kIOMemoryBufferPurgeable;
+
+ // must create the entry before any pages are allocated
+
+ // set flags for entry + object create
+ memEntryCacheMode |= MAP_MEM_NAMED_CREATE;
+
+ if (options & kIOMemoryPurgeable)
+ memEntryCacheMode |= MAP_MEM_PURGABLE;
}
else
{
+ memEntryCacheMode |= MAP_MEM_NAMED_REUSE;
vmmap = kernel_map;
// Buffer shouldn't auto prepare they should be prepared explicitly
@@ -225,18 +445,11 @@
capacity, highestMask, alignment, contig);
}
else if (needZero
- && ((capacity + alignment) <= (page_size - gIOPageAllocChunkBytes)))
+ && ((capacity + alignment) <= (page_size - kIOPageAllocChunkBytes)))
{
_internalFlags |= kInternalFlagPageAllocated;
needZero = false;
- _buffer = (void *) iopa_alloc(&gIOBMDPageAllocator, &IOBMDPageProc, capacity, alignment);
- if (_buffer)
- {
- IOStatisticsAlloc(kIOStatisticsMallocAligned, capacity);
-#if IOALLOCDEBUG
- OSAddAtomic(capacity, &debug_iomalloc_size);
-#endif
- }
+ _buffer = (void *) iopa_alloc(capacity, alignment);
}
else if (alignment > 1)
{
@@ -254,14 +467,26 @@
}
if( (options & (kIOMemoryPageable | kIOMapCacheMask))) {
+ ipc_port_t sharedMem;
vm_size_t size = round_page(capacity);
- // initWithOptions will create memory entry
- iomdOptions |= kIOMemoryPersistent;
+ kr = mach_make_memory_entry(vmmap,
+ &size, (vm_offset_t)_buffer,
+ memEntryCacheMode, &sharedMem,
+ NULL );
+
+ if( (KERN_SUCCESS == kr) && (size != round_page(capacity))) {
+ ipc_port_release_send( sharedMem );
+ kr = kIOReturnVMError;
+ }
+ if( KERN_SUCCESS != kr)
+ return( false );
+
+ _memEntry = (void *) sharedMem;
if( options & kIOMemoryPageable) {
#if IOALLOCDEBUG
- OSAddAtomicLong(size, &debug_iomallocpageable_size);
+ debug_iomallocpageable_size += size;
#endif
mapTask = inTask;
if (NULL == inTask)
@@ -275,8 +500,7 @@
while (startAddr < endAddr)
{
- UInt8 dummyVar = *startAddr;
- (void) dummyVar;
+ *startAddr;
startAddr += page_size;
}
}
@@ -302,7 +526,7 @@
return( false );
}
reserved->map = createMappingInTask(mapTask, 0,
- kIOMapAnywhere | (options & kIOMapPrefault) | (options & kIOMapCacheMask), 0, 0);
+ kIOMapAnywhere | (options & kIOMapCacheMask), 0, 0);
if (!reserved->map)
{
_buffer = 0;
@@ -494,7 +718,7 @@
if (options & kIOMemoryPageable)
{
#if IOALLOCDEBUG
- OSAddAtomicLong(-(round_page(size)), &debug_iomallocpageable_size);
+ debug_iomallocpageable_size -= round_page(size);
#endif
}
else if (buffer)
@@ -507,16 +731,7 @@
}
else if (kInternalFlagPageAllocated & internalFlags)
{
- uintptr_t page;
- page = iopa_free(&gIOBMDPageAllocator, (uintptr_t) buffer, size);
- if (page)
- {
- kmem_free(kernel_map, page, page_size);
- }
-#if IOALLOCDEBUG
- OSAddAtomic(-size, &debug_iomalloc_size);
-#endif
- IOStatisticsAlloc(kIOStatisticsFreeAligned, size);
+ iopa_free((uintptr_t) buffer, size);
}
else if (alignment > 1)
{
@@ -554,7 +769,6 @@
void IOBufferMemoryDescriptor::setLength(vm_size_t length)
{
assert(length <= _capacity);
- if (length > _capacity) return;
_length = length;
_ranges.v64->length = length;