Loading...
iokit/Kernel/IOBufferMemoryDescriptor.cpp xnu-2422.100.13 xnu-2050.22.13
--- xnu/xnu-2422.100.13/iokit/Kernel/IOBufferMemoryDescriptor.cpp
+++ xnu/xnu-2050.22.13/iokit/Kernel/IOBufferMemoryDescriptor.cpp
@@ -71,26 +71,208 @@
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
+#if 0
+#undef assert
+#define assert(ex)  \
+	((ex) ? (void)0 : Assert(__FILE__, __LINE__, # ex))
+#endif
+
+enum
+{
+    kIOPageAllocChunkBytes = (PAGE_SIZE / 64),
+    kIOPageAllocSignature  = 'iopa'
+};
+
+struct io_pagealloc_t
+{
+    queue_chain_t link;
+    uint64_t      avail;
+    uint32_t      signature;
+};
+typedef struct io_pagealloc_t io_pagealloc_t;
+
+typedef char io_pagealloc_t_assert[(sizeof(io_pagealloc_t) <= kIOPageAllocChunkBytes) ? 1 : -1];
+
+IOSimpleLock * gIOPageAllocLock;
+queue_head_t   gIOPageAllocList;
+vm_size_t      gIOPageAllocCount;
+vm_size_t      gIOPageAllocBytes;
+
+static io_pagealloc_t * 
+iopa_allocpage(void)
+{
+    kern_return_t    kr;
+    io_pagealloc_t * pa;
+    vm_address_t     vmaddr = 0;
+
+    int options = 0; // KMA_LOMEM;
+    kr = kernel_memory_allocate(kernel_map, &vmaddr,
+				page_size, 0, options);
+    if (KERN_SUCCESS != kr) return (0);
+
+    bzero((void *) vmaddr, page_size);
+    pa = (typeof(pa)) (vmaddr + page_size - kIOPageAllocChunkBytes);
+
+    pa->signature = kIOPageAllocSignature;
+    pa->avail     = -2ULL;
+
+    return (pa);
+}
+
+static void 
+iopa_freepage(io_pagealloc_t * pa)
+{
+    kmem_free( kernel_map, trunc_page((uintptr_t) pa), page_size);
+}
+
+static uintptr_t
+iopa_allocinpage(io_pagealloc_t * pa, uint32_t count, uint64_t align)
+{
+    uint32_t n, s;
+    uint64_t avail = pa->avail;
+
+    assert(avail);
+
+    // find strings of count 1 bits in avail
+    for (n = count; n > 1; n -= s)
+    {
+    	s = n >> 1;
+    	avail = avail & (avail << s);
+    }
+    // and aligned
+    avail &= align;
+
+    if (avail)
+    {
+	n = __builtin_clzll(avail);
+	pa->avail &= ~((-1ULL << (64 - count)) >> n);
+	if (!pa->avail && pa->link.next)
+	{
+	    remque(&pa->link);
+	    pa->link.next = 0;
+	}
+	return (n * kIOPageAllocChunkBytes + trunc_page((uintptr_t) pa));
+    }
+
+    return (0);
+}
+
+static uint32_t 
+log2up(uint32_t size)
+{
+    if (size <= 1) size = 0;
+    else size = 32 - __builtin_clz(size - 1);
+    return (size);
+}
+
+static uintptr_t 
+iopa_alloc(vm_size_t bytes, uint32_t balign)
+{
+    static const uint64_t align_masks[] = {
+	0xFFFFFFFFFFFFFFFF,
+	0xAAAAAAAAAAAAAAAA,
+	0x8888888888888888,
+	0x8080808080808080,
+	0x8000800080008000,
+	0x8000000080000000,
+	0x8000000000000000,
+    };
+    io_pagealloc_t * pa;
+    uintptr_t        addr = 0;
+    uint32_t         count;
+    uint64_t         align;
+
+    if (!bytes) bytes = 1;
+    count = (bytes + kIOPageAllocChunkBytes - 1) / kIOPageAllocChunkBytes;
+    align = align_masks[log2up((balign + kIOPageAllocChunkBytes - 1) / kIOPageAllocChunkBytes)];
+
+    IOSimpleLockLock(gIOPageAllocLock);
+    pa = (typeof(pa)) queue_first(&gIOPageAllocList);
+    while (!queue_end(&gIOPageAllocList, &pa->link))
+    {
+	addr = iopa_allocinpage(pa, count, align);
+	if (addr)
+	{
+	    gIOPageAllocBytes += bytes;
+	    break;
+	}
+	pa = (typeof(pa)) queue_next(&pa->link);
+    }
+    IOSimpleLockUnlock(gIOPageAllocLock);
+    if (!addr)
+    {
+        pa = iopa_allocpage();
+	if (pa)
+	{
+	    addr = iopa_allocinpage(pa, count, align);
+	    IOSimpleLockLock(gIOPageAllocLock);
+	    if (pa->avail) enqueue_head(&gIOPageAllocList, &pa->link);
+	    gIOPageAllocCount++;
+	    if (addr) gIOPageAllocBytes += bytes;
+	    IOSimpleLockUnlock(gIOPageAllocLock);
+	}
+    }
+
+    if (addr)
+    {
+        assert((addr & ((1 << log2up(balign)) - 1)) == 0);
+    	IOStatisticsAlloc(kIOStatisticsMallocAligned, bytes);
+#if IOALLOCDEBUG
+	debug_iomalloc_size += bytes;
+#endif
+    }
+
+    return (addr);
+}
+
+static void 
+iopa_free(uintptr_t addr, vm_size_t bytes)
+{
+    io_pagealloc_t * pa;
+    uint32_t         count;
+    uintptr_t        chunk;
+
+    if (!bytes) bytes = 1;
+
+    chunk = (addr & page_mask);
+    assert(0 == (chunk & (kIOPageAllocChunkBytes - 1)));
+
+    pa = (typeof(pa)) (addr | (page_size - kIOPageAllocChunkBytes));
+    assert(kIOPageAllocSignature == pa->signature);
+
+    count = (bytes + kIOPageAllocChunkBytes - 1) / kIOPageAllocChunkBytes;
+    chunk /= kIOPageAllocChunkBytes;
+
+    IOSimpleLockLock(gIOPageAllocLock);
+    if (!pa->avail)
+    {
+	assert(!pa->link.next);
+	enqueue_tail(&gIOPageAllocList, &pa->link);
+    }
+    pa->avail |= ((-1ULL << (64 - count)) >> chunk);
+    if (pa->avail != -2ULL) pa = 0;
+    else
+    {
+        remque(&pa->link);
+        pa->link.next = 0;
+        pa->signature = 0;
+	gIOPageAllocCount--;
+    }
+    gIOPageAllocBytes -= bytes;
+    IOSimpleLockUnlock(gIOPageAllocLock);
+    if (pa) iopa_freepage(pa);
+
+#if IOALLOCDEBUG
+    debug_iomalloc_size -= bytes;
+#endif
+    IOStatisticsAlloc(kIOStatisticsFreeAligned, bytes);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
 #define super IOGeneralMemoryDescriptor
 OSDefineMetaClassAndStructors(IOBufferMemoryDescriptor,
 				IOGeneralMemoryDescriptor);
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-static uintptr_t IOBMDPageProc(iopa_t * a)
-{
-    kern_return_t kr;
-    vm_address_t  vmaddr  = 0;
-    int           options = 0; // KMA_LOMEM;
-
-    kr = kernel_memory_allocate(kernel_map, &vmaddr,
-				page_size, 0, options);
-
-    if (KERN_SUCCESS != kr) vmaddr = 0;
-    else 		    bzero((void *) vmaddr, page_size);
-
-    return ((uintptr_t) vmaddr);
-}
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
@@ -267,14 +449,7 @@
 	{
             _internalFlags |= kInternalFlagPageAllocated;
             needZero        = false;
-            _buffer         = (void *) iopa_alloc(&gIOBMDPageAllocator, &IOBMDPageProc, capacity, alignment);
-	    if (_buffer)
-	    {
-		IOStatisticsAlloc(kIOStatisticsMallocAligned, capacity);
-#if IOALLOCDEBUG
-		debug_iomalloc_size += capacity;
-#endif
-	    }
+            _buffer         = (void *) iopa_alloc(capacity, alignment);
 	}
 	else if (alignment > 1)
 	{
@@ -325,8 +500,7 @@
 
 	    while (startAddr < endAddr)
 	    {
-		UInt8 dummyVar = *startAddr;
-		(void) dummyVar;
+		*startAddr;
 		startAddr += page_size;
  	    }
 	}
@@ -557,16 +731,7 @@
 	}
 	else if (kInternalFlagPageAllocated & internalFlags)
 	{
-	    uintptr_t page;
-            page = iopa_free(&gIOBMDPageAllocator, (uintptr_t) buffer, size);
-	    if (page)
-	    {
-		kmem_free(kernel_map, page, page_size);
-	    }
-#if IOALLOCDEBUG
-	    debug_iomalloc_size -= size;
-#endif
-	    IOStatisticsAlloc(kIOStatisticsFreeAligned, size);
+            iopa_free((uintptr_t) buffer, size);
 	}
         else if (alignment > 1)
 	{