Loading...
libkern/c++/OSData.cpp xnu-8019.41.5 xnu-8792.61.2
--- xnu/xnu-8019.41.5/libkern/c++/OSData.cpp
+++ xnu/xnu-8792.61.2/libkern/c++/OSData.cpp
@@ -53,69 +53,63 @@
 
 #define EXTERNAL ((unsigned int) -1)
 
-static SECURITY_READ_ONLY_LATE(vm_map_t) alloc_map;
-__startup_func
-static void
-getAllocMap(void)
-{
-	alloc_map = KHEAP_DATA_BUFFERS->kh_fallback_map;
-}
-STARTUP(ZALLOC, STARTUP_RANK_LAST, getAllocMap);
-
 bool
 OSData::initWithCapacity(unsigned int inCapacity)
 {
-	void *_data = NULL;
-
-	if (data) {
-		OSCONTAINER_ACCUMSIZE(-((size_t)capacity));
-		if (!inCapacity || (capacity < inCapacity)) {
-			// clean out old data's storage if it isn't big enough
-			if (capacity < page_size) {
-				kfree_data_container(data, capacity);
-			} else {
-				kmem_free(alloc_map, (vm_offset_t)data, capacity);
-			}
-			data = NULL;
+	struct kalloc_result kr;
+	bool success = true;
+
+	if (!super::init()) {
+		return false;
+	}
+
+	/*
+	 * OSData use of Z_MAY_COPYINMAP serves 2 purpposes:
+	 *
+	 * - It makes sure than when it goes to the VM, it uses its own object
+	 *   rather than the kernel object so that vm_map_copyin() can be used.
+	 *
+	 * - On Intel, it goes to the VM for any size >= PAGE_SIZE to maintain
+	 *   old (inefficient) ABI. On arm64 it will use kalloc_data() instead
+	 *   until the vm_map_copy_t msg_ool_size_small threshold for copies.
+	 */
+
+	if (inCapacity == 0) {
+		if (capacity) {
+			OSCONTAINER_ACCUMSIZE(-(size_t)capacity);
+			/* can't use kfree() as we need to pass Z_MAY_COPYINMAP */
+			__kheap_realloc(KHEAP_DATA_BUFFERS, data, capacity, 0,
+			    Z_VM_TAG_BT(Z_WAITOK_ZERO | Z_FULLSIZE | Z_MAY_COPYINMAP,
+			    VM_KERN_MEMORY_LIBKERN), (void *)&this->data);
+			data     = nullptr;
 			capacity = 0;
 		}
-	}
-
-	if (!super::init()) {
-		return false;
-	}
-
-	if (inCapacity && !data) {
-		if (inCapacity < page_size) {
-			data = (void *)kalloc_data_container(inCapacity, Z_WAITOK);
+	} else if (inCapacity <= capacity) {
+		/*
+		 * Nothing to change
+		 */
+	} else {
+		kr = kalloc_ext(KHEAP_DATA_BUFFERS, inCapacity,
+		    Z_VM_TAG_BT(Z_WAITOK_ZERO | Z_FULLSIZE | Z_MAY_COPYINMAP,
+		    VM_KERN_MEMORY_LIBKERN), (void *)&this->data);
+
+		if (kr.addr) {
+			size_t delta = 0;
+
+			data     = kr.addr;
+			delta   -= capacity;
+			capacity = (uint32_t)MIN(kr.size, UINT32_MAX);
+			delta   += capacity;
+			OSCONTAINER_ACCUMSIZE(delta);
 		} else {
-			kern_return_t kr;
-			if (round_page_overflow(inCapacity, &inCapacity)) {
-				kr = KERN_RESOURCE_SHORTAGE;
-			} else {
-				kr = kmem_alloc_flags(alloc_map, (vm_offset_t *)&_data, inCapacity,
-				    IOMemoryTag(alloc_map), KMA_ATOMIC);
-				data = _data;
-			}
-			if (KERN_SUCCESS != kr) {
-				data = NULL;
-			}
-		}
-		if (!data) {
-			return false;
-		}
-		capacity = inCapacity;
-	}
-	OSCONTAINER_ACCUMSIZE(capacity);
+			success = false;
+		}
+	}
 
 	length = 0;
-	if (inCapacity < 16) {
-		capacityIncrement = 16;
-	} else {
-		capacityIncrement = inCapacity;
-	}
-
-	return true;
+	capacityIncrement = MAX(16, inCapacity);
+
+	return success;
 }
 
 bool
@@ -230,11 +224,10 @@
 OSData::free()
 {
 	if ((capacity != EXTERNAL) && data && capacity) {
-		if (capacity < page_size) {
-			kfree_data_container(data, capacity);
-		} else {
-			kmem_free(alloc_map, (vm_offset_t)data, capacity);
-		}
+		/* can't use kfree() as we need to pass Z_MAY_COPYINMAP */
+		__kheap_realloc(KHEAP_DATA_BUFFERS, data, capacity, 0,
+		    Z_VM_TAG_BT(Z_WAITOK_ZERO | Z_FULLSIZE | Z_MAY_COPYINMAP,
+		    VM_KERN_MEMORY_LIBKERN), (void *)&this->data);
 		OSCONTAINER_ACCUMSIZE( -((size_t)capacity));
 	} else if (capacity == EXTERNAL) {
 		DeallocFunction freemem = reserved ? reserved->deallocFunction : NULL;
@@ -276,10 +269,8 @@
 unsigned int
 OSData::ensureCapacity(unsigned int newCapacity)
 {
-	unsigned char * newData;
+	struct kalloc_result kr;
 	unsigned int finalCapacity;
-	void * copydata;
-	kern_return_t kr;
 
 	if (newCapacity <= capacity) {
 		return capacity;
@@ -293,52 +284,50 @@
 		return capacity;
 	}
 
-	copydata = data;
-
-	if (finalCapacity >= page_size) {
-		// round up
-		finalCapacity = round_page_32(finalCapacity);
-		// integer overflow check
-		if (finalCapacity < newCapacity) {
-			return capacity;
-		}
-		if (capacity >= page_size) {
-			copydata = NULL;
-			kr = kmem_realloc(alloc_map,
-			    (vm_offset_t)data,
-			    capacity,
-			    (vm_offset_t *)&newData,
-			    finalCapacity,
-			    IOMemoryTag(alloc_map));
-		} else {
-			kr = kmem_alloc_flags(alloc_map, (vm_offset_t *)&newData,
-			    finalCapacity, IOMemoryTag(alloc_map), KMA_ATOMIC);
-		}
-		if (KERN_SUCCESS != kr) {
-			newData = NULL;
-		}
-	} else {
-		newData = (unsigned char *)kalloc_data_container(finalCapacity, Z_WAITOK);
-	}
-
-	if (newData) {
-		bzero(newData + capacity, finalCapacity - capacity);
-		if (copydata) {
-			bcopy(copydata, newData, capacity);
-		}
-		if (data) {
-			if (capacity < page_size) {
-				kfree_data_container(data, capacity);
-			} else {
-				kmem_free(alloc_map, (vm_offset_t)data, capacity);
-			}
-		}
-		OSCONTAINER_ACCUMSIZE(((size_t)finalCapacity) - ((size_t)capacity));
-		data = (void *) newData;
-		capacity = finalCapacity;
+	kr = krealloc_ext((void *)KHEAP_DATA_BUFFERS, data, capacity, finalCapacity,
+	    Z_VM_TAG_BT(Z_WAITOK_ZERO | Z_FULLSIZE | Z_MAY_COPYINMAP,
+	    VM_KERN_MEMORY_LIBKERN), (void *)&this->data);
+
+	if (kr.addr) {
+		size_t delta = 0;
+
+		data     = kr.addr;
+		delta   -= capacity;
+		capacity = (uint32_t)MIN(kr.size, UINT32_MAX);
+		delta   += capacity;
+		OSCONTAINER_ACCUMSIZE(delta);
 	}
 
 	return capacity;
+}
+
+bool
+OSData::clipForCopyout()
+{
+	unsigned int newCapacity = (uint32_t)round_page(length);
+	__assert_only struct kalloc_result kr;
+
+	/*
+	 * OSData allocations are atomic, which means that if copyoutkdata()
+	 * is used on them, and that there are fully unused pages at the end
+	 * of the OSData buffer, then vm_map_copyin() will try to clip the VM
+	 * entry which will panic.
+	 *
+	 * In order to avoid this, trim down the unused pages.
+	 *
+	 * We know this operation never fails and keeps the allocation
+	 * address stable.
+	 */
+	if (length >= msg_ool_size_small && newCapacity < capacity) {
+		kr = krealloc_ext((void *)KHEAP_DATA_BUFFERS,
+		    data, capacity, newCapacity,
+		    Z_VM_TAG_BT(Z_WAITOK_ZERO | Z_FULLSIZE | Z_MAY_COPYINMAP,
+		    VM_KERN_MEMORY_LIBKERN), (void *)&this->data);
+		assert(kr.addr == data);
+		OSCONTAINER_ACCUMSIZE(((size_t)newCapacity) - ((size_t)capacity));
+		capacity = newCapacity;
+	}
+	return true;
 }
 
 bool