Loading...
--- xnu/xnu-8019.41.5/libkern/c++/OSData.cpp
+++ xnu/xnu-8792.61.2/libkern/c++/OSData.cpp
@@ -53,69 +53,63 @@
#define EXTERNAL ((unsigned int) -1)
-static SECURITY_READ_ONLY_LATE(vm_map_t) alloc_map;
-__startup_func
-static void
-getAllocMap(void)
-{
- alloc_map = KHEAP_DATA_BUFFERS->kh_fallback_map;
-}
-STARTUP(ZALLOC, STARTUP_RANK_LAST, getAllocMap);
-
bool
OSData::initWithCapacity(unsigned int inCapacity)
{
- void *_data = NULL;
-
- if (data) {
- OSCONTAINER_ACCUMSIZE(-((size_t)capacity));
- if (!inCapacity || (capacity < inCapacity)) {
- // clean out old data's storage if it isn't big enough
- if (capacity < page_size) {
- kfree_data_container(data, capacity);
- } else {
- kmem_free(alloc_map, (vm_offset_t)data, capacity);
- }
- data = NULL;
+ struct kalloc_result kr;
+ bool success = true;
+
+ if (!super::init()) {
+ return false;
+ }
+
+ /*
+ * OSData use of Z_MAY_COPYINMAP serves 2 purpposes:
+ *
+ * - It makes sure than when it goes to the VM, it uses its own object
+ * rather than the kernel object so that vm_map_copyin() can be used.
+ *
+ * - On Intel, it goes to the VM for any size >= PAGE_SIZE to maintain
+ * old (inefficient) ABI. On arm64 it will use kalloc_data() instead
+ * until the vm_map_copy_t msg_ool_size_small threshold for copies.
+ */
+
+ if (inCapacity == 0) {
+ if (capacity) {
+ OSCONTAINER_ACCUMSIZE(-(size_t)capacity);
+ /* can't use kfree() as we need to pass Z_MAY_COPYINMAP */
+ __kheap_realloc(KHEAP_DATA_BUFFERS, data, capacity, 0,
+ Z_VM_TAG_BT(Z_WAITOK_ZERO | Z_FULLSIZE | Z_MAY_COPYINMAP,
+ VM_KERN_MEMORY_LIBKERN), (void *)&this->data);
+ data = nullptr;
capacity = 0;
}
- }
-
- if (!super::init()) {
- return false;
- }
-
- if (inCapacity && !data) {
- if (inCapacity < page_size) {
- data = (void *)kalloc_data_container(inCapacity, Z_WAITOK);
+ } else if (inCapacity <= capacity) {
+ /*
+ * Nothing to change
+ */
+ } else {
+ kr = kalloc_ext(KHEAP_DATA_BUFFERS, inCapacity,
+ Z_VM_TAG_BT(Z_WAITOK_ZERO | Z_FULLSIZE | Z_MAY_COPYINMAP,
+ VM_KERN_MEMORY_LIBKERN), (void *)&this->data);
+
+ if (kr.addr) {
+ size_t delta = 0;
+
+ data = kr.addr;
+ delta -= capacity;
+ capacity = (uint32_t)MIN(kr.size, UINT32_MAX);
+ delta += capacity;
+ OSCONTAINER_ACCUMSIZE(delta);
} else {
- kern_return_t kr;
- if (round_page_overflow(inCapacity, &inCapacity)) {
- kr = KERN_RESOURCE_SHORTAGE;
- } else {
- kr = kmem_alloc_flags(alloc_map, (vm_offset_t *)&_data, inCapacity,
- IOMemoryTag(alloc_map), KMA_ATOMIC);
- data = _data;
- }
- if (KERN_SUCCESS != kr) {
- data = NULL;
- }
- }
- if (!data) {
- return false;
- }
- capacity = inCapacity;
- }
- OSCONTAINER_ACCUMSIZE(capacity);
+ success = false;
+ }
+ }
length = 0;
- if (inCapacity < 16) {
- capacityIncrement = 16;
- } else {
- capacityIncrement = inCapacity;
- }
-
- return true;
+ capacityIncrement = MAX(16, inCapacity);
+
+ return success;
}
bool
@@ -230,11 +224,10 @@
OSData::free()
{
if ((capacity != EXTERNAL) && data && capacity) {
- if (capacity < page_size) {
- kfree_data_container(data, capacity);
- } else {
- kmem_free(alloc_map, (vm_offset_t)data, capacity);
- }
+ /* can't use kfree() as we need to pass Z_MAY_COPYINMAP */
+ __kheap_realloc(KHEAP_DATA_BUFFERS, data, capacity, 0,
+ Z_VM_TAG_BT(Z_WAITOK_ZERO | Z_FULLSIZE | Z_MAY_COPYINMAP,
+ VM_KERN_MEMORY_LIBKERN), (void *)&this->data);
OSCONTAINER_ACCUMSIZE( -((size_t)capacity));
} else if (capacity == EXTERNAL) {
DeallocFunction freemem = reserved ? reserved->deallocFunction : NULL;
@@ -276,10 +269,8 @@
unsigned int
OSData::ensureCapacity(unsigned int newCapacity)
{
- unsigned char * newData;
+ struct kalloc_result kr;
unsigned int finalCapacity;
- void * copydata;
- kern_return_t kr;
if (newCapacity <= capacity) {
return capacity;
@@ -293,52 +284,50 @@
return capacity;
}
- copydata = data;
-
- if (finalCapacity >= page_size) {
- // round up
- finalCapacity = round_page_32(finalCapacity);
- // integer overflow check
- if (finalCapacity < newCapacity) {
- return capacity;
- }
- if (capacity >= page_size) {
- copydata = NULL;
- kr = kmem_realloc(alloc_map,
- (vm_offset_t)data,
- capacity,
- (vm_offset_t *)&newData,
- finalCapacity,
- IOMemoryTag(alloc_map));
- } else {
- kr = kmem_alloc_flags(alloc_map, (vm_offset_t *)&newData,
- finalCapacity, IOMemoryTag(alloc_map), KMA_ATOMIC);
- }
- if (KERN_SUCCESS != kr) {
- newData = NULL;
- }
- } else {
- newData = (unsigned char *)kalloc_data_container(finalCapacity, Z_WAITOK);
- }
-
- if (newData) {
- bzero(newData + capacity, finalCapacity - capacity);
- if (copydata) {
- bcopy(copydata, newData, capacity);
- }
- if (data) {
- if (capacity < page_size) {
- kfree_data_container(data, capacity);
- } else {
- kmem_free(alloc_map, (vm_offset_t)data, capacity);
- }
- }
- OSCONTAINER_ACCUMSIZE(((size_t)finalCapacity) - ((size_t)capacity));
- data = (void *) newData;
- capacity = finalCapacity;
+ kr = krealloc_ext((void *)KHEAP_DATA_BUFFERS, data, capacity, finalCapacity,
+ Z_VM_TAG_BT(Z_WAITOK_ZERO | Z_FULLSIZE | Z_MAY_COPYINMAP,
+ VM_KERN_MEMORY_LIBKERN), (void *)&this->data);
+
+ if (kr.addr) {
+ size_t delta = 0;
+
+ data = kr.addr;
+ delta -= capacity;
+ capacity = (uint32_t)MIN(kr.size, UINT32_MAX);
+ delta += capacity;
+ OSCONTAINER_ACCUMSIZE(delta);
}
return capacity;
+}
+
+bool
+OSData::clipForCopyout()
+{
+ unsigned int newCapacity = (uint32_t)round_page(length);
+ __assert_only struct kalloc_result kr;
+
+ /*
+ * OSData allocations are atomic, which means that if copyoutkdata()
+ * is used on them, and that there are fully unused pages at the end
+ * of the OSData buffer, then vm_map_copyin() will try to clip the VM
+ * entry which will panic.
+ *
+ * In order to avoid this, trim down the unused pages.
+ *
+ * We know this operation never fails and keeps the allocation
+ * address stable.
+ */
+ if (length >= msg_ool_size_small && newCapacity < capacity) {
+ kr = krealloc_ext((void *)KHEAP_DATA_BUFFERS,
+ data, capacity, newCapacity,
+ Z_VM_TAG_BT(Z_WAITOK_ZERO | Z_FULLSIZE | Z_MAY_COPYINMAP,
+ VM_KERN_MEMORY_LIBKERN), (void *)&this->data);
+ assert(kr.addr == data);
+ OSCONTAINER_ACCUMSIZE(((size_t)newCapacity) - ((size_t)capacity));
+ capacity = newCapacity;
+ }
+ return true;
}
bool