Loading...
--- xnu/xnu-6153.41.3/libkern/c++/OSData.cpp
+++ xnu/xnu-8792.61.2/libkern/c++/OSData.cpp
@@ -29,11 +29,9 @@
#include <string.h>
-__BEGIN_DECLS
#include <vm/vm_kern.h>
-__END_DECLS
-
-#define LIBKERN_SMART_POINTERS
+
+#define IOKIT_ENABLE_SHARED_PTR
#include <libkern/c++/OSData.h>
#include <libkern/c++/OSSerialize.h>
@@ -43,8 +41,8 @@
#define super OSObject
-OSDefineMetaClassAndStructors(OSData, OSObject)
-OSMetaClassDefineReservedUsed(OSData, 0); // setDeallocFunction
+OSDefineMetaClassAndStructorsWithZone(OSData, OSObject, ZC_ZFREE_CLEARMEM)
+OSMetaClassDefineReservedUsedX86(OSData, 0); // setDeallocFunction
OSMetaClassDefineReservedUnused(OSData, 1);
OSMetaClassDefineReservedUnused(OSData, 2);
OSMetaClassDefineReservedUnused(OSData, 3);
@@ -58,53 +56,60 @@
bool
OSData::initWithCapacity(unsigned int inCapacity)
{
- if (data) {
- OSCONTAINER_ACCUMSIZE(-((size_t)capacity));
- if (!inCapacity || (capacity < inCapacity)) {
- // clean out old data's storage if it isn't big enough
- if (capacity < page_size) {
- kfree(data, capacity);
- } else {
- kmem_free(kernel_map, (vm_offset_t)data, capacity);
- }
- data = NULL;
+ struct kalloc_result kr;
+ bool success = true;
+
+ if (!super::init()) {
+ return false;
+ }
+
+ /*
+ * OSData use of Z_MAY_COPYINMAP serves 2 purpposes:
+ *
+ * - It makes sure than when it goes to the VM, it uses its own object
+ * rather than the kernel object so that vm_map_copyin() can be used.
+ *
+ * - On Intel, it goes to the VM for any size >= PAGE_SIZE to maintain
+ * old (inefficient) ABI. On arm64 it will use kalloc_data() instead
+ * until the vm_map_copy_t msg_ool_size_small threshold for copies.
+ */
+
+ if (inCapacity == 0) {
+ if (capacity) {
+ OSCONTAINER_ACCUMSIZE(-(size_t)capacity);
+ /* can't use kfree() as we need to pass Z_MAY_COPYINMAP */
+ __kheap_realloc(KHEAP_DATA_BUFFERS, data, capacity, 0,
+ Z_VM_TAG_BT(Z_WAITOK_ZERO | Z_FULLSIZE | Z_MAY_COPYINMAP,
+ VM_KERN_MEMORY_LIBKERN), (void *)&this->data);
+ data = nullptr;
capacity = 0;
}
- }
-
- if (!super::init()) {
- return false;
- }
-
- if (inCapacity && !data) {
- if (inCapacity < page_size) {
- data = (void *) kalloc_container(inCapacity);
+ } else if (inCapacity <= capacity) {
+ /*
+ * Nothing to change
+ */
+ } else {
+ kr = kalloc_ext(KHEAP_DATA_BUFFERS, inCapacity,
+ Z_VM_TAG_BT(Z_WAITOK_ZERO | Z_FULLSIZE | Z_MAY_COPYINMAP,
+ VM_KERN_MEMORY_LIBKERN), (void *)&this->data);
+
+ if (kr.addr) {
+ size_t delta = 0;
+
+ data = kr.addr;
+ delta -= capacity;
+ capacity = (uint32_t)MIN(kr.size, UINT32_MAX);
+ delta += capacity;
+ OSCONTAINER_ACCUMSIZE(delta);
} else {
- kern_return_t kr;
- if (round_page_overflow(inCapacity, &inCapacity)) {
- kr = KERN_RESOURCE_SHORTAGE;
- } else {
- kr = kmem_alloc(kernel_map, (vm_offset_t *)&data, inCapacity, IOMemoryTag(kernel_map));
- }
- if (KERN_SUCCESS != kr) {
- data = NULL;
- }
- }
- if (!data) {
- return false;
- }
- capacity = inCapacity;
- }
- OSCONTAINER_ACCUMSIZE(capacity);
+ success = false;
+ }
+ }
length = 0;
- if (inCapacity < 16) {
- capacityIncrement = 16;
- } else {
- capacityIncrement = inCapacity;
- }
-
- return true;
+ capacityIncrement = MAX(16, inCapacity);
+
+ return success;
}
bool
@@ -155,10 +160,10 @@
}
}
-OSDataPtr
+OSSharedPtr<OSData>
OSData::withCapacity(unsigned int inCapacity)
{
- OSDataPtr me = OSDataPtr::alloc();
+ OSSharedPtr<OSData> me = OSMakeShared<OSData>();
if (me && !me->initWithCapacity(inCapacity)) {
return nullptr;
@@ -167,10 +172,10 @@
return me;
}
-OSDataPtr
+OSSharedPtr<OSData>
OSData::withBytes(const void *bytes, unsigned int inLength)
{
- OSDataPtr me = OSDataPtr::alloc();
+ OSSharedPtr<OSData> me = OSMakeShared<OSData>();
if (me && !me->initWithBytes(bytes, inLength)) {
return nullptr;
@@ -178,10 +183,10 @@
return me;
}
-OSDataPtr
+OSSharedPtr<OSData>
OSData::withBytesNoCopy(void *bytes, unsigned int inLength)
{
- OSDataPtr me = OSDataPtr::alloc();
+ OSSharedPtr<OSData> me = OSMakeShared<OSData>();
if (me && !me->initWithBytesNoCopy(bytes, inLength)) {
return nullptr;
@@ -190,10 +195,10 @@
return me;
}
-OSDataPtr
+OSSharedPtr<OSData>
OSData::withData(const OSData *inData)
{
- OSDataPtr me = OSDataPtr::alloc();
+ OSSharedPtr<OSData> me = OSMakeShared<OSData>();
if (me && !me->initWithData(inData)) {
return nullptr;
@@ -202,11 +207,11 @@
return me;
}
-OSDataPtr
+OSSharedPtr<OSData>
OSData::withData(const OSData *inData,
unsigned int start, unsigned int inLength)
{
- OSDataPtr me = OSDataPtr::alloc();
+ OSSharedPtr<OSData> me = OSMakeShared<OSData>();
if (me && !me->initWithData(inData, start, inLength)) {
return nullptr;
@@ -219,11 +224,10 @@
OSData::free()
{
if ((capacity != EXTERNAL) && data && capacity) {
- if (capacity < page_size) {
- kfree(data, capacity);
- } else {
- kmem_free(kernel_map, (vm_offset_t)data, capacity);
- }
+ /* can't use kfree() as we need to pass Z_MAY_COPYINMAP */
+ __kheap_realloc(KHEAP_DATA_BUFFERS, data, capacity, 0,
+ Z_VM_TAG_BT(Z_WAITOK_ZERO | Z_FULLSIZE | Z_MAY_COPYINMAP,
+ VM_KERN_MEMORY_LIBKERN), (void *)&this->data);
OSCONTAINER_ACCUMSIZE( -((size_t)capacity));
} else if (capacity == EXTERNAL) {
DeallocFunction freemem = reserved ? reserved->deallocFunction : NULL;
@@ -232,7 +236,7 @@
}
}
if (reserved) {
- kfree(reserved, sizeof(ExpansionData));
+ kfree_type(ExpansionData, reserved);
}
super::free();
}
@@ -265,10 +269,8 @@
unsigned int
OSData::ensureCapacity(unsigned int newCapacity)
{
- unsigned char * newData;
+ struct kalloc_result kr;
unsigned int finalCapacity;
- void * copydata;
- kern_return_t kr;
if (newCapacity <= capacity) {
return capacity;
@@ -282,51 +284,50 @@
return capacity;
}
- copydata = data;
-
- if (finalCapacity >= page_size) {
- // round up
- finalCapacity = round_page_32(finalCapacity);
- // integer overflow check
- if (finalCapacity < newCapacity) {
- return capacity;
- }
- if (capacity >= page_size) {
- copydata = NULL;
- kr = kmem_realloc(kernel_map,
- (vm_offset_t)data,
- capacity,
- (vm_offset_t *)&newData,
- finalCapacity,
- IOMemoryTag(kernel_map));
- } else {
- kr = kmem_alloc(kernel_map, (vm_offset_t *)&newData, finalCapacity, IOMemoryTag(kernel_map));
- }
- if (KERN_SUCCESS != kr) {
- newData = NULL;
- }
- } else {
- newData = (unsigned char *) kalloc_container(finalCapacity);
- }
-
- if (newData) {
- bzero(newData + capacity, finalCapacity - capacity);
- if (copydata) {
- bcopy(copydata, newData, capacity);
- }
- if (data) {
- if (capacity < page_size) {
- kfree(data, capacity);
- } else {
- kmem_free(kernel_map, (vm_offset_t)data, capacity);
- }
- }
- OSCONTAINER_ACCUMSIZE(((size_t)finalCapacity) - ((size_t)capacity));
- data = (void *) newData;
- capacity = finalCapacity;
+ kr = krealloc_ext((void *)KHEAP_DATA_BUFFERS, data, capacity, finalCapacity,
+ Z_VM_TAG_BT(Z_WAITOK_ZERO | Z_FULLSIZE | Z_MAY_COPYINMAP,
+ VM_KERN_MEMORY_LIBKERN), (void *)&this->data);
+
+ if (kr.addr) {
+ size_t delta = 0;
+
+ data = kr.addr;
+ delta -= capacity;
+ capacity = (uint32_t)MIN(kr.size, UINT32_MAX);
+ delta += capacity;
+ OSCONTAINER_ACCUMSIZE(delta);
}
return capacity;
+}
+
+bool
+OSData::clipForCopyout()
+{
+ unsigned int newCapacity = (uint32_t)round_page(length);
+ __assert_only struct kalloc_result kr;
+
+ /*
+ * OSData allocations are atomic, which means that if copyoutkdata()
+ * is used on them, and that there are fully unused pages at the end
+ * of the OSData buffer, then vm_map_copyin() will try to clip the VM
+ * entry which will panic.
+ *
+ * In order to avoid this, trim down the unused pages.
+ *
+ * We know this operation never fails and keeps the allocation
+ * address stable.
+ */
+ if (length >= msg_ool_size_small && newCapacity < capacity) {
+ kr = krealloc_ext((void *)KHEAP_DATA_BUFFERS,
+ data, capacity, newCapacity,
+ Z_VM_TAG_BT(Z_WAITOK_ZERO | Z_FULLSIZE | Z_MAY_COPYINMAP,
+ VM_KERN_MEMORY_LIBKERN), (void *)&this->data);
+ assert(kr.addr == data);
+ OSCONTAINER_ACCUMSIZE(((size_t)newCapacity) - ((size_t)capacity));
+ capacity = newCapacity;
+ }
+ return true;
}
bool
@@ -582,11 +583,10 @@
OSData::setDeallocFunction(DeallocFunction func)
{
if (!reserved) {
- reserved = (typeof(reserved))kalloc_container(sizeof(ExpansionData));
+ reserved = (typeof(reserved))kalloc_type(ExpansionData, (zalloc_flags_t)(Z_WAITOK | Z_ZERO));
if (!reserved) {
return;
}
- bzero(reserved, sizeof(ExpansionData));
}
reserved->deallocFunction = func;
}
@@ -595,11 +595,10 @@
OSData::setSerializable(bool serializable)
{
if (!reserved) {
- reserved = (typeof(reserved))kalloc_container(sizeof(ExpansionData));
+ reserved = (typeof(reserved))kalloc_type(ExpansionData, (zalloc_flags_t)(Z_WAITOK | Z_ZERO));
if (!reserved) {
return;
}
- bzero(reserved, sizeof(ExpansionData));
}
reserved->disableSerialization = (!serializable);
}