Loading...
--- xnu/xnu-344/iokit/Kernel/IOMemoryDescriptor.cpp
+++ xnu/xnu-201/iokit/Kernel/IOMemoryDescriptor.cpp
@@ -34,10 +34,6 @@
#include <IOKit/IOKitDebug.h>
#include <libkern/c++/OSContainers.h>
-#include <libkern/c++/OSDictionary.h>
-#include <libkern/c++/OSArray.h>
-#include <libkern/c++/OSSymbol.h>
-#include <libkern/c++/OSNumber.h>
#include <sys/cdefs.h>
__BEGIN_DECLS
@@ -45,10 +41,7 @@
#include <device/device_port.h>
void bcopy_phys(char *from, char *to, int size);
void pmap_enter(pmap_t pmap, vm_offset_t va, vm_offset_t pa,
- vm_prot_t prot, unsigned int flags, boolean_t wired);
-#ifndef i386
-struct phys_entry *pmap_find_physentry(vm_offset_t pa);
-#endif
+ vm_prot_t prot, boolean_t wired);
void ipc_port_release_send(ipc_port_t port);
vm_offset_t vm_map_get_phys_page(vm_map_t map, vm_offset_t offset);
@@ -58,9 +51,6 @@
int device_handle,
vm_size_t size,
int flags);
-void
-device_pager_deallocate(
- memory_object_t);
kern_return_t
device_pager_populate_object(
memory_object_t pager,
@@ -68,23 +58,12 @@
vm_offset_t phys_addr,
vm_size_t size);
-/*
- * Page fault handling based on vm_map (or entries therein)
- */
-extern kern_return_t vm_fault(
- vm_map_t map,
- vm_offset_t vaddr,
- vm_prot_t fault_type,
- boolean_t change_wiring,
- int interruptible,
- pmap_t caller_pmap,
- vm_offset_t caller_pmap_addr);
-
__END_DECLS
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-OSDefineMetaClassAndAbstractStructors( IOMemoryDescriptor, OSObject )
+OSDefineMetaClass( IOMemoryDescriptor, OSObject )
+OSDefineAbstractStructors( IOMemoryDescriptor, OSObject )
#define super IOMemoryDescriptor
@@ -100,16 +79,6 @@
IOIteratePageableMapsCallback callback, void * ref);
}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-static IORecursiveLock * gIOMemoryLock;
-
-#define LOCK IORecursiveLockLock( gIOMemoryLock)
-#define UNLOCK IORecursiveLockUnlock( gIOMemoryLock)
-#define SLEEP IORecursiveLockSleep( gIOMemoryLock, (void *)this, THREAD_UNINT)
-#define WAKEUP \
- IORecursiveLockWakeup( gIOMemoryLock, (void *)this, /* one-thread */ false)
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -155,40 +124,22 @@
vm_object_offset_t offset,
vm_size_t size)
{
- struct ExpansionData {
- void * devicePager;
- unsigned int pagerContig:1;
- unsigned int unused:31;
- IOMemoryDescriptor * memory;
- };
- kern_return_t kr;
- ExpansionData * ref = (ExpansionData *) device_handle;
- IOMemoryDescriptor * memDesc;
-
- LOCK;
- memDesc = ref->memory;
- if( memDesc)
- kr = memDesc->handleFault( device_pager, 0, 0,
- offset, size, kIOMapDefaultCache /*?*/);
- else
- kr = KERN_ABORTED;
- UNLOCK;
-
- return( kr );
+ IOMemoryDescriptor * memDesc = (IOMemoryDescriptor *) device_handle;
+
+ assert( OSDynamicCast( IOMemoryDescriptor, memDesc ));
+
+ return( memDesc->handleFault( device_pager, 0, 0,
+ offset, size, kIOMapDefaultCache /*?*/));
}
kern_return_t device_close(
int device_handle)
{
- struct ExpansionData {
- void * devicePager;
- unsigned int pagerContig:1;
- unsigned int unused:31;
- IOMemoryDescriptor * memory;
- };
- ExpansionData * ref = (ExpansionData *) device_handle;
-
- IODelete( ref, ExpansionData, 1 );
+ IOMemoryDescriptor * memDesc = (IOMemoryDescriptor *) device_handle;
+
+ assert( OSDynamicCast( IOMemoryDescriptor, memDesc ));
+
+ memDesc->release();
return( kIOReturnSuccess );
}
@@ -461,23 +412,12 @@
*/
void IOGeneralMemoryDescriptor::free()
{
- LOCK;
- if( reserved)
- reserved->memory = 0;
- UNLOCK;
-
while (_wireCount)
complete();
if (_kernPtrAligned)
unmapFromKernel();
if (_ranges.v && _rangesIsAllocated)
IODelete(_ranges.v, IOVirtualRange, _rangesCount);
-
- if( reserved && reserved->devicePager)
- device_pager_deallocate( reserved->devicePager );
-
- // memEntry holds a ref on the device pager which owns reserved (ExpansionData)
- // so no reserved access after this point
if( _memEntry)
ipc_port_release_send( (ipc_port_t) _memEntry );
super::free();
@@ -559,7 +499,6 @@
/* DEPRECATED */ _kernPtrAligned + off,
/* DEPRECATED */ phys_addr,
/* DEPRECATED */ VM_PROT_READ|VM_PROT_WRITE,
-/* DEPRECATED */ VM_WIMG_USE_DEFAULT,
/* DEPRECATED */ TRUE);
/* DEPRECATED */ }
/* DEPRECATED */ _kernPtrAtIndex = rangeIndex;
@@ -599,12 +538,11 @@
IOPhysicalAddress IOMemoryDescriptor::getSourceSegment( IOByteCount offset,
IOByteCount * length )
{
- IOPhysicalAddress physAddr = 0;
-
- if( prepare() == kIOReturnSuccess) {
- physAddr = getPhysicalSegment( offset, length );
- complete();
- }
+ IOPhysicalAddress physAddr;
+
+ prepare();
+ physAddr = getPhysicalSegment( offset, length );
+ complete();
return( physAddr );
}
@@ -772,11 +710,6 @@
return bytesCopied;
}
-
-extern "C" {
-// osfmk/device/iokit_rpc.c
-extern unsigned int IOTranslateCacheBits(struct phys_entry *pp);
-};
/* DEPRECATED */ void IOGeneralMemoryDescriptor::setPosition(IOByteCount position)
/* DEPRECATED */ {
@@ -1171,14 +1104,14 @@
- trunc_page(_ranges.v[index].address);
if( _task) {
-#ifndef i386
- vm_size_t actualSize = size;
- kr = mach_make_memory_entry( get_task_map(_task),
+#if NOTYET
+ vm_object_offset_t actualSize = size;
+ kr = mach_make_memory_entry_64( get_task_map(_task),
&actualSize, _ranges.v[0].address,
VM_PROT_READ | VM_PROT_WRITE, &sharedMem,
NULL );
- if( (KERN_SUCCESS == kr) && (actualSize != round_page(size))) {
+ if( (KERN_SUCCESS == kr) && (actualSize != size)) {
#if IOASSERT
IOLog("mach_make_memory_entry_64 (%08lx) size (%08lx:%08lx)\n",
_ranges.v[0].address, (UInt32)actualSize, size);
@@ -1188,18 +1121,12 @@
}
if( KERN_SUCCESS != kr)
-#endif /* i386 */
+#endif /* NOTYET */
sharedMem = MACH_PORT_NULL;
} else do {
memory_object_t pager;
- unsigned int flags=0;
- struct phys_entry *pp;
- IOPhysicalAddress pa;
- IOPhysicalLength segLen;
-
- pa = getPhysicalSegment( sourceOffset, &segLen );
if( !reserved) {
reserved = IONew( ExpansionData, 1 );
@@ -1207,77 +1134,32 @@
continue;
}
reserved->pagerContig = (1 == _rangesCount);
- reserved->memory = this;
-
-#ifndef i386
- switch(options & kIOMapCacheMask ) { /*What cache mode do we need*/
-
- case kIOMapDefaultCache:
- default:
- if((pp = pmap_find_physentry(pa))) {/* Find physical address */
- /* Use physical attributes as default */
- flags = IOTranslateCacheBits(pp);
-
- }
- else { /* If no physical, just hard code attributes */
- flags = DEVICE_PAGER_CACHE_INHIB |
- DEVICE_PAGER_COHERENT | DEVICE_PAGER_GUARDED;
- }
- break;
-
- case kIOMapInhibitCache:
- flags = DEVICE_PAGER_CACHE_INHIB |
- DEVICE_PAGER_COHERENT | DEVICE_PAGER_GUARDED;
- break;
-
- case kIOMapWriteThruCache:
- flags = DEVICE_PAGER_WRITE_THROUGH |
- DEVICE_PAGER_COHERENT | DEVICE_PAGER_GUARDED;
- break;
-
- case kIOMapCopybackCache:
- flags = DEVICE_PAGER_COHERENT;
- break;
- }
-
- flags |= reserved->pagerContig ? DEVICE_PAGER_CONTIGUOUS : 0;
-#else
- flags = reserved->pagerContig ? DEVICE_PAGER_CONTIGUOUS : 0;
-#endif
-
- pager = device_pager_setup( (memory_object_t) 0, (int) reserved,
- size, flags);
+
+ pager = device_pager_setup( (memory_object_t) 0, (int) this, size,
+ reserved->pagerContig ? DEVICE_PAGER_CONTIGUOUS : 0 );
assert( pager );
if( pager) {
+ retain(); // pager has a ref
kr = mach_memory_object_memory_entry_64( (host_t) 1, false /*internal*/,
size, VM_PROT_READ | VM_PROT_WRITE, pager, &sharedMem );
assert( KERN_SUCCESS == kr );
if( KERN_SUCCESS != kr) {
- device_pager_deallocate( pager );
+// chris?
+// ipc_port_release_send( (ipc_port_t) pager );
pager = MACH_PORT_NULL;
sharedMem = MACH_PORT_NULL;
}
}
- if( pager && sharedMem)
- reserved->devicePager = pager;
- else {
- IODelete( reserved, ExpansionData, 1 );
- reserved = 0;
- }
+ reserved->devicePager = pager;
} while( false );
_memEntry = (void *) sharedMem;
}
-#ifndef i386
- if( 0 == sharedMem)
- kr = kIOReturnVMError;
- else
-#endif
- kr = super::doMap( addressMap, atAddress,
+ kr = super::doMap( addressMap, atAddress,
options, sourceOffset, length );
return( kr );
@@ -1308,11 +1190,15 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-OSDefineMetaClassAndAbstractStructors( IOMemoryMap, OSObject )
-
-/* inline function implementation */
-IOPhysicalAddress IOMemoryMap::getPhysicalAddress()
- { return( getPhysicalSegment( 0, 0 )); }
+static IORecursiveLock * gIOMemoryLock;
+
+#define LOCK IORecursiveLockLock( gIOMemoryLock)
+#define UNLOCK IORecursiveLockUnlock( gIOMemoryLock)
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+OSDefineMetaClass( IOMemoryMap, OSObject )
+OSDefineAbstractStructors( IOMemoryMap, OSObject )
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -1329,11 +1215,8 @@
vm_map_t addressMap;
IOOptionBits options;
-protected:
- virtual void taggedRelease(const void *tag = 0) const;
+public:
virtual void free();
-
-public:
// IOMemoryMap methods
virtual IOVirtualAddress getVirtualAddress();
@@ -1349,7 +1232,7 @@
IOByteCount * length);
// for IOMemoryDescriptor use
- _IOMemoryMap * copyCompatible(
+ _IOMemoryMap * isCompatible(
IOMemoryDescriptor * owner,
task_t intoTask,
IOVirtualAddress toAddress,
@@ -1357,13 +1240,13 @@
IOByteCount offset,
IOByteCount length );
- bool initCompatible(
+ bool init(
IOMemoryDescriptor * memory,
IOMemoryMap * superMap,
IOByteCount offset,
IOByteCount length );
- bool initWithDescriptor(
+ bool init(
IOMemoryDescriptor * memory,
task_t intoTask,
IOVirtualAddress toAddress,
@@ -1384,7 +1267,7 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-bool _IOMemoryMap::initCompatible(
+bool _IOMemoryMap::init(
IOMemoryDescriptor * _memory,
IOMemoryMap * _superMap,
IOByteCount _offset,
@@ -1414,7 +1297,7 @@
return( true );
}
-bool _IOMemoryMap::initWithDescriptor(
+bool _IOMemoryMap::init(
IOMemoryDescriptor * _memory,
task_t intoTask,
IOVirtualAddress toAddress,
@@ -1433,7 +1316,7 @@
addressMap = get_task_map(intoTask);
if( !addressMap)
return( false);
- vm_map_reference(addressMap);
+ kernel_vm_map_reference(addressMap);
_memory->retain();
memory = _memory;
@@ -1519,7 +1402,6 @@
return( err );
}
-
IOReturn IOMemoryDescriptor::doMap(
vm_map_t addressMap,
IOVirtualAddress * atAddress,
@@ -1612,15 +1494,21 @@
if( !addressMap) {
+ LOCK;
+
if( kIOMemoryRedirected & _flags) {
#ifdef DEBUG
- IOLog("sleep mem redirect %p, %lx\n", this, sourceOffset);
+ IOLog("sleep mem redirect %x, %lx\n", address, sourceOffset);
#endif
do {
- SLEEP;
+ assert_wait( (event_t) this, THREAD_UNINT );
+ UNLOCK;
+ thread_block((void (*)(void)) 0);
+ LOCK;
} while( kIOMemoryRedirected & _flags );
}
+ UNLOCK;
return( kIOReturnSuccess );
}
@@ -1649,18 +1537,11 @@
segLen - pageOffset);
#endif
-
-
-
-
-#ifdef i386
- /* i386 doesn't support faulting on device memory yet */
if( addressMap && (kIOReturnSuccess == err))
err = IOMapPages( addressMap, address, physAddr, segLen, options );
assert( KERN_SUCCESS == err );
if( err)
break;
-#endif
if( pager) {
if( reserved && reserved->pagerContig) {
@@ -1669,7 +1550,7 @@
allPhys = getPhysicalSegment( 0, &allLen );
assert( allPhys );
- err = device_pager_populate_object( pager, 0, trunc_page(allPhys), round_page(allLen) );
+ err = device_pager_populate_object( pager, 0, trunc_page(allPhys), round_page(allPhys + allLen) );
} else {
@@ -1684,31 +1565,6 @@
if( err)
break;
}
-#ifndef i386
- /* *** ALERT *** */
- /* *** Temporary Workaround *** */
-
- /* This call to vm_fault causes an early pmap level resolution */
- /* of the mappings created above. Need for this is in absolute */
- /* violation of the basic tenet that the pmap layer is a cache. */
- /* Further, it implies a serious I/O architectural violation on */
- /* the part of some user of the mapping. As of this writing, */
- /* the call to vm_fault is needed because the NVIDIA driver */
- /* makes a call to pmap_extract. The NVIDIA driver needs to be */
- /* fixed as soon as possible. The NVIDIA driver should not */
- /* need to query for this info as it should know from the doMap */
- /* call where the physical memory is mapped. When a query is */
- /* necessary to find a physical mapping, it should be done */
- /* through an iokit call which includes the mapped memory */
- /* handle. This is required for machine architecture independence.*/
-
- if(!(kIOMemoryRedirected & _flags)) {
- vm_fault(addressMap, address, 3, FALSE, FALSE, NULL, 0);
- }
-
- /* *** Temporary Workaround *** */
- /* *** ALERT *** */
-#endif
sourceOffset += segLen - pageOffset;
address += segLen;
bytes -= segLen;
@@ -1770,7 +1626,7 @@
_flags |= kIOMemoryRedirected;
else {
_flags &= ~kIOMemoryRedirected;
- WAKEUP;
+ thread_wakeup( (event_t) this);
}
UNLOCK;
@@ -1802,7 +1658,7 @@
if( logical && addressMap
&& (get_task_map( safeTask) != addressMap)
&& (0 == (options & kIOMapStatic))) {
-
+
IOUnmapPages( addressMap, logical, length );
if( !redirect) {
err = vm_deallocate( addressMap, logical, length );
@@ -1812,7 +1668,7 @@
} else
err = kIOReturnSuccess;
#ifdef DEBUG
- IOLog("IOMemoryMap::redirect(%d, %p) %x:%lx from %p\n", redirect, this, logical, length, addressMap);
+ IOLog("IOMemoryMap::redirect(%d, %x) %x from %p\n", redirect, err, logical, addressMap);
#endif
}
UNLOCK;
@@ -1854,15 +1710,6 @@
addressTask = 0;
logical = 0;
UNLOCK;
-}
-
-// Overload the release mechanism. All mappings must be a member
-// of a memory descriptors _mappings set. This means that we
-// always have 2 references on a mapping. When either of these mappings
-// are released we need to free ourselves.
-void _IOMemoryMap::taggedRelease(const void *tag = 0) const
-{
- super::taggedRelease(tag, 2);
}
void _IOMemoryMap::free()
@@ -1910,7 +1757,7 @@
return( memory );
}
-_IOMemoryMap * _IOMemoryMap::copyCompatible(
+_IOMemoryMap * _IOMemoryMap::isCompatible(
IOMemoryDescriptor * owner,
task_t task,
IOVirtualAddress toAddress,
@@ -1922,10 +1769,7 @@
if( (!task) || (task != getAddressTask()))
return( 0 );
- if( (options ^ _options) & kIOMapReadOnly)
- return( 0 );
- if( (kIOMapDefaultCache != (_options & kIOMapCacheMask))
- && ((options ^ _options) & kIOMapCacheMask))
+ if( (options ^ _options) & (kIOMapCacheMask | kIOMapReadOnly))
return( 0 );
if( (0 == (_options & kIOMapAnywhere)) && (logical != toAddress))
@@ -1946,7 +1790,7 @@
} else {
mapping = new _IOMemoryMap;
if( mapping
- && !mapping->initCompatible( owner, this, _offset, _length )) {
+ && !mapping->init( owner, this, _offset, _length )) {
mapping->release();
mapping = 0;
}
@@ -1984,6 +1828,9 @@
{
if( _mappings)
_mappings->release();
+
+ if( reserved)
+ IODelete( reserved, ExpansionData, 1 );
super::free();
}
@@ -2000,7 +1847,7 @@
LOCK;
if( map
- && !map->initWithDescriptor( this, intoTask, mapAddress,
+ && !map->init( this, intoTask, mapAddress,
options | kIOMapStatic, 0, getLength() )) {
map->release();
map = 0;
@@ -2054,7 +1901,7 @@
while( (mapping = (_IOMemoryMap *) iter->getNextObject())) {
- if( (mapping = mapping->copyCompatible(
+ if( (mapping = mapping->isCompatible(
owner, intoTask, toAddress,
options | kIOMapReference,
offset, length )))
@@ -2073,11 +1920,10 @@
mapping = new _IOMemoryMap;
if( mapping
- && !mapping->initWithDescriptor( owner, intoTask, toAddress, options,
+ && !mapping->init( owner, intoTask, toAddress, options,
offset, length )) {
-#ifdef DEBUG
+
IOLog("Didn't make map %08lx : %08lx\n", offset, length );
-#endif
mapping->release();
mapping = 0;
}
@@ -2097,16 +1943,19 @@
if( mapping) {
if( 0 == _mappings)
_mappings = OSSet::withCapacity(1);
- if( _mappings )
- _mappings->setObject( mapping );
+ if( _mappings && _mappings->setObject( mapping ))
+ mapping->release(); /* really */
}
}
void IOMemoryDescriptor::removeMapping(
IOMemoryMap * mapping )
{
- if( _mappings)
+ if( _mappings) {
+ mapping->retain();
+ mapping->retain();
_mappings->removeObject( mapping);
+ }
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -2335,140 +2184,6 @@
bool asReference = false)
{
return( false );
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-bool IOGeneralMemoryDescriptor::serialize(OSSerialize * s) const
-{
- OSSymbol const *keys[2];
- OSObject *values[2];
- OSDictionary *dict;
- IOVirtualRange *vcopy;
- unsigned int index, nRanges;
- bool result;
-
- if (s == NULL) return false;
- if (s->previouslySerialized(this)) return true;
-
- // Pretend we are an array.
- if (!s->addXMLStartTag(this, "array")) return false;
-
- nRanges = _rangesCount;
- vcopy = (IOVirtualRange *) IOMalloc(sizeof(IOVirtualRange) * nRanges);
- if (vcopy == 0) return false;
-
- keys[0] = OSSymbol::withCString("address");
- keys[1] = OSSymbol::withCString("length");
-
- result = false;
- values[0] = values[1] = 0;
-
- // From this point on we can go to bail.
-
- // Copy the volatile data so we don't have to allocate memory
- // while the lock is held.
- LOCK;
- if (nRanges == _rangesCount) {
- for (index = 0; index < nRanges; index++) {
- vcopy[index] = _ranges.v[index];
- }
- } else {
- // The descriptor changed out from under us. Give up.
- UNLOCK;
- result = false;
- goto bail;
- }
- UNLOCK;
-
- for (index = 0; index < nRanges; index++)
- {
- values[0] = OSNumber::withNumber(_ranges.v[index].address, sizeof(_ranges.v[index].address) * 8);
- if (values[0] == 0) {
- result = false;
- goto bail;
- }
- values[1] = OSNumber::withNumber(_ranges.v[index].length, sizeof(_ranges.v[index].length) * 8);
- if (values[1] == 0) {
- result = false;
- goto bail;
- }
- OSDictionary *dict = OSDictionary::withObjects((const OSObject **)values, (const OSSymbol **)keys, 2);
- if (dict == 0) {
- result = false;
- goto bail;
- }
- values[0]->release();
- values[1]->release();
- values[0] = values[1] = 0;
-
- result = dict->serialize(s);
- dict->release();
- if (!result) {
- goto bail;
- }
- }
- result = s->addXMLEndTag("array");
-
- bail:
- if (values[0])
- values[0]->release();
- if (values[1])
- values[1]->release();
- if (keys[0])
- keys[0]->release();
- if (keys[1])
- keys[1]->release();
- if (vcopy)
- IOFree(vcopy, sizeof(IOVirtualRange) * nRanges);
- return result;
-}
-
-bool IOSubMemoryDescriptor::serialize(OSSerialize * s) const
-{
- if (!s) {
- return (false);
- }
- if (s->previouslySerialized(this)) return true;
-
- // Pretend we are a dictionary.
- // We must duplicate the functionality of OSDictionary here
- // because otherwise object references will not work;
- // they are based on the value of the object passed to
- // previouslySerialized and addXMLStartTag.
-
- if (!s->addXMLStartTag(this, "dict")) return false;
-
- char const *keys[3] = {"offset", "length", "parent"};
-
- OSObject *values[3];
- values[0] = OSNumber::withNumber(_start, sizeof(_start) * 8);
- if (values[0] == 0)
- return false;
- values[1] = OSNumber::withNumber(_length, sizeof(_length) * 8);
- if (values[1] == 0) {
- values[0]->release();
- return false;
- }
- values[2] = _parent;
-
- bool result = true;
- for (int i=0; i<3; i++) {
- if (!s->addString("<key>") ||
- !s->addString(keys[i]) ||
- !s->addXMLEndTag("key") ||
- !values[i]->serialize(s)) {
- result = false;
- break;
- }
- }
- values[0]->release();
- values[1]->release();
- if (!result) {
- return false;
- }
-
- return s->addXMLEndTag("dict");
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -2489,7 +2204,3 @@
OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 13);
OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 14);
OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 15);
-
-/* inline function implementation */
-IOPhysicalAddress IOMemoryDescriptor::getPhysicalAddress()
- { return( getPhysicalSegment( 0, 0 )); }