Loading...
--- xnu/xnu-792/iokit/Kernel/IOMemoryDescriptor.cpp
+++ xnu/xnu-517/iokit/Kernel/IOMemoryDescriptor.cpp
@@ -1,21 +1,24 @@
/*
- * Copyright (c) 1998-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
*
* @APPLE_LICENSE_HEADER_START@
*
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License"). You may not use this file except in compliance with the
- * License. Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved.
*
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this
+ * file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
*
* @APPLE_LICENSE_HEADER_END@
*/
@@ -37,29 +40,21 @@
#include <IOKit/IOKitDebug.h>
-#include "IOKitKernelInternal.h"
-
#include <libkern/c++/OSContainers.h>
#include <libkern/c++/OSDictionary.h>
#include <libkern/c++/OSArray.h>
#include <libkern/c++/OSSymbol.h>
#include <libkern/c++/OSNumber.h>
-
-#include <sys/uio.h>
+#include <sys/cdefs.h>
__BEGIN_DECLS
#include <vm/pmap.h>
-#include <vm/vm_pageout.h>
-#include <vm/vm_shared_memory_server.h>
#include <mach/memory_object_types.h>
#include <device/device_port.h>
#ifndef i386
-#include <mach/vm_prot.h>
-#include <vm/vm_fault.h>
struct phys_entry *pmap_find_physentry(ppnum_t pa);
#endif
-
extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
void ipc_port_release_send(ipc_port_t port);
@@ -91,8 +86,26 @@
unsigned int *page_list_count,
int *flags);
+/*
+ * Page fault handling based on vm_map (or entries therein)
+ */
+extern kern_return_t vm_fault(
+ vm_map_t map,
+ vm_offset_t vaddr,
+ vm_prot_t fault_type,
+ boolean_t change_wiring,
+ int interruptible,
+ pmap_t caller_pmap,
+ vm_offset_t caller_pmap_addr);
+
unsigned int IOTranslateCacheBits(struct phys_entry *pp);
+vm_map_t IOPageableMapForAddress( vm_address_t address );
+
+typedef kern_return_t (*IOIteratePageableMapsCallback)(vm_map_t map, void * ref);
+
+kern_return_t IOIteratePageableMaps(vm_size_t size,
+ IOIteratePageableMapsCallback callback, void * ref);
__END_DECLS
#define kIOMaximumMappedIOByteCount (512*1024*1024)
@@ -117,115 +130,6 @@
#define SLEEP IORecursiveLockSleep( gIOMemoryLock, (void *)this, THREAD_UNINT)
#define WAKEUP \
IORecursiveLockWakeup( gIOMemoryLock, (void *)this, /* one-thread */ false)
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-class _IOMemoryMap : public IOMemoryMap
-{
- OSDeclareDefaultStructors(_IOMemoryMap)
-public:
- IOMemoryDescriptor * memory;
- IOMemoryMap * superMap;
- IOByteCount offset;
- IOByteCount length;
- IOVirtualAddress logical;
- task_t addressTask;
- vm_map_t addressMap;
- IOOptionBits options;
- upl_t redirUPL;
- ipc_port_t redirEntry;
- IOMemoryDescriptor * owner;
-
-protected:
- virtual void taggedRelease(const void *tag = 0) const;
- virtual void free();
-
-public:
-
- // IOMemoryMap methods
- virtual IOVirtualAddress getVirtualAddress();
- virtual IOByteCount getLength();
- virtual task_t getAddressTask();
- virtual IOMemoryDescriptor * getMemoryDescriptor();
- virtual IOOptionBits getMapOptions();
-
- virtual IOReturn unmap();
- virtual void taskDied();
-
- virtual IOReturn redirect(IOMemoryDescriptor * newBackingMemory,
- IOOptionBits options,
- IOByteCount offset = 0);
-
- virtual IOPhysicalAddress getPhysicalSegment(IOByteCount offset,
- IOByteCount * length);
-
- // for IOMemoryDescriptor use
- _IOMemoryMap * copyCompatible(
- IOMemoryDescriptor * owner,
- task_t intoTask,
- IOVirtualAddress toAddress,
- IOOptionBits options,
- IOByteCount offset,
- IOByteCount length );
-
- bool initCompatible(
- IOMemoryDescriptor * memory,
- IOMemoryMap * superMap,
- IOByteCount offset,
- IOByteCount length );
-
- bool initWithDescriptor(
- IOMemoryDescriptor * memory,
- task_t intoTask,
- IOVirtualAddress toAddress,
- IOOptionBits options,
- IOByteCount offset,
- IOByteCount length );
-
- IOReturn redirect(
- task_t intoTask, bool redirect );
-};
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-// Some data structures and accessor macros used by the initWithOptions
-// Function
-
-enum ioPLBlockFlags {
- kIOPLOnDevice = 0x00000001,
- kIOPLExternUPL = 0x00000002,
-};
-
-struct typePersMDData
-{
- const IOGeneralMemoryDescriptor *fMD;
- ipc_port_t fMemEntry;
-};
-
-struct ioPLBlock {
- upl_t fIOPL;
- vm_address_t fIOMDOffset; // The offset of this iopl in descriptor
- vm_offset_t fPageInfo; // Pointer to page list or index into it
- ppnum_t fMappedBase; // Page number of first page in this iopl
- unsigned int fPageOffset; // Offset within first page of iopl
- unsigned int fFlags; // Flags
-};
-
-struct ioGMDData {
- IOMapper *fMapper;
- unsigned int fPageCnt;
- upl_page_info_t fPageList[];
- ioPLBlock fBlocks[];
-};
-
-#define getDataP(osd) ((ioGMDData *) (osd)->getBytesNoCopy())
-#define getIOPLList(d) ((ioPLBlock *) &(d->fPageList[d->fPageCnt]))
-#define getNumIOPL(osd, d) \
- (((osd)->getLength() - ((char *) getIOPLList(d) - (char *) d)) / sizeof(ioPLBlock))
-#define getPageList(d) (&(d->fPageList[0]))
-#define computeDataSize(p, u) \
- (sizeof(ioGMDData) + p * sizeof(upl_page_info_t) + u * sizeof(ioPLBlock))
-
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -254,12 +158,8 @@
LOCK;
memDesc = ref->memory;
if( memDesc)
- {
- memDesc->retain();
kr = memDesc->handleFault( device_pager, 0, 0,
offset, size, kIOMapDefaultCache /*?*/);
- memDesc->release();
- }
else
kr = KERN_ABORTED;
UNLOCK;
@@ -282,26 +182,7 @@
return( kIOReturnSuccess );
}
-}; // end extern "C"
-
-// Note this inline function uses C++ reference arguments to return values
-// This means that pointers are not passed and NULLs don't have to be
-// checked for as a NULL reference is illegal.
-static inline void
-getAddrLenForInd(user_addr_t &addr, IOPhysicalLength &len, // Output variables
- UInt32 type, IOGeneralMemoryDescriptor::Ranges r, UInt32 ind)
-{
- assert(kIOMemoryTypePhysical == type || kIOMemoryTypeUIO == type
- || kIOMemoryTypeVirtual == type);
- if (kIOMemoryTypeUIO == type) {
- user_size_t us;
- uio_getiov((uio_t) r.uio, ind, &addr, &us); len = us;
- }
- else {
- IOVirtualRange cur = r.v[ind];
- addr = cur.address;
- len = cur.length;
- }
+
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -445,87 +326,6 @@
self = 0;
}
return self;
-}
-
-IOMemoryDescriptor * IOMemoryDescriptor::
- withPersistentMemoryDescriptor(IOMemoryDescriptor *originalMD)
-{
- IOGeneralMemoryDescriptor *origGenMD =
- OSDynamicCast(IOGeneralMemoryDescriptor, originalMD);
-
- if (origGenMD)
- return IOGeneralMemoryDescriptor::
- withPersistentMemoryDescriptor(origGenMD);
- else
- return 0;
-}
-
-IOMemoryDescriptor * IOGeneralMemoryDescriptor::
- withPersistentMemoryDescriptor(IOGeneralMemoryDescriptor *originalMD)
-{
- ipc_port_t sharedMem = (ipc_port_t) originalMD->createNamedEntry();
-
- if (!sharedMem)
- return 0;
-
- if (sharedMem == originalMD->_memEntry) {
- originalMD->retain(); // Add a new reference to ourselves
- ipc_port_release_send(sharedMem); // Remove extra send right
- return originalMD;
- }
-
- IOGeneralMemoryDescriptor * self = new IOGeneralMemoryDescriptor;
- typePersMDData initData = { originalMD, sharedMem };
-
- if (self
- && !self->initWithOptions(&initData, 1, 0, 0, kIOMemoryTypePersistentMD, 0)) {
- self->release();
- self = 0;
- }
- return self;
-}
-
-void *IOGeneralMemoryDescriptor::createNamedEntry()
-{
- kern_return_t error;
- ipc_port_t sharedMem;
-
- IOOptionBits type = _flags & kIOMemoryTypeMask;
-
- user_addr_t range0Addr;
- IOByteCount range0Len;
- getAddrLenForInd(range0Addr, range0Len, type, _ranges, 0);
- range0Addr = trunc_page_64(range0Addr);
-
- vm_size_t size = ptoa_32(_pages);
- vm_address_t kernelPage = (vm_address_t) range0Addr;
-
- vm_map_t theMap = ((_task == kernel_task)
- && (kIOMemoryBufferPageable & _flags))
- ? IOPageableMapForAddress(kernelPage)
- : get_task_map(_task);
-
- memory_object_size_t actualSize = size;
- vm_prot_t prot = VM_PROT_READ | VM_PROT_WRITE;
- if (_memEntry)
- prot |= MAP_MEM_NAMED_REUSE;
-
- error = mach_make_memory_entry_64(theMap,
- &actualSize, range0Addr, prot, &sharedMem, (ipc_port_t) _memEntry);
-
- if (KERN_SUCCESS == error) {
- if (actualSize == size) {
- return sharedMem;
- } else {
-#if IOASSERT
- IOLog("IOGMD::mach_make_memory_entry_64 (%08llx) size (%08lx:%08x)\n",
- (UInt64)range0Addr, (UInt32)actualSize, size);
-#endif
- ipc_port_release_send( sharedMem );
- }
- }
-
- return MACH_PORT_NULL;
}
/*
@@ -605,15 +405,16 @@
if (task) {
mdOpts |= kIOMemoryTypeVirtual;
-
- // Auto-prepare if this is a kernel memory descriptor as very few
- // clients bother to prepare() kernel memory.
- // But it was not enforced so what are you going to do?
if (task == kernel_task)
mdOpts |= kIOMemoryAutoPrepare;
}
else
mdOpts |= kIOMemoryTypePhysical;
+
+ // @@@ gvdl: Need to remove this
+ // Auto-prepare if this is a kernel memory descriptor as very few
+ // clients bother to prepare() kernel memory.
+ // But it has been enforced so what are you going to do?
return initWithOptions(ranges, count, 0, task, mdOpts, /* mapper */ 0);
}
@@ -622,8 +423,8 @@
* initWithOptions:
*
* IOMemoryDescriptor. The buffer is made up of several virtual address ranges,
- * from a given task, several physical ranges, an UPL from the ubc
- * system or a uio (may be 64bit) from the BSD subsystem.
+ * from a given task or several physical ranges or finally an UPL from the ubc
+ * system.
*
* Passing the ranges as a reference will avoid an extra allocation.
*
@@ -632,6 +433,36 @@
* I/O Kit classes, although it is supported here.
*/
+enum ioPLBlockFlags {
+ kIOPLOnDevice = 0x00000001,
+ kIOPLExternUPL = 0x00000002,
+};
+
+struct ioPLBlock {
+ upl_t fIOPL;
+ vm_address_t fIOMDOffset; // The offset of this iopl in descriptor
+ vm_offset_t fPageInfo; // Pointer to page list or index into it
+ ppnum_t fMappedBase; // Page number of first page in this iopl
+ unsigned int fPageOffset; // Offset within first page of iopl
+ unsigned int fFlags; // Flags
+};
+
+struct ioGMDData {
+ IOMapper *fMapper;
+ unsigned int fPageCnt;
+ upl_page_info_t fPageList[0]; // @@@ gvdl need to get rid of this
+ // should be able to use upl directly
+ ioPLBlock fBlocks[0];
+};
+
+#define getDataP(osd) ((ioGMDData *) (osd)->getBytesNoCopy())
+#define getIOPLList(d) ((ioPLBlock *) &(d->fPageList[d->fPageCnt]))
+#define getNumIOPL(d,len) \
+ ((len - ((char *) getIOPLList(d) - (char *) d)) / sizeof(ioPLBlock))
+#define getPageList(d) (&(d->fPageList[0]))
+#define computeDataSize(p, u) \
+ (sizeof(ioGMDData) + p * sizeof(upl_page_info_t) + u * sizeof(ioPLBlock))
+
bool
IOGeneralMemoryDescriptor::initWithOptions(void * buffers,
UInt32 count,
@@ -640,36 +471,8 @@
IOOptionBits options,
IOMapper * mapper)
{
- IOOptionBits type = options & kIOMemoryTypeMask;
-
- // Grab the original MD's configuation data to initialse the
- // arguments to this function.
- if (kIOMemoryTypePersistentMD == type) {
-
- typePersMDData *initData = (typePersMDData *) buffers;
- const IOGeneralMemoryDescriptor *orig = initData->fMD;
- ioGMDData *dataP = getDataP(orig->_memoryEntries);
-
- // Only accept persistent memory descriptors with valid dataP data.
- assert(orig->_rangesCount == 1);
- if ( !(orig->_flags & kIOMemoryPersistent) || !dataP)
- return false;
-
- _memEntry = initData->fMemEntry; // Grab the new named entry
- options = orig->_flags | kIOMemoryAsReference;
- _singleRange = orig->_singleRange; // Initialise our range
- buffers = &_singleRange;
- count = 1;
-
- // Now grab the original task and whatever mapper was previously used
- task = orig->_task;
- mapper = dataP->fMapper;
-
- // We are ready to go through the original initialisation now
- }
-
- switch (type) {
- case kIOMemoryTypeUIO:
+
+ switch (options & kIOMemoryTypeMask) {
case kIOMemoryTypeVirtual:
assert(task);
if (!task)
@@ -679,11 +482,11 @@
case kIOMemoryTypePhysical: // Neither Physical nor UPL should have a task
mapper = kIOMapperNone;
-
case kIOMemoryTypeUPL:
assert(!task);
break;
default:
+panic("IOGMD::iWO(): bad type"); // @@@ gvdl: for testing
return false; /* bad argument */
}
@@ -708,8 +511,6 @@
unmapFromKernel();
if (_ranges.v && _rangesIsAllocated)
IODelete(_ranges.v, IOVirtualRange, _rangesCount);
- if (_memEntry)
- { ipc_port_release_send((ipc_port_t) _memEntry); _memEntry = 0; }
}
else {
if (!super::init())
@@ -725,8 +526,6 @@
gIOSystemMapper = mapper = IOMapper::gSystem;
}
- // Remove the dynamic internal use flags from the initial setting
- options &= ~(kIOMemoryPreparedReadOnly);
_flags = options;
_task = task;
@@ -737,7 +536,7 @@
_cachedPhysicalAddress = 0;
_cachedVirtualAddress = 0;
- if (kIOMemoryTypeUPL == type) {
+ if ( (options & kIOMemoryTypeMask) == kIOMemoryTypeUPL) {
ioGMDData *dataP;
unsigned int dataSize = computeDataSize(/* pages */ 0, /* upls */ 1);
@@ -768,6 +567,8 @@
iopl.fFlags = pageList->device | kIOPLExternUPL;
iopl.fIOMDOffset = 0;
if (!pageList->device) {
+ // @@@ gvdl: Ask JoeS are the pages contiguious with the list?
+ // or there a chance that we may be inserting 0 phys_addrs?
// Pre-compute the offset into the UPL's page list
pageList = &pageList[atop_32(offset)];
offset &= PAGE_MASK;
@@ -785,60 +586,50 @@
_memoryEntries->appendBytes(&iopl, sizeof(iopl));
}
- else {
- // kIOMemoryTypeVirtual | kIOMemoryTypeUIO | kIOMemoryTypePhysical
-
- // Initialize the memory descriptor
- if (options & kIOMemoryAsReference) {
- _rangesIsAllocated = false;
-
- // Hack assignment to get the buffer arg into _ranges.
- // I'd prefer to do _ranges = (Ranges) buffers, but that doesn't
- // work, C++ sigh.
- // This also initialises the uio & physical ranges.
- _ranges.v = (IOVirtualRange *) buffers;
- }
- else {
- assert(kIOMemoryTypeUIO != type);
-
- _rangesIsAllocated = true;
- _ranges.v = IONew(IOVirtualRange, count);
- if (!_ranges.v)
+ else { /* kIOMemoryTypeVirtual | kIOMemoryTypePhysical */
+ IOVirtualRange *ranges = (IOVirtualRange *) buffers;
+
+ /*
+ * Initialize the memory descriptor.
+ */
+
+ _length = 0;
+ _pages = 0;
+ for (unsigned ind = 0; ind < count; ind++) {
+ IOVirtualRange cur = ranges[ind];
+
+ _length += cur.length;
+ _pages += atop_32(cur.address + cur.length + PAGE_MASK)
+ - atop_32(cur.address);
+ }
+
+ _ranges.v = 0;
+ _rangesIsAllocated = !(options & kIOMemoryAsReference);
+ _rangesCount = count;
+
+ if (options & kIOMemoryAsReference)
+ _ranges.v = ranges;
+ else {
+ _ranges.v = IONew(IOVirtualRange, count);
+ if (!_ranges.v)
return false;
- bcopy(buffers, _ranges.v, count * sizeof(IOVirtualRange));
- }
-
- // Find starting address within the vector of ranges
- Ranges vec = _ranges;
- UInt32 length = 0;
- UInt32 pages = 0;
- for (unsigned ind = 0; ind < count; ind++) {
- user_addr_t addr;
- UInt32 len;
-
- // addr & len are returned by this function
- getAddrLenForInd(addr, len, type, vec, ind);
- pages += (atop_64(addr + len + PAGE_MASK) - atop_64(addr));
- len += length;
- assert(len > length); // Check for 32 bit wrap around
- length = len;
- }
- _length = length;
- _pages = pages;
- _rangesCount = count;
+ bcopy(/* from */ ranges, _ranges.v,
+ count * sizeof(IOVirtualRange));
+ }
// Auto-prepare memory at creation time.
// Implied completion when descriptor is free-ed
- if (kIOMemoryTypePhysical == type)
- _wireCount++; // Physical MDs are, by definition, wired
- else { /* kIOMemoryTypeVirtual | kIOMemoryTypeUIO */
+ if ( (options & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
+ _wireCount++; // Physical MDs are start out wired
+ else { /* kIOMemoryTypeVirtual */
ioGMDData *dataP;
- unsigned dataSize = computeDataSize(_pages, /* upls */ count * 2);
+ unsigned int dataSize =
+ computeDataSize(_pages, /* upls */ _rangesCount * 2);
if (!_memoryEntries) {
_memoryEntries = OSData::withCapacity(dataSize);
if (!_memoryEntries)
- return false;
+ return false;
}
else if (!_memoryEntries->initWithCapacity(dataSize))
return false;
@@ -848,8 +639,38 @@
dataP->fMapper = mapper;
dataP->fPageCnt = _pages;
- if ( (kIOMemoryPersistent & _flags) && !_memEntry)
- _memEntry = createNamedEntry();
+ if (kIOMemoryPersistent & _flags)
+ {
+ kern_return_t error;
+ ipc_port_t sharedMem;
+
+ vm_size_t size = _pages << PAGE_SHIFT;
+ vm_address_t startPage;
+
+ startPage = trunc_page_32(_ranges.v[0].address);
+
+ vm_map_t theMap = ((_task == kernel_task) && (kIOMemoryBufferPageable & _flags))
+ ? IOPageableMapForAddress(startPage)
+ : get_task_map(_task);
+
+ vm_size_t actualSize = size;
+ error = mach_make_memory_entry( theMap,
+ &actualSize, startPage,
+ VM_PROT_READ | VM_PROT_WRITE, &sharedMem,
+ NULL );
+
+ if (KERN_SUCCESS == error) {
+ if (actualSize == round_page_32(size)) {
+ _memEntry = (void *) sharedMem;
+ } else {
+#if IOASSERT
+ IOLog("mach_make_memory_entry_64 (%08x) size (%08lx:%08x)\n",
+ startPage, (UInt32)actualSize, size);
+#endif
+ ipc_port_release_send( sharedMem );
+ }
+ }
+ }
if ((_flags & kIOMemoryAutoPrepare)
&& prepare() != kIOReturnSuccess)
@@ -1071,10 +892,11 @@
length += cur.length;
}
- // @@@ gvdl: should be assert(address);
+ // @@@ gvdl: should assert(address);
// but can't as NVidia GeForce creates a bogus physical mem
- assert(address
- || /* nvidia */ (!_ranges.p[0].address && 1 == _rangesCount));
+ {
+ assert(address || /*nvidia*/(!_ranges.p[0].address && 1 == _rangesCount));
+ }
assert(length);
}
else do {
@@ -1091,7 +913,7 @@
ioGMDData * dataP = getDataP(_memoryEntries);
const ioPLBlock *ioplList = getIOPLList(dataP);
- UInt ind, numIOPLs = getNumIOPL(_memoryEntries, dataP);
+ UInt ind, numIOPLs = getNumIOPL(dataP, _memoryEntries->getLength());
upl_page_info_t *pageList = getPageList(dataP);
assert(numIOPLs > 0);
@@ -1211,50 +1033,38 @@
return phys64;
}
-IOPhysicalAddress IOGeneralMemoryDescriptor::
-getSourceSegment(IOByteCount offset, IOByteCount *lengthOfSegment)
+IOPhysicalAddress IOGeneralMemoryDescriptor::getSourceSegment
+ (IOByteCount offset, IOByteCount *lengthOfSegment)
{
IOPhysicalAddress address = 0;
IOPhysicalLength length = 0;
- IOOptionBits type = _flags & kIOMemoryTypeMask;
assert(offset <= _length);
- if ( type == kIOMemoryTypeUPL)
+ if ( (_flags & kIOMemoryTypeMask) == kIOMemoryTypeUPL)
return super::getSourceSegment( offset, lengthOfSegment );
- else if ( offset < _length ) // (within bounds?)
+
+ if ( offset < _length ) // (within bounds?)
{
unsigned rangesIndex = 0;
- Ranges vec = _ranges;
- user_addr_t addr;
-
- // Find starting address within the vector of ranges
- for (;;) {
- getAddrLenForInd(addr, length, type, vec, rangesIndex);
- if (offset < length)
- break;
- offset -= length; // (make offset relative)
- rangesIndex++;
- }
-
- // Now that we have the starting range,
- // lets find the last contiguous range
- addr += offset;
- length -= offset;
-
- for ( ++rangesIndex; rangesIndex < _rangesCount; rangesIndex++ ) {
- user_addr_t newAddr;
- IOPhysicalLength newLen;
-
- getAddrLenForInd(newAddr, newLen, type, vec, rangesIndex);
- if (addr + length != newAddr)
- break;
- length += newLen;
- }
- if (addr)
- address = (IOPhysicalAddress) addr; // Truncate address to 32bit
- else
- length = 0;
+
+ for ( ; offset >= _ranges.v[rangesIndex].length; rangesIndex++ )
+ {
+ offset -= _ranges.v[rangesIndex].length; // (make offset relative)
+ }
+
+ address = _ranges.v[rangesIndex].address + offset;
+ length = _ranges.v[rangesIndex].length - offset;
+
+ for ( ++rangesIndex; rangesIndex < _rangesCount; rangesIndex++ )
+ {
+ if ( address + length != _ranges.v[rangesIndex].address ) break;
+
+ length += _ranges.v[rangesIndex].length; // (coalesce ranges)
+ }
+
+ assert(address);
+ if ( address == 0 ) length = 0;
}
if ( lengthOfSegment ) *lengthOfSegment = length;
@@ -1275,124 +1085,6 @@
/* DEPRECATED */ }
/* DEPRECATED */ /* USE INSTEAD: map(), readBytes(), writeBytes() */
-
-
-IOReturn IOMemoryDescriptor::setPurgeable( IOOptionBits newState,
- IOOptionBits * oldState )
-{
- IOReturn err = kIOReturnSuccess;
- vm_purgable_t control;
- int state;
-
- do
- {
- if (!_memEntry)
- {
- err = kIOReturnNotReady;
- break;
- }
-
- control = VM_PURGABLE_SET_STATE;
- switch (newState)
- {
- case kIOMemoryPurgeableKeepCurrent:
- control = VM_PURGABLE_GET_STATE;
- break;
-
- case kIOMemoryPurgeableNonVolatile:
- state = VM_PURGABLE_NONVOLATILE;
- break;
- case kIOMemoryPurgeableVolatile:
- state = VM_PURGABLE_VOLATILE;
- break;
- case kIOMemoryPurgeableEmpty:
- state = VM_PURGABLE_EMPTY;
- break;
- default:
- err = kIOReturnBadArgument;
- break;
- }
-
- if (kIOReturnSuccess != err)
- break;
-
- err = mach_memory_entry_purgable_control((ipc_port_t) _memEntry, control, &state);
-
- if (oldState)
- {
- if (kIOReturnSuccess == err)
- {
- switch (state)
- {
- case VM_PURGABLE_NONVOLATILE:
- state = kIOMemoryPurgeableNonVolatile;
- break;
- case VM_PURGABLE_VOLATILE:
- state = kIOMemoryPurgeableVolatile;
- break;
- case VM_PURGABLE_EMPTY:
- state = kIOMemoryPurgeableEmpty;
- break;
- default:
- state = kIOMemoryPurgeableNonVolatile;
- err = kIOReturnNotReady;
- break;
- }
- *oldState = state;
- }
- }
- }
- while (false);
-
- return (err);
-}
-
-extern "C" void dcache_incoherent_io_flush64(addr64_t pa, unsigned int count);
-extern "C" void dcache_incoherent_io_store64(addr64_t pa, unsigned int count);
-
-IOReturn IOMemoryDescriptor::performOperation( IOOptionBits options,
- IOByteCount offset, IOByteCount length )
-{
- IOByteCount remaining;
- void (*func)(addr64_t pa, unsigned int count) = 0;
-
- switch (options)
- {
- case kIOMemoryIncoherentIOFlush:
- func = &dcache_incoherent_io_flush64;
- break;
- case kIOMemoryIncoherentIOStore:
- func = &dcache_incoherent_io_store64;
- break;
- }
-
- if (!func)
- return (kIOReturnUnsupported);
-
- remaining = length = min(length, getLength() - offset);
- while (remaining)
- // (process another target segment?)
- {
- addr64_t dstAddr64;
- IOByteCount dstLen;
-
- dstAddr64 = getPhysicalSegment64(offset, &dstLen);
- if (!dstAddr64)
- break;
-
- // Clip segment length to remaining
- if (dstLen > remaining)
- dstLen = remaining;
-
- (*func)(dstAddr64, dstLen);
-
- offset += dstLen;
- remaining -= dstLen;
- }
-
- return (remaining ? kIOReturnUnderrun : kIOReturnSuccess);
-}
-
#ifdef __ppc__
extern vm_offset_t static_memory_end;
#define io_kernel_static_end static_memory_end
@@ -1403,12 +1095,14 @@
static kern_return_t
io_get_kernel_static_upl(
- vm_map_t /* map */,
+ vm_map_t map,
vm_address_t offset,
vm_size_t *upl_size,
upl_t *upl,
upl_page_info_array_t page_list,
- unsigned int *count)
+ unsigned int *count,
+ int *flags,
+ int force_data_sync)
{
unsigned int pageCount, page;
ppnum_t phys;
@@ -1437,7 +1131,6 @@
IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)
{
- IOOptionBits type = _flags & kIOMemoryTypeMask;
IOReturn error = kIOReturnNoMemory;
ioGMDData *dataP;
ppnum_t mapBase = 0;
@@ -1445,7 +1138,6 @@
ipc_port_t sharedMem = (ipc_port_t) _memEntry;
assert(!_wireCount);
- assert(kIOMemoryTypeVirtual == type || kIOMemoryTypeUIO == type);
if (_pages >= gIOMaximumMappedIOPageCount)
return kIOReturnNoResources;
@@ -1479,47 +1171,36 @@
}
uplFlags |= UPL_SET_IO_WIRE | UPL_SET_LITE;
- // Find the appropriate vm_map for the given task
+ //
+ // Check user read/write access to the data buffer.
+ //
+ unsigned int pageIndex = 0;
+ IOByteCount mdOffset = 0;
vm_map_t curMap;
if (_task == kernel_task && (kIOMemoryBufferPageable & _flags))
curMap = 0;
else
{ curMap = get_task_map(_task); }
- // Iterate over the vector of virtual ranges
- Ranges vec = _ranges;
- unsigned int pageIndex = 0;
- IOByteCount mdOffset = 0;
for (UInt range = 0; range < _rangesCount; range++) {
ioPLBlock iopl;
- user_addr_t startPage;
+ IOVirtualRange curRange = _ranges.v[range];
+ vm_address_t startPage;
IOByteCount numBytes;
- // Get the startPage address and length of vec[range]
- getAddrLenForInd(startPage, numBytes, type, vec, range);
- iopl.fPageOffset = (short) startPage & PAGE_MASK;
- numBytes += iopl.fPageOffset;
- startPage = trunc_page_64(startPage);
-
+ startPage = trunc_page_32(curRange.address);
+ iopl.fPageOffset = (short) curRange.address & PAGE_MASK;
if (mapper)
iopl.fMappedBase = mapBase + pageIndex;
else
iopl.fMappedBase = 0;
-
- // Iterate over the current range, creating UPLs
+ numBytes = iopl.fPageOffset + curRange.length;
+
while (numBytes) {
dataP = getDataP(_memoryEntries);
- vm_address_t kernelStart = (vm_address_t) startPage;
- vm_map_t theMap;
- if (curMap)
- theMap = curMap;
- else if (!sharedMem) {
- assert(_task == kernel_task);
- theMap = IOPageableMapForAddress(kernelStart);
- }
- else
- theMap = NULL;
-
+ vm_map_t theMap =
+ (curMap)? curMap
+ : IOPageableMapForAddress(startPage);
upl_page_info_array_t pageInfo = getPageList(dataP);
int ioplFlags = uplFlags;
upl_page_list_ptr_t baseInfo = &pageInfo[pageIndex];
@@ -1527,32 +1208,36 @@
vm_size_t ioplSize = round_page_32(numBytes);
unsigned int numPageInfo = atop_32(ioplSize);
- if (theMap == kernel_map && kernelStart < io_kernel_static_end) {
+ if ((theMap == kernel_map) && (startPage < io_kernel_static_end))
+ {
error = io_get_kernel_static_upl(theMap,
- kernelStart,
- &ioplSize,
- &iopl.fIOPL,
- baseInfo,
- &numPageInfo);
- }
- else if (sharedMem) {
+ startPage,
+ &ioplSize,
+ &iopl.fIOPL,
+ baseInfo,
+ &numPageInfo,
+ &ioplFlags,
+ false);
+
+ } else if (sharedMem && (kIOMemoryPersistent & _flags)) {
+
error = memory_object_iopl_request(sharedMem,
- ptoa_32(pageIndex),
- &ioplSize,
- &iopl.fIOPL,
- baseInfo,
- &numPageInfo,
- &ioplFlags);
- }
- else {
- assert(theMap);
- error = vm_map_create_upl(theMap,
- startPage,
- &ioplSize,
- &iopl.fIOPL,
- baseInfo,
- &numPageInfo,
- &ioplFlags);
+ ptoa_32(pageIndex),
+ &ioplSize,
+ &iopl.fIOPL,
+ baseInfo,
+ &numPageInfo,
+ &ioplFlags);
+
+ } else {
+ error = vm_map_get_upl(theMap,
+ startPage,
+ &ioplSize,
+ &iopl.fIOPL,
+ baseInfo,
+ &numPageInfo,
+ &ioplFlags,
+ false);
}
assert(ioplSize);
@@ -1583,17 +1268,14 @@
if ((_flags & kIOMemoryAutoPrepare) && iopl.fIOPL)
{
- upl_commit(iopl.fIOPL, 0, 0);
- upl_deallocate(iopl.fIOPL);
+ kernel_upl_commit(iopl.fIOPL, 0, 0);
iopl.fIOPL = 0;
}
if (!_memoryEntries->appendBytes(&iopl, sizeof(iopl))) {
// Clean up partial created and unsaved iopl
- if (iopl.fIOPL) {
- upl_abort(iopl.fIOPL, 0);
- upl_deallocate(iopl.fIOPL);
- }
+ if (iopl.fIOPL)
+ kernel_upl_abort(iopl.fIOPL, 0);
goto abortExit;
}
@@ -1620,17 +1302,14 @@
abortExit:
{
dataP = getDataP(_memoryEntries);
- UInt done = getNumIOPL(_memoryEntries, dataP);
+ UInt done = getNumIOPL(dataP, _memoryEntries->getLength());
ioPLBlock *ioplList = getIOPLList(dataP);
for (UInt range = 0; range < done; range++)
{
- if (ioplList[range].fIOPL) {
- upl_abort(ioplList[range].fIOPL, 0);
- upl_deallocate(ioplList[range].fIOPL);
- }
+ if (ioplList[range].fIOPL)
+ kernel_upl_abort(ioplList[range].fIOPL, 0);
}
- (void) _memoryEntries->initWithBytes(dataP, sizeof(ioGMDData)); // == setLength()
if (mapper && mapBase)
mapper->iovmFree(mapBase, _pages);
@@ -1650,11 +1329,9 @@
*/
IOReturn IOGeneralMemoryDescriptor::prepare(IODirection forDirection)
{
- IOReturn error = kIOReturnSuccess;
- IOOptionBits type = _flags & kIOMemoryTypeMask;
-
- if (!_wireCount
- && (kIOMemoryTypeVirtual == type || kIOMemoryTypeUIO == type) ) {
+ IOReturn error = kIOReturnSuccess;
+
+ if (!_wireCount && (_flags & kIOMemoryTypeMask) == kIOMemoryTypeVirtual) {
error = wireVirtual(forDirection);
if (error)
return error;
@@ -1683,27 +1360,23 @@
_wireCount--;
if (!_wireCount) {
- IOOptionBits type = _flags & kIOMemoryTypeMask;
-
- if (kIOMemoryTypePhysical == type) {
+ if ((_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical) {
/* kIOMemoryTypePhysical */
// DO NOTHING
}
else {
ioGMDData * dataP = getDataP(_memoryEntries);
ioPLBlock *ioplList = getIOPLList(dataP);
- UInt count = getNumIOPL(_memoryEntries, dataP);
+ UInt count = getNumIOPL(dataP, _memoryEntries->getLength());
if (dataP->fMapper && _pages && ioplList[0].fMappedBase)
dataP->fMapper->iovmFree(ioplList[0].fMappedBase, _pages);
// Only complete iopls that we created which are for TypeVirtual
- if (kIOMemoryTypeVirtual == type || kIOMemoryTypeUIO == type) {
+ if ( (_flags & kIOMemoryTypeMask) == kIOMemoryTypeVirtual) {
for (UInt ind = 0; ind < count; ind++)
- if (ioplList[ind].fIOPL) {
- upl_commit(ioplList[ind].fIOPL, 0, 0);
- upl_deallocate(ioplList[ind].fIOPL);
- }
+ if (ioplList[ind].fIOPL)
+ kernel_upl_commit(ioplList[ind].fIOPL, 0, 0);
}
(void) _memoryEntries->initWithBytes(dataP, sizeof(ioGMDData)); // == setLength()
@@ -1722,51 +1395,37 @@
kern_return_t kr;
ipc_port_t sharedMem = (ipc_port_t) _memEntry;
- IOOptionBits type = _flags & kIOMemoryTypeMask;
- Ranges vec = _ranges;
-
- user_addr_t range0Addr = 0;
- IOByteCount range0Len = 0;
-
- if (vec.v)
- getAddrLenForInd(range0Addr, range0Len, type, vec, 0);
-
// mapping source == dest? (could be much better)
- if( _task
- && (addressMap == get_task_map(_task)) && (options & kIOMapAnywhere)
- && (1 == _rangesCount) && (0 == sourceOffset)
- && range0Addr && (length <= range0Len) ) {
- if (sizeof(user_addr_t) > 4 && ((UInt64) range0Addr) >> 32)
- return kIOReturnOverrun; // Doesn't fit in 32bit return field
- else {
- *atAddress = range0Addr;
+ if( _task && (addressMap == get_task_map(_task)) && (options & kIOMapAnywhere)
+ && (1 == _rangesCount) && (0 == sourceOffset)
+ && (length <= _ranges.v[0].length) ) {
+ *atAddress = _ranges.v[0].address;
return( kIOReturnSuccess );
- }
}
if( 0 == sharedMem) {
- vm_size_t size = ptoa_32(_pages);
+ vm_size_t size = _pages << PAGE_SHIFT;
if( _task) {
#ifndef i386
- memory_object_size_t actualSize = size;
- kr = mach_make_memory_entry_64(get_task_map(_task),
- &actualSize, range0Addr,
+ vm_size_t actualSize = size;
+ kr = mach_make_memory_entry( get_task_map(_task),
+ &actualSize, _ranges.v[0].address,
VM_PROT_READ | VM_PROT_WRITE, &sharedMem,
NULL );
if( (KERN_SUCCESS == kr) && (actualSize != round_page_32(size))) {
#if IOASSERT
- IOLog("mach_make_memory_entry_64 (%08llx) size (%08lx:%08x)\n",
- range0Addr, (UInt32) actualSize, size);
+ IOLog("mach_make_memory_entry_64 (%08x) size (%08lx:%08x)\n",
+ _ranges.v[0].address, (UInt32)actualSize, size);
#endif
kr = kIOReturnVMError;
ipc_port_release_send( sharedMem );
}
if( KERN_SUCCESS != kr)
-#endif /* !i386 */
+#endif /* i386 */
sharedMem = MACH_PORT_NULL;
} else do {
@@ -1843,7 +1502,6 @@
_memEntry = (void *) sharedMem;
}
-
#ifndef i386
if( 0 == sharedMem)
kr = kIOReturnVMError;
@@ -1861,19 +1519,22 @@
IOByteCount length )
{
// could be much better
- if( _task && (addressMap == get_task_map(_task)) && (1 == _rangesCount)) {
-
- IOOptionBits type = _flags & kIOMemoryTypeMask;
- user_addr_t range0Addr;
- IOByteCount range0Len;
-
- getAddrLenForInd(range0Addr, range0Len, type, _ranges, 0);
- if (logical == range0Addr && length <= range0Len)
+ if( _task && (addressMap == get_task_map(_task)) && (1 == _rangesCount)
+ && (logical == _ranges.v[0].address)
+ && (length <= _ranges.v[0].length) )
return( kIOReturnSuccess );
- }
return( super::doUnmap( addressMap, logical, length ));
}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+extern "C" {
+// osfmk/device/iokit_rpc.c
+extern kern_return_t IOMapPages( vm_map_t map, vm_offset_t va, vm_offset_t pa,
+ vm_size_t length, unsigned int mapFlags);
+extern kern_return_t IOUnmapPages(vm_map_t map, vm_offset_t va, vm_size_t length);
+};
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -1883,6 +1544,68 @@
IOPhysicalAddress IOMemoryMap::getPhysicalAddress()
{ return( getPhysicalSegment( 0, 0 )); }
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+class _IOMemoryMap : public IOMemoryMap
+{
+ OSDeclareDefaultStructors(_IOMemoryMap)
+
+ IOMemoryDescriptor * memory;
+ IOMemoryMap * superMap;
+ IOByteCount offset;
+ IOByteCount length;
+ IOVirtualAddress logical;
+ task_t addressTask;
+ vm_map_t addressMap;
+ IOOptionBits options;
+
+protected:
+ virtual void taggedRelease(const void *tag = 0) const;
+ virtual void free();
+
+public:
+
+ // IOMemoryMap methods
+ virtual IOVirtualAddress getVirtualAddress();
+ virtual IOByteCount getLength();
+ virtual task_t getAddressTask();
+ virtual IOMemoryDescriptor * getMemoryDescriptor();
+ virtual IOOptionBits getMapOptions();
+
+ virtual IOReturn unmap();
+ virtual void taskDied();
+
+ virtual IOPhysicalAddress getPhysicalSegment(IOByteCount offset,
+ IOByteCount * length);
+
+ // for IOMemoryDescriptor use
+ _IOMemoryMap * copyCompatible(
+ IOMemoryDescriptor * owner,
+ task_t intoTask,
+ IOVirtualAddress toAddress,
+ IOOptionBits options,
+ IOByteCount offset,
+ IOByteCount length );
+
+ bool initCompatible(
+ IOMemoryDescriptor * memory,
+ IOMemoryMap * superMap,
+ IOByteCount offset,
+ IOByteCount length );
+
+ bool initWithDescriptor(
+ IOMemoryDescriptor * memory,
+ task_t intoTask,
+ IOVirtualAddress toAddress,
+ IOOptionBits options,
+ IOByteCount offset,
+ IOByteCount length );
+
+ IOReturn redirect(
+ task_t intoTask, bool redirect );
+};
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#undef super
#define super IOMemoryMap
@@ -1929,29 +1652,21 @@
IOByteCount _offset,
IOByteCount _length )
{
- bool ok;
- bool redir = ((kIOMapUnique|kIOMapReference) == ((kIOMapUnique|kIOMapReference) & _options));
-
- if ((!_memory) || (!intoTask))
+ bool ok;
+
+ if( (!_memory) || (!intoTask) || !super::init())
return( false);
if( (_offset + _length) > _memory->getLength())
return( false);
- if (!redir)
- {
- if (!super::init())
- return(false);
- addressMap = get_task_map(intoTask);
- if( !addressMap)
- return( false);
- vm_map_reference(addressMap);
- addressTask = intoTask;
- logical = toAddress;
- options = _options;
- }
+ addressMap = get_task_map(intoTask);
+ if( !addressMap)
+ return( false);
+ vm_map_reference(addressMap);
_memory->retain();
+ memory = _memory;
offset = _offset;
if( _length)
@@ -1959,34 +1674,25 @@
else
length = _memory->getLength();
+ addressTask = intoTask;
+ logical = toAddress;
+ options = _options;
+
if( options & kIOMapStatic)
ok = true;
else
- ok = (kIOReturnSuccess == _memory->doMap( addressMap, &toAddress,
- _options, offset, length ));
- if (ok || redir)
- {
- if (memory)
- memory->release();
- memory = _memory;
- logical = toAddress;
- }
- else
- {
- _memory->release();
- if (!redir)
- {
- logical = 0;
- memory = 0;
- vm_map_deallocate(addressMap);
- addressMap = 0;
- }
- }
-
+ ok = (kIOReturnSuccess == memory->doMap( addressMap, &logical,
+ options, offset, length ));
+ if( !ok) {
+ logical = 0;
+ memory->release();
+ memory = 0;
+ vm_map_deallocate(addressMap);
+ addressMap = 0;
+ }
return( ok );
}
-/* LP64todo - these need to expand */
struct IOMemoryDescriptorMapAllocRef
{
ipc_port_t sharedMem;
@@ -2100,75 +1806,28 @@
if( 0 == length)
length = getLength();
- sourceAddr = getSourceSegment( sourceOffset, NULL );
- pageOffset = sourceAddr - trunc_page_32( sourceAddr );
-
- ref.size = round_page_32( length + pageOffset );
-
- if ((kIOMapReference|kIOMapUnique) == ((kIOMapReference|kIOMapUnique) & options))
- {
- upl_t redirUPL2;
- vm_size_t size;
- int flags;
-
- _IOMemoryMap * mapping = (_IOMemoryMap *) *atAddress;
- ref.mapped = mapping->getVirtualAddress();
-
- if (!_memEntry)
- {
- err = kIOReturnNotReadable;
- continue;
- }
-
- size = length;
- flags = UPL_COPYOUT_FROM | UPL_SET_INTERNAL
- | UPL_SET_LITE | UPL_SET_IO_WIRE | UPL_BLOCK_ACCESS;
-
- if (KERN_SUCCESS != memory_object_iopl_request((ipc_port_t) _memEntry, 0, &size, &redirUPL2,
- NULL, NULL,
- &flags))
- redirUPL2 = NULL;
-
- err = upl_transpose(redirUPL2, mapping->redirUPL);
- if (kIOReturnSuccess != err)
- {
- IOLog("upl_transpose(%x)\n", err);
- err = kIOReturnSuccess;
- }
-
- if (redirUPL2)
- {
- upl_commit(redirUPL2, NULL, 0);
- upl_deallocate(redirUPL2);
- redirUPL2 = 0;
- }
- {
- // swap the memEntries since they now refer to different vm_objects
- void * me = _memEntry;
- _memEntry = mapping->memory->_memEntry;
- mapping->memory->_memEntry = me;
- }
- }
- else
- {
-
- logical = *atAddress;
- if( options & kIOMapAnywhere)
- // vm_map looks for addresses above here, even when VM_FLAGS_ANYWHERE
- ref.mapped = 0;
- else {
- ref.mapped = trunc_page_32( logical );
- if( (logical - ref.mapped) != pageOffset) {
- err = kIOReturnVMError;
- continue;
- }
- }
-
- if( ref.sharedMem && (addressMap == kernel_map) && (kIOMemoryBufferPageable & _flags))
- err = IOIteratePageableMaps( ref.size, &IOMemoryDescriptorMapAlloc, &ref );
- else
- err = IOMemoryDescriptorMapAlloc( addressMap, &ref );
- }
+ sourceAddr = getSourceSegment( sourceOffset, NULL );
+ assert( sourceAddr );
+ pageOffset = sourceAddr - trunc_page_32( sourceAddr );
+
+ ref.size = round_page_32( length + pageOffset );
+
+ logical = *atAddress;
+ if( options & kIOMapAnywhere)
+ // vm_map looks for addresses above here, even when VM_FLAGS_ANYWHERE
+ ref.mapped = 0;
+ else {
+ ref.mapped = trunc_page_32( logical );
+ if( (logical - ref.mapped) != pageOffset) {
+ err = kIOReturnVMError;
+ continue;
+ }
+ }
+
+ if( ref.sharedMem && (addressMap == kernel_map) && (kIOMemoryBufferPageable & _flags))
+ err = IOIteratePageableMaps( ref.size, &IOMemoryDescriptorMapAlloc, &ref );
+ else
+ err = IOMemoryDescriptorMapAlloc( addressMap, &ref );
if( err != KERN_SUCCESS)
continue;
@@ -2310,11 +1969,7 @@
/* handle. This is required for machine architecture independence.*/
if(!(kIOMemoryRedirected & _flags)) {
- vm_fault(addressMap,
- (vm_map_offset_t)address,
- VM_PROT_READ|VM_PROT_WRITE,
- FALSE, THREAD_UNINT, NULL,
- (vm_map_offset_t)0);
+ vm_fault(addressMap, address, 3, FALSE, FALSE, NULL, 0);
}
/* *** Temporary Workaround *** */
@@ -2360,30 +2015,27 @@
return( err );
}
-IOReturn IOMemoryDescriptor::redirect( task_t safeTask, bool doRedirect )
-{
- IOReturn err = kIOReturnSuccess;
+IOReturn IOMemoryDescriptor::redirect( task_t safeTask, bool redirect )
+{
+ IOReturn err;
_IOMemoryMap * mapping = 0;
OSIterator * iter;
LOCK;
- if( doRedirect)
- _flags |= kIOMemoryRedirected;
- else
- _flags &= ~kIOMemoryRedirected;
-
do {
if( (iter = OSCollectionIterator::withCollection( _mappings))) {
- while( (mapping = (_IOMemoryMap *) iter->getNextObject()))
- mapping->redirect( safeTask, doRedirect );
-
- iter->release();
- }
+ while( (mapping = (_IOMemoryMap *) iter->getNextObject()))
+ mapping->redirect( safeTask, redirect );
+
+ iter->release();
+ }
} while( false );
- if (!doRedirect)
- {
+ if( redirect)
+ _flags |= kIOMemoryRedirected;
+ else {
+ _flags &= ~kIOMemoryRedirected;
WAKEUP;
}
@@ -2392,35 +2044,33 @@
// temporary binary compatibility
IOSubMemoryDescriptor * subMem;
if( (subMem = OSDynamicCast( IOSubMemoryDescriptor, this)))
- err = subMem->redirect( safeTask, doRedirect );
+ err = subMem->redirect( safeTask, redirect );
else
- err = kIOReturnSuccess;
+ err = kIOReturnSuccess;
return( err );
}
-IOReturn IOSubMemoryDescriptor::redirect( task_t safeTask, bool doRedirect )
-{
- return( _parent->redirect( safeTask, doRedirect ));
-}
-
-IOReturn _IOMemoryMap::redirect( task_t safeTask, bool doRedirect )
+IOReturn IOSubMemoryDescriptor::redirect( task_t safeTask, bool redirect )
+{
+ return( _parent->redirect( safeTask, redirect ));
+}
+
+IOReturn _IOMemoryMap::redirect( task_t safeTask, bool redirect )
{
IOReturn err = kIOReturnSuccess;
if( superMap) {
-// err = ((_IOMemoryMap *)superMap)->redirect( safeTask, doRedirect );
+// err = ((_IOMemoryMap *)superMap)->redirect( safeTask, redirect );
} else {
LOCK;
if( logical && addressMap
- && (!safeTask || (get_task_map(safeTask) != addressMap))
- && (0 == (options & kIOMapStatic)))
- {
- IOUnmapPages( addressMap, logical, length );
- if(!doRedirect && safeTask
- && ((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical))
- {
+ && (get_task_map( safeTask) != addressMap)
+ && (0 == (options & kIOMapStatic))) {
+
+ IOUnmapPages( addressMap, logical, length );
+ if( !redirect) {
err = vm_deallocate( addressMap, logical, length );
err = memory->doMap( addressMap, &logical,
(options & ~kIOMapAnywhere) /*| kIOMapReserve*/,
@@ -2428,16 +2078,11 @@
} else
err = kIOReturnSuccess;
#ifdef DEBUG
- IOLog("IOMemoryMap::redirect(%d, %p) %x:%lx from %p\n", doRedirect, this, logical, length, addressMap);
+ IOLog("IOMemoryMap::redirect(%d, %p) %x:%lx from %p\n", redirect, this, logical, length, addressMap);
#endif
}
UNLOCK;
}
-
- if (((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
- && safeTask
- && (doRedirect != (0 != (memory->_flags & kIOMemoryRedirected))))
- memory->redirect(safeTask, doRedirect);
return( err );
}
@@ -2499,20 +2144,8 @@
memory->release();
}
- if (owner && (owner != memory))
- {
- LOCK;
- owner->removeMapping(this);
- UNLOCK;
- }
-
if( superMap)
superMap->release();
-
- if (redirUPL) {
- upl_commit(redirUPL, NULL, 0);
- upl_deallocate(redirUPL);
- }
super::free();
}
@@ -2557,8 +2190,6 @@
if( (!task) || (!addressMap) || (addressMap != get_task_map(task)))
return( 0 );
- if( options & kIOMapUnique)
- return( 0 );
if( (options ^ _options) & kIOMapReadOnly)
return( 0 );
if( (kIOMapDefaultCache != (_options & kIOMapCacheMask))
@@ -2593,12 +2224,12 @@
}
IOPhysicalAddress _IOMemoryMap::getPhysicalSegment( IOByteCount _offset,
- IOPhysicalLength * _length)
+ IOPhysicalLength * length)
{
IOPhysicalAddress address;
LOCK;
- address = memory->getPhysicalSegment( offset + _offset, _length );
+ address = memory->getPhysicalSegment( offset + _offset, length );
UNLOCK;
return( address );
@@ -2633,24 +2264,24 @@
IOVirtualAddress mapAddress,
IOOptionBits options )
{
- _IOMemoryMap * newMap;
-
- newMap = new _IOMemoryMap;
+ _IOMemoryMap * map;
+
+ map = new _IOMemoryMap;
LOCK;
- if( newMap
- && !newMap->initWithDescriptor( this, intoTask, mapAddress,
+ if( map
+ && !map->initWithDescriptor( this, intoTask, mapAddress,
options | kIOMapStatic, 0, getLength() )) {
- newMap->release();
- newMap = 0;
- }
-
- addMapping( newMap);
+ map->release();
+ map = 0;
+ }
+
+ addMapping( map);
UNLOCK;
- return( newMap);
+ return( map);
}
IOMemoryMap * IOMemoryDescriptor::map(
@@ -2673,69 +2304,6 @@
length = getLength();
return( makeMapping( this, intoTask, toAddress, options, offset, length ));
-}
-
-IOReturn _IOMemoryMap::redirect(IOMemoryDescriptor * newBackingMemory,
- IOOptionBits options,
- IOByteCount offset)
-{
- IOReturn err = kIOReturnSuccess;
- IOMemoryDescriptor * physMem = 0;
-
- LOCK;
-
- if (logical && addressMap) do
- {
- if ((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
- {
- physMem = memory;
- physMem->retain();
- }
-
- if (!redirUPL)
- {
- vm_size_t size = length;
- int flags = UPL_COPYOUT_FROM | UPL_SET_INTERNAL
- | UPL_SET_LITE | UPL_SET_IO_WIRE | UPL_BLOCK_ACCESS;
- if (KERN_SUCCESS != memory_object_iopl_request((ipc_port_t) memory->_memEntry, 0, &size, &redirUPL,
- NULL, NULL,
- &flags))
- redirUPL = 0;
-
- if (physMem)
- {
- IOUnmapPages( addressMap, logical, length );
- physMem->redirect(0, true);
- }
- }
-
- if (newBackingMemory)
- {
- if (newBackingMemory != memory)
- {
- if (this != newBackingMemory->makeMapping(newBackingMemory, addressTask, (IOVirtualAddress) this,
- options | kIOMapUnique | kIOMapReference,
- offset, length))
- err = kIOReturnError;
- }
- if (redirUPL)
- {
- upl_commit(redirUPL, NULL, 0);
- upl_deallocate(redirUPL);
- redirUPL = 0;
- }
- if (physMem)
- physMem->redirect(0, false);
- }
- }
- while (false);
-
- UNLOCK;
-
- if (physMem)
- physMem->release();
-
- return (err);
}
IOMemoryMap * IOMemoryDescriptor::makeMapping(
@@ -2746,102 +2314,37 @@
IOByteCount offset,
IOByteCount length )
{
- IOMemoryDescriptor * mapDesc = 0;
_IOMemoryMap * mapping = 0;
OSIterator * iter;
LOCK;
- do
- {
- if (kIOMapUnique & options)
- {
- IOPhysicalAddress phys;
- IOByteCount physLen;
-
- if (owner != this)
- continue;
-
- if ((_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
- {
- phys = getPhysicalSegment(offset, &physLen);
- if (!phys || (physLen < length))
- continue;
-
- mapDesc = IOMemoryDescriptor::withPhysicalAddress(
- phys, length, _direction);
- if (!mapDesc)
- continue;
- offset = 0;
- }
- else
- {
- mapDesc = this;
- mapDesc->retain();
- }
-
- if (kIOMapReference & options)
- {
- mapping = (_IOMemoryMap *) toAddress;
- mapping->retain();
-
-#if 1
- uint32_t pageOffset1 = mapDesc->getSourceSegment( offset, NULL );
- pageOffset1 -= trunc_page_32( pageOffset1 );
-
- uint32_t pageOffset2 = mapping->getVirtualAddress();
- pageOffset2 -= trunc_page_32( pageOffset2 );
-
- if (pageOffset1 != pageOffset2)
- IOLog("::redirect can't map offset %x to addr %x\n",
- pageOffset1, mapping->getVirtualAddress());
-#endif
-
-
- if (!mapping->initWithDescriptor( mapDesc, intoTask, toAddress, options,
- offset, length ))
- {
-#ifdef DEBUG
- IOLog("Didn't redirect map %08lx : %08lx\n", offset, length );
-#endif
- }
-
- if (mapping->owner)
- mapping->owner->removeMapping(mapping);
- continue;
- }
- }
- else
- {
- // look for an existing mapping
- if( (iter = OSCollectionIterator::withCollection( _mappings))) {
-
- while( (mapping = (_IOMemoryMap *) iter->getNextObject())) {
-
- if( (mapping = mapping->copyCompatible(
- owner, intoTask, toAddress,
- options | kIOMapReference,
- offset, length )))
- break;
- }
- iter->release();
- }
-
-
- if (mapping)
- mapping->retain();
-
- if( mapping || (options & kIOMapReference))
- continue;
-
- mapDesc = owner;
- mapDesc->retain();
- }
+ do {
+ // look for an existing mapping
+ if( (iter = OSCollectionIterator::withCollection( _mappings))) {
+
+ while( (mapping = (_IOMemoryMap *) iter->getNextObject())) {
+
+ if( (mapping = mapping->copyCompatible(
+ owner, intoTask, toAddress,
+ options | kIOMapReference,
+ offset, length )))
+ break;
+ }
+ iter->release();
+ if( mapping)
+ continue;
+ }
+
+
+ if( mapping || (options & kIOMapReference))
+ continue;
+
owner = this;
mapping = new _IOMemoryMap;
if( mapping
- && !mapping->initWithDescriptor( mapDesc, intoTask, toAddress, options,
+ && !mapping->initWithDescriptor( owner, intoTask, toAddress, options,
offset, length )) {
#ifdef DEBUG
IOLog("Didn't make map %08lx : %08lx\n", offset, length );
@@ -2850,22 +2353,11 @@
mapping = 0;
}
- if (mapping)
- mapping->retain();
-
} while( false );
- if (mapping)
- {
- mapping->owner = owner;
- owner->addMapping( mapping);
- mapping->release();
- }
+ owner->addMapping( mapping);
UNLOCK;
-
- if (mapDesc)
- mapDesc->release();
return( mapping);
}
@@ -2967,19 +2459,6 @@
return( address );
}
-
-IOReturn IOSubMemoryDescriptor::doMap(
- vm_map_t addressMap,
- IOVirtualAddress * atAddress,
- IOOptionBits options,
- IOByteCount sourceOffset,
- IOByteCount length )
-{
- if( sourceOffset >= _length)
- return( kIOReturnOverrun );
- return (_parent->doMap(addressMap, atAddress, options, sourceOffset + _start, length));
-}
-
IOPhysicalAddress IOSubMemoryDescriptor::getSourceSegment( IOByteCount offset,
IOByteCount * length )
{
@@ -3042,36 +2521,6 @@
UNLOCK;
return( byteCount );
-}
-
-IOReturn IOSubMemoryDescriptor::setPurgeable( IOOptionBits newState,
- IOOptionBits * oldState )
-{
- IOReturn err;
-
- LOCK;
- err = _parent->setPurgeable( newState, oldState );
- UNLOCK;
-
- return( err );
-}
-
-IOReturn IOSubMemoryDescriptor::performOperation( IOOptionBits options,
- IOByteCount offset, IOByteCount length )
-{
- IOReturn err;
-
- assert(offset <= _length);
-
- if( offset >= _length)
- return( kIOReturnOverrun );
-
- LOCK;
- err = _parent->performOperation( options, _start + offset,
- min(length, _length - offset) );
- UNLOCK;
-
- return( err );
}
IOReturn IOSubMemoryDescriptor::prepare(
@@ -3106,10 +2555,9 @@
IOByteCount offset,
IOByteCount length )
{
- IOMemoryMap * mapping = 0;
-
- if (!(kIOMapUnique & options))
- mapping = (IOMemoryMap *) _parent->makeMapping(
+ IOMemoryMap * mapping;
+
+ mapping = (IOMemoryMap *) _parent->makeMapping(
_parent, intoTask,
toAddress - (_start + offset),
options | kIOMapReference,
@@ -3182,15 +2630,10 @@
{
OSSymbol const *keys[2];
OSObject *values[2];
- struct SerData {
- user_addr_t address;
- user_size_t length;
- } *vcopy;
+ IOVirtualRange *vcopy;
unsigned int index, nRanges;
bool result;
- IOOptionBits type = _flags & kIOMemoryTypeMask;
-
if (s == NULL) return false;
if (s->previouslySerialized(this)) return true;
@@ -3198,7 +2641,7 @@
if (!s->addXMLStartTag(this, "array")) return false;
nRanges = _rangesCount;
- vcopy = (SerData *) IOMalloc(sizeof(SerData) * nRanges);
+ vcopy = (IOVirtualRange *) IOMalloc(sizeof(IOVirtualRange) * nRanges);
if (vcopy == 0) return false;
keys[0] = OSSymbol::withCString("address");
@@ -3213,12 +2656,8 @@
// while the lock is held.
LOCK;
if (nRanges == _rangesCount) {
- Ranges vec = _ranges;
for (index = 0; index < nRanges; index++) {
- user_addr_t addr; IOByteCount len;
- getAddrLenForInd(addr, len, type, vec, index);
- vcopy[index].address = addr;
- vcopy[index].length = len;
+ vcopy[index] = _ranges.v[index];
}
} else {
// The descriptor changed out from under us. Give up.
@@ -3230,15 +2669,12 @@
for (index = 0; index < nRanges; index++)
{
- user_addr_t addr = vcopy[index].address;
- IOByteCount len = (IOByteCount) vcopy[index].length;
- values[0] =
- OSNumber::withNumber(addr, (((UInt64) addr) >> 32)? 64 : 32);
+ values[0] = OSNumber::withNumber(_ranges.v[index].address, sizeof(_ranges.v[index].address) * 8);
if (values[0] == 0) {
result = false;
goto bail;
}
- values[1] = OSNumber::withNumber(len, sizeof(len) * 8);
+ values[1] = OSNumber::withNumber(_ranges.v[index].length, sizeof(_ranges.v[index].length) * 8);
if (values[1] == 0) {
result = false;
goto bail;
@@ -3326,8 +2762,8 @@
OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 0);
OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 1);
OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 2);
-OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 3);
-OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 4);
+OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 3);
+OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 4);
OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 5);
OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 6);
OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 7);