Loading...
--- dyld/dyld-733.6/dyld3/shared-cache/update_dyld_sim_shared_cache.cpp
+++ dyld/dyld-519.2.2/dyld3/shared-cache/update_dyld_sim_shared_cache.cpp
@@ -1,6 +1,6 @@
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*-
*
- * Copyright (c) 2018 Apple Inc. All rights reserved.
+ * Copyright (c) 2016 Apple Inc. All rights reserved.
*
* @APPLE_LICENSE_HEADER_START@
*
@@ -27,7 +27,6 @@
#include <sys/mman.h>
#include <mach/mach.h>
#include <mach/mach_time.h>
-#include <mach-o/dyld.h>
#include <limits.h>
#include <stdarg.h>
#include <stdio.h>
@@ -48,7 +47,6 @@
#include <dscsym.h>
#include <dispatch/dispatch.h>
#include <pthread/pthread.h>
-#include <CoreFoundation/CoreFoundation.h>
#include <algorithm>
#include <vector>
@@ -57,186 +55,166 @@
#include <iostream>
#include <fstream>
+#include "MachOParser.h"
#include "FileUtils.h"
#include "StringUtils.h"
#include "DyldSharedCache.h"
-#include "MachOFile.h"
-#include "MachOAnalyzer.h"
-#include "ClosureFileSystemPhysical.h"
+
+
struct MappedMachOsByCategory
{
- const dyld3::GradedArchs& archs;
+ std::string archName;
std::vector<DyldSharedCache::MappedMachO> dylibsForCache;
std::vector<DyldSharedCache::MappedMachO> otherDylibsAndBundles;
std::vector<DyldSharedCache::MappedMachO> mainExecutables;
};
-static const char* sAllowedPrefixes[] = {
+static const char* sSearchDirs[] = {
+ "/bin",
+ "/sbin",
"/usr",
"/System",
-// don't look at main executables until simulator supports dyld3
-// "/bin",
-// "/sbin",
};
-static const char* sDontUsePrefixes[] = {
+static const char* sSkipDirs[] = {
"/usr/share",
- "/usr/local",
+ "/usr/local/include",
};
-static const char* sMacOSHostLibs[] = {
+
+static const char* sMacOsAdditions[] = {
"/usr/lib/system/libsystem_kernel.dylib",
"/usr/lib/system/libsystem_platform.dylib",
"/usr/lib/system/libsystem_pthread.dylib",
};
-static const char* sMacOSBinaries[] = {
- "/sbin/launchd_sim_trampoline",
- "/usr/sbin/iokitsimd",
- "/usr/lib/system/host/liblaunch_sim.dylib",
-};
static bool verbose = false;
-
-static bool addIfMachO(const dyld3::closure::FileSystem& fileSystem, const std::string& runtimePath, const struct stat& statBuf, std::vector<MappedMachOsByCategory>& files, dyld3::Platform platform)
-{
- // don't precompute closure info for any debug or profile dylibs
- if ( endsWith(runtimePath, "_profile.dylib") || endsWith(runtimePath, "_debug.dylib") || endsWith(runtimePath, "_asan.dylib") || endsWith(runtimePath, "_profile") || endsWith(runtimePath, "_debug") )
+static bool addIfMachO(const std::string& simRuntimeRootPath, const std::string& runtimePath, const struct stat& statBuf, dyld3::Platform platform, std::vector<MappedMachOsByCategory>& files)
+{
+ // read start of file to determine if it is mach-o or a fat file
+ std::string fullPath = simRuntimeRootPath + runtimePath;
+ int fd = ::open(fullPath.c_str(), O_RDONLY);
+ if ( fd < 0 )
return false;
- if ( startsWith(runtimePath, "/usr/lib/system/introspection/") )
- return false;
-
bool result = false;
- for (MappedMachOsByCategory& file : files) {
+ const void* wholeFile = ::mmap(NULL, statBuf.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
+ if ( wholeFile != MAP_FAILED ) {
Diagnostics diag;
- char realerPath[MAXPATHLEN];
- dyld3::closure::LoadedFileInfo loadedFileInfo = dyld3::MachOAnalyzer::load(diag, fileSystem, runtimePath.c_str(), file.archs, platform, realerPath);
- const dyld3::MachOAnalyzer* ma = (const dyld3::MachOAnalyzer*)loadedFileInfo.fileContent;
-
- if ( ma != nullptr ) {
- bool sipProtected = false; // isProtectedBySIP(fd);
- bool issetuid = false;
- const uint64_t sliceLen = loadedFileInfo.sliceLen;
- if ( ma->isDynamicExecutable() ) {
- #if 0 // dyld3 not enabled for simulator yet, so don't collect main executables
- issetuid = (statBuf.st_mode & (S_ISUID|S_ISGID));
- file.mainExecutables.emplace_back(runtimePath, ma, sliceLen, issetuid, sipProtected, loadedFileInfo.sliceOffset, statBuf.st_mtime, statBuf.st_ino);
- result = true;
- #endif
- }
- else if ( ma->canBePlacedInDyldCache(runtimePath.c_str(), ^(const char* msg) {}) ) {
- file.dylibsForCache.emplace_back(runtimePath, ma, sliceLen, issetuid, sipProtected, loadedFileInfo.sliceOffset, statBuf.st_mtime, statBuf.st_ino);
- result = true;
- }
- }
- }
-
+ bool usedWholeFile = false;
+ for (MappedMachOsByCategory& file : files) {
+ size_t sliceOffset;
+ size_t sliceLength;
+ bool fatButMissingSlice;
+ const void* slice = MAP_FAILED;
+ if ( dyld3::FatUtil::isFatFileWithSlice(diag, wholeFile, statBuf.st_size, file.archName, sliceOffset, sliceLength, fatButMissingSlice) ) {
+ slice = ::mmap(NULL, sliceLength, PROT_READ, MAP_PRIVATE, fd, sliceOffset);
+ if ( slice != MAP_FAILED ) {
+ //fprintf(stderr, "mapped slice at %p size=0x%0lX, offset=0x%0lX for %s\n", p, len, offset, fullPath.c_str());
+ if ( !dyld3::MachOParser::isValidMachO(diag, file.archName, platform, slice, sliceLength, fullPath.c_str(), false) ) {
+ ::munmap((void*)slice, sliceLength);
+ slice = MAP_FAILED;
+ }
+ }
+ }
+ else if ( !fatButMissingSlice && dyld3::MachOParser::isValidMachO(diag, file.archName, platform, wholeFile, statBuf.st_size, fullPath.c_str(), false) ) {
+ slice = wholeFile;
+ sliceLength = statBuf.st_size;
+ sliceOffset = 0;
+ usedWholeFile = true;
+ //fprintf(stderr, "mapped whole file at %p size=0x%0lX for %s\n", p, len, inputPath.c_str());
+ }
+ if ( slice != MAP_FAILED ) {
+ const mach_header* mh = (mach_header*)slice;
+ dyld3::MachOParser parser(mh);
+ if ( parser.platform() != platform ) {
+ fprintf(stderr, "skipped wrong platform binary: %s\n", fullPath.c_str());
+ result = false;
+ }
+ else {
+ bool sip = true; // assume anything found in the simulator runtime is a platform binary
+ if ( parser.isDynamicExecutable() ) {
+ bool issetuid = (statBuf.st_mode & (S_ISUID|S_ISGID));
+ file.mainExecutables.emplace_back(runtimePath, mh, sliceLength, issetuid, sip, sliceOffset, statBuf.st_mtime, statBuf.st_ino);
+ }
+ else {
+ if ( parser.canBePlacedInDyldCache(runtimePath) ) {
+ file.dylibsForCache.emplace_back(runtimePath, mh, sliceLength, false, sip, sliceOffset, statBuf.st_mtime, statBuf.st_ino);
+ }
+ else {
+ file.otherDylibsAndBundles.emplace_back(runtimePath, mh, sliceLength, false, sip, sliceOffset, statBuf.st_mtime, statBuf.st_ino);
+ }
+ }
+ result = true;
+ }
+ }
+ }
+ if ( !usedWholeFile )
+ ::munmap((void*)wholeFile, statBuf.st_size);
+ }
+ ::close(fd);
return result;
}
-static void findAllFiles(const dyld3::closure::FileSystem& fileSystem, const std::vector<std::string>& pathPrefixes, std::vector<MappedMachOsByCategory>& files, dyld3::Platform platform)
+static void findAllFiles(const std::string& simRuntimeRootPath, dyld3::Platform platform, std::vector<MappedMachOsByCategory>& files)
{
std::unordered_set<std::string> skipDirs;
- for (const char* s : sDontUsePrefixes)
+ for (const char* s : sSkipDirs)
skipDirs.insert(s);
-
- __block std::unordered_set<std::string> alreadyUsed;
- bool multiplePrefixes = (pathPrefixes.size() > 1);
- for (const std::string& prefix : pathPrefixes) {
- // get all files from overlay for this search dir
- for (const char* searchDir : sAllowedPrefixes ) {
- iterateDirectoryTree(prefix, searchDir, ^(const std::string& dirPath) { return (skipDirs.count(dirPath) != 0); }, ^(const std::string& path, const struct stat& statBuf) {
- // ignore files that don't have 'x' bit set (all runnable mach-o files do)
- const bool hasXBit = ((statBuf.st_mode & S_IXOTH) == S_IXOTH);
- if ( !hasXBit && !endsWith(path, ".dylib") )
- return;
-
- // ignore files too small (must have at least a page of TEXT and LINKEDIT)
- if ( statBuf.st_size < 0x2000 )
- return;
-
- // don't add paths already found using previous prefix
- if ( multiplePrefixes && (alreadyUsed.count(path) != 0) )
- return;
-
- // don't add binaries built for the host Mac OS platform
- for (std::string s : sMacOSBinaries) {
- if (s.compare(path) == 0)
- return;
- }
- // if the file is mach-o, add to list
- if ( addIfMachO(fileSystem, path, statBuf, files, platform) ) {
- if ( multiplePrefixes )
- alreadyUsed.insert(path);
- }
- });
- }
- }
-}
-
-static void addMacOSHostLibs(std::vector<MappedMachOsByCategory>& allFileSets, dyld3::Platform platform)
-{
- dyld3::closure::FileSystemPhysical fileSystem;
- for (const char* path : sMacOSHostLibs) {
+
+ for (const char* searchDir : sSearchDirs ) {
+ iterateDirectoryTree(simRuntimeRootPath, searchDir, ^(const std::string& dirPath) { return (skipDirs.count(dirPath) != 0); }, ^(const std::string& path, const struct stat& statBuf) {
+ // ignore files that don't have 'x' bit set (all runnable mach-o files do)
+ const bool hasXBit = ((statBuf.st_mode & S_IXOTH) == S_IXOTH);
+ if ( !hasXBit && !endsWith(path, ".dylib") )
+ return;
+
+ // ignore files too small
+ if ( statBuf.st_size < 0x3000 )
+ return;
+
+ // if the file is mach-o add to list
+ addIfMachO(simRuntimeRootPath, path, statBuf, platform, files);
+ });
+ }
+}
+
+static void addMacOSAdditions(std::vector<MappedMachOsByCategory>& allFileSets)
+{
+ for (const char* addPath : sMacOsAdditions) {
struct stat statBuf;
- if ( stat(path, &statBuf) == 0 ) {
- addIfMachO(fileSystem, path, statBuf, allFileSets, dyld3::Platform::macOS);
- }
- }
-}
-
-static void addMacOSBinaries(const dyld3::closure::FileSystem& fileSystem, const std::vector<std::string>& pathPrefixes, std::vector<MappedMachOsByCategory>& files)
-{
- for (const std::string& prefix : pathPrefixes) {
- for (std::string path : sMacOSBinaries) {
- std::string fullPath = prefix + path;
- struct stat statBuf;
- if ( stat(fullPath.c_str(), &statBuf) == 0 ) {
- addIfMachO(fileSystem, path, statBuf, files, dyld3::Platform::macOS);
- }
- }
- }
-}
+ if ( stat(addPath, &statBuf) == 0 )
+ addIfMachO("", addPath, statBuf, dyld3::Platform::macOS, allFileSets);
+ }
+}
+
static bool dontCache(const std::string& simRuntimeRootPath, const std::string& archName,
const std::unordered_set<std::string>& pathsWithDuplicateInstallName,
- const DyldSharedCache::MappedMachO& aFile, bool warn,
- const std::unordered_set<std::string>& skipDylibs)
-{
- if ( skipDylibs.count(aFile.runtimePath) )
- return true;
+ const DyldSharedCache::MappedMachO& aFile, bool warn)
+{
if ( startsWith(aFile.runtimePath, "/usr/lib/system/introspection/") )
return true;
if ( startsWith(aFile.runtimePath, "/usr/local/") )
return true;
-
- // anything inside a .app bundle is specific to app, so should not be in shared cache
+
+ // anything inside a .app bundle is specific to app, so should be in shared cache
if ( aFile.runtimePath.find(".app/") != std::string::npos )
return true;
-
- if ( archName == "i386" ) {
- if ( startsWith(aFile.runtimePath, "/System/Library/CoreServices/") )
- return true;
- if ( startsWith(aFile.runtimePath, "/System/Library/Extensions/") )
- return true;
- }
-
+
if ( aFile.runtimePath.find("//") != std::string::npos ) {
- if (warn) fprintf(stderr, "update_dyld_sim_shared_cache: warning: %s use of bad install name %s\n", archName.c_str(), aFile.runtimePath.c_str());
+ if (warn) fprintf(stderr, "update_dyld_sim_shared_cache: warning: %s double-slash in install name %s\n", archName.c_str(), aFile.runtimePath.c_str());
+ }
+
+ dyld3::MachOParser parser(aFile.mh);
+ const char* installName = parser.installName();
+ if ( (pathsWithDuplicateInstallName.count(aFile.runtimePath) != 0) && (aFile.runtimePath != installName) ) {
+ if (warn) fprintf(stderr, "update_dyld_sim_shared_cache: warning: %s skipping because of duplicate install name %s\n", archName.c_str(), aFile.runtimePath.c_str());
return true;
}
-
- const char* installName = aFile.mh->installName();
- if ( (pathsWithDuplicateInstallName.count(aFile.runtimePath) != 0) && (aFile.runtimePath != installName) ) {
- // <rdar://problem/46431467> if a dylib moves and a symlink is installed into its place, bom iterator will see both and issue a warning
- struct stat statBuf;
- bool isSymLink = ( (lstat(aFile.runtimePath.c_str(), &statBuf) == 0) && S_ISLNK(statBuf.st_mode) );
- if (!isSymLink && warn) fprintf(stderr, "update_dyld_sim_shared_cache: warning: %s skipping because of duplicate install name %s\n", archName.c_str(), aFile.runtimePath.c_str());
- return true;
- }
-
+
if ( aFile.runtimePath != installName ) {
// see if install name is a symlink to actual path
std::string fullInstall = simRuntimeRootPath + installName;
@@ -250,30 +228,22 @@
return false;
}
}
- // <rdar://problem/38000411> also if runtime path is a symlink to install name
- std::string fullRuntime = simRuntimeRootPath + aFile.runtimePath;
- if ( realpath(fullRuntime.c_str(), resolvedPath) != NULL ) {
- std::string resolvedSymlink = resolvedPath;
- if ( !simRuntimeRootPath.empty() ) {
- resolvedSymlink = resolvedSymlink.substr(simRuntimeRootPath.size());
- }
- if ( resolvedSymlink == installName ) {
- return false;
- }
- }
if (warn) fprintf(stderr, "update_dyld_sim_shared_cache: warning: %s skipping because of bad install name %s\n", archName.c_str(), aFile.runtimePath.c_str());
return true;
}
+
return false;
}
-static void pruneCachedDylibs(const std::string& volumePrefix, const std::unordered_set<std::string>& skipDylibs, MappedMachOsByCategory& fileSet)
+static void pruneCachedDylibs(const std::string& simRuntimeRootPath, MappedMachOsByCategory& fileSet)
{
std::unordered_set<std::string> pathsWithDuplicateInstallName;
-
+
std::unordered_map<std::string, std::string> installNameToFirstPath;
for (DyldSharedCache::MappedMachO& aFile : fileSet.dylibsForCache) {
- const char* installName = aFile.mh->installName();
+ //fprintf(stderr, "dylib: %s\n", aFile.runtimePath.c_str());
+ dyld3::MachOParser parser(aFile.mh);
+ const char* installName = parser.installName();
auto pos = installNameToFirstPath.find(installName);
if ( pos == installNameToFirstPath.end() ) {
installNameToFirstPath[installName] = aFile.runtimePath;
@@ -283,27 +253,15 @@
pathsWithDuplicateInstallName.insert(installNameToFirstPath[installName]);
}
}
-
+
for (DyldSharedCache::MappedMachO& aFile : fileSet.dylibsForCache) {
- if ( dontCache(volumePrefix, fileSet.archs.name(), pathsWithDuplicateInstallName, aFile, true, skipDylibs) ){
- // <rdar://problem/46423929> don't build dlopen closures for symlinks to something in the dyld cache
- if ( pathsWithDuplicateInstallName.count(aFile.runtimePath) == 0 )
- fileSet.otherDylibsAndBundles.push_back(aFile);
- }
- }
+ if ( dontCache(simRuntimeRootPath, fileSet.archName, pathsWithDuplicateInstallName, aFile, true) )
+ fileSet.otherDylibsAndBundles.push_back(aFile);
+ }
fileSet.dylibsForCache.erase(std::remove_if(fileSet.dylibsForCache.begin(), fileSet.dylibsForCache.end(),
- [&](const DyldSharedCache::MappedMachO& aFile) { return dontCache(volumePrefix, fileSet.archs.name(), pathsWithDuplicateInstallName, aFile, false, skipDylibs); }),
- fileSet.dylibsForCache.end());
-}
-
-static void pruneOtherDylibs(const std::string& volumePrefix, MappedMachOsByCategory& fileSet)
-{
- // other OS dylibs should not contain dylibs that are embedded in some .app bundle
- fileSet.otherDylibsAndBundles.erase(std::remove_if(fileSet.otherDylibsAndBundles.begin(), fileSet.otherDylibsAndBundles.end(),
- [&](const DyldSharedCache::MappedMachO& aFile) { return (aFile.runtimePath.find(".app/") != std::string::npos); }),
- fileSet.otherDylibsAndBundles.end());
-}
-
+ [&](const DyldSharedCache::MappedMachO& aFile) { return dontCache(simRuntimeRootPath, fileSet.archName, pathsWithDuplicateInstallName, aFile, false); }),
+ fileSet.dylibsForCache.end());
+}
static bool existingCacheUpToDate(const std::string& existingCache, const std::vector<DyldSharedCache::MappedMachO>& currentDylibs)
{
@@ -311,22 +269,17 @@
int fd = ::open(existingCache.c_str(), O_RDONLY);
if ( fd < 0 )
return false;
- struct stat statbuf;
- if ( ::fstat(fd, &statbuf) == -1 ) {
- ::close(fd);
- return false;
- }
-
+
// build map of found dylibs
std::unordered_map<std::string, const DyldSharedCache::MappedMachO*> currentDylibMap;
for (const DyldSharedCache::MappedMachO& aFile : currentDylibs) {
//fprintf(stderr, "0x%0llX 0x%0llX %s\n", aFile.inode, aFile.modTime, aFile.runtimePath.c_str());
currentDylibMap[aFile.runtimePath] = &aFile;
}
-
+
// make sure all dylibs in existing cache have same mtime and inode as found dylib
__block bool foundMismatch = false;
- const uint64_t cacheMapLen = statbuf.st_size;
+ const uint64_t cacheMapLen = 0x40000000;
void *p = ::mmap(NULL, cacheMapLen, PROT_READ, MAP_PRIVATE, fd, 0);
if ( p != MAP_FAILED ) {
const DyldSharedCache* cache = (DyldSharedCache*)p;
@@ -378,17 +331,16 @@
} \
} while ( 0 )
-int main(int argc, const char* argv[], const char* envp[])
+int main(int argc, const char* argv[])
{
std::string rootPath;
+ std::string dylibListFile;
bool force = false;
- bool dylibsRemoved = false;
std::string cacheDir;
- std::string dylibOrderFile;
- std::string dirtyDataOrderFile;
- dyld3::Platform platform = dyld3::Platform::iOS_simulator;
- std::unordered_set<std::string> skipDylibs;
-
+ std::unordered_set<std::string> archStrs;
+
+ dyld3::Platform platform = dyld3::Platform::iOS;
+
// parse command line options
for (int i = 1; i < argc; ++i) {
const char* arg = argv[i];
@@ -398,40 +350,29 @@
else if (strcmp(arg, "-verbose") == 0) {
verbose = true;
}
- else if ((strcmp(arg, "-root") == 0) || (strcmp(arg, "--root") == 0)) {
- TERMINATE_IF_LAST_ARG("-root missing path argument\n");
+ else if (strcmp(arg, "-tvOS") == 0) {
+ platform = dyld3::Platform::tvOS;
+ }
+ else if (strcmp(arg, "-iOS") == 0) {
+ platform = dyld3::Platform::iOS;
+ }
+ else if (strcmp(arg, "-watchOS") == 0) {
+ platform = dyld3::Platform::watchOS;
+ }
+ else if ( strcmp(arg, "-runtime_dir") == 0 ) {
+ TERMINATE_IF_LAST_ARG("-runtime_dir missing path argument\n");
rootPath = argv[++i];
}
else if (strcmp(arg, "-cache_dir") == 0) {
TERMINATE_IF_LAST_ARG("-cache_dir missing path argument\n");
cacheDir = argv[++i];
}
- else if (strcmp(arg, "-iOS") == 0) {
- platform = dyld3::Platform::iOS_simulator;
- }
- else if (strcmp(arg, "-watchOS") == 0) {
- platform = dyld3::Platform::watchOS_simulator;
- }
- else if (strcmp(arg, "-tvOS") == 0) {
- platform = dyld3::Platform::tvOS_simulator;
- }
- else if (strcmp(arg, "-dylibs_removed_in_mastering") == 0) {
- dylibsRemoved = true;
- }
- else if (strcmp(arg, "-dylib_order_file") == 0) {
- TERMINATE_IF_LAST_ARG("-dylib_order_file missing path argument\n");
- dylibOrderFile = argv[++i];
- }
- else if (strcmp(arg, "-dirty_data_order_file") == 0) {
- TERMINATE_IF_LAST_ARG("-dirty_data_order_file missing path argument\n");
- dirtyDataOrderFile = argv[++i];
+ else if (strcmp(arg, "-arch") == 0) {
+ TERMINATE_IF_LAST_ARG("-arch missing argument\n");
+ archStrs.insert(argv[++i]);
}
else if (strcmp(arg, "-force") == 0) {
force = true;
- }
- else if (strcmp(arg, "-skip") == 0) {
- TERMINATE_IF_LAST_ARG("-skip missing argument\n");
- skipDylibs.insert(argv[++i]);
}
else {
//usage();
@@ -439,138 +380,98 @@
return 1;
}
}
-
- if ( rootPath.empty()) {
- fprintf(stderr, "-root should be specified\n");
+
+ if ( cacheDir.empty() ) {
+ fprintf(stderr, "missing -cache_dir <path> option to specify directory in which to write cache file(s)\n");
return 1;
}
- if (cacheDir.empty()) {
- fprintf(stderr, "-cache_dir should be specified\n");
+
+ if ( rootPath.empty() ) {
+ fprintf(stderr, "missing -runtime_dir <path> option to specify directory which is root of simulator runtime)\n");
return 1;
}
- // canonicalize rootPath
- char resolvedPath[PATH_MAX];
- if ( realpath(rootPath.c_str(), resolvedPath) != NULL ) {
- rootPath = resolvedPath;
- }
-
- // Find the boot volume so that we can ensure all overlays are on the same volume
- struct stat rootStatBuf;
- if ( stat(rootPath.c_str(), &rootStatBuf) != 0 ) {
- fprintf(stderr, "update_dyld_sim_shared_cache: error: could not stat root file system because '%s'\n", strerror(errno));
- return 1;
- }
-
- std::vector<std::string> pathPrefixes;
-
- pathPrefixes.push_back(rootPath);
-
- // build FileSystem object
- const char* fsRoot = rootPath.empty() ? nullptr : rootPath.c_str();
- dyld3::closure::FileSystemPhysical fileSystem(fsRoot, nullptr);
-
-
+ else {
+ // canonicalize rootPath
+ char resolvedPath[PATH_MAX];
+ if ( realpath(rootPath.c_str(), resolvedPath) != NULL ) {
+ rootPath = resolvedPath;
+ }
+ }
+
int err = mkpath_np(cacheDir.c_str(), S_IRWXU | S_IRGRP|S_IXGRP | S_IROTH|S_IXOTH);
if ( (err != 0) && (err != EEXIST) ) {
- fprintf(stderr, "update_dyld_sim_shared_cache: could not access cache dir: mkpath_np(%s) failed errno=%d\n", cacheDir.c_str(), err);
+ fprintf(stderr, "mkpath_np fail: %d", err);
return 1;
}
+ if ( archStrs.empty() ) {
+ switch ( platform ) {
+ case dyld3::Platform::iOS:
+ archStrs.insert("x86_64");
+ break;
+ case dyld3::Platform::tvOS:
+ archStrs.insert("x86_64");
+ break;
+ case dyld3::Platform::watchOS:
+ archStrs.insert("i386");
+ break;
+ case dyld3::Platform::unknown:
+ case dyld3::Platform::macOS:
+ assert(0 && "macOS does not have a simulator");
+ break;
+ case dyld3::Platform::bridgeOS:
+ assert(0 && "bridgeOS does not have a simulator");
+ break;
+ }
+ }
+
uint64_t t1 = mach_absolute_time();
+ // find all mach-o files for requested architectures
__block std::vector<MappedMachOsByCategory> allFileSets;
- switch ( platform ) {
- case dyld3::Platform::iOS_simulator:
- allFileSets.push_back({dyld3::GradedArchs::x86_64});
- break;
- case dyld3::Platform::watchOS_simulator:
- allFileSets.push_back({dyld3::GradedArchs::i386});
- break;
- case dyld3::Platform::tvOS_simulator:
- allFileSets.push_back({dyld3::GradedArchs::x86_64});
- break;
- default:
- assert(0 && "invalid platform");
- break;
- }
- findAllFiles(fileSystem, pathPrefixes, allFileSets, platform);
- addMacOSHostLibs(allFileSets, platform);
- addMacOSBinaries(fileSystem, pathPrefixes, allFileSets);
-
- // nothing in OS uses i386 dylibs, so only dylibs used by third party apps need to be in cache
+ if ( archStrs.count("x86_64") )
+ allFileSets.push_back({"x86_64"});
+ if ( archStrs.count("i386") )
+ allFileSets.push_back({"i386"});
+ findAllFiles(rootPath, platform, allFileSets);
+ addMacOSAdditions(allFileSets);
for (MappedMachOsByCategory& fileSet : allFileSets) {
- pruneCachedDylibs(rootPath, skipDylibs, fileSet);
- pruneOtherDylibs(rootPath, fileSet);
+ pruneCachedDylibs(rootPath, fileSet);
}
uint64_t t2 = mach_absolute_time();
- if ( verbose ) {
- fprintf(stderr, "time to scan file system and construct lists of mach-o files: %ums\n", absolutetime_to_milliseconds(t2-t1));
- }
-
- // build caches in parallel on machines with at leat 4GB of RAM
- uint64_t memSize = 0;
- size_t sz = sizeof(memSize);;
- bool buildInParallel = false;
- if ( sysctlbyname("hw.memsize", &memSize, &sz, NULL, 0) == 0 ) {
- if ( memSize >= 0x100000000ULL )
- buildInParallel = true;
- }
- dispatch_queue_t dqueue = buildInParallel ? dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0)
- : dispatch_queue_create("serial-queue", DISPATCH_QUEUE_SERIAL);
-
- // build all caches
+
+ fprintf(stderr, "time to scan file system and construct lists of mach-o files: %ums\n", absolutetime_to_milliseconds(t2-t1));
+
+ // build all caches in parallel
__block bool cacheBuildFailure = false;
- __block bool wroteSomeCacheFile = false;
- dispatch_apply(allFileSets.size(), dqueue, ^(size_t index) {
+ dispatch_apply(allFileSets.size(), dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^(size_t index) {
MappedMachOsByCategory& fileSet = allFileSets[index];
- const std::string outFile = cacheDir + "/dyld_sim_shared_cache_" + fileSet.archs.name();
-
+ const std::string outFile = cacheDir + "/dyld_shared_cache_" + fileSet.archName;
+ __block std::unordered_set<std::string> knownMissingDylib;
+
DyldSharedCache::MappedMachO (^loader)(const std::string&) = ^DyldSharedCache::MappedMachO(const std::string& runtimePath) {
- if ( skipDylibs.count(runtimePath) )
- return DyldSharedCache::MappedMachO();
-
- for (const std::string& prefix : pathPrefixes) {
- std::string fullPath = prefix + runtimePath;
- struct stat statBuf;
- if ( stat(fullPath.c_str(), &statBuf) == 0 ) {
- char truePath[PATH_MAX];
- if ( realpath(fullPath.c_str(), truePath) != NULL ) {
- std::string resolvedSymlink = truePath;
- if ( !rootPath.empty() ) {
- resolvedSymlink = resolvedSymlink.substr(rootPath.size());
- }
- if ( (runtimePath != resolvedSymlink) && !contains(runtimePath, "InputContext") ) { //HACK remove InputContext when fixed
- // path requested is a symlink path, check if real path already loaded
- for (const DyldSharedCache::MappedMachO& aDylibMapping : fileSet.dylibsForCache) {
- if ( aDylibMapping.runtimePath == resolvedSymlink ) {
- if ( verbose )
- fprintf(stderr, "verifySelfContained, redirect %s to %s\n", runtimePath.c_str(), aDylibMapping.runtimePath.c_str());
- return aDylibMapping;
- }
- }
- }
- }
-
- std::vector<MappedMachOsByCategory> mappedFiles;
- mappedFiles.push_back({fileSet.archs});
- if ( addIfMachO(fileSystem, runtimePath, statBuf, mappedFiles, platform) ) {
- if ( !mappedFiles.back().dylibsForCache.empty() ) {
- if ( verbose )
- fprintf(stderr, "verifySelfContained, add %s\n", mappedFiles.back().dylibsForCache.back().runtimePath.c_str());
- return mappedFiles.back().dylibsForCache.back();
- }
- }
- }
+ std::string fullPath = rootPath + runtimePath;
+ struct stat statBuf;
+ if ( stat(fullPath.c_str(), &statBuf) == 0 ) {
+ std::vector<MappedMachOsByCategory> mappedFiles;
+ mappedFiles.push_back({fileSet.archName});
+ if ( addIfMachO(rootPath, runtimePath, statBuf, platform, mappedFiles) ) {
+ if ( !mappedFiles.back().dylibsForCache.empty() )
+ return mappedFiles.back().dylibsForCache.back();
+ }
+ }
+ if ( knownMissingDylib.count(runtimePath) == 0 ) {
+ fprintf(stderr, "update_dyld_sim_shared_cache: warning: %s could not use in dylid cache: %s\n", fileSet.archName.c_str(), runtimePath.c_str());
+ knownMissingDylib.insert(runtimePath);
}
return DyldSharedCache::MappedMachO();
};
size_t startCount = fileSet.dylibsForCache.size();
std::vector<std::pair<DyldSharedCache::MappedMachO, std::set<std::string>>> excludes;
- std::unordered_set<std::string> badZippered;
- DyldSharedCache::verifySelfContained(fileSet.dylibsForCache, badZippered, loader, excludes);
+ DyldSharedCache::verifySelfContained(fileSet.dylibsForCache, loader, excludes);
for (size_t i=startCount; i < fileSet.dylibsForCache.size(); ++i) {
- fprintf(stderr, "update_dyld_sim_shared_cache: warning: %s not in initial scan, but adding required dylib %s\n", fileSet.archs.name(), fileSet.dylibsForCache[i].runtimePath.c_str());
+ fprintf(stderr, "update_dyld_sim_shared_cache: warning: %s not found in initial scan, but adding required dylib %s\n", fileSet.archName.c_str(), fileSet.dylibsForCache[i].runtimePath.c_str());
}
for (auto& exclude : excludes) {
std::string reasons = "(\"";
@@ -581,51 +482,58 @@
}
}
reasons += "\")";
- fprintf(stderr, "update_dyld_shared_cache: warning: %s rejected from cached dylibs: %s (%s)\n", fileSet.archs.name(), exclude.first.runtimePath.c_str(), reasons.c_str());
+ fprintf(stderr, "update_dyld_shared_cache: warning: %s rejected from cached dylibs: %s (%s)\n", fileSet.archName.c_str(), exclude.first.runtimePath.c_str(), reasons.c_str());
fileSet.otherDylibsAndBundles.push_back(exclude.first);
}
-
+
// check if cache is already up to date
if ( !force ) {
if ( existingCacheUpToDate(outFile, fileSet.dylibsForCache) )
return;
}
-
-
+ fprintf(stderr, "make %s cache with %lu dylibs, %lu other dylibs, %lu programs\n", fileSet.archName.c_str(), fileSet.dylibsForCache.size(), fileSet.otherDylibsAndBundles.size(), fileSet.mainExecutables.size());
+
// build cache new cache file
DyldSharedCache::CreateOptions options;
- options.outputFilePath = outFile;
- options.outputMapFilePath = cacheDir + "/dyld_sim_shared_cache_" + fileSet.archs.name() + ".map";
- options.archs = &fileSet.archs;
+ options.archName = fileSet.archName;
options.platform = platform;
options.excludeLocalSymbols = false;
options.optimizeStubs = false;
options.optimizeObjC = true;
options.codeSigningDigestMode = DyldSharedCache::SHA256only;
- options.dylibsRemovedDuringMastering = dylibsRemoved;
+ options.dylibsRemovedDuringMastering = false;
options.inodesAreSameAsRuntime = true;
options.cacheSupportsASLR = false;
options.forSimulator = true;
- options.isLocallyBuiltCache = true;
options.verbose = verbose;
options.evictLeafDylibsOnOverflow = true;
- options.dylibOrdering = parseOrderFile(dylibOrderFile);
- options.dirtyDataSegmentOrdering = parseOrderFile(dirtyDataOrderFile);
- DyldSharedCache::CreateResults results = DyldSharedCache::create(options, fileSystem, fileSet.dylibsForCache, fileSet.otherDylibsAndBundles, fileSet.mainExecutables);
-
+ options.pathPrefixes = { rootPath };
+ DyldSharedCache::CreateResults results = DyldSharedCache::create(options, fileSet.dylibsForCache, fileSet.otherDylibsAndBundles, fileSet.mainExecutables);
+
// print any warnings
for (const std::string& warn : results.warnings) {
- fprintf(stderr, "update_dyld_sim_shared_cache: warning: %s %s\n", fileSet.archs.name(), warn.c_str());
- }
- if ( results.errorMessage.empty() ) {
- wroteSomeCacheFile = true;
- }
- else {
+ fprintf(stderr, "update_dyld_sim_shared_cache: warning: %s %s\n", fileSet.archName.c_str(), warn.c_str());
+ }
+ if ( !results.errorMessage.empty() ) {
+ // print error (if one)
fprintf(stderr, "update_dyld_sim_shared_cache: %s\n", results.errorMessage.c_str());
cacheBuildFailure = true;
}
+ else {
+ // save new cache file to disk and write new .map file
+ assert(results.cacheContent != nullptr);
+ if ( !safeSave(results.cacheContent, results.cacheLength, outFile) )
+ cacheBuildFailure = true;
+ if ( !cacheBuildFailure ) {
+ std::string mapStr = results.cacheContent->mapFile();
+ std::string outFileMap = cacheDir + "/dyld_shared_cache_" + fileSet.archName + ".map";
+ safeSave(mapStr.c_str(), mapStr.size(), outFileMap);
+ }
+ // free created cache buffer
+ vm_deallocate(mach_task_self(), (vm_address_t)results.cacheContent, results.cacheLength);
+ }
});
-
+
// we could unmap all input files, but tool is about to quit
return (cacheBuildFailure ? 1 : 0);