Loading...
--- dyld/dyld-1340/cache_builder/dyld_shared_cache_builder.mm
+++ /dev/null
@@ -1,1412 +0,0 @@
-/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*-
- *
- * Copyright (c) 2016 Apple Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- *
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
- *
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
- * limitations under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
- */
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/mman.h>
-#include <sys/resource.h>
-#include <mach/mach.h>
-#include <mach/mach_time.h>
-#include <limits.h>
-#include <stdarg.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <math.h>
-#include <fcntl.h>
-#include <dlfcn.h>
-#include <signal.h>
-#include <errno.h>
-#include <sysexits.h>
-#include <sys/uio.h>
-#include <unistd.h>
-#include <sys/param.h>
-#include <sys/sysctl.h>
-#include <sys/resource.h>
-#include <dirent.h>
-#include <libgen.h>
-#include <pthread.h>
-#include <fts.h>
-
-#include <vector>
-#include <array>
-#include <list>
-#include <set>
-#include <map>
-#include <unordered_set>
-#include <algorithm>
-#include <fstream>
-#include <regex>
-
-#include <spawn.h>
-
-#include <Bom/Bom.h>
-#include <Foundation/NSData.h>
-#include <Foundation/NSDictionary.h>
-#include <Foundation/NSPropertyList.h>
-#include <Foundation/NSString.h>
-
-#include "Defines.h"
-#include "Diagnostics.h"
-#include "DyldSharedCache.h"
-#include "FileUtils.h"
-#include "JSONReader.h"
-#include "JSONWriter.h"
-#include "Platform.h"
-#include "StringUtils.h"
-#include "mrm_shared_cache_builder.h"
-
-#if !__has_feature(objc_arc)
-#error The use of libdispatch in this files requires it to be compiled with ARC in order to avoid leaks
-#endif
-
-extern char** environ;
-
-static dispatch_queue_t build_queue;
-
-static int runCommandAndWait(Diagnostics& diags, const char* args[])
-{
- pid_t pid;
- int status;
- int res = posix_spawn(&pid, args[0], nullptr, nullptr, (char**)args, environ);
- if (res != 0)
- diags.error("Failed to spawn %s: %s (%d)", args[0], strerror(res), res);
-
- do {
- res = waitpid(pid, &status, 0);
- } while (res == -1 && errno == EINTR);
- if (res != -1) {
- if (WIFEXITED(status)) {
- res = WEXITSTATUS(status);
- } else {
- res = -1;
- }
- }
-
- return res;
-}
-
-static void processRoots(std::list<std::string>& roots, const char *tempRootsDir)
-{
- std::list<std::string> processedRoots;
- struct stat sb;
- int res = 0;
- const char* args[8];
-
- for (const auto& root : roots) {
- res = stat(root.c_str(), &sb);
-
- if (res == 0 && S_ISDIR(sb.st_mode)) {
- processedRoots.push_back(root);
- continue;
- }
-
- char tempRootDir[MAXPATHLEN];
- strlcpy(tempRootDir, tempRootsDir, MAXPATHLEN);
- strlcat(tempRootDir, "/XXXXXXXX", MAXPATHLEN);
- mkdtemp(tempRootDir);
-
- if (endsWith(root, ".cpio") || endsWith(root, ".cpio.gz") || endsWith(root, ".cpgz") || endsWith(root, ".cpio.bz2") || endsWith(root, ".cpbz2") || endsWith(root, ".pax") || endsWith(root, ".pax.gz") || endsWith(root, ".pgz") || endsWith(root, ".pax.bz2") || endsWith(root, ".pbz2")) {
- args[0] = (char*)"/usr/bin/ditto";
- args[1] = (char*)"-x";
- args[2] = (char*)root.c_str();
- args[3] = tempRootDir;
- args[4] = nullptr;
- } else if (endsWith(root, ".tar")) {
- args[0] = (char*)"/usr/bin/tar";
- args[1] = (char*)"xf";
- args[2] = (char*)root.c_str();
- args[3] = (char*)"-C";
- args[4] = tempRootDir;
- args[5] = nullptr;
- } else if (endsWith(root, ".tar.gz") || endsWith(root, ".tgz")) {
- args[0] = (char*)"/usr/bin/tar";
- args[1] = (char*)"xzf";
- args[2] = (char*)root.c_str();
- args[3] = (char*)"-C";
- args[4] = tempRootDir;
- args[5] = nullptr;
- } else if (endsWith(root, ".tar.bz2")
- || endsWith(root, ".tbz2")
- || endsWith(root, ".tbz")) {
- args[0] = (char*)"/usr/bin/tar";
- args[1] = (char*)"xjf";
- args[2] = (char*)root.c_str();
- args[3] = (char*)"-C";
- args[4] = tempRootDir;
- args[5] = nullptr;
- } else if (endsWith(root, ".zip")) {
- args[0] = (char*)"/usr/bin/ditto";
- args[1] = (char*)"-xk";
- args[2] = (char*)root.c_str();
- args[3] = tempRootDir;
- args[4] = nullptr;
- } else {
- fprintf(stderr, "unknown archive type: %s\n", root.c_str());
- exit(EX_DATAERR);
- }
-
- Diagnostics diags;
- if (res != runCommandAndWait(diags, args)) {
- fprintf(stderr, "could not expand archive %s: %s (%d) because '%s'\n",
- root.c_str(), strerror(res), res,
- diags.hasError() ? diags.errorMessageCStr() : "unknown error");
- exit(EX_DATAERR);
- }
- for (auto& existingRoot : processedRoots) {
- if (existingRoot == tempRootDir)
- continue;
- }
-
- processedRoots.push_back(tempRootDir);
- }
-
- roots = processedRoots;
-}
-
-static void writeRootList(const std::string& dstRoot, const std::list<std::string>& roots)
-{
- if (roots.size() == 0)
- return;
-
- std::string rootFile = dstRoot + "/roots.txt";
- FILE* froots = ::fopen(rootFile.c_str(), "w");
- if (froots == NULL)
- return;
-
- for (auto& root : roots) {
- fprintf(froots, "%s\n", root.c_str());
- }
-
- ::fclose(froots);
-}
-
-struct FilteredCopyOptions {
- Diagnostics* diags = nullptr;
- std::set<std::string>* cachePaths = nullptr;
- std::set<std::string>* dylibsFoundInRoots = nullptr;
-};
-
-static BOMCopierCopyOperation filteredCopyIncludingPaths(BOMCopier copier, const char* path, BOMFSObjType type, off_t size)
-{
- std::string absolutePath = &path[1];
- const FilteredCopyOptions *userData = (const FilteredCopyOptions*)BOMCopierUserData(copier);
-
- // Don't copy from the artifact if the dylib is actally in a -root
- if ( userData->dylibsFoundInRoots->count(absolutePath) != 0 ) {
- userData->diags->verbose("Skipping copying dylib from shared cache artifact as it is in a -root: '%s'\n", absolutePath.c_str());
- return BOMCopierSkipFile;
- }
-
- for (const std::string& cachePath : *userData->cachePaths) {
- if (startsWith(cachePath, absolutePath)) {
- userData->diags->verbose("Copying dylib from shared cache artifact: '%s'\n", absolutePath.c_str());
- return BOMCopierContinue;
- }
- }
- if (userData->cachePaths->count(absolutePath)) {
- userData->diags->verbose("Copying dylib from shared cache artifact: '%s'\n", absolutePath.c_str());
- return BOMCopierContinue;
- }
- return BOMCopierSkipFile;
-}
-
-static Disposition stringToDisposition(Diagnostics& diags, const std::string& str) {
- if (diags.hasError())
- return Unknown;
- if (str == "Unknown")
- return Unknown;
- if (str == "InternalDevelopment")
- return InternalDevelopment;
- if (str == "Customer")
- return Customer;
- if (str == "InternalMinDevelopment")
- return InternalMinDevelopment;
- if (str == "SymbolsCache")
- return SymbolsCache;
- return Unknown;
-}
-
-static Platform stringToPlatform(Diagnostics& diags, const std::string& str) {
- if (diags.hasError())
- return unknown;
- if (str == "unknown")
- return unknown;
- if ( (str == "macOS") || (str == "osx") )
- return macOS;
- if (str == "iOS")
- return iOS;
- if (str == "tvOS")
- return tvOS;
- if (str == "watchOS")
- return watchOS;
- if (str == "bridgeOS")
- return bridgeOS;
- if (str == "iOSMac")
- return iOSMac;
- if (str == "UIKitForMac")
- return iOSMac;
- if (str == "iOS_simulator")
- return iOS_simulator;
- if (str == "tvOS_simulator")
- return tvOS_simulator;
- if (str == "watchOS_simulator")
- return watchOS_simulator;
- if (str == "driverKit")
- return driverKit;
- if (str == "macOSExclaveKit")
- return macOSExclaveKit;
- if (str == "iOSExclaveKit")
- return iOSExclaveKit;
- if ( std::isdigit(str.front()) ) {
- // Also allow platforms to be specified as an integer
- return (Platform)atoi(str.c_str());
- }
- if ( startsWith(str, "platform") ) {
- std::string_view strView = str;
- strView.remove_prefix(8);
- if ( std::isdigit(strView.front()) ) {
- // Also allow platforms to be specified as an integer
- return (Platform)atoi(strView.data());
- }
- }
- return unknown;
-}
-
-static FileFlags stringToFileFlags(Diagnostics& diags, const std::string& str) {
- if (diags.hasError())
- return NoFlags;
- if (str == "NoFlags")
- return NoFlags;
- if (str == "MustBeInCache")
- return MustBeInCache;
- if (str == "ShouldBeExcludedFromCacheIfUnusedLeaf")
- return ShouldBeExcludedFromCacheIfUnusedLeaf;
- if (str == "RequiredClosure")
- return RequiredClosure;
- if (str == "DylibOrderFile")
- return DylibOrderFile;
- if (str == "DirtyDataOrderFile")
- return DirtyDataOrderFile;
- if (str == "ObjCOptimizationsFile")
- return ObjCOptimizationsFile;
- if (str == "SwiftGenericMetadataFile")
- return SwiftGenericMetadataFile;
- if (str == "OptimizationFile")
- return OptimizationFile;
- return NoFlags;
-}
-
-struct SharedCacheBuilderOptions {
- Diagnostics diags;
- std::list<std::string> roots;
- std::string dylibCacheDir;
- std::string artifactDir;
- std::string release;
- bool emitDevCaches = true;
- bool emitCustomerCaches = true;
- bool emitElidedDylibs = true;
- bool listConfigs = false;
- bool copyRoots = false;
- bool debug = false;
- bool debugIMPCaches = false;
- bool debugCacheLayout = false;
- bool useMRM = false;
- bool timePasses = false;
- bool printStats = false;
- bool printRemovedFiles = false;
- bool emitJSONMap = false;
- std::string dstRoot;
- std::string buildAllPath;
- std::string resultPath;
- std::string baselineDifferenceResultPath;
- std::list<std::string> baselineCacheMapPaths;
- std::string baselineCacheMapDirPath;
- bool baselineCopyRoots = false;
- bool printCDHashes = false;
- std::set<std::string> cmdLineArchs;
-};
-
-typedef std::tuple<std::string, std::string, FileFlags, std::string> InputFile;
-
-static void loadMRMFiles(Diagnostics& diags,
- MRMSharedCacheBuilder* sharedCacheBuilder,
- const std::vector<InputFile>& inputFiles,
- std::vector<std::pair<const void*, size_t>>& mappedFiles,
- const std::set<std::string>& baselineCacheFiles) {
-
- for (const InputFile& inputFile : inputFiles) {
- const std::string& buildPath = std::get<0>(inputFile);
- const std::string& runtimePath = std::get<1>(inputFile);
- FileFlags fileFlags = std::get<2>(inputFile);
- const std::string& projectName = std::get<3>(inputFile);
-
- struct stat stat_buf;
- int fd = ::open(buildPath.c_str(), O_RDONLY, 0);
- if (fd == -1) {
- if (baselineCacheFiles.count(runtimePath)) {
- diags.error("can't open file '%s', errno=%d\n", buildPath.c_str(), errno);
- return;
- } else {
- // Don't spam with paths we know will be missing
- if ( buildPath.starts_with("./System/Library/Templates/Data/") )
- continue;
- diags.verbose("can't open file '%s', errno=%d\n", buildPath.c_str(), errno);
- continue;
- }
- }
-
- if (fstat(fd, &stat_buf) == -1) {
- if (baselineCacheFiles.count(runtimePath)) {
- diags.error("can't stat open file '%s', errno=%d\n", buildPath.c_str(), errno);
- ::close(fd);
- return;
- } else {
- diags.verbose("can't stat open file '%s', errno=%d\n", buildPath.c_str(), errno);
- ::close(fd);
- continue;
- }
- }
-
- const void* buffer = mmap(NULL, (size_t)stat_buf.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
- if (buffer == MAP_FAILED) {
- diags.error("mmap() for file at %s failed, errno=%d\n", buildPath.c_str(), errno);
- ::close(fd);
- }
- ::close(fd);
-
- mappedFiles.emplace_back(buffer, (size_t)stat_buf.st_size);
-
- addFile_v2(sharedCacheBuilder, runtimePath.c_str(), (uint8_t*)buffer, (size_t)stat_buf.st_size, fileFlags, projectName.c_str());
- }
-}
-
-static void unloadMRMFiles(std::vector<std::pair<const void*, size_t>>& mappedFiles) {
- for (auto mappedFile : mappedFiles)
- ::munmap((void*)mappedFile.first, mappedFile.second);
-}
-
-static ssize_t write64(int fildes, const void *buf, size_t nbyte)
-{
- unsigned char* uchars = (unsigned char*)buf;
- ssize_t total = 0;
-
- while (nbyte)
- {
- /*
- * If we were writing socket- or stream-safe code we'd chuck the
- * entire buf to write(2) and then gracefully re-request bytes that
- * didn't get written. But write(2) will return EINVAL if you ask it to
- * write more than 2^31-1 bytes. So instead we actually need to throttle
- * the input to write.
- *
- * Historically code using write(2) to write to disk will assert that
- * that all of the requested bytes were written. It seems harmless to
- * re-request bytes as one does when writing to streams, with the
- * compromise that we will return immediately when write(2) returns 0
- * bytes written.
- */
- size_t limit = 0x7FFFFFFF;
- size_t towrite = nbyte < limit ? nbyte : limit;
- ssize_t wrote = write(fildes, uchars, towrite);
- if (-1 == wrote)
- {
- return -1;
- }
- else if (0 == wrote)
- {
- break;
- }
- else
- {
- nbyte -= wrote;
- uchars += wrote;
- total += wrote;
- }
- }
-
- return total;
-}
-
-static void printRemovedFiles(bool cacheBuildSuccess, MRMSharedCacheBuilder* sharedCacheBuilder,
- const SharedCacheBuilderOptions& options)
-{
- if ( !cacheBuildSuccess || !options.printRemovedFiles )
- return;
-
- uint64_t fileResultCount = 0;
- if (const char* const* fileResults = getFilesToRemove(sharedCacheBuilder, &fileResultCount)) {
- for (uint64_t i = 0; i != fileResultCount; ++i)
- printf("Removed: %s\n", fileResults[i]);
- }
-}
-
-static void writeMRMResults(bool cacheBuildSuccess, MRMSharedCacheBuilder* sharedCacheBuilder,
- const SharedCacheBuilderOptions& options)
-{
- if (!cacheBuildSuccess) {
- uint64_t errorCount = 0;
- if (const char* const* errors = getErrors(sharedCacheBuilder, &errorCount)) {
- for (uint64_t i = 0, e = errorCount; i != e; ++i) {
- const char* errorMessage = errors[i];
- fprintf(stderr, "ERROR: %s\n", errorMessage);
- }
- }
- }
-
- // Now emit each cache we generated, or the errors for them.
- uint64_t cacheResultCount = 0;
- if (const CacheResult* const* cacheResults = getCacheResults(sharedCacheBuilder, &cacheResultCount)) {
- for (uint64_t i = 0, e = cacheResultCount; i != e; ++i) {
- const CacheResult& result = *(cacheResults[i]);
- // Always print the warnings if we have roots, even if there are errors
- // But not if we have -build_all, as its too noisy
- bool emitWarnings = (result.numErrors == 0) || !options.roots.empty() || options.debug;
- if ( options.dstRoot.empty() )
- emitWarnings = false;
- if ( emitWarnings ) {
- for (uint64_t warningIndex = 0; warningIndex != result.numWarnings; ++warningIndex) {
- fprintf(stderr, "[%s] WARNING: %s\n", result.loggingPrefix, result.warnings[warningIndex]);
- }
- }
- if (result.numErrors) {
- for (uint64_t errorIndex = 0; errorIndex != result.numErrors; ++errorIndex) {
- fprintf(stderr, "[%s] ERROR: %s\n", result.loggingPrefix, result.errors[errorIndex]);
- }
- cacheBuildSuccess = false;
- }
- }
- }
-
- if (!cacheBuildSuccess) {
- return;
- }
-
- // If we built caches, then write everything out.
- // TODO: Decide if we should we write any good caches anyway?
- if ( cacheBuildSuccess ) {
- uint64_t fileResultCount = 0;
- if (const FileResult* const* fileResults = getFileResults(sharedCacheBuilder, &fileResultCount)) {
- for (uint64_t i = 0, e = fileResultCount; i != e; ++i) {
- const FileResult* fileResultPtr = fileResults[i];
-
- assert(fileResultPtr->version == 1);
- const FileResult_v1& fileResult = *(const FileResult_v1*)fileResultPtr;
-
- switch (fileResult.behavior) {
- case AddFile:
- break;
- case ChangeFile:
- continue;
- }
-
- if ( options.printCDHashes ) {
- fprintf(stderr, "[%s/%s] %s cdhash: %s\n", fileResult.hashArch, fileResult.hashType, fileResult.path, fileResult.hash);
- }
-
- if ( (fileResult.data != nullptr) && !options.dstRoot.empty() ) {
- const std::string path = options.dstRoot + fileResult.path;
- std::string pathTemplate = path + "-XXXXXX";
- size_t templateLen = strlen(pathTemplate.c_str())+2;
- char pathTemplateSpace[templateLen];
- strlcpy(pathTemplateSpace, pathTemplate.c_str(), templateLen);
- int fd = mkstemp(pathTemplateSpace);
- if ( fd != -1 ) {
- ::ftruncate(fd, fileResult.size);
- uint64_t writtenSize = write64(fd, fileResult.data, fileResult.size);
- if ( writtenSize == fileResult.size ) {
- ::fchmod(fd, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH); // mkstemp() makes file "rw-------", switch it to "rw-r--r--"
- if ( ::rename(pathTemplateSpace, path.c_str()) == 0) {
- ::close(fd);
- continue; // success
- }
- }
- else {
- fprintf(stderr, "ERROR: could not write file %s\n", pathTemplateSpace);
- cacheBuildSuccess = false;
- }
- ::close(fd);
- ::unlink(pathTemplateSpace);
- }
- else {
- fprintf(stderr, "ERROR: could not open file %s\n", pathTemplateSpace);
- cacheBuildSuccess = false;
- }
- }
- }
- }
-
- // Give up if we couldn't write the caches
- if (!cacheBuildSuccess) {
- return;
- }
- }
-
- if ( options.emitJSONMap && !options.dstRoot.empty() ) {
- if (const CacheResult* const* cacheResults = getCacheResults(sharedCacheBuilder, &cacheResultCount)) {
- for (uint64_t i = 0, e = cacheResultCount; i != e; ++i) {
- const CacheResult& result = *(cacheResults[i]);
-
- const std::string path = options.dstRoot + "/" + result.loggingPrefix + ".json";
- std::string pathTemplate = path + "-XXXXXX";
- size_t templateLen = strlen(pathTemplate.c_str())+2;
- char pathTemplateSpace[templateLen];
- strlcpy(pathTemplateSpace, pathTemplate.c_str(), templateLen);
- int fd = mkstemp(pathTemplateSpace);
- if ( fd != -1 ) {
- size_t jsonLength = strlen(result.mapJSON) + 1;
- ::ftruncate(fd, jsonLength);
- uint64_t writtenSize = write64(fd, result.mapJSON, jsonLength);
- if ( writtenSize == jsonLength ) {
- ::fchmod(fd, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH); // mkstemp() makes file "rw-------", switch it to "rw-r--r--"
- if ( ::rename(pathTemplateSpace, path.c_str()) == 0) {
- ::close(fd);
- continue; // success
- }
- }
- else {
- fprintf(stderr, "ERROR: could not write map file %s\n", pathTemplateSpace);
- cacheBuildSuccess = false;
- }
- ::close(fd);
- ::unlink(pathTemplateSpace);
- }
- else {
- fprintf(stderr, "ERROR: could not open map file %s\n", pathTemplateSpace);
- cacheBuildSuccess = false;
- }
- }
- }
- }
-
- // If we ask for -stats, then also emit the stat files
- if ( options.printStats && !options.dstRoot.empty() ) {
- uint64_t statResultCount = 0;
- if ( const char* const* statResults = getCacheStats(sharedCacheBuilder, &statResultCount) ) {
- for (uint64_t i = 0; i != statResultCount; ++i) {
- std::string_view statString = statResults[i];
-
- const std::string path = options.dstRoot + "/" + "stats." + std::to_string(i) + ".json";
- std::string pathTemplate = path + "-XXXXXX";
- size_t templateLen = strlen(pathTemplate.c_str()) + 2;
- char pathTemplateSpace[templateLen];
- strlcpy(pathTemplateSpace, pathTemplate.c_str(), templateLen);
- int fd = mkstemp(pathTemplateSpace);
- if ( fd != -1 ) {
- size_t jsonLength = statString.size();
- ::ftruncate(fd, jsonLength);
- uint64_t writtenSize = write64(fd, statString.data(), jsonLength);
- if ( writtenSize == jsonLength ) {
- ::fchmod(fd, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH); // mkstemp() makes file "rw-------", switch it to "rw-r--r--"
- if ( ::rename(pathTemplateSpace, path.c_str()) == 0) {
- ::close(fd);
- continue; // success
- }
- }
- else {
- fprintf(stderr, "ERROR: could not write map file %s\n", pathTemplateSpace);
- cacheBuildSuccess = false;
- }
- ::close(fd);
- ::unlink(pathTemplateSpace);
- }
- else {
- fprintf(stderr, "ERROR: could not open map file %s\n", pathTemplateSpace);
- cacheBuildSuccess = false;
- }
- }
- }
- }
-}
-
-static void buildCacheFromJSONManifest(Diagnostics& diags, const SharedCacheBuilderOptions& options,
- const std::string& jsonManifestPath) {
- json::Node manifestNode = json::readJSON(diags, jsonManifestPath.c_str(), false /* useJSON5 */);
- if (diags.hasError())
- return;
-
- // Top level node should be a map of the options, files, and symlinks.
- if (manifestNode.map.empty()) {
- diags.error("Expected map for JSON manifest node\n");
- return;
- }
-
- // Parse the nodes in the top level manifest node
- const json::Node& versionNode = json::getRequiredValue(diags, manifestNode, "version");
- uint64_t manifestVersion = json::parseRequiredInt(diags, versionNode);
- if (diags.hasError())
- return;
-
- const uint64_t supportedManifestVersion = 1;
- if (manifestVersion != supportedManifestVersion) {
- diags.error("JSON manfiest version of %lld is unsupported. Supported version is %lld\n",
- manifestVersion, supportedManifestVersion);
- return;
- }
- const json::Node& buildOptionsNode = json::getRequiredValue(diags, manifestNode, "buildOptions");
- const json::Node& filesNode = json::getRequiredValue(diags, manifestNode, "files");
- const json::Node* symlinksNode = json::getOptionalValue(diags, manifestNode, "symlinks");
-
- // Parse the archs
- const json::Node& archsNode = json::getRequiredValue(diags, buildOptionsNode, "archs");
- if (diags.hasError())
- return;
- if (archsNode.array.empty()) {
- diags.error("Build options archs node is not an array\n");
- return;
- }
- std::set<std::string> jsonArchs;
- const char* archs[archsNode.array.size()];
- uint64_t numArchs = 0;
- for (const json::Node& archNode : archsNode.array) {
- const char* archName = json::parseRequiredString(diags, archNode).c_str();
- jsonArchs.insert(archName);
- if ( options.cmdLineArchs.empty() || options.cmdLineArchs.count(archName) ) {
- archs[numArchs++] = archName;
- }
- }
-
- // Check that the command line archs are in the JSON list
- if ( !options.cmdLineArchs.empty() ) {
- for (const std::string& cmdLineArch : options.cmdLineArchs) {
- if ( !jsonArchs.count(cmdLineArch) ) {
- std::string validArchs = "";
- for (const std::string& jsonArch : jsonArchs) {
- if ( !validArchs.empty() ) {
- validArchs += ", ";
- }
- validArchs += jsonArch;
- }
- diags.error("Command line -arch '%s' is not valid for this device. Valid archs are (%s)\n", cmdLineArch.c_str(), validArchs.c_str());
- return;
- }
- }
- }
-
- // Parse the rest of the options node.
- BuildOptions_v3 buildOptions;
- buildOptions.version = json::parseRequiredInt(diags, json::getRequiredValue(diags, buildOptionsNode, "version"));
- buildOptions.updateName = json::parseRequiredString(diags, json::getRequiredValue(diags, buildOptionsNode, "updateName")).c_str();
- buildOptions.deviceName = json::parseRequiredString(diags, json::getRequiredValue(diags, buildOptionsNode, "deviceName")).c_str();
- buildOptions.disposition = stringToDisposition(diags, json::parseRequiredString(diags, json::getRequiredValue(diags, buildOptionsNode, "disposition")));
- buildOptions.platform = stringToPlatform(diags, json::parseRequiredString(diags, json::getRequiredValue(diags, buildOptionsNode, "platform")));
- buildOptions.archs = archs;
- buildOptions.numArchs = numArchs;
- buildOptions.verboseDiagnostics = options.debug;
- buildOptions.verboseIMPCaches = options.debugIMPCaches;
- buildOptions.verboseCacheLayout = options.debugCacheLayout;
- buildOptions.isLocallyBuiltCache = true;
-
- // optimizeForSize was added in version 2
- buildOptions.optimizeForSize = false;
- if ( buildOptions.version >= 2 ) {
- buildOptions.optimizeForSize = json::parseRequiredBool(diags, json::getRequiredValue(diags, buildOptionsNode, "optimizeForSize"));
- }
-
- // timePasses was added in version 3
- buildOptions.filesRemovedFromDisk = true;
- buildOptions.timePasses = false;
- buildOptions.printStats = false;
- if ( buildOptions.version == 2 ) {
- // HACK:! Bump to version 3 so that timePasses/printStats are picked up.
- buildOptions.version = 3;
- buildOptions.timePasses = options.timePasses;
- buildOptions.printStats = options.printStats;
- } else if ( buildOptions.version >= 3 ) {
- const json::Node* filesRemovedNode = json::getOptionalValue(diags, buildOptionsNode, "filesRemovedFromDisk");
- const json::Node* timePassesNode = json::getOptionalValue(diags, buildOptionsNode, "timePasses");
- const json::Node* printStatsNode = json::getOptionalValue(diags, buildOptionsNode, "printStats");
- if ( filesRemovedNode != nullptr )
- buildOptions.filesRemovedFromDisk = json::parseRequiredBool(diags, *filesRemovedNode);
- if ( timePassesNode != nullptr )
- buildOptions.timePasses = json::parseRequiredBool(diags, *timePassesNode);
- if ( printStatsNode != nullptr )
- buildOptions.printStats = json::parseRequiredBool(diags, *printStatsNode);
- }
-
- if (diags.hasError())
- return;
-
- // Override the disposition if we don't want certain caches.
- switch (buildOptions.disposition) {
- case Unknown:
- // Nothing we can do here as we can't assume what caches are built here.
- break;
- case InternalDevelopment:
- if (!options.emitDevCaches && !options.emitCustomerCaches) {
- diags.error("both -no_customer_cache and -no_development_cache passed\n");
- break;
- }
- if (!options.emitDevCaches) {
- // This builds both caches, but we don't want dev
- buildOptions.disposition = Customer;
- }
- if (!options.emitCustomerCaches) {
- // This builds both caches, but we don't want customer
- buildOptions.disposition = InternalMinDevelopment;
- }
- break;
- case Customer:
- if (!options.emitCustomerCaches) {
- diags.error("Cannot request no customer cache for Customer as that is already only a customer cache\n");
- }
- break;
- case InternalMinDevelopment:
- if (!options.emitDevCaches) {
- diags.error("Cannot request no dev cache for InternalMinDevelopment as that is already only a dev cache\n");
- }
- break;
- case SymbolsCache:
- break;
- }
-
- if (diags.hasError())
- return;
-
- struct MRMSharedCacheBuilder* sharedCacheBuilder = createSharedCacheBuilder((const BuildOptions_v1*)&buildOptions);
-
- // Parse the files
- if (filesNode.array.empty()) {
- diags.error("Build options files node is not an array\n");
- return;
- }
-
- std::vector<InputFile> inputFiles;
- std::set<std::string> dylibsFoundInRoots;
- for (const json::Node& fileNode : filesNode.array) {
- std::string path = json::parseRequiredString(diags, json::getRequiredValue(diags, fileNode, "path")).c_str();
- FileFlags fileFlags = stringToFileFlags(diags, json::parseRequiredString(diags, json::getRequiredValue(diags, fileNode, "flags")));
-
- std::string_view projectName;
- if ( const json::Node* projectNode = json::getOptionalValue(diags, fileNode, "project") )
- projectName = projectNode->value;
-
- // We can optionally have a sourcePath entry which is the path to get the source content from instead of the install path
- std::string sourcePath;
- const json::Node* sourcePathNode = json::getOptionalValue(diags, fileNode, "sourcePath");
- if ( sourcePathNode != nullptr ) {
- if (!sourcePathNode->array.empty()) {
- diags.error("sourcePath node cannot be an array\n");
- return;
- }
- if (!sourcePathNode->map.empty()) {
- diags.error("sourcePath node cannot be a map\n");
- return;
- }
- sourcePath = sourcePathNode->value;
- } else {
- sourcePath = path;
- }
-
- std::string buildPath = sourcePath;
-
- // Check if one of the -root's has this path
- bool foundInOverlay = false;
- for (const std::string& overlay : options.roots) {
- struct stat sb;
- std::string filePath = overlay + path;
- if (!stat(filePath.c_str(), &sb)) {
- foundInOverlay = true;
- diags.verbose("Taking '%s' from overlay '%s' instead of dylib cache\n", path.c_str(), overlay.c_str());
- inputFiles.push_back({ filePath, path, fileFlags, std::string(projectName) });
- dylibsFoundInRoots.insert(path);
- break;
- }
- }
-
- if (foundInOverlay)
- continue;
-
- // Build paths are relative to the build artifact root directory.
- switch (fileFlags) {
- case NoFlags:
- case MustBeInCache:
- case ShouldBeExcludedFromCacheIfUnusedLeaf:
- case RequiredClosure:
- case DylibOrderFile:
- case DirtyDataOrderFile:
- case ObjCOptimizationsFile:
- case SwiftGenericMetadataFile:
- case OptimizationFile:
- buildPath = "." + buildPath;
- break;
- }
- inputFiles.push_back({ buildPath, path, fileFlags, std::string(projectName) });
- }
-
- if (diags.hasError())
- return;
-
- // Parse the baseline from the map(s) if we have it
- BaselineCachesChecker baselineCaches({ &archs[0], &archs[numArchs] }, mach_o::Platform(buildOptions.platform));
-
- // If we have a maps directory, use it
- if ( !options.baselineCacheMapDirPath.empty() ) {
- if ( mach_o::Error err = baselineCaches.addBaselineMaps(options.baselineCacheMapDirPath) ) {
- diags.error("%s", err.message());
- return;
- }
- } else {
- for ( const std::string& baselineCacheMapPath : options.baselineCacheMapPaths ) {
- if ( mach_o::Error err = baselineCaches.addBaselineMap(baselineCacheMapPath) ) {
- diags.error("%s", err.message());
- return;
- }
- }
- }
-
- std::vector<std::pair<const void*, size_t>> mappedFiles;
- {
- uint64_t startTimeNanos = clock_gettime_nsec_np(CLOCK_UPTIME_RAW);
- loadMRMFiles(diags, sharedCacheBuilder, inputFiles, mappedFiles, baselineCaches.unionBaselineDylibs());
- uint64_t endTimeNanos = clock_gettime_nsec_np(CLOCK_UPTIME_RAW);
-
- if ( options.timePasses ) {
- uint64_t timeMillis = (endTimeNanos - startTimeNanos) / 1000000;
- fprintf(stderr, "loadMRMFiles: time = %lldms\n", timeMillis);
- }
- }
-
- if (diags.hasError())
- return;
-
- // Parse the symlinks if we have them
- if (symlinksNode) {
- if (symlinksNode->array.empty()) {
- diags.error("Build options symlinks node is not an array\n");
- return;
- }
- for (const json::Node& symlinkNode : symlinksNode->array) {
- std::string fromPath = json::parseRequiredString(diags, json::getRequiredValue(diags, symlinkNode, "path")).c_str();
- const std::string& toPath = json::parseRequiredString(diags, json::getRequiredValue(diags, symlinkNode, "target")).c_str();
- addSymlink(sharedCacheBuilder, fromPath.c_str(), toPath.c_str());
- }
- }
-
- if (diags.hasError())
- return;
-
- // Don't create a directory if we are skipping writes, which means we have no dstRoot set
- if (!options.dstRoot.empty()) {
- if ( buildOptions.platform == macOS ) {
- (void)mkpath_np((options.dstRoot + MACOSX_MRM_DYLD_SHARED_CACHE_DIR).c_str(), 0755);
- } else if (buildOptions.platform == driverKit ) {
- (void)mkpath_np((options.dstRoot + DRIVERKIT_DYLD_SHARED_CACHE_DIR).c_str(), 0755);
- } else if ( mach_o::Platform(buildOptions.platform).isExclaveKit() ) {
- (void)mkpath_np((options.dstRoot + EXCLAVEKIT_DYLD_SHARED_CACHE_DIR).c_str(), 0755);
- } else if ( buildOptions.disposition == SymbolsCache ) {
- // symbols cache always uses /System/Library/dyld, even on iOS
- (void)mkpath_np((options.dstRoot + MACOSX_MRM_DYLD_SHARED_CACHE_DIR).c_str(), 0755);
- } else {
- (void)mkpath_np((options.dstRoot + IPHONE_DYLD_SHARED_CACHE_DIR).c_str(), 0755);
- }
- }
-
- // Actually build the cache.
- bool cacheBuildSuccess = runSharedCacheBuilder(sharedCacheBuilder);
-
- // Compare this cache to the baseline cache and see if we have any roots to copy over
- if (!options.baselineDifferenceResultPath.empty() || options.baselineCopyRoots) {
- std::set<std::string> dylibsInNewCaches;
- if (cacheBuildSuccess) {
- uint64_t fileResultCount = 0;
- if (const char* const* fileResults = getFilesToRemove(sharedCacheBuilder, &fileResultCount)) {
- for (uint64_t i = 0; i != fileResultCount; ++i)
- dylibsInNewCaches.insert(fileResults[i]);
- }
- }
-
- if ( options.baselineCopyRoots && cacheBuildSuccess ) {
- uint64_t cacheResultCount = 0;
- if ( const CacheResult* const* cacheResults = getCacheResults(sharedCacheBuilder, &cacheResultCount) ) {
- for ( uint64_t i = 0; i != cacheResultCount; ++i ) {
- const CacheResult& result = *(cacheResults[i]);
- if ( result.mapJSON == nullptr )
- continue;
- std::string_view mapString = result.mapJSON;
- if ( mapString.empty() )
- continue;
-
- if ( mach_o::Error err = baselineCaches.addNewMap(mapString) ) {
- diags.error("%s", err.message());
- return;
- }
- }
- }
-
- uint64_t fileResultCount = 0;
- if ( const char* const* fileResults = getFilesToRemove(sharedCacheBuilder, &fileResultCount) )
- baselineCaches.setFilesFromNewCaches({ fileResults, fileResultCount });
-
- // Work out the set of dylibs in the old caches but not the new ones
- std::set<std::string> dylibsMissingFromNewCaches = baselineCaches.dylibsMissingFromNewCaches();
- if ( !dylibsMissingFromNewCaches.empty() ) {
- BOMCopier copier = BOMCopierNewWithSys(BomSys_default());
- FilteredCopyOptions userData = { &diags, &dylibsMissingFromNewCaches, &dylibsFoundInRoots };
- BOMCopierSetUserData(copier, (void*)&userData);
- BOMCopierSetCopyFileStartedHandler(copier, filteredCopyIncludingPaths);
- std::string dylibCacheRootDir = realFilePath(options.dylibCacheDir);
- if (dylibCacheRootDir == "") {
- fprintf(stderr, "Could not find dylib Root directory to copy baseline roots from\n");
- exit(EX_NOINPUT);
- }
- BOMCopierCopy(copier, dylibCacheRootDir.c_str(), options.dstRoot.c_str());
- BOMCopierFree(copier);
-
- for (const std::string& dylibMissingFromNewCache : dylibsMissingFromNewCaches) {
- diags.verbose("Dylib missing from new cache: '%s'\n", dylibMissingFromNewCache.c_str());
- }
- }
- }
-
- if (!options.baselineDifferenceResultPath.empty()) {
- auto cppToObjStr = [](const std::string& str) {
- return [NSString stringWithUTF8String:str.c_str()];
- };
-
- // Work out the set of dylibs in the cache and taken from any -roots
- NSMutableArray<NSString*>* dylibsFromRoots = [NSMutableArray array];
- for (auto& root : options.roots) {
- for (const std::string& dylibInstallName : dylibsInNewCaches) {
- struct stat sb;
- std::string filePath = root + "/" + dylibInstallName;
- if (!stat(filePath.c_str(), &sb)) {
- [dylibsFromRoots addObject:cppToObjStr(dylibInstallName)];
- }
- }
- }
-
- // Work out the set of dylibs in the new cache but not in the baseline cache.
- NSMutableArray<NSString*>* dylibsMissingFromBaselineCache = [NSMutableArray array];
- for (const std::string& newDylib : dylibsInNewCaches) {
- if ( !baselineCaches.unionBaselineDylibs().count(newDylib) )
- [dylibsMissingFromBaselineCache addObject:cppToObjStr(newDylib)];
- }
-
- NSMutableDictionary* cacheDict = [[NSMutableDictionary alloc] init];
- cacheDict[@"root-paths-in-cache"] = dylibsFromRoots;
- cacheDict[@"device-paths-to-delete"] = dylibsMissingFromBaselineCache;
-
- NSError* error = nil;
- NSData* outData = [NSPropertyListSerialization dataWithPropertyList:cacheDict
- format:NSPropertyListBinaryFormat_v1_0
- options:0
- error:&error];
- (void)[outData writeToFile:cppToObjStr(options.baselineDifferenceResultPath) atomically:YES];
- }
- }
-
- printRemovedFiles(cacheBuildSuccess, sharedCacheBuilder, options);
-
- writeMRMResults(cacheBuildSuccess, sharedCacheBuilder, options);
-
- destroySharedCacheBuilder(sharedCacheBuilder);
-
- unloadMRMFiles(mappedFiles);
-
- // On failure, add an error to the diagnostic so that the caller can see that the build failed
- if ( !cacheBuildSuccess ) {
- diags.error("see other errors");
- }
-}
-
-static std::string realPathOrExit(const char* argName, const char* argValue)
-{
- std::string realpath = realPath(argValue);
- if ( realpath.empty() || !fileExists(realpath) ) {
- fprintf(stderr, "%s path doesn't exist: %s\n", argName, argValue);
- exit(EX_NOINPUT);
- }
- return realpath;
-}
-
-static const char* leafName(std::string_view str)
-{
- const char* start = strrchr(str.data(), '/');
- if ( start != nullptr )
- return &start[1];
- else
- return str.data();
-}
-
-int main(int argc, const char* argv[])
-{
- __block SharedCacheBuilderOptions options;
- std::string jsonManifestPath;
- char* tempRootsDir = strdup("/tmp/dyld_shared_cache_builder.XXXXXX");
-
- mkdtemp(tempRootsDir);
-
- for (int i = 1; i < argc; ++i) {
- const char* arg = argv[i];
- if (arg[0] == '-') {
- if (strcmp(arg, "-debug") == 0) {
- options.debug = true;
- } else if (strcmp(arg, "-debug-imp-caches") == 0) {
- options.debugIMPCaches = true;
- } else if (strcmp(arg, "-debug-cache-layout") == 0) {
- options.debugCacheLayout = true;
- } else if (strcmp(arg, "-list_configs") == 0) {
- options.listConfigs = true;
- } else if (strcmp(arg, "-root") == 0) {
- std::string realpath = realPathOrExit("-root", argv[++i]);
- if ( std::find(options.roots.begin(), options.roots.end(), realpath) == options.roots.end() ) {
- // Push roots on to the front so that each -root overrides previous entries
- options.roots.push_front(realpath);
- }
- } else if (strcmp(arg, "-copy_roots") == 0) {
- options.copyRoots = true;
- } else if (strcmp(arg, "-dylib_cache") == 0) {
- options.dylibCacheDir = realPathOrExit("-dylib_cache", argv[++i]);
- } else if (strcmp(arg, "-artifact") == 0) {
- options.artifactDir = realPathOrExit("-artifact", argv[++i]);
- } else if (strcmp(arg, "-no_overflow_dylibs") == 0) {
- options.emitElidedDylibs = false;
- } else if (strcmp(arg, "-no_development_cache") == 0) {
- options.emitDevCaches = false;
- } else if (strcmp(arg, "-development_cache") == 0) {
- options.emitDevCaches = true;
- } else if (strcmp(arg, "-no_customer_cache") == 0) {
- options.emitCustomerCaches = false;
- } else if (strcmp(arg, "-customer_cache") == 0) {
- options.emitCustomerCaches = true;
- } else if (strcmp(arg, "-overflow_dylibs") == 0) {
- options.emitElidedDylibs = true;
- } else if (strcmp(arg, "-mrm") == 0) {
- options.useMRM = true;
- } else if (strcmp(arg, "-time-passes") == 0) {
- options.timePasses = true;
- } else if (strcmp(arg, "-stats") == 0) {
- options.printStats = true;
- } else if (strcmp(arg, "-removed_files") == 0) {
- options.printRemovedFiles = true;
- } else if (strcmp(arg, "-emit_json") == 0) {
- // unused
- } else if (strcmp(arg, "-emit_json_map") == 0) {
- options.emitJSONMap = true;
- } else if (strcmp(arg, "-json_manifest") == 0) {
- jsonManifestPath = realPathOrExit("-json_manifest", argv[++i]);
- } else if (strcmp(arg, "-build_all") == 0) {
- options.buildAllPath = realPathOrExit("-build_all", argv[++i]);
- } else if (strcmp(arg, "-dst_root") == 0) {
- options.dstRoot = realPath(argv[++i]);
- } else if (strcmp(arg, "-release") == 0) {
- options.release = argv[++i];
- } else if (strcmp(arg, "-results") == 0) {
- options.resultPath = realPath(argv[++i]);
- } else if (strcmp(arg, "-baseline_diff_results") == 0) {
- options.baselineDifferenceResultPath = realPath(argv[++i]);
- } else if (strcmp(arg, "-baseline_copy_roots") == 0) {
- options.baselineCopyRoots = true;
- } else if (strcmp(arg, "-print_cdhashes") == 0) {
- options.printCDHashes = true;
- } else if (strcmp(arg, "-baseline_cache_map") == 0) {
- std::string path = realPathOrExit("-baseline_cache_map", argv[++i]);
- options.baselineCacheMapPaths.push_back(path);
- } else if (strcmp(arg, "-baseline_cache_maps") == 0) {
- std::string path = realPathOrExit("-baseline_cache_maps", argv[++i]);
- options.baselineCacheMapDirPath = path;
- } else if (strcmp(arg, "-arch") == 0) {
- if ( ++i < argc ) {
- options.cmdLineArchs.insert(argv[i]);
- }
- else {
- fprintf(stderr, "-arch missing architecture name");
- exit(EX_USAGE);
- }
- } else if (strcmp(arg, "-help") == 0) {
- // no usage() to show, but having this allows clients to probe
- // whether flags are supported by seeing if `-flag2check -help`
- // exits with EXIT_SUCCESS or EX_USAGE
- exit(EXIT_SUCCESS);
- } else {
- //usage();
- fprintf(stderr, "unknown option: %s\n", arg);
- exit(EX_USAGE);
- }
- } else {
- fprintf(stderr, "unknown option: %s\n", arg);
- exit(EX_USAGE);
- }
- }
- (void)options.emitElidedDylibs; // not implemented yet
-
- time_t mytime = time(0);
- fprintf(stderr, "Started: %s", asctime(localtime(&mytime)));
- processRoots(options.roots, tempRootsDir);
-
- struct rlimit rl = { OPEN_MAX, OPEN_MAX };
- (void)setrlimit(RLIMIT_NOFILE, &rl);
-
- if (options.dylibCacheDir.empty() && options.artifactDir.empty() && options.release.empty()) {
- fprintf(stderr, "you must specify either -dylib_cache, -artifact or -release\n");
- exit(EX_USAGE);
- } else if (!options.dylibCacheDir.empty() && !options.release.empty()) {
- fprintf(stderr, "you may not use -dylib_cache and -release at the same time\n");
- exit(EX_USAGE);
- } else if (!options.dylibCacheDir.empty() && !options.artifactDir.empty()) {
- fprintf(stderr, "you may not use -dylib_cache and -artifact at the same time\n");
- exit(EX_USAGE);
- }
-
- if (jsonManifestPath.empty() && options.buildAllPath.empty()) {
- fprintf(stderr, "Must specify a -json_manifest path OR a -build_all path\n");
- exit(EX_USAGE);
- }
-
- if (!options.buildAllPath.empty()) {
- if (!options.dstRoot.empty()) {
- fprintf(stderr, "Cannot combine -dst_root and -build_all\n");
- exit(EX_USAGE);
- }
- if (!jsonManifestPath.empty()) {
- fprintf(stderr, "Cannot combine -json_manifest and -build_all\n");
- exit(EX_USAGE);
- }
- if (!options.baselineDifferenceResultPath.empty()) {
- fprintf(stderr, "Cannot combine -baseline_diff_results and -build_all\n");
- exit(EX_USAGE);
- }
- if (options.baselineCopyRoots) {
- fprintf(stderr, "Cannot combine -baseline_copy_roots and -build_all\n");
- exit(EX_USAGE);
- }
- if (!options.baselineCacheMapPaths.empty()) {
- fprintf(stderr, "Cannot combine -baseline_cache_map and -build_all\n");
- exit(EX_USAGE);
- }
- if (!options.baselineCacheMapDirPath.empty()) {
- fprintf(stderr, "Cannot combine -baseline_cache_maps and -build_all\n");
- exit(EX_USAGE);
- }
- } else if (!options.listConfigs) {
- if (options.dstRoot.empty()) {
- fprintf(stderr, "Must specify a valid -dst_root OR -list_configs\n");
- exit(EX_USAGE);
- }
-
- if (jsonManifestPath.empty()) {
- fprintf(stderr, "Must specify a -json_manifest path OR -list_configs\n");
- exit(EX_USAGE);
- }
- }
-
- // Some options don't work with a JSON manifest
- if (!jsonManifestPath.empty()) {
- if (!options.resultPath.empty()) {
- fprintf(stderr, "Cannot use -results with -json_manifest\n");
- exit(EX_USAGE);
- }
- if (!options.baselineDifferenceResultPath.empty() && options.baselineCacheMapPaths.empty() && options.baselineCacheMapDirPath.empty()) {
- fprintf(stderr, "Must use -baseline_cache_map/-baseline_cache_maps with -baseline_diff_results when using -json_manifest\n");
- exit(EX_USAGE);
- }
- if (options.baselineCopyRoots && options.baselineCacheMapPaths.empty() && options.baselineCacheMapDirPath.empty()) {
- fprintf(stderr, "Must use -baseline_cache_map/-baseline_cache_maps with -baseline_copy_roots when using -json_manifest\n");
- exit(EX_USAGE);
- }
- } else {
- if (!options.baselineCacheMapPaths.empty()) {
- fprintf(stderr, "Cannot use -baseline_cache_map without -json_manifest\n");
- exit(EX_USAGE);
- }
- if (!options.baselineCacheMapDirPath.empty()) {
- fprintf(stderr, "Cannot use -baseline_cache_maps without -json_manifest\n");
- exit(EX_USAGE);
- }
- }
-
- if (!options.baselineCacheMapPaths.empty()) {
- if (options.baselineDifferenceResultPath.empty() && !options.baselineCopyRoots) {
- fprintf(stderr, "Must use -baseline_cache_map with -baseline_diff_results or -baseline_copy_roots\n");
- exit(EX_USAGE);
- }
- }
-
- if (!options.baselineCacheMapDirPath.empty()) {
- if (options.baselineDifferenceResultPath.empty() && !options.baselineCopyRoots) {
- fprintf(stderr, "Must use -baseline_cache_maps with -baseline_diff_results or -baseline_copy_roots\n");
- exit(EX_USAGE);
- }
- }
-
- // Find all the JSON files if we use -build_all
- __block std::vector<std::string> jsonPaths;
- if (!options.buildAllPath.empty()) {
- struct stat stat_buf;
- if (stat(options.buildAllPath.c_str(), &stat_buf) != 0) {
- fprintf(stderr, "Could not find -build_all path '%s'\n", options.buildAllPath.c_str());
- exit(EX_NOINPUT);
- }
-
- if ( (stat_buf.st_mode & S_IFMT) != S_IFDIR ) {
- fprintf(stderr, "-build_all path is not a directory '%s'\n", options.buildAllPath.c_str());
- exit(EX_DATAERR);
- }
-
- auto processFile = ^(const std::string& path, const struct stat& statBuf) {
- if ( !endsWith(path, ".json") )
- return;
-
- jsonPaths.push_back(path);
- };
-
- iterateDirectoryTree("", options.buildAllPath,
- ^(const std::string& dirPath) { return false; },
- processFile, true /* process files */, true /* recurse */);
-
- if (jsonPaths.empty()) {
- fprintf(stderr, "Didn't find any .json files inside -build_all path: %s\n", options.buildAllPath.c_str());
- exit(EX_DATAERR);
- }
-
- if (options.listConfigs) {
- for (const std::string& path : jsonPaths) {
- fprintf(stderr, "Found config: %s\n", path.c_str());
- }
- exit(EXIT_SUCCESS);
- }
- }
-
- if (!options.artifactDir.empty()) {
- // Find the dylib cache dir from inside the artifact dir
- struct stat stat_buf;
- if (stat(options.artifactDir.c_str(), &stat_buf) != 0) {
- fprintf(stderr, "Could not find artifact path '%s'\n", options.artifactDir.c_str());
- exit(EX_NOINPUT);
- }
- std::string dir = options.artifactDir + "/AppleInternal/Developer/DylibCaches";
- if (stat(dir.c_str(), &stat_buf) != 0) {
- fprintf(stderr, "Could not find artifact path '%s'\n", dir.c_str());
- exit(EX_DATAERR);
- }
-
- if (!options.release.empty()) {
- // Use the given release
- options.dylibCacheDir = dir + "/" + options.release + ".dlc";
- } else {
- // Find a release directory
- __block std::vector<std::string> subDirectories;
- iterateDirectoryTree("", dir, ^(const std::string& dirPath) {
- subDirectories.push_back(dirPath);
- return false;
- }, nullptr, false, false);
-
- if (subDirectories.empty()) {
- fprintf(stderr, "Could not find dlc subdirectories inside '%s'\n", dir.c_str());
- exit(EX_DATAERR);
- }
-
- if (subDirectories.size() > 1) {
- fprintf(stderr, "Found too many subdirectories inside artifact path '%s'. Use -release to select one\n", dir.c_str());
- exit(EX_DATAERR);
- }
-
- options.dylibCacheDir = subDirectories.front();
- }
- }
-
- if (options.dylibCacheDir.empty()) {
- options.dylibCacheDir = std::string("/AppleInternal/Developer/DylibCaches/") + options.release + ".dlc";
- }
-
- //Move into the dir so we can use relative path manifests
- if ( int result = chdir(options.dylibCacheDir.c_str()); result == -1 ) {
- fprintf(stderr, "Couldn't cd in to dylib cache directory of '%s' because: %s\n",
- options.dylibCacheDir.c_str(), strerror(errno));
- }
-
- if (!options.buildAllPath.empty()) {
- bool requiresConcurrencyLimit = false;
- dispatch_semaphore_t concurrencyLimit = NULL;
- // Try build 1 cache per 8GB of RAM
- uint64_t memSize = 0;
- size_t sz = sizeof(memSize);
- if ( sysctlbyname("hw.memsize", &memSize, &sz, NULL, 0) == 0 ) {
- uint64_t maxThreads = std::max(memSize / 0x200000000ULL, 1ULL);
- fprintf(stderr, "Detected %lldGb or less of memory, limiting concurrency to %lld threads\n",
- memSize / (1 << 30), maxThreads);
- requiresConcurrencyLimit = true;
- concurrencyLimit = dispatch_semaphore_create(maxThreads);
- }
-
- __block int finishedCount = 0;
- std::atomic_bool failedToBuildCache = { false };
- __block auto& failedToBuildCacheRef = failedToBuildCache;
- dispatch_apply(jsonPaths.size(), DISPATCH_APPLY_AUTO, ^(size_t index) {
- // Horrible hack to limit concurrency in low spec build machines.
- if (requiresConcurrencyLimit) { dispatch_semaphore_wait(concurrencyLimit, DISPATCH_TIME_FOREVER); }
-
- const std::string& jsonPath = jsonPaths[index];
- Diagnostics diags(options.debug);
- buildCacheFromJSONManifest(diags, options, jsonPath);
-
- if (diags.hasError()) {
- fprintf(stderr, "dyld_shared_cache_builder: error: %s\n", diags.errorMessage().c_str());
- failedToBuildCacheRef = true;
- }
-
- time_t endTime = time(0);
- std::string timeString = asctime(localtime(&endTime));
- timeString.pop_back();
- fprintf(stderr, "Finished[% 4d/% 4d]: %s %s\n",
- ++finishedCount, (int)jsonPaths.size(), timeString.c_str(), leafName(jsonPath));
-
- if (requiresConcurrencyLimit) { dispatch_semaphore_signal(concurrencyLimit); }
- });
-
- if ( failedToBuildCacheRef )
- return EXIT_FAILURE;
- } else {
- Diagnostics diags(options.debug);
- buildCacheFromJSONManifest(diags, options, jsonManifestPath);
-
- if (diags.hasError()) {
- fprintf(stderr, "dyld_shared_cache_builder: error: %s\n", diags.errorMessage().c_str());
- return EXIT_FAILURE;
- }
- }
-
- Diagnostics diags;
- const char* args[8];
- args[0] = (char*)"/bin/rm";
- args[1] = (char*)"-rf";
- args[2] = (char*)tempRootsDir;
- args[3] = nullptr;
- (void)runCommandAndWait(diags, args);
-
- if (diags.hasError()) {
- // errors from our final rm -rf should just be warnings
- fprintf(stderr, "dyld_shared_cache_builder: warning: %s\n", diags.errorMessage().c_str());
- }
-
- for (const std::string& warn : diags.warnings()) {
- fprintf(stderr, "dyld_shared_cache_builder: warning: %s\n", warn.c_str());
- }
-
- // Finally, write the roots.txt to tell us which roots we pulled in
- if (!options.dstRoot.empty())
- writeRootList(options.dstRoot + "/System/Library/Caches/com.apple.dyld", options.roots);
-
- return EXIT_SUCCESS;
-}