Loading...
--- /dev/null
+++ dyld/dyld-1340/common/SymbolsCache.cpp
@@ -0,0 +1,2775 @@
+/*
+ * Copyright (c) 2022 Apple Inc. All rights reserved.
+ *
+ * @APPLE_LICENSE_HEADER_START@
+ *
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this
+ * file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_LICENSE_HEADER_END@
+ */
+
+/*
+
+ This file suppports building and verifying against a symbols database.
+
+ The database contains imports and exports for all shared cache eligible mach-o files
+ in a given build. It also contains the list of re-exported dylibs.
+
+ The main tables are:
+ - BINARY: Contains the path, install name, arch, etc, for a given dylib (and in future exe)
+ - SYMBOL: Maps from ID to symbol name. Used only to deduplicate symbol strings
+ - SYMBOL_ID_REF: Corresponds to imported (referenced) symbols. Is a tuple of symbol ID, and the binary IDs of the client and target dylibs
+ - SYMBOL_ID_DEF: Corresponds to exported (defined) symbols. Is a tuple of symbol ID and the dylib which defines the symbol
+ - REEXPORT: Corresponds to LC_REEXPORT_DYLIB's. Contains tuples of umbrella and client dylib.
+
+ The symbols cache can contain arbitrary arch and platform for binaries. A single database is expected
+ to contain all platforms, such as the main OS but also driverKit, etc.
+
+ To verify binaries against a database, the key check is whether a new binary removes a symbol still in use by
+ a binary in the cache. That is, does the new binary cause a SYMBOL_ID_REF to become invalid. Verification
+ is passed all new binaries, so only binaries in the database, and not in the roots passed in, will be verified.
+
+ Re-exports are special. Instead of storing all re-exports on the umbrella dylib (ie, promoting all UIKitCore SYMBOL_ID_DEF's
+ up to UIKit), the actual re-export edges are just recorded. It is the task of the verify step to walk all re-exports when
+ looking to resolve symbols. This is recursive to support arbitrary tiers of re-exports
+
+ */
+
+#include "SymbolsCache.h"
+#include "ClosureFileSystem.h"
+#include "FileUtils.h"
+#include "Image.h"
+#include "JSONReader.h"
+#include "MachOFile.h"
+#include "Misc.h"
+#include "Version32.h"
+
+#include <assert.h>
+#include <list>
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_set>
+#include <vector>
+
+#include <sqlite3.h>
+asm(".linker_option \"-lsqlite3\"");
+
+const uint32_t SchemaMajorVersion = 1;
+
+// 1 - the first version
+// 2 - added UUID to Binary table
+// 3 - added Project to Binary table
+const uint32_t SchemaMinorVersion = 3;
+
+const uint32_t MinSupportedSchemaVersion = 1;
+const uint32_t MaxSupportedSchemaVersion = 1;
+
+using mach_o::Error;
+using mach_o::Fixup;
+using mach_o::Header;
+using mach_o::Image;
+using mach_o::Platform;
+using mach_o::PlatformAndVersions;
+using mach_o::Symbol;
+using mach_o::Version32;
+typedef SymbolsCacheBinary::ImportedSymbol ImportedSymbol;
+
+SymbolsCacheBinary::SymbolsCacheBinary(std::string path, Platform platform, std::string arch,
+ std::string uuid, std::string projectName)
+ : path(path), platform(platform), arch(arch), uuid(uuid), projectName(projectName)
+{
+}
+
+SymbolsCache::SymbolsCache()
+{
+}
+
+SymbolsCache::SymbolsCache(std::string_view dbPath)
+ : dbPath(dbPath)
+{
+}
+
+SymbolsCache::~SymbolsCache()
+{
+ if ( !dbPath.empty() && (symbolsDB != nullptr) )
+ sqlite3_close(symbolsDB);
+}
+
+static Error getSchemaVersion(sqlite3* symbolsDB, Version32& version);
+
+Error SymbolsCache::open()
+{
+ bool checkSchemaVersion = false;
+ if ( dbPath.empty() ) {
+ if ( int result = sqlite3_open(":memory:", &symbolsDB) ) {
+ return Error("could not open symbols database due to: %s", sqlite3_errmsg(symbolsDB));
+ }
+ } else {
+ // If the database exists on disk, then check its compatible
+ if ( fileExists(dbPath) )
+ checkSchemaVersion = true;
+
+ if ( int result = sqlite3_open(dbPath.c_str(), &symbolsDB) ) {
+ return Error("Could not open symbols database at '%s' due to: %s",
+ dbPath.c_str(), sqlite3_errmsg(symbolsDB));
+ }
+ }
+
+ if ( checkSchemaVersion ) {
+ Version32 version;
+ if ( Error err = getSchemaVersion(this->symbolsDB, version) )
+ return err;
+
+ if ( (version.major() < MinSupportedSchemaVersion) || (version.major() > MaxSupportedSchemaVersion) ) {
+ return Error("Database schema (%d) is not supported. Only supported schemas are [%d..%d]",
+ version.major(), MinSupportedSchemaVersion, MaxSupportedSchemaVersion);
+ }
+ }
+
+ return Error();
+}
+
+Error SymbolsCache::createTables()
+{
+ assert(symbolsDB != nullptr);
+
+ // Create table for metadata
+ {
+ const char* query = "CREATE TABLE IF NOT EXISTS METADATA("
+ "SCHEMA_VERSION INTEGER NOT NULL, "
+ "SCHEMA_MINOR_VERSION INTEGER NOT NULL, "
+ "UNIQUE(SCHEMA_VERSION, SCHEMA_MINOR_VERSION) ON CONFLICT REPLACE"
+ ");";
+
+ char* errorMessage = nullptr;
+ if ( int result = sqlite3_exec(symbolsDB, query, NULL, 0, &errorMessage) ) {
+ Error err = Error("Could not create table 'METADATA' because: %s", (const char*)errorMessage);
+ sqlite3_free(errorMessage);
+ return err;
+ }
+ }
+
+ // Create table for binaries
+ {
+ const char* query = "CREATE TABLE IF NOT EXISTS BINARY("
+ "ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "
+ "PATH TEXT NOT NULL, "
+ "INSTALL_NAME TEXT, "
+ "PLATFORM INTEGER NOT NULL, "
+ "ARCH TEXT NOT NULL, "
+ "UUID TEXT, "
+ "PROJECT_NAME TEXT, "
+ "UNIQUE(PATH, PLATFORM, ARCH) ON CONFLICT REPLACE"
+ ");";
+
+ char* errorMessage = nullptr;
+ if ( int result = sqlite3_exec(symbolsDB, query, NULL, 0, &errorMessage) ) {
+ Error err = Error("Could not create table 'BINARY' because: %s", (const char*)errorMessage);
+ sqlite3_free(errorMessage);
+ return err;
+ }
+ }
+
+ // Create table for symbols
+ {
+ const char* query = "CREATE TABLE IF NOT EXISTS SYMBOL("
+ "ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "
+ "NAME TEXT UNIQUE NOT NULL);";
+
+ char* errorMessage = nullptr;
+ if ( int result = sqlite3_exec(symbolsDB, query, NULL, 0, &errorMessage) ) {
+ Error err = Error("Could not create table 'SYMBOL' because: %s", (const char*)errorMessage);
+ sqlite3_free(errorMessage);
+ return err;
+ }
+
+ const char* query2 = "CREATE INDEX IF NOT EXISTS SYMBOL_INDEX ON SYMBOL(NAME)";
+
+ char* errorMessage2 = nullptr;
+ if ( int result = sqlite3_exec(symbolsDB, query2, NULL, 0, &errorMessage2) ) {
+ Error err = Error("Could not create index 'SYMBOL' because: %s", (const char*)errorMessage);
+ sqlite3_free(errorMessage2);
+ return err;
+ }
+ }
+
+ // Create table for symbols references
+ {
+ const char* query = "CREATE TABLE IF NOT EXISTS SYMBOL_ID_REF("
+ "ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "
+ "DEF_BINARY_ID INTEGER REFERENCES BINARY NOT NULL, "
+ "REF_BINARY_ID INTEGER REFERENCES BINARY NOT NULL, "
+ "SYMBOL_ID INTEGER REFERENCES SYMBOL NOT NULL, "
+ "UNIQUE(DEF_BINARY_ID, REF_BINARY_ID, SYMBOL_ID) ON CONFLICT REPLACE);";
+
+ char* errorMessage = nullptr;
+ if ( int result = sqlite3_exec(symbolsDB, query, NULL, 0, &errorMessage) ) {
+ Error err = Error("Could not create table 'SYMBOL_ID_REF' because: %s", (const char*)errorMessage);
+ sqlite3_free(errorMessage);
+ return err;
+ }
+ }
+
+ // Create view for symbols references
+ {
+ const char* query = "CREATE VIEW IF NOT EXISTS SYMBOL_REF(DEF_BINARY_ID, REF_BINARY_ID, SYMBOL_NAME) AS "
+ "SELECT SYMBOL_ID_REF.DEF_BINARY_ID, SYMBOL_ID_REF.REF_BINARY_ID, SYMBOL.NAME AS SYMBOL_NAME "
+ "FROM SYMBOL_ID_REF JOIN SYMBOL "
+ "ON SYMBOL_ID_REF.SYMBOL_ID = SYMBOL.ID;";
+
+ char* errorMessage = nullptr;
+ if ( int result = sqlite3_exec(symbolsDB, query, NULL, 0, &errorMessage) ) {
+ Error err = Error("Could not create view 'SYMBOL_REF' because: %s", (const char*)errorMessage);
+ sqlite3_free(errorMessage);
+ return err;
+ }
+ }
+
+ // Create table for symbols definitions
+ {
+ const char* query = "CREATE TABLE IF NOT EXISTS SYMBOL_ID_DEF("
+ "ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "
+ "DEF_BINARY_ID INTEGER REFERENCES BINARY NOT NULL, "
+ "SYMBOL_ID INTEGER REFERENCES SYMBOL NOT NULL, "
+ "UNIQUE(DEF_BINARY_ID, SYMBOL_ID) ON CONFLICT REPLACE);";
+
+ char* errorMessage = nullptr;
+ if ( int result = sqlite3_exec(symbolsDB, query, NULL, 0, &errorMessage) ) {
+ Error err = Error("Could not create table 'SYMBOL_ID_DEF' because: %s", (const char*)errorMessage);
+ sqlite3_free(errorMessage);
+ return err;
+ }
+ }
+
+ // Create view for symbols definitions
+ {
+ const char* query = "CREATE VIEW IF NOT EXISTS SYMBOL_DEF(DEF_BINARY_ID, SYMBOL_NAME) AS "
+ "SELECT SYMBOL_ID_DEF.DEF_BINARY_ID, SYMBOL.NAME AS SYMBOL_NAME "
+ "FROM SYMBOL_ID_DEF JOIN SYMBOL "
+ "ON SYMBOL_ID_DEF.SYMBOL_ID = SYMBOL.ID;";
+
+ char* errorMessage = nullptr;
+ if ( int result = sqlite3_exec(symbolsDB, query, NULL, 0, &errorMessage) ) {
+ Error err = Error("Could not create view 'SYMBOL_DEF' because: %s", (const char*)errorMessage);
+ sqlite3_free(errorMessage);
+ return err;
+ }
+ }
+
+ // Create table for re-exports
+ {
+ const char* query = "CREATE TABLE IF NOT EXISTS REEXPORT("
+ "ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "
+ "BINARY_ID INTEGER REFERENCES BINARY NOT NULL, "
+ "DEP_BINARY_ID INTEGER REFERENCES BINARY NOT NULL, "
+ "UNIQUE(BINARY_ID, DEP_BINARY_ID) ON CONFLICT REPLACE);";
+
+ char* errorMessage = nullptr;
+ if ( int result = sqlite3_exec(symbolsDB, query, NULL, 0, &errorMessage) ) {
+ Error err = Error("Could not create table 'REEXPORT' because: %s", (const char*)errorMessage);
+ sqlite3_free(errorMessage);
+ return err;
+ }
+ }
+
+ return Error();
+}
+
+static Error columnExists(sqlite3* symbolsDB, std::string_view tableName, std::string_view columnName, bool& exists)
+{
+ const char* selectQuery = "SELECT COUNT(*) FROM pragma_table_info(?) WHERE name=?";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, selectQuery, -1, &statement, 0) ) {
+ Error err = Error("Could not prepare statement for table 'pragma_table_info' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_text(statement, 1, tableName.data(), -1, SQLITE_TRANSIENT) ) {
+ Error err = Error("Could not bind text for table 'pragma_table_info' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_text(statement, 2, columnName.data(), -1, SQLITE_TRANSIENT) ) {
+ Error err = Error("Could not bind text for table 'pragma_table_info' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ // Get results
+ std::vector<int64_t> results;
+ while( sqlite3_step(statement) == SQLITE_ROW ) {
+ results.push_back(sqlite3_column_int64(statement, 0));
+ }
+
+ sqlite3_finalize(statement);
+
+ if ( results.empty() )
+ return Error::none();
+
+ if ( results.size() > 1 ) {
+ return Error("Too many pragma_table_info results");
+ }
+
+ exists = results.front() != 0;
+
+ return Error::none();
+}
+
+static Error getSchemaVersion(sqlite3* symbolsDB, Version32& version)
+{
+ bool minorVersionExists = false;
+ if ( Error err = columnExists(symbolsDB, "METADATA", "SCHEMA_MINOR_VERSION", minorVersionExists) )
+ return err;
+
+ if ( minorVersionExists ) {
+ const char* selectQuery = "SELECT SCHEMA_VERSION, SCHEMA_MINOR_VERSION FROM METADATA";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, selectQuery, -1, &statement, 0) ) {
+ Error err = Error("Could not prepare statement for table 'METADATA' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ // Get results
+ std::vector<std::pair<int64_t, int64_t>> results;
+ while( sqlite3_step(statement) == SQLITE_ROW ) {
+ results.push_back({ sqlite3_column_int64(statement, 0), sqlite3_column_int64(statement, 1) });
+ }
+
+ sqlite3_finalize(statement);
+
+ if ( results.empty() ) {
+ version = Version32(1, 0);
+ return Error::none();
+ }
+
+ if ( results.size() > 1 ) {
+ return Error("Too many schema version results");
+ }
+
+ version = Version32(results.front().first, results.front().second);
+
+ return Error::none();
+ } else {
+ const char* selectQuery = "SELECT SCHEMA_VERSION FROM METADATA";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, selectQuery, -1, &statement, 0) ) {
+ Error err = Error("Could not prepare statement for table 'METADATA' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ // Get results
+ std::vector<int64_t> results;
+ while( sqlite3_step(statement) == SQLITE_ROW ) {
+ results.push_back(sqlite3_column_int64(statement, 0));
+ }
+
+ sqlite3_finalize(statement);
+
+ if ( results.empty() ) {
+ version = Version32(1, 0);
+ return Error::none();
+ }
+
+ if ( results.size() > 1 ) {
+ return Error("Too many schema version results");
+ }
+
+ version = Version32(results.front(), 0);
+
+ return Error::none();
+ }
+}
+
+static Error getDylibID(sqlite3* symbolsDB, std::string_view installName,
+ Platform platform, std::string_view arch,
+ std::optional<int64_t>& binaryID)
+{
+ const char* selectQuery = "SELECT ID FROM BINARY WHERE INSTALL_NAME = ? AND PLATFORM = ? AND ARCH = ?";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, selectQuery, -1, &statement, 0) ) {
+ Error err = Error("Could not prepare statement for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_text(statement, 1, installName.data(), -1, SQLITE_TRANSIENT) ) {
+ Error err = Error("Could not bind text for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_int(statement, 2, platform.value()) ) {
+ Error err = Error("Could not bind int for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_text(statement, 3, arch.data(), -1, SQLITE_TRANSIENT) ) {
+ Error err = Error("Could not bind text for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ // Get results
+ std::vector<int64_t> results;
+ while( sqlite3_step(statement) == SQLITE_ROW ) {
+ results.push_back(sqlite3_column_int64(statement, 0));
+ }
+
+ sqlite3_finalize(statement);
+
+ if ( results.empty() )
+ return Error::none();
+
+ if ( results.size() > 1 ) {
+ return Error("Too many binary results for dylib: %s", installName.data());
+ }
+
+ binaryID = results.front();
+
+ return Error::none();
+}
+
+static Error getBinaryUUID(sqlite3* symbolsDB, const std::string_view path,
+ const Platform platform, const std::string_view arch,
+ std::string& binaryUUID)
+{
+ // Check if the DB is new enough to have the UUID column. It appeared in 1.2
+ {
+ Version32 schemaVersion;
+ if ( Error err = getSchemaVersion(symbolsDB, schemaVersion) )
+ return err;
+
+ if ( schemaVersion < Version32(1, 2) )
+ return Error();
+ }
+
+ const char* selectQuery = "SELECT UUID FROM BINARY WHERE PATH = ? AND PLATFORM = ? AND ARCH = ?";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, selectQuery, -1, &statement, 0) ) {
+ Error err = Error("Could not prepare statement for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_text(statement, 1, path.data(), -1, SQLITE_TRANSIENT) ) {
+ Error err = Error("Could not bind text for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_int(statement, 2, platform.value()) ) {
+ Error err = Error("Could not bind int for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_text(statement, 3, arch.data(), -1, SQLITE_TRANSIENT) ) {
+ Error err = Error("Could not bind text for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ // Get results
+ std::vector<std::string> results;
+ while( sqlite3_step(statement) == SQLITE_ROW ) {
+ if ( sqlite3_column_type(statement, 0) != SQLITE_NULL )
+ results.push_back((const char*)sqlite3_column_text(statement, 0));
+ }
+
+ sqlite3_finalize(statement);
+
+ if ( results.empty() )
+ return Error::none();
+
+ if ( results.size() > 1 ) {
+ return Error("Too many binary results for binary UUID: %s", path.data());
+ }
+
+ binaryUUID = results.front();
+
+ return Error::none();
+}
+
+static Error getBinaryProject(sqlite3* symbolsDB, const std::string_view path,
+ const Platform platform, const std::string_view arch,
+ std::string& projectName)
+{
+ // Check if the DB is new enough. The Project column appeared in version 3
+ {
+ Version32 schemaVersion;
+ if ( Error err = getSchemaVersion(symbolsDB, schemaVersion) )
+ return err;
+
+ if ( schemaVersion < Version32(1, 3) )
+ return Error();
+ }
+
+ const char* selectQuery = "SELECT PROJECT_NAME FROM BINARY WHERE PATH = ? AND PLATFORM = ? AND ARCH = ?";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, selectQuery, -1, &statement, 0) ) {
+ Error err = Error("Could not prepare statement for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_text(statement, 1, path.data(), -1, SQLITE_TRANSIENT) ) {
+ Error err = Error("Could not bind text for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_int(statement, 2, platform.value()) ) {
+ Error err = Error("Could not bind int for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_text(statement, 3, arch.data(), -1, SQLITE_TRANSIENT) ) {
+ Error err = Error("Could not bind text for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ // Get results
+ std::vector<std::string> results;
+ while( sqlite3_step(statement) == SQLITE_ROW ) {
+ if ( sqlite3_column_type(statement, 0) != SQLITE_NULL )
+ results.push_back((const char*)sqlite3_column_text(statement, 0));
+ }
+
+ sqlite3_finalize(statement);
+
+ if ( results.empty() )
+ return Error::none();
+
+ if ( results.size() > 1 ) {
+ return Error("Too many binary results for binary project name: %s", path.data());
+ }
+
+ projectName = results.front();
+
+ return Error::none();
+}
+
+static Error getBinaryInstallName(sqlite3* symbolsDB, const std::string_view path,
+ const Platform platform, const std::string_view arch,
+ std::string& installName)
+{
+ const char* selectQuery = "SELECT INSTALL_NAME FROM BINARY WHERE PATH = ? AND PLATFORM = ? AND ARCH = ?";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, selectQuery, -1, &statement, 0) ) {
+ Error err = Error("Could not prepare statement for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_text(statement, 1, path.data(), -1, SQLITE_TRANSIENT) ) {
+ Error err = Error("Could not bind text for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_int(statement, 2, platform.value()) ) {
+ Error err = Error("Could not bind int for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_text(statement, 3, arch.data(), -1, SQLITE_TRANSIENT) ) {
+ Error err = Error("Could not bind text for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ // Get results
+ std::vector<std::string> results;
+ while( sqlite3_step(statement) == SQLITE_ROW ) {
+ if ( sqlite3_column_type(statement, 0) != SQLITE_NULL )
+ results.push_back((const char*)sqlite3_column_text(statement, 0));
+ }
+
+ sqlite3_finalize(statement);
+
+ if ( results.empty() )
+ return Error::none();
+
+ if ( results.size() > 1 ) {
+ return Error("Too many binary results for binary install name: %s", path.data());
+ }
+
+ installName = results.front();
+
+ return Error::none();
+}
+
+static Error getBinaryID(sqlite3* symbolsDB, std::string_view path, std::string_view installName,
+ Platform platform, std::string_view arch,
+ std::optional<int64_t>& binaryID)
+{
+ const char* selectQuery = "SELECT ID FROM BINARY WHERE PATH = ? AND INSTALL_NAME = ? AND PLATFORM = ? AND ARCH = ?";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, selectQuery, -1, &statement, 0) ) {
+ Error err = Error("Could not prepare statement for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_text(statement, 1, path.data(), -1, SQLITE_TRANSIENT) ) {
+ Error err = Error("Could not bind text for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( installName.empty() ) {
+ if ( int result = sqlite3_bind_null(statement, 2) ) {
+ Error err = Error("Could not bind null for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+ } else {
+ if ( int result = sqlite3_bind_text(statement, 2, installName.data(), -1, SQLITE_TRANSIENT) ) {
+ Error err = Error("Could not bind text for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+ }
+
+ if ( int result = sqlite3_bind_int(statement, 3, platform.value()) ) {
+ Error err = Error("Could not bind int for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_text(statement, 4, arch.data(), -1, SQLITE_TRANSIENT) ) {
+ Error err = Error("Could not bind text for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ // Get results
+ std::vector<int64_t> results;
+ while( sqlite3_step(statement) == SQLITE_ROW ) {
+ results.push_back(sqlite3_column_int64(statement, 0));
+ }
+
+ sqlite3_finalize(statement);
+
+ if ( results.empty() )
+ return Error::none();
+
+ if ( results.size() > 1 ) {
+ return Error("Too many binary results for: %s", path.data());
+ }
+
+ binaryID = results.front();
+
+ return Error::none();
+}
+
+static Error addBinary(sqlite3* symbolsDB, std::string_view path, std::string_view installName,
+ Platform platform, std::string_view arch, std::string_view uuid, std::string_view projectName,
+ int64_t& binaryID)
+{
+ const char* insertQuery = "INSERT INTO BINARY(PATH, INSTALL_NAME, PLATFORM, ARCH, UUID, PROJECT_NAME) VALUES(?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING RETURNING BINARY.ID";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, insertQuery, -1, &statement, 0) ) {
+ Error err = Error("Could not prepare statement for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_text(statement, 1, path.data(), -1, SQLITE_TRANSIENT) ) {
+ Error err = Error("Could not bind text for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( installName.empty() ) {
+ if ( int result = sqlite3_bind_null(statement, 2) ) {
+ Error err = Error("Could not bind null for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+ } else {
+ if ( int result = sqlite3_bind_text(statement, 2, installName.data(), -1, SQLITE_TRANSIENT) ) {
+ Error err = Error("Could not bind text for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+ }
+
+ if ( int result = sqlite3_bind_int(statement, 3, platform.value()) ) {
+ Error err = Error("Could not bind int for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_text(statement, 4, arch.data(), -1, SQLITE_TRANSIENT) ) {
+ Error err = Error("Could not bind text for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( uuid.empty() ) {
+ if ( int result = sqlite3_bind_null(statement, 5) ) {
+ Error err = Error("Could not bind null for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+ } else {
+ if ( int result = sqlite3_bind_text(statement, 5, uuid.data(), -1, SQLITE_TRANSIENT) ) {
+ Error err = Error("Could not bind text for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+ }
+
+ if ( projectName.empty() ) {
+ if ( int result = sqlite3_bind_null(statement, 6) ) {
+ Error err = Error("Could not bind null for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+ } else {
+ if ( int result = sqlite3_bind_text(statement, 6, projectName.data(), -1, SQLITE_TRANSIENT) ) {
+ Error err = Error("Could not bind text for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+ }
+
+ // Get results
+ std::vector<int64_t> results;
+ while( int result = sqlite3_step(statement) ) {
+ if ( result == SQLITE_DONE )
+ break;
+ if ( result == SQLITE_ROW) {
+ results.push_back(sqlite3_column_int64(statement, 0));
+ } else {
+ Error err = Error("Could not insert into table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+ }
+
+ sqlite3_reset(statement);
+ sqlite3_finalize(statement);
+
+ if ( results.empty() ) {
+ std::optional<int64_t> maybeBinaryID;
+ if ( Error err = getBinaryID(symbolsDB, path, installName, platform, arch, maybeBinaryID) ) {
+ return err;
+ }
+
+ // Its ok to skip binaries the database doesn't know about.
+ if ( !maybeBinaryID.has_value() )
+ return Error("No result for binary with path: %s", path.data());
+
+ binaryID = maybeBinaryID.value();
+ } else {
+ if ( results.size() > 1 ) {
+ return Error("Too many binary results for binary: %s", installName.data());
+ }
+
+ binaryID = results.front();
+ }
+
+ return Error::none();
+}
+
+typedef std::pair<int64_t, std::string> SymbolIDAndString;
+static Error addSymbolStrings(sqlite3* symbolsDB,
+ std::span<const std::string> strings,
+ std::vector<SymbolIDAndString>& results)
+{
+ const char* insertQuery = "INSERT INTO SYMBOL(NAME) "
+ "VALUES("
+ "?"
+ ") "
+ "ON CONFLICT DO NOTHING RETURNING SYMBOL.ID, SYMBOL.NAME";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, insertQuery, -1, &statement, 0) ) {
+ return Error("Could not prepare statement for table 'SYMBOL' because: %s", (const char*)strerror(result));
+ }
+
+ for ( std::string_view str : strings ) {
+ if ( int result = sqlite3_bind_text(statement, 1, str.data(), -1, SQLITE_TRANSIENT) ) {
+ return Error("Could not bind text for table 'SYMBOL' because: %s", (const char*)strerror(result));
+ }
+
+ // printf("inserting: %s %s\n", installName, symbolName);
+
+ // Get results
+ while( int result = sqlite3_step(statement) ) {
+ if ( result == SQLITE_DONE )
+ break;
+ if ( result == SQLITE_ROW) {
+ results.push_back({ sqlite3_column_int64(statement, 0), (const char*)sqlite3_column_text(statement, 1) });
+ } else {
+ Error err = Error("Could not insert into table 'SYMBOL' because: %s", (const char*)strerror(result));
+ return err;
+ }
+ }
+
+ sqlite3_reset(statement);
+ }
+ sqlite3_finalize(statement);
+
+ return Error::none();
+}
+
+static Error addExports(sqlite3* symbolsDB, int64_t binaryID,
+ std::span<const std::string> exports,
+ const SymbolsCache::SymbolNameCache& symbolNameCache)
+{
+ const char* insertQuery = "INSERT INTO SYMBOL_ID_DEF(DEF_BINARY_ID, SYMBOL_ID) "
+ "VALUES("
+ "?, "
+ "?"
+ ")";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, insertQuery, -1, &statement, 0) ) {
+ return Error("Could not prepare statement for table 'SYMBOL_ID_DEF' because: %s", (const char*)strerror(result));
+ }
+
+ for ( std::string_view symbolName : exports ) {
+ auto it = symbolNameCache.find(symbolName.data());
+ if ( it == symbolNameCache.end() )
+ return Error("Could not find symbol name for '%s", symbolName.data());
+
+ if ( int result = sqlite3_bind_int64(statement, 1, binaryID) ) {
+ return Error("Could not bind int for table 'SYMBOL_ID_DEF' because: %s", (const char*)strerror(result));
+ }
+
+ if ( int result = sqlite3_bind_int64(statement, 2, it->second) ) {
+ return Error("Could not bind int for table 'SYMBOL_ID_DEF' because: %s", (const char*)strerror(result));
+ }
+
+ if ( int result = sqlite3_step(statement); result != SQLITE_DONE ) {
+ return Error("Could not insert into table 'SYMBOL_ID_DEF' because: %s", (const char*)strerror(result));
+ }
+ sqlite3_reset(statement);
+ }
+ sqlite3_finalize(statement);
+
+ return Error::none();
+}
+
+static Error addImports(sqlite3* symbolsDB, int64_t refBinaryID,
+ Platform platform, std::string_view arch,
+ std::span<const ImportedSymbol> imports,
+ const SymbolsCache::SymbolNameCache& symbolNameCache)
+{
+ // Add dependent binaries and record their binary IDs
+ std::vector<int64_t> targetBinaryIDs;
+ {
+ for ( ImportedSymbol importedSymbol : imports ) {
+ // The target is an install name string or the binary ID we need
+ if ( const int64_t* targetBinaryID = std::get_if<int64_t>(&importedSymbol.targetBinary) ) {
+ targetBinaryIDs.push_back(*targetBinaryID);
+ continue;
+ }
+
+ std::string_view installNameView = std::get<std::string>(importedSymbol.targetBinary);
+ int64_t targetBinaryID = 0;
+ if ( Error err = addBinary(symbolsDB, installNameView, installNameView, platform, arch, "", "", targetBinaryID) )
+ return err;
+
+ targetBinaryIDs.push_back(targetBinaryID);
+ }
+ }
+
+ // Add symbol refs (imports)
+ {
+ const char* insertQuery = "INSERT INTO SYMBOL_ID_REF(DEF_BINARY_ID, REF_BINARY_ID, SYMBOL_ID) "
+ "VALUES("
+ "?, "
+ "?, "
+ "? "
+ ")";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, insertQuery, -1, &statement, 0) ) {
+ return Error("Could not prepare statement for table 'SYMBOL_ID_REF' because: %s", (const char*)strerror(result));
+ }
+
+ assert(imports.size() == targetBinaryIDs.size());
+ for ( uint32_t symbolIndex = 0; symbolIndex != imports.size(); ++symbolIndex ) {
+ const ImportedSymbol& importedSymbol = imports[symbolIndex];
+ int64_t targetBinaryID = targetBinaryIDs[symbolIndex];
+
+ auto it = symbolNameCache.find(importedSymbol.symbolName.data());
+ if ( it == symbolNameCache.end() )
+ return Error("Could not find symbol name for '%s", importedSymbol.symbolName.data());
+
+ if ( int result = sqlite3_bind_int64(statement, 1, targetBinaryID) ) {
+ return Error("Could not bind int for table 'SYMBOL_ID_REF' because: %s", (const char*)strerror(result));
+ }
+
+ if ( int result = sqlite3_bind_int64(statement, 2, refBinaryID) ) {
+ return Error("Could not bind int for table 'SYMBOL_ID_REF' because: %s", (const char*)strerror(result));
+ }
+
+ if ( int result = sqlite3_bind_int64(statement, 3, it->second) ) {
+ return Error("Could not bind int for table 'SYMBOL_ID_REF' because: %s", (const char*)strerror(result));
+ }
+
+ // printf("inserting: %s %s\n", installName, symbolName);
+
+ if ( int result = sqlite3_step(statement); result != SQLITE_DONE ) {
+ return Error("Could not insert into table 'SYMBOL_ID_REF' because: %s", (const char*)strerror(result));
+ }
+ sqlite3_reset(statement);
+ }
+ sqlite3_finalize(statement);
+ }
+
+ return Error::none();
+}
+
+static Error addReexports(sqlite3* symbolsDB, int64_t binaryID,
+ Platform platform, std::string_view arch,
+ std::span<const SymbolsCacheBinary::TargetBinary> reexports)
+{
+ // Add dependent binaries and record their binary IDs
+ std::vector<int64_t> targetBinaryIDs;
+ {
+ for ( const SymbolsCacheBinary::TargetBinary& reexport : reexports ) {
+ // The target is an install name string or the binary ID we need
+ if ( const int64_t* targetBinaryID = std::get_if<int64_t>(&reexport) ) {
+ targetBinaryIDs.push_back(*targetBinaryID);
+ continue;
+ }
+
+ std::string_view installNameView = std::get<std::string>(reexport);
+ int64_t targetBinaryID = 0;
+ if ( Error err = addBinary(symbolsDB, installNameView, installNameView, platform, arch, "", "", targetBinaryID) )
+ return err;
+
+ targetBinaryIDs.push_back(targetBinaryID);
+ }
+ }
+
+ // Add symbol refs (imports)
+ {
+ const char* insertQuery = "INSERT INTO REEXPORT(BINARY_ID, DEP_BINARY_ID) "
+ "VALUES("
+ "?, "
+ "?"
+ ")";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, insertQuery, -1, &statement, 0) ) {
+ return Error("Could not prepare statement for table 'REEXPORT' because: %s", (const char*)strerror(result));
+ }
+
+ assert(reexports.size() == targetBinaryIDs.size());
+ for ( int64_t targetBinaryID : targetBinaryIDs ) {
+ if ( int result = sqlite3_bind_int64(statement, 1, binaryID) ) {
+ return Error("Could not bind int for table 'REEXPORT' because: %s", (const char*)strerror(result));
+ }
+
+ if ( int result = sqlite3_bind_int64(statement, 2, targetBinaryID) ) {
+ return Error("Could not bind int for table 'REEXPORT' because: %s", (const char*)strerror(result));
+ }
+
+ // printf("inserting: %s %s\n", installName, symbolName);
+
+ if ( int result = sqlite3_step(statement); result != SQLITE_DONE ) {
+ return Error("Could not insert into table 'REEXPORT' because: %s", (const char*)strerror(result));
+ }
+ sqlite3_reset(statement);
+ }
+ sqlite3_finalize(statement);
+ }
+
+ return Error::none();
+}
+
+static Error addMetadata(sqlite3* symbolsDB)
+{
+ const char* insertQuery = "INSERT INTO METADATA(SCHEMA_VERSION, SCHEMA_MINOR_VERSION) VALUES(?, ?) ON CONFLICT DO NOTHING";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, insertQuery, -1, &statement, 0) ) {
+ Error err = Error("Could not prepare statement for table 'METADATA' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_int(statement, 1, SchemaMajorVersion) ) {
+ Error err = Error("Could not bind text for table 'METADATA' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_int(statement, 2, SchemaMinorVersion) ) {
+ Error err = Error("Could not bind text for table 'METADATA' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_step(statement); result != SQLITE_DONE ) {
+ return Error("Could not insert into table 'METADATA' because: %s", (const char*)strerror(result));
+ }
+
+ sqlite3_reset(statement);
+ sqlite3_finalize(statement);
+
+ return Error::none();
+}
+
+Error SymbolsCache::create()
+{
+ if ( Error err = open() )
+ return err;
+ if ( Error err = createTables() )
+ return err;
+ if ( Error err = addMetadata(this->symbolsDB) )
+ return err;
+ return Error();
+}
+
+namespace {
+
+struct Slice
+{
+ const Header* sliceHeader;
+ size_t sliceLength;
+ Platform platform;
+};
+
+struct CallbackOnError
+{
+ typedef void (^Callback)();
+ CallbackOnError(Callback callback) : callback(callback) { }
+ ~CallbackOnError() {
+ if ( callback )
+ callback();
+ }
+
+ Callback callback;
+};
+
+}
+
+static Error getSlicesToAdd(const SymbolsCache::ArchPlatforms& archPlatforms,
+ const dyld3::closure::FileSystem& fileSystem,
+ const void* buffer, uint64_t bufferSize, std::string_view path,
+ std::vector<Slice>& slices)
+{
+ if ( path.ends_with(".metallib") )
+ return Error::none();
+
+ Error parseErr = mach_o::forEachHeader({ (uint8_t*)buffer, bufferSize }, path,
+ ^(const Header *hdr, size_t sliceLength, bool &stop) {
+ std::span<const Platform> supportedPlatforms;
+ if ( archPlatforms.empty() ) {
+ // support all platforms if there are no archs
+ } else if ( auto it = archPlatforms.find(hdr->archName()); it != archPlatforms.end() ) {
+ supportedPlatforms = it->second;
+ } else {
+ return;
+ }
+
+ PlatformAndVersions pvs = hdr->platformAndVersions();
+ if ( pvs.platform.empty() )
+ return;
+
+ // HACK: Pretend zippered are macOS, so that the database doesn't have to care about zippering
+ Platform platform;
+ if ( (pvs.platform == Platform::zippered) || (pvs.platform == Platform::macCatalyst) )
+ platform = Platform::macOS;
+ else
+ platform = pvs.platform;
+
+ if ( !supportedPlatforms.empty() && (std::find(supportedPlatforms.begin(), supportedPlatforms.end(), platform) == supportedPlatforms.end()) )
+ return;
+
+ if ( !hdr->isDylib() && !hdr->isDynamicExecutable() )
+ return;
+
+ if ( hdr->isDylib() ) {
+ std::string_view installName = hdr->installName();
+ std::string_view dylibPath = path;
+ if ( installName != dylibPath ) {
+ // We now typically require that install names and paths match. However symlinks may allow us to bring in a path which
+ // doesn't match its install name.
+ // For example:
+ // /usr/lib/libstdc++.6.0.9.dylib is a real file with install name /usr/lib/libstdc++.6.dylib
+ // /usr/lib/libstdc++.6.dylib is a symlink to /usr/lib/libstdc++.6.0.9.dylib
+ // So long as we add both paths (with one as an alias) then this will work, even if dylibs are removed from disk
+ // but the symlink remains.
+ // Apply the same symlink crawling for dylibs that will install their contents to Cryptex paths but will have
+ // install names with the cryptex paths removed.
+ char resolvedSymlinkPath[PATH_MAX];
+ if ( fileSystem.getRealPath(installName.data(), resolvedSymlinkPath) ) {
+ if ( resolvedSymlinkPath == dylibPath ) {
+ // Symlink is the install name and points to the on-disk dylib
+ //fprintf(stderr, "Symlink works: %s == %s\n", inputFile.path, installName.c_str());
+ dylibPath = installName;
+ }
+ } else {
+ // HACK: The build record doesn't have symlinks or anything to allow the above realpath code
+ // to reason about the cryptex. So just look for it specifically
+ if ( dylibPath == (std::string("/System/Cryptexes/OS") + std::string(installName)) )
+ dylibPath = installName;
+ }
+ }
+
+ const dyld3::MachOFile* mf = (const dyld3::MachOFile*)hdr;
+ if ( !mf->canBePlacedInDyldCache(dylibPath.data(), false /* check objc */, ^(const char* format, ...){ }) )
+ return;
+ }
+
+ slices.push_back({ hdr, sliceLength, platform });
+ });
+
+ if ( parseErr ) {
+ return parseErr;
+ }
+
+ return Error::none();
+}
+
+static std::string_view leafName(std::string_view str)
+{
+ size_t pos = str.rfind('/');
+ if ( pos == std::string_view::npos )
+ return str;
+ return str.substr(pos+1);
+}
+
+static mach_o::Error makeBinaryFromJSON(const SymbolsCache::ArchPlatforms& archPlatforms,
+ const json::Node& rootNode, std::string_view path,
+ std::string_view projectName,
+ bool allowExecutables,
+ std::vector<SymbolsCacheBinary>& binaries)
+{
+ using json::Node;
+
+ // In XBS we expect trace files to be decompressed along with some helpful preamble. The key for that is
+ // a node called "api-version" so if we see that, we know this file has a certain structure
+ Diagnostics diags;
+ if ( json::getOptionalValue(diags, rootNode, "api-version") ) {
+ // Walk the trace-files[] and then the contents[]
+ const Node& traceFilesNode = json::getRequiredValue(diags, rootNode, "trace-files");
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+
+ for ( const Node& traceFileNode : traceFilesNode.array ) {
+ const Node& contentsNode = json::getRequiredValue(diags, traceFileNode, "contents");
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+
+ for ( const Node& contentNode : contentsNode.array ) {
+ if ( Error err = makeBinaryFromJSON(archPlatforms, contentNode, path, projectName, allowExecutables, binaries) )
+ return err;
+ }
+ }
+
+ return Error::none();
+ }
+
+ const Node& versionNode = json::getRequiredValue(diags, rootNode, "version");
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+
+ uint64_t jsonVersion = json::parseRequiredInt(diags, versionNode);
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+
+ if ( jsonVersion > 2 ) {
+ // Is it ok to silently return? It allows old tools to ignore new JSON so maybe what we want
+ return Error::none();
+ }
+
+ // Skip binaries which aren't cache eligible
+ const Node* sharedCacheEligibleNode = json::getOptionalValue(diags, rootNode, "shared-cache-eligible");
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+
+ if ( (sharedCacheEligibleNode != nullptr) && sharedCacheEligibleNode->value != "yes" )
+ return Error::none();
+
+ const Node& archNode = json::getRequiredValue(diags, rootNode, "arch");
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+
+ const std::string& archName = json::parseRequiredString(diags, archNode);
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+
+ std::span<const Platform> supportedPlatforms;
+ if ( archPlatforms.empty() ) {
+ // support all platforms if there are no archs
+ } else if ( auto it = archPlatforms.find(archName); it != archPlatforms.end() ) {
+ supportedPlatforms = it->second;
+ } else {
+ return Error::none();
+ }
+
+ const Node& platformsNode = json::getRequiredValue(diags, rootNode, "platforms");
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+
+ if ( platformsNode.array.empty() )
+ return Error::none();
+
+ Platform platform;
+ for ( const Node& platformNode : platformsNode.array ) {
+ const Node& nameNode = json::getRequiredValue(diags, platformNode, "name");
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+
+ const std::string& platformName = json::parseRequiredString(diags, nameNode);
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+
+ Platform foundPlatform = Platform::byName(platformName);
+
+ // HACK: Pretend zippered are macOS, so that the database doesn't have to care about zippering
+ if ( (foundPlatform == Platform::zippered) || (foundPlatform == Platform::macCatalyst) )
+ foundPlatform = Platform::macOS;
+
+ if ( !supportedPlatforms.empty() && (std::find(supportedPlatforms.begin(), supportedPlatforms.end(), foundPlatform) == supportedPlatforms.end()) )
+ continue;
+
+ platform = foundPlatform;
+ }
+
+ if ( Error err = platform.valid() )
+ return Error::none();
+
+ const Node* installNameNode = json::getOptionalValue(diags, rootNode, "install-name");
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+
+ const Node* finalPathNode = json::getOptionalValue(diags, rootNode, "final-output-path");
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+
+ if ( !installNameNode && !allowExecutables )
+ return Error::none();
+
+ if ( !installNameNode && !finalPathNode )
+ return Error::none();
+
+ std::string_view installName;
+ if ( installNameNode != nullptr ) {
+ installName = json::parseRequiredString(diags, *installNameNode);
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+ }
+
+ std::string_view finalPath;
+ if ( finalPathNode != nullptr ) {
+ finalPath = json::parseRequiredString(diags, *finalPathNode);
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+ } else {
+ finalPath = installName;
+ }
+
+ const Node* uuidNode = json::getOptionalValue(diags, rootNode, "uuid");
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+
+ std::string_view uuid;
+ if ( uuidNode ) {
+ uuid = json::parseRequiredString(diags, *uuidNode);
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+ }
+
+ std::vector<SymbolsCacheBinary::ImportedSymbol> importedSymbols;
+ std::vector<SymbolsCacheBinary::TargetBinary> reexports;
+ const Node* linkedDylibsNode = json::getOptionalValue(diags, rootNode, "linked-dylibs");
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+
+ if ( (linkedDylibsNode != nullptr) && !linkedDylibsNode->array.empty() ) {
+ for ( const Node& linkedDylibNode : linkedDylibsNode->array ) {
+ const Node& targetInstallNameNode = json::getRequiredValue(diags, linkedDylibNode, "install-name");
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+
+ std::string_view targetInstallName = json::parseRequiredString(diags, targetInstallNameNode);
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+
+ if ( !Header::isSharedCacheEligiblePath(targetInstallName.data()) )
+ continue;
+
+ const Node& importedSymbolsNode = json::getRequiredValue(diags, linkedDylibNode, "imported-symbols");
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+
+ if ( !importedSymbolsNode.array.empty() ) {
+ importedSymbols.reserve(importedSymbolsNode.array.size());
+ for ( const Node& importedSymbol : importedSymbolsNode.array ) {
+ importedSymbols.push_back({ std::string(targetInstallName), importedSymbol.value });
+ }
+ }
+
+ const Node& attributesNode = json::getRequiredValue(diags, linkedDylibNode, "attributes");
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+
+ if ( !attributesNode.array.empty() ) {
+ for ( const Node& attributeNode : attributesNode.array ) {
+ if ( attributeNode.value == "re-export" )
+ reexports.push_back(std::string(targetInstallName));
+ }
+ }
+ }
+ }
+
+ __block std::vector<std::string> exportedSymbols;
+ if ( !installName.empty() && Header::isSharedCacheEligiblePath(installName.data()) ) {
+ const Node* exportedSymbolsNode = json::getOptionalValue(diags, rootNode, "exports");
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+
+ if ( (exportedSymbolsNode != nullptr) && !exportedSymbolsNode->array.empty() ) {
+ exportedSymbols.reserve(exportedSymbolsNode->array.size());
+ for ( const Node& exportedSymbol : exportedSymbolsNode->array )
+ exportedSymbols.push_back(exportedSymbol.value);
+ }
+ }
+
+ SymbolsCacheBinary binary(std::string(finalPath), platform, archName,
+ std::string(uuid), std::string(projectName));
+ binary.installName = installName;
+ binary.exportedSymbols = std::move(exportedSymbols);
+ binary.importedSymbols = std::move(importedSymbols);
+ binary.reexportedLibraries = std::move(reexports);
+ binary.inputFileName = leafName(path);
+
+ binaries.push_back(std::move(binary));
+ return Error::none();
+}
+
+Error SymbolsCache::makeBinariesFromJSON(const ArchPlatforms& archPlatforms,
+ const void* buffer, uint64_t bufferSize, std::string_view path,
+ std::string_view projectName, bool allowExecutables,
+ std::vector<SymbolsCacheBinary>& binaries)
+{
+ using json::Node;
+
+ // The buffer is likely in the "JSON lines" format. If so, parse each line as its own JSON
+ {
+ std::string_view wholeString((const char*)buffer, bufferSize);
+ while ( !wholeString.empty() ) {
+ auto nextNewLinePos = wholeString.find('\n');
+ if ( nextNewLinePos == std::string_view::npos )
+ break;
+ std::string_view line = wholeString.substr(0, nextNewLinePos);
+ wholeString = wholeString.substr(line.size() + 1);
+ if ( line.empty() )
+ continue;
+
+ if ( line.starts_with('{') && line.ends_with('}') ) {
+ Diagnostics diags;
+ Node rootNode = json::readJSON(diags, line.data(), line.size(), false /* useJSON5 */);
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+
+ if ( Error err = makeBinaryFromJSON(archPlatforms, rootNode, path, projectName, allowExecutables, binaries) )
+ return err;
+ } else {
+ break;
+ }
+ }
+
+ // If we processed the whole file as JSON lines, then nothing else to do
+ if ( wholeString.empty() )
+ return Error::none();
+ }
+
+ Diagnostics diags;
+ Node rootNode = json::readJSON(diags, buffer, bufferSize, false /* useJSON5 */);
+ if ( diags.hasError() )
+ return Error("Could not parse JSON '%s' because: %s", path.data(), diags.errorMessageCStr());
+
+ return makeBinaryFromJSON(archPlatforms, rootNode, path, projectName, allowExecutables, binaries);
+}
+
+Error SymbolsCache::makeBinaries(const ArchPlatforms& archPlatforms,
+ const dyld3::closure::FileSystem& fileSystem,
+ const void* buffer, uint64_t bufferSize, std::string_view path,
+ std::string_view projectName,
+ std::vector<SymbolsCacheBinary>& binaries)
+{
+ if ( path.ends_with(".json") )
+ return makeBinariesFromJSON(archPlatforms, buffer, bufferSize, path, projectName, false, binaries);
+
+ std::vector<Slice> slices;
+ if ( Error err = getSlicesToAdd(archPlatforms, fileSystem, buffer, bufferSize, path, slices) )
+ return err;
+
+ if ( slices.empty() )
+ return Error::none();
+
+ for ( const Slice& slice : slices ) {
+ const Header* mh = slice.sliceHeader;
+ Platform platform = slice.platform;
+ const char* sliceArch = mh->archName();
+
+ Image image(slice.sliceHeader, slice.sliceLength, Image::MappingKind::wholeSliceMapped);
+
+ // printf("Processing: %s", &path[0]);
+
+ // Add def binary
+ std::string_view binaryInstallName;
+ if ( const char* installName = mh->installName() )
+ binaryInstallName = installName;
+
+ // Add defs (exports)
+ __block std::vector<std::string> exportedSymbols;
+ if ( const char* installName = mh->installName(); (installName != nullptr) && (installName[0] == '/') ) {
+ if ( image.hasExportsTrie() ) {
+ image.exportsTrie().forEachExportedSymbol(^(const Symbol& symbol, bool& stopExport) {
+ exportedSymbols.push_back(symbol.name().c_str());
+ });
+ }
+ }
+
+ // Add symbol refs (imports)
+ __block std::vector<SymbolsCacheBinary::ImportedSymbol> importedSymbols;
+ image.forEachBindTarget(^(const Fixup::BindTarget& targetInfo, bool& stop) {
+ // TODO: We should be able to check weak-defs too, by looking at all binaries in the
+ // dependency tree of this binary.
+ if ( targetInfo.libOrdinal <= 0 )
+ return;
+ const char* depLoadPath = mh->linkedDylibLoadPath(targetInfo.libOrdinal-1);
+
+ importedSymbols.push_back({ depLoadPath, targetInfo.symbolName.c_str() });
+ });
+
+ // Add re-exports
+ __block std::vector<SymbolsCacheBinary::TargetBinary> reexports;
+ if ( const char* installName = mh->installName(); (installName != nullptr) && (installName[0] == '/') ) {
+ mh->forEachLinkedDylib(^(const char* loadPath, mach_o::LinkedDylibAttributes kind, Version32 compatVersion, Version32 curVersion,
+ bool synthesizedLink, bool& stop) {
+ if ( kind.reExport )
+ reexports.push_back(loadPath);
+ });
+ }
+
+ // Get UUID
+ std::string uuidString;
+ uuid_t uuid;
+ if ( mh->getUuid(uuid) ) {
+ uuid_string_t uuidStrBuffer;
+ uuid_unparse(uuid, uuidStrBuffer);
+ uuidString = uuidStrBuffer;
+ }
+
+ std::string_view binaryPath = mh->isDylib() ? binaryInstallName : path;
+
+ SymbolsCacheBinary binary(std::string(binaryPath), platform, sliceArch,
+ std::string(uuidString), std::string(projectName));
+ binary.installName = binaryInstallName;
+ binary.exportedSymbols = std::move(exportedSymbols);
+ binary.importedSymbols = std::move(importedSymbols);
+ binary.reexportedLibraries = std::move(reexports);
+
+ binaries.push_back(std::move(binary));
+ }
+
+ return Error::none();
+}
+
+Error SymbolsCache::serialize(const uint8_t*& buffer, uint64_t& bufferSize)
+{
+ sqlite3_exec(symbolsDB, "VACUUM", 0, 0, 0);
+
+ sqlite3_int64 resultSize = 0;
+ unsigned char* resultBuffer = sqlite3_serialize(symbolsDB, "main", &resultSize, 0);
+ if ( !resultBuffer )
+ return("Could not serialize symbols database");
+
+ buffer = resultBuffer;
+ bufferSize = resultSize;
+
+ return Error();
+}
+
+// Testing
+Error SymbolsCache::startTransaction()
+{
+ char* errorMessage = nullptr;
+ if ( int result = sqlite3_exec(symbolsDB, "BEGIN", NULL, 0, &errorMessage) ) {
+ Error err = Error("Could not 'BEGIN' because: %s", (const char*)errorMessage);
+ sqlite3_free(errorMessage);
+ return err;
+ }
+
+ return Error::none();
+}
+
+Error SymbolsCache::endTransaction()
+{
+ char* errorMessage = nullptr;
+ Error err = Error::none();
+ if ( int result = sqlite3_exec(symbolsDB, "COMMIT", NULL, 0, &errorMessage) ) {
+ err = Error("Could not 'COMMIT' because: %s", (const char*)errorMessage);
+ sqlite3_free(errorMessage);
+ }
+ return err;
+}
+
+Error SymbolsCache::rollbackTransaction()
+{
+ char* errorMessage = nullptr;
+ Error err = Error::none();
+ if ( int result = sqlite3_exec(symbolsDB, "ROLLBACK", NULL, 0, &errorMessage) ) {
+ err = Error("Could not 'ROLLBACK' because: %s", (const char*)errorMessage);
+ sqlite3_free(errorMessage);
+ }
+ return err;
+}
+
+Error SymbolsCache::addExecutableFile(std::string_view path, Platform platform, std::string_view arch,
+ std::string_view uuid, std::string_view projectName,
+ int64_t& binaryID)
+{
+ return ::addBinary(this->symbolsDB, path, "", platform, arch, uuid, projectName, binaryID);
+}
+
+Error SymbolsCache::addDylibFile(std::string_view path, std::string_view installName,
+ Platform platform, std::string_view arch, std::string_view uuid,
+ std::string_view projectName,
+ int64_t& binaryID)
+{
+ return ::addBinary(this->symbolsDB, path, installName, platform, arch, uuid, projectName, binaryID);
+}
+
+Error SymbolsCache::addBinaries(std::vector<SymbolsCacheBinary>& binaries)
+{
+ // Add all entries to the BINARY table
+ {
+ if ( mach_o::Error err = this->startTransaction() )
+ return err;
+
+ __block Error rollbackError = Error::none();
+ CallbackOnError callbackOnError(^() { rollbackError = this->rollbackTransaction(); });
+
+ for ( SymbolsCacheBinary& binary : binaries ) {
+ int64_t binaryID = 0;
+ if ( binary.installName.empty() ) {
+ if ( Error err = this->addExecutableFile(binary.path, binary.platform, binary.arch, binary.uuid, binary.projectName, binaryID) )
+ return err;
+ } else {
+ if ( Error err = this->addDylibFile(binary.path, binary.installName, binary.platform, binary.arch, binary.uuid, binary.projectName, binaryID) )
+ return err;
+ }
+
+ binary.binaryID = binaryID;
+ }
+
+ if ( mach_o::Error err = this->endTransaction() )
+ return err;
+
+ // If we succeeded then don't rollback
+ callbackOnError.callback = nullptr;
+
+ if ( rollbackError )
+ return std::move(rollbackError);
+ }
+
+ // Add all entries to the SYMBOL table
+ {
+ if ( mach_o::Error err = this->startTransaction() )
+ return err;
+
+ __block Error rollbackError = Error::none();
+ CallbackOnError callbackOnError(^() { rollbackError = this->rollbackTransaction(); });
+
+ for ( SymbolsCacheBinary& binary : binaries ) {
+ if ( !binary.exportedSymbols.empty() ) {
+ std::vector<SymbolIDAndString> results;
+ if ( Error err = addSymbolStrings(this->symbolsDB, binary.exportedSymbols, results) )
+ return err;
+
+ if ( !results.empty() ) {
+ for ( const SymbolIDAndString& symbolIDAndString : results )
+ this->symbolNameCache[symbolIDAndString.second] = symbolIDAndString.first;
+ }
+ }
+
+ if ( !binary.importedSymbols.empty() ) {
+ std::vector<std::string> symbolNames;
+ for ( const SymbolsCacheBinary::ImportedSymbol& importedSymbol : binary.importedSymbols )
+ symbolNames.push_back(importedSymbol.symbolName);
+
+ std::vector<SymbolIDAndString> results;
+ if ( Error err = addSymbolStrings(this->symbolsDB, symbolNames, results) )
+ return err;
+
+ if ( !results.empty() ) {
+ for ( const SymbolIDAndString& symbolIDAndString : results )
+ this->symbolNameCache[symbolIDAndString.second] = symbolIDAndString.first;
+ }
+ }
+ }
+
+ if ( mach_o::Error err = this->endTransaction() )
+ return err;
+
+ // If we succeeded then don't rollback
+ callbackOnError.callback = nullptr;
+
+ if ( rollbackError )
+ return std::move(rollbackError);
+ }
+
+ // Add all imports (SYMBOL_REF), exports(SYMBOL_DEF) and reexports
+ {
+ if ( mach_o::Error err = this->startTransaction() )
+ return err;
+
+ __block Error rollbackError = Error::none();
+ CallbackOnError callbackOnError(^() { rollbackError = this->rollbackTransaction(); });
+
+ for ( SymbolsCacheBinary& binary : binaries ) {
+ if ( !binary.exportedSymbols.empty() ) {
+ if ( Error err = addExports(this->symbolsDB, binary.binaryID.value(), binary.exportedSymbols, this->symbolNameCache) )
+ return err;
+ }
+
+ if ( !binary.importedSymbols.empty() ) {
+ if ( Error err = addImports(this->symbolsDB, binary.binaryID.value(), binary.platform, binary.arch, binary.importedSymbols, this->symbolNameCache) )
+ return err;
+ }
+
+ if ( !binary.reexportedLibraries.empty() ) {
+ if ( Error err = addReexports(this->symbolsDB, binary.binaryID.value(), binary.platform, binary.arch, binary.reexportedLibraries) )
+ return err;
+ }
+ }
+
+ if ( mach_o::Error err = this->endTransaction() )
+ return err;
+
+ // If we succeeded then don't rollback
+ callbackOnError.callback = nullptr;
+
+ if ( rollbackError )
+ return std::move(rollbackError);
+ }
+
+ return Error::none();
+}
+
+bool SymbolsCache::containsExecutable(std::string_view path) const
+{
+ const char* selectQuery = "SELECT PATH FROM BINARY WHERE PATH = ?";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, selectQuery, -1, &statement, 0) ) {
+ return false;
+ }
+
+ if ( int result = sqlite3_bind_text(statement, 1, path.data(), -1, SQLITE_TRANSIENT) ) {
+ return false;
+ }
+
+ // Get results
+ int count = 0;
+ while( sqlite3_step(statement) == SQLITE_ROW ) {
+ // printf("Got result: %s\n", sqlite3_column_text(statement, 0));
+ ++count;
+ }
+
+ sqlite3_finalize(statement);
+
+ return (count != 0);
+}
+
+bool SymbolsCache::containsDylib(std::string_view path, std::string_view installName) const
+{
+ const char* selectQuery = "SELECT PATH FROM BINARY WHERE PATH = ? AND INSTALL_NAME = ?";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, selectQuery, -1, &statement, 0) ) {
+ return false;
+ }
+
+ if ( int result = sqlite3_bind_text(statement, 1, path.data(), -1, SQLITE_TRANSIENT) ) {
+ return false;
+ }
+
+ if ( int result = sqlite3_bind_text(statement, 2, installName.data(), -1, SQLITE_TRANSIENT) ) {
+ return false;
+ }
+
+ // Get results
+ int count = 0;
+ while( sqlite3_step(statement) == SQLITE_ROW ) {
+ // printf("Got result: %s\n", sqlite3_column_text(statement, 0));
+ ++count;
+ }
+
+ sqlite3_finalize(statement);
+
+ return (count != 0);
+}
+
+mach_o::Error SymbolsCache::getAllBinaries(std::vector<SymbolsCacheBinary>& binaries) const
+{
+ bool canGetProjectName = false;
+
+ // Check if the DB is new enough. The Project column appeared in version 3
+ {
+ Version32 schemaVersion;
+ if ( Error err = getSchemaVersion(symbolsDB, schemaVersion) )
+ return err;
+
+ canGetProjectName = schemaVersion >= Version32(1, 3);
+ }
+
+ const char* selectQueryOld = "SELECT BINARY.PATH, BINARY.ARCH, BINARY.PLATFORM "
+ "FROM BINARY "
+ "ORDER BY BINARY.PATH";
+ const char* selectQueryNew = "SELECT BINARY.PATH, BINARY.ARCH, BINARY.PLATFORM, BINARY.UUID, BINARY.PROJECT_NAME "
+ "FROM BINARY "
+ "ORDER BY BINARY.PATH";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, canGetProjectName ? selectQueryNew : selectQueryOld, -1, &statement, 0) ) {
+ Error err = Error("Could not prepare statement for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ // Get results
+ while( sqlite3_step(statement) == SQLITE_ROW ) {
+ const char* path = (const char*)sqlite3_column_text(statement, 0);
+ const char* arch = (const char*)sqlite3_column_text(statement, 1);
+ int64_t platform = sqlite3_column_int64(statement, 2);
+ const char* uuid = nullptr;
+ const char* projectName = nullptr;
+
+ if ( canGetProjectName ) {
+ if ( sqlite3_column_type(statement, 3) != SQLITE_NULL )
+ uuid = (const char*)sqlite3_column_text(statement, 3);
+ if ( sqlite3_column_type(statement, 4) != SQLITE_NULL )
+ projectName = (const char*)sqlite3_column_text(statement, 4);
+ }
+ binaries.push_back({
+ path, Platform((uint32_t)platform), arch,
+ (uuid != nullptr ? uuid : ""),
+ (projectName != nullptr ? projectName : "")
+ });
+ }
+
+ sqlite3_finalize(statement);
+
+ return Error::none();
+}
+
+std::vector<SymbolsCacheBinary::ImportedSymbol> SymbolsCache::getImports(std::string_view path) const
+{
+ const char* selectQuery = "SELECT DEF_BINARY.INSTALL_NAME, SYMBOL_REF.SYMBOL_NAME "
+ "FROM SYMBOL_REF "
+ "JOIN BINARY AS REF_BINARY ON SYMBOL_REF.REF_BINARY_ID = REF_BINARY.ID "
+ "JOIN BINARY AS DEF_BINARY ON SYMBOL_REF.DEF_BINARY_ID = DEF_BINARY.ID "
+ "WHERE REF_BINARY.PATH = ?";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, selectQuery, -1, &statement, 0) ) {
+ return { };
+ }
+
+ if ( int result = sqlite3_bind_text(statement, 1, path.data(), -1, SQLITE_TRANSIENT) ) {
+ return { };
+ }
+
+ // Get results
+ std::vector<SymbolsCacheBinary::ImportedSymbol> imports;
+ while( sqlite3_step(statement) == SQLITE_ROW ) {
+ SymbolsCacheBinary::ImportedSymbol imp;
+ imp.targetBinary = (const char*)sqlite3_column_text(statement, 0);
+ imp.symbolName = (const char*)sqlite3_column_text(statement, 1);
+ imports.push_back(imp);
+ }
+
+ sqlite3_finalize(statement);
+
+ return imports;
+}
+
+Error SymbolsCache::getAllImports(std::vector<SymbolsCache::ImportedSymbol>& imports) const
+{
+ const char* selectQuery = "SELECT REF_BINARY.ARCH, REF_BINARY.PATH, DEF_BINARY.INSTALL_NAME, SYMBOL_REF.SYMBOL_NAME "
+ "FROM SYMBOL_REF "
+ "JOIN BINARY AS REF_BINARY ON SYMBOL_REF.REF_BINARY_ID = REF_BINARY.ID "
+ "JOIN BINARY AS DEF_BINARY ON SYMBOL_REF.DEF_BINARY_ID = DEF_BINARY.ID "
+ "ORDER BY REF_BINARY.PATH, DEF_BINARY.INSTALL_NAME, SYMBOL_NAME";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, selectQuery, -1, &statement, 0) ) {
+ Error err = Error("Could not prepare statement for table 'SYMBOL_REF' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ // Get results
+ while( sqlite3_step(statement) == SQLITE_ROW ) {
+ const char* archName = (const char*)sqlite3_column_text(statement, 0);
+ const char* clientPath = (const char*)sqlite3_column_text(statement, 1);
+ const char* installName = (const char*)sqlite3_column_text(statement, 2);
+ const char* symbolName = (const char*)sqlite3_column_text(statement, 3);
+ imports.push_back({ archName, clientPath, installName, symbolName });
+ }
+
+ sqlite3_finalize(statement);
+
+ return Error::none();
+}
+
+static Error getBinaryIDForPath(sqlite3* symbolsDB, std::string_view path,
+ std::optional<int64_t>& binaryID)
+{
+ const char* selectQuery = "SELECT ID FROM BINARY WHERE PATH = ?";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, selectQuery, -1, &statement, 0) ) {
+ Error err = Error("Could not prepare statement for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_text(statement, 1, path.data(), -1, SQLITE_TRANSIENT) ) {
+ Error err = Error("Could not bind text for table 'BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ // Get results
+ std::vector<int64_t> results;
+ while( sqlite3_step(statement) == SQLITE_ROW ) {
+ results.push_back(sqlite3_column_int64(statement, 0));
+ }
+
+ sqlite3_finalize(statement);
+
+ if ( results.empty() )
+ return Error::none();
+
+ if ( results.size() > 1 ) {
+ return Error("Too many binary results for: %s", path.data());
+ }
+
+ binaryID = results.front();
+
+ return Error::none();
+}
+
+static Error getExports(sqlite3* symbolsDB, int64_t binaryID,
+ std::vector<std::string>& exports)
+{
+ const char* selectQuery = "SELECT SYMBOL_NAME FROM SYMBOL_DEF WHERE SYMBOL_DEF.DEF_BINARY_ID = ?";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, selectQuery, -1, &statement, 0) ) {
+ Error err = Error("Could not prepare statement for table 'SYMBOL_DEF' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_int64(statement, 1, binaryID) ) {
+ Error err = Error("Could not bind int for table 'SYMBOL_DEF' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ // Get results
+ while( sqlite3_step(statement) == SQLITE_ROW ) {
+ exports.push_back((const char*)sqlite3_column_text(statement, 0));
+ }
+
+ sqlite3_finalize(statement);
+
+ return Error::none();
+}
+
+std::vector<std::string> SymbolsCache::getExports(std::string_view path) const
+{
+ std::optional<int64_t> binaryID;
+ if ( Error err = getBinaryIDForPath(this->symbolsDB, path, binaryID) ) {
+ return { };
+ }
+
+ // Its ok to skip binaries the database doesn't know about.
+ if ( !binaryID.has_value() )
+ return { };
+
+ // Get the exports from the database
+ std::vector<std::string> exports;
+ if ( Error err = ::getExports(this->symbolsDB, binaryID.value(), exports) ) {
+ return { };
+ }
+
+ return exports;
+}
+
+Error SymbolsCache::getAllExports(std::vector<ExportedSymbol>& exports) const
+{
+ const char* selectQuery = "SELECT BINARY.ARCH, BINARY.INSTALL_NAME, SYMBOL_DEF.SYMBOL_NAME "
+ "FROM SYMBOL_DEF JOIN BINARY ON SYMBOL_DEF.DEF_BINARY_ID = BINARY.ID "
+ "ORDER BY INSTALL_NAME, SYMBOL_NAME";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, selectQuery, -1, &statement, 0) ) {
+ Error err = Error("Could not prepare statement for table 'SYMBOL_DEF' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ // Get results
+ while( sqlite3_step(statement) == SQLITE_ROW ) {
+ const char* archName = (const char*)sqlite3_column_text(statement, 0);
+ const char* installName = (const char*)sqlite3_column_text(statement, 1);
+ const char* symbolName = (const char*)sqlite3_column_text(statement, 2);
+ exports.push_back({ archName, installName, symbolName });
+ }
+
+ sqlite3_finalize(statement);
+
+ return Error::none();
+}
+
+static Error getReexports(sqlite3* symbolsDB, int64_t binaryID,
+ std::vector<std::string>& reexports)
+{
+ const char* selectQuery = "SELECT INSTALL_NAME "
+ "FROM BINARY JOIN REEXPORT ON BINARY.ID = REEXPORT.DEP_BINARY_ID "
+ "WHERE REEXPORT.BINARY_ID = ?";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, selectQuery, -1, &statement, 0) ) {
+ Error err = Error("Could not prepare statement for join 'BINARY/REEXPORT' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_int64(statement, 1, binaryID) ) {
+ Error err = Error("Could not bind int for join 'BINARY/REEXPORT' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ // Get results
+ while( sqlite3_step(statement) == SQLITE_ROW ) {
+ reexports.push_back((const char*)sqlite3_column_text(statement, 0));
+ }
+
+ sqlite3_finalize(statement);
+
+ return Error::none();
+}
+
+std::vector<std::string> SymbolsCache::getReexports(std::string_view path) const
+{
+ std::optional<int64_t> binaryID;
+ if ( Error err = getBinaryIDForPath(this->symbolsDB, path, binaryID) ) {
+ return { };
+ }
+
+ // Its ok to skip binaries the database doesn't know about.
+ if ( !binaryID.has_value() )
+ return { };
+
+ // Get the reexports from the database
+ std::vector<std::string> reexports;
+ if ( Error err = ::getReexports(this->symbolsDB, binaryID.value(), reexports) ) {
+ return { };
+ }
+
+ return reexports;
+}
+
+static Error getUsesOfExport(sqlite3* symbolsDB, int64_t binaryID,
+ std::string_view exportedSymbol,
+ std::vector<std::string>& clientBinaryPaths)
+{
+ const char* selectQuery = "SELECT BINARY.PATH "
+ "FROM SYMBOL_REF JOIN BINARY "
+ "ON SYMBOL_REF.REF_BINARY_ID = BINARY.ID "
+ "WHERE SYMBOL_REF.DEF_BINARY_ID = ? AND SYMBOL_REF.SYMBOL_NAME = ?";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, selectQuery, -1, &statement, 0) ) {
+ Error err = Error("Could not prepare statement for join 'SYMBOL_REF/BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_int64(statement, 1, binaryID) ) {
+ Error err = Error("Could not bind int for join 'SYMBOL_REF/BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ if ( int result = sqlite3_bind_text(statement, 2, exportedSymbol.data(), -1, SQLITE_TRANSIENT) ) {
+ Error err = Error("Could not bind text for join 'SYMBOL_REF/BINARY' because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ // Get results
+ while( sqlite3_step(statement) == SQLITE_ROW ) {
+ clientBinaryPaths.push_back((const char*)sqlite3_column_text(statement, 0));
+ }
+
+ sqlite3_finalize(statement);
+
+ return Error::none();
+}
+
+struct BinaryKey
+{
+ std::string_view installNameOrPath;
+ Platform platform;
+ std::string_view arch;
+};
+
+static bool operator==(const BinaryKey& a, const BinaryKey& b) {
+ return (a.installNameOrPath == b.installNameOrPath) && (a.platform == b.platform) && (a.arch == b.arch);
+}
+
+namespace std
+{
+
+template<>
+struct std::hash<BinaryKey>
+{
+ uint64_t operator()(const BinaryKey& val) const
+ {
+ uint64_t hash = 0;
+ hash = hash ^ std::hash<std::string_view>{}(val.installNameOrPath) << 0;
+ hash = hash ^ std::hash<uint32_t>{}(val.platform.value()) << 32;
+ hash = hash ^ std::hash<std::string_view>{}(val.arch) << 48;
+ return hash;
+ }
+};
+
+} // namespace std
+
+Error SymbolsCache::checkNewBinaries(bool warnOnRemovedSymbols, ExecutableMode executableMode,
+ std::vector<SymbolsCacheBinary>&& binaries,
+ const BinaryProjects& binaryProjects,
+ std::vector<ResultBinary>& results,
+ std::vector<mach_o::Error>& internalWarnings,
+ std::vector<ExportsChangedBinary>* changedExports) const
+{
+ // Split out in to OS dylibs vs other binaries
+ // We only want to verify the exports from OS binaries
+ std::vector<SymbolsCacheBinary> osDylibs;
+ std::vector<SymbolsCacheBinary*> otherBinaries;
+ std::unordered_map<BinaryKey, SymbolsCacheBinary*> osDylibMap;
+ std::unordered_map<BinaryKey, SymbolsCacheBinary*> newClientsMap;
+
+ for ( SymbolsCacheBinary& binary : binaries ) {
+ if ( binary.installName.starts_with('/') )
+ osDylibs.push_back(binary);
+ else
+ otherBinaries.push_back(&binary);
+ }
+
+ for ( SymbolsCacheBinary& binary : osDylibs ) {
+ osDylibMap[{ binary.installName, binary.platform, binary.arch }] = &binary;
+ newClientsMap[{ binary.path, binary.platform, binary.arch }] = &binary;
+
+ if ( binary.path.starts_with("/System/Cryptexes/OS/") ) {
+ constexpr int prefixLen = std::string_view("/System/Cryptexes/OS").size();
+ if ( binary.path.substr(prefixLen) == binary.installName )
+ newClientsMap[{ binary.installName, binary.platform, binary.arch }] = &binary;
+ }
+ }
+
+ for ( SymbolsCacheBinary* binary : otherBinaries )
+ newClientsMap[{ binary->path, binary->platform, binary->arch }] = binary;
+
+ // Early exit if no binaries with new exports. Not sure if we'd ever want to verify
+ // the other binaries anyway. In theory their imports should be valid as they were just rebuilt
+ if ( osDylibs.empty() )
+ return Error::none();
+
+ // Promote re-exports to make it look like the top-level dylib exports them. This will line up with
+ // imports from other binaries which are looking for the exports in the top-level dylib
+ {
+ std::list<SymbolsCacheBinary*> worklist;
+ for ( SymbolsCacheBinary& binary : osDylibs )
+ worklist.push_back(&binary);
+
+ std::unordered_map<BinaryKey, SymbolsCacheBinary*> processedBinaries;
+ std::unordered_map<BinaryKey, std::unique_ptr<SymbolsCacheBinary>> databaseBinaries;
+ while ( !worklist.empty() ) {
+ SymbolsCacheBinary* binary = worklist.front();
+ worklist.pop_front();
+
+ if ( binary->installName.empty() )
+ continue;
+
+ // If we have no re-exports, then this binary is done
+ if ( binary->reexportedLibraries.empty() ) {
+ processedBinaries[{ binary->installName, binary->platform, binary->arch }] = binary;
+ continue;
+ }
+
+ // Check if we need to put this binary back in the worklist to wait on deps
+ bool waitOnDeps = false;
+ for ( const SymbolsCacheBinary::TargetBinary& reexportTarget : binary->reexportedLibraries ) {
+ std::string reexport = std::get<std::string>(reexportTarget);
+ if ( !processedBinaries.count({ reexport, binary->platform, binary->arch }) ) {
+ // Unprocessed dep. Let see if its even a dep we know about
+ if ( osDylibMap.find({ reexport, binary->platform, binary->arch }) != osDylibMap.end() ) {
+ // new binary. We'll get to it later, so just put this back in the queue
+ waitOnDeps = true;
+ break;
+ } else if ( databaseBinaries.count({ reexport, binary->platform, binary->arch }) ) {
+ // We know about this binary, but didn't process it yet
+ waitOnDeps = true;
+ break;
+ } else {
+ // unknown binary. Let see if its in the database
+ std::optional<int64_t> binaryID;
+ if ( Error err = getDylibID(this->symbolsDB, reexport, binary->platform, binary->arch, binaryID) ) {
+ continue;
+ }
+
+ // Its ok to skip binaries the database doesn't know about.
+ if ( !binaryID.has_value() )
+ continue;
+
+ // Get the exports from the database
+ std::vector<std::string> exports;
+ if ( Error err = ::getExports(this->symbolsDB, binaryID.value(), exports) ) {
+ // FIXME: What should we do here? For now log the error and skip the binary
+ internalWarnings.push_back(Error("Skipping re-exported binary due to getExports(): %s", err.message()));
+ continue;
+ }
+
+ // Get the exports from the database
+ std::vector<std::string> reexports;
+ if ( Error err = ::getReexports(this->symbolsDB, binaryID.value(), reexports) ) {
+ // FIXME: What should we do here? For now log the error and skip the binary
+ internalWarnings.push_back(Error("Skipping re-exported binary due to getReexports(): %s", err.message()));
+ continue;
+ }
+
+ std::unique_ptr<SymbolsCacheBinary> newBinary = std::make_unique<SymbolsCacheBinary>(reexport, binary->platform, binary->arch, "", "");
+ newBinary->path = reexport;
+ newBinary->installName = reexport;
+ newBinary->exportedSymbols = std::move(exports);
+
+ for ( const std::string& reexportedLibrary : reexports )
+ newBinary->reexportedLibraries.push_back(reexportedLibrary);
+
+ worklist.push_back(newBinary.get());
+ databaseBinaries[{ newBinary->installName, binary->platform, binary->arch }] = std::move(newBinary);
+
+ waitOnDeps = true;
+ break;
+ }
+ }
+ }
+
+ if ( waitOnDeps ) {
+ worklist.push_back(binary);
+ continue;
+ }
+
+ // All deps that we could find should be done. Promote their symbols up to this binary
+ for ( SymbolsCacheBinary::TargetBinary reexportTarget : binary->reexportedLibraries ) {
+ std::string_view reexport = std::get<std::string>(reexportTarget);
+ if ( auto it = processedBinaries.find({ reexport, binary->platform, binary->arch }); it != processedBinaries.end() ) {
+ binary->exportedSymbols.insert(binary->exportedSymbols.end(),
+ it->second->exportedSymbols.begin(), it->second->exportedSymbols.end());
+ }
+ }
+ processedBinaries[{ binary->installName, binary->platform, binary->arch }] = binary;
+ }
+ }
+
+ std::map<std::string, std::string_view> rootsPathsForErrorCases;
+
+ // For each OS dylib, compare its exports against the exports in the database. If it removes a
+ // symbol then error out if that symbol has refs
+ for ( SymbolsCacheBinary& binary : osDylibs ) {
+ std::optional<int64_t> binaryID;
+ if ( Error err = getDylibID(this->symbolsDB, binary.installName, binary.platform, binary.arch, binaryID) ) {
+ // FIXME: What should we do here? For now log the error and skip the binary
+ internalWarnings.push_back(Error("Skipping binary due to getDylibID(): %s", err.message()));
+ continue;
+ }
+
+ // Its ok to skip binaries the database doesn't know about.
+ if ( !binaryID.has_value() ) {
+ if ( verbose )
+ printf("Skipping binary as it doesn't exist in the database: %s\n", binary.installName.c_str());
+ continue;
+ }
+
+ // Get the exports from the database
+ std::vector<std::string> exports;
+ if ( Error err = ::getExports(this->symbolsDB, binaryID.value(), exports) ) {
+ // FIXME: What should we do here? For now log the error and skip the binary
+ internalWarnings.push_back(Error("Skipping binary due to getExports(): %s", err.message()));
+ continue;
+ }
+
+ // Add in symbols from re-exports
+ {
+ // Get the exports from the database
+ std::vector<std::string> reexports;
+ if ( Error err = ::getReexports(this->symbolsDB, binaryID.value(), reexports) ) {
+ // FIXME: What should we do here? For now log the error and skip the binary
+ internalWarnings.push_back(Error("Skipping re-exported binary due to getReexports(): %s", err.message()));
+ continue;
+ }
+
+ if ( !reexports.empty() ) {
+ std::list<std::string> worklist;
+ worklist.insert(worklist.end(), reexports.begin(), reexports.end());
+
+ std::set<std::string> processedBinaries;
+ std::vector<int64_t> reexportedBinaries;
+ while ( !worklist.empty() ) {
+ std::string reexport = worklist.front();
+ worklist.pop_front();
+
+ if ( processedBinaries.count(reexport) )
+ continue;
+
+ // unknown binary. Let see if its in the database
+ std::optional<int64_t> reexportBinaryID;
+ if ( Error err = getDylibID(this->symbolsDB, reexport, binary.platform, binary.arch, reexportBinaryID) ) {
+ continue;
+ }
+
+ // Its ok to skip binaries the database doesn't know about.
+ if ( !reexportBinaryID.has_value() )
+ continue;
+
+ processedBinaries.insert(reexport);
+ reexportedBinaries.push_back(reexportBinaryID.value());
+
+ // See if there are more re-exports to add
+ std::vector<std::string> nextReexports;
+ if ( Error err = ::getReexports(this->symbolsDB, reexportBinaryID.value(), nextReexports) ) {
+ // FIXME: What should we do here? For now log the error and skip the binary
+ internalWarnings.push_back(Error("Skipping re-exported binary due to getReexports(): %s", err.message()));
+ continue;
+ }
+
+ worklist.insert(worklist.end(), nextReexports.begin(), nextReexports.end());
+ }
+
+ for ( int64_t reexportedBinaryID : reexportedBinaries ) {
+ std::vector<std::string> reexportedExports;
+ if ( Error err = ::getExports(this->symbolsDB, reexportedBinaryID, reexportedExports) ) {
+ // FIXME: What should we do here? For now log the error and skip the binary
+ internalWarnings.push_back(Error("Skipping binary due to getExports(): %s", err.message()));
+ continue;
+ }
+
+ exports.insert(exports.end(), reexportedExports.begin(), reexportedExports.end());
+ }
+ }
+ }
+
+ std::string binaryProject;
+ if ( Error err = getBinaryProject(this->symbolsDB, binary.path, binary.platform, binary.arch, binaryProject) ) {
+ // No project is ok. We can continue without it
+ }
+
+ // Work out if any exports were removed
+ std::set<std::string_view> removedExports;
+ removedExports.insert(exports.begin(), exports.end());
+ for ( std::string_view exp : binary.exportedSymbols )
+ removedExports.erase(exp);
+
+ if ( changedExports != nullptr ) {
+ // Find out if we added exports
+ std::set<std::string_view> addedExports;
+ addedExports.insert(binary.exportedSymbols.begin(), binary.exportedSymbols.end());
+ for ( std::string_view exp : exports )
+ addedExports.erase(exp);
+
+ for ( std::string_view exp : removedExports ) {
+ ExportsChangedBinary result;
+ result.installName = binary.path;
+ result.arch = binary.arch;
+ result.uuid = binary.uuid;
+ result.projectName = binaryProject;
+ result.symbolName = exp;
+ result.wasAdded = false;
+
+ changedExports->push_back(std::move(result));
+ }
+
+ for ( std::string_view exp : addedExports ) {
+ ExportsChangedBinary result;
+ result.installName = binary.path;
+ result.arch = binary.arch;
+ result.uuid = binary.uuid;
+ result.projectName = binaryProject;
+ result.symbolName = exp;
+ result.wasAdded = true;
+
+ changedExports->push_back(std::move(result));
+ }
+ }
+
+ if ( removedExports.empty() ) {
+ if ( verbose )
+ printf("Skipping binary as it didn't remove any used exports: %s\n", binary.installName.c_str());
+ continue;
+ }
+
+ // HACK!: A few B&I projects build multiple copies of the same binary, and those are confused for each other
+ // For now skip errors from these projects until we can handle them. They'll still be caught when using a mach-o, just not JSON
+ if ( binary.inputFileName.ends_with(".json") ) {
+ if ( binary.installName == "/System/Library/Frameworks/AudioToolbox.framework/AudioToolbox" )
+ continue;
+ if ( binary.installName == "/usr/lib/libNFC_HAL.dylib" )
+ continue;
+ if ( binary.installName == "/System/Library/PrivateFrameworks/WiFiPeerToPeer.framework/WiFiPeerToPeer" )
+ continue;
+ if ( binary.installName == "/usr/lib/libz.1.dylib" )
+ continue;
+
+ // Filter out LAR and _tests projects
+ // Note project name looks something like: dyld_tests-version.json
+ if ( binary.inputFileName.find("_tests-") != std::string_view::npos )
+ continue;
+ if ( binary.inputFileName.find("Tests-") != std::string_view::npos )
+ continue;
+ if ( binary.inputFileName.find("_lar-") != std::string_view::npos )
+ continue;
+ }
+
+ // If we removed exports, now we need to see if they have uses
+ for ( std::string_view exp : removedExports ) {
+ std::vector<std::string> clientPaths;
+ if ( Error err = getUsesOfExport(this->symbolsDB, binaryID.value(), exp, clientPaths) ) {
+ // FIXME: What should we do here? For now log the error and skip the binary export
+ internalWarnings.push_back(Error("Skipping binary export due to getUsesOfExport(): %s", err.message()));
+ continue;
+ }
+
+ // No uses. Skip this one
+ if ( clientPaths.empty() ) {
+ if ( warnOnRemovedSymbols )
+ internalWarnings.push_back(Error("Binary '%s' removing unused export: '%s'", binary.path.data(), exp.data()));
+ continue;
+ }
+
+ for ( std::string_view path : clientPaths ) {
+ // If this client was also rebuilt, then filter it out if it doesn't use this symbol any more
+ std::string clientUUID;
+ std::string clientRootPath;
+ std::string clientProject;
+ bool warnOnClient = false;
+
+ // Skip executables and non-shared cache dylibs if we aren't verifying them
+ {
+ std::string clientInstallName;
+ if ( Error err = getBinaryInstallName(this->symbolsDB, path, binary.platform, binary.arch, clientInstallName) ) {
+ // Skip binaries if their install name generates some kind of error
+ internalWarnings.push_back(Error("Skipping binary export due to getBinaryInstallName(): %s", err.message()));
+ continue;
+ }
+
+ bool isCacheEligible = false;
+ if ( !clientInstallName.empty() ) {
+ isCacheEligible = Header::isSharedCacheEligiblePath(clientInstallName.data());
+ }
+
+ switch ( executableMode ) {
+ case ExecutableMode::off:
+ // This means we're verifying only shared cache dylibs. Skip everything else
+ if ( !isCacheEligible )
+ continue;
+ break;
+ case ExecutableMode::warn:
+ // If we later find issues with this client, record them as errors
+ // if its from the shared cache, but warnings otherwise
+ if ( !isCacheEligible )
+ warnOnClient = true;
+ break;
+ case ExecutableMode::error:
+ // anu issues found here will be errors
+ break;
+ }
+ }
+
+ if ( Error err = getBinaryProject(this->symbolsDB, path, binary.platform, binary.arch, clientProject) ) {
+ // No project is ok. We can continue without it
+ }
+ if ( auto it = newClientsMap.find({ path, binary.platform, binary.arch }); it != newClientsMap.end() ) {
+ // FIXME: Do we need a map?
+ SymbolsCacheBinary* clientBinary = it->second;
+ auto importIt = std::find_if(clientBinary->importedSymbols.begin(),
+ clientBinary->importedSymbols.end(),
+ [&](const SymbolsCacheBinary::ImportedSymbol& elt) {
+ if ( elt.symbolName != exp )
+ return false;
+ if ( const std::string* installName = std::get_if<std::string>(&elt.targetBinary) )
+ return *installName == binary.installName;
+ return false;
+ });
+ if ( importIt == clientBinary->importedSymbols.end() ) {
+ // No uses of this export, skip this one
+ continue;
+ }
+ clientRootPath = clientBinary->rootPath;
+ clientUUID = clientBinary->uuid;
+ } else {
+ // See if the broken client is actually a project we have a root for. If so, ignore it
+ // as perhaps it was deleted or moved
+ if ( !binaryProjects.empty() && !clientProject.empty() ) {
+ if ( binaryProjects.count(clientProject) )
+ continue;
+ }
+
+ // See if we can get a UUID from the database
+ if ( Error err = getBinaryUUID(this->symbolsDB, path, binary.platform, binary.arch, clientUUID) ) {
+ // No UUID is ok. We can continue without it
+ }
+ }
+
+ ResultBinary result;
+ result.installName = binary.path;
+ result.arch = binary.arch;
+ result.uuid = binary.uuid;
+ result.rootPath = binary.rootPath;
+ result.projectName = binaryProject;
+ result.warn = warnOnClient;
+
+ result.client.path = path;
+ result.client.uuid = clientUUID;
+ result.client.rootPath = clientRootPath;
+ result.client.projectName = clientProject;
+ result.client.symbolName = exp;
+
+ results.push_back(std::move(result));
+ }
+ }
+ }
+
+ return Error::none();
+}
+
+Error SymbolsCache::dump() const
+{
+ std::vector<std::string> tableNames;
+ {
+ const char* selectQuery = "SELECT tbl_name FROM sqlite_master WHERE type = 'table';";
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, selectQuery, -1, &statement, 0) ) {
+ Error err = Error("Could not prepare statement for tables because: %s", (const char*)strerror(result));
+ return err;
+ }
+
+ // Get results
+ while( sqlite3_step(statement) == SQLITE_ROW ) {
+ tableNames.push_back((const char*)sqlite3_column_text(statement, 0));
+ }
+
+ sqlite3_finalize(statement);
+ }
+
+ if ( tableNames.empty() ) {
+ printf("Empty database\n");
+ return Error::none();
+ }
+
+ for ( std::string_view tableName : tableNames ) {
+ printf("Table: %s\n", tableName.data());
+
+ std::string selectQuery = std::string("SELECT * FROM ") + tableName.data();
+ sqlite3_stmt *statement = nullptr;
+ if ( int result = sqlite3_prepare_v2(symbolsDB, selectQuery.c_str(), -1, &statement, 0) ) {
+ Error err = Error("Could not prepare statement for table '%s' because: %s",
+ tableName.data(), (const char*)strerror(result));
+ return err;
+ }
+
+ // Get results
+ while( sqlite3_step(statement) == SQLITE_ROW ) {
+ int numColumns = sqlite3_column_count(statement);
+ bool needsComma = false;
+ for ( int i = 0; i != numColumns; ++i ) {
+ if ( needsComma )
+ printf(", ");
+ printf("%s", (const char*)sqlite3_column_text(statement, i));
+ needsComma = true;
+ }
+ printf("\n");
+ }
+ printf("\n");
+
+ sqlite3_finalize(statement);
+ }
+
+ return Error::none();
+}
+
+//
+// MARK: --- helper methods to output results ---
+//
+
+void printResultSummary(std::span<ResultBinary> verifyResults, bool bniOutput,
+ FILE* summaryLogFile)
+{
+ std::set<std::string> errorClientProjects;
+ std::set<std::string> warnClientProjects;
+
+ // Get the projects which are errors, then the list which are only warnings
+ for ( const ResultBinary& result : verifyResults ) {
+ if ( result.warn )
+ continue;
+
+ if ( result.client.projectName.empty())
+ continue;
+
+ errorClientProjects.insert(result.client.projectName);
+ }
+
+ // Get the projects which are errors, then the list which are only warnings
+ for ( const ResultBinary& result : verifyResults ) {
+ if ( !result.warn )
+ continue;
+
+ if ( result.client.projectName.empty())
+ continue;
+
+ // Skip projects also in the error list
+ if ( errorClientProjects.count(result.client.projectName) )
+ continue;
+
+ warnClientProjects.insert(result.client.projectName);
+ }
+
+ if ( errorClientProjects.empty() && warnClientProjects.empty() )
+ return;
+
+ fprintf(summaryLogFile, "--- Summary ---\n\n");
+
+ if ( !errorClientProjects.empty() )
+ fprintf(summaryLogFile, "Error: some projects have removed symbols\n\n");
+ else
+ fprintf(summaryLogFile, "Warning: some projects have removed symbols\n\n");
+
+ fprintf(summaryLogFile, "Expected resolution is to rebuild dependencies\n\n");
+
+ auto printProjects = [&](const std::set<std::string>& clientProjects) {
+ if ( bniOutput ) {
+ fprintf(summaryLogFile, "Run command: xbs dispatch addProjects");
+ for ( std::string_view project : clientProjects )
+ fprintf(summaryLogFile, " %s", project.data());
+ } else {
+ fprintf(summaryLogFile, "Add the following to your submission notes, or container\n");
+ fprintf(summaryLogFile, " REBUILD_DEPENDENCIES=");
+ bool needsComma = false;
+ for ( std::string_view project : clientProjects ) {
+ if ( needsComma )
+ fprintf(summaryLogFile, ",");
+ else
+ needsComma = true;
+ fprintf(summaryLogFile, "%s", project.data());
+ }
+ }
+ fprintf(summaryLogFile, "\n\n");
+ };
+
+ if ( !errorClientProjects.empty() )
+ printProjects(errorClientProjects);
+
+ if ( !warnClientProjects.empty() )
+ printProjects(warnClientProjects);
+}
+
+void printResultsSymbolDetails(std::span<ResultBinary> verifyResults, FILE* detailsLogFile)
+{
+ struct ProjectResult
+ {
+ struct Client
+ {
+ std::string path;
+ std::string uuid;
+ std::set<std::string> symbols;
+ };
+
+ struct ClientProject
+ {
+ // map from path to its results
+ std::map<std::string, Client> clients;
+ };
+
+ struct Dylib
+ {
+ std::string uuid;
+
+ // map from project name to its clients
+ std::map<std::string, ClientProject> clientProjects;
+ };
+
+ // map from install name to its results
+ std::map<std::string, Dylib> dylibs;
+ };
+
+ // map from project name to its results
+ // loop twice. First iteration prints errors, second prints warnings
+ for ( bool errors : { true, false } ) {
+ std::map<std::string, ProjectResult> failingProjects;
+ for ( const ResultBinary& result : verifyResults ) {
+ if ( result.warn ) {
+ // result is just a warning. Ok if we are generating warnings, but not if errors
+ if ( errors )
+ continue;
+ } else {
+ // result is an error. Ok if we are generating errors, but not if warnings
+ if ( !errors )
+ continue;
+ }
+
+ std::string projectName = result.projectName.empty() ? "<unknown project>" : result.projectName;
+ std::string clientProjectName = result.client.projectName.empty() ? "<unknown project>" : result.client.projectName;
+
+ ProjectResult& projectResult = failingProjects[projectName];
+ ProjectResult::Dylib& dylib = projectResult.dylibs[result.installName];
+
+ dylib.uuid = result.uuid;
+ ProjectResult::ClientProject& clientProject = dylib.clientProjects[clientProjectName];
+ ProjectResult::Client& client = clientProject.clients[result.client.path];
+ client.path = result.client.path;
+ client.uuid = result.client.uuid;
+ client.symbols.insert(result.client.symbolName);
+ }
+
+ if ( failingProjects.empty() )
+ continue;
+
+ fprintf(detailsLogFile, "--- Detailed symbol information (%s) ---\n\n",
+ errors ? "errors" : "warnings");
+
+ for ( std::pair<std::string, ProjectResult> result : failingProjects ) {
+ fprintf(detailsLogFile, "%s:\n", result.first.data());
+ for ( std::pair<std::string, ProjectResult::Dylib> dylib : result.second.dylibs ) {
+ std::string dylibUUID;
+ if ( !dylib.second.uuid.empty())
+ dylibUUID = " (" + dylib.second.uuid + ")";
+ fprintf(detailsLogFile, " %s%s:\n", dylib.first.data(), dylibUUID.data());
+
+ for ( std::pair<std::string, ProjectResult::ClientProject> clientProject : dylib.second.clientProjects ) {
+ fprintf(detailsLogFile, " %s:\n", clientProject.first.data());
+ for ( std::pair<std::string, ProjectResult::Client> client : clientProject.second.clients ) {
+ std::string clientUUID;
+ if ( !client.second.uuid.empty())
+ clientUUID = " (" + client.second.uuid + ")";
+ fprintf(detailsLogFile, " %s%s:\n", client.first.data(), clientUUID.data());
+ for ( std::string_view symbolName : client.second.symbols )
+ fprintf(detailsLogFile, " %s\n", symbolName.data());
+ }
+ }
+ }
+ fprintf(detailsLogFile, "\n");
+ }
+ }
+}
+
+void printResultsInternalInformation(std::span<ResultBinary> verifyResults,
+ std::span<std::pair<std::string, std::string>> rootErrors,
+ FILE* detailsLogFile)
+{
+ std::set<std::string> usedRootPaths;
+ for ( const ResultBinary& result : verifyResults ) {
+ if ( !result.rootPath.empty() )
+ usedRootPaths.insert(result.rootPath);
+ if ( !result.client.rootPath.empty() )
+ usedRootPaths.insert(result.client.rootPath);
+ }
+
+ if ( !usedRootPaths.empty() || !rootErrors.empty() ) {
+ fprintf(detailsLogFile, "--- Internal information ---\n\n");
+ }
+
+ if ( !usedRootPaths.empty() ) {
+ fprintf(detailsLogFile, "Note, the following root paths were used in the above errors:\n");
+ for ( std::string_view usedRootPath : usedRootPaths ) {
+ fprintf(detailsLogFile, " %s\n", usedRootPath.data());
+ }
+ fprintf(detailsLogFile, "\n");
+ }
+
+ if ( !rootErrors.empty() ) {
+ fprintf(detailsLogFile, "Note, the following root paths were inaccessible:\n");
+ for ( const auto& rootPathAndError : rootErrors ) {
+ fprintf(detailsLogFile, " %s due to '%s'\n", rootPathAndError.first.data(), rootPathAndError.second.data());
+ }
+ fprintf(detailsLogFile, "\n");
+ }
+}
+
+void printResultsJSON(std::span<ResultBinary> verifyResults,
+ std::span<ExportsChangedBinary> exportsChanged,
+ FILE* jsonFile)
+{
+ fprintf(jsonFile, "{\n");
+
+ {
+ fprintf(jsonFile, " \"removed-used-symbols\" : [\n");
+
+ bool needsComma = false;
+ for ( const ResultBinary& binary : verifyResults ) {
+ if ( needsComma )
+ fprintf(jsonFile, ",\n");
+ else
+ needsComma = true;
+
+ bool defInSharedCache = Header::isSharedCacheEligiblePath(binary.installName.c_str());
+ bool useInSharedCache = Header::isSharedCacheEligiblePath(binary.client.path.c_str());
+
+ fprintf(jsonFile, " {\n");
+
+ fprintf(jsonFile, " \"arch\" : \"%s\",\n", binary.arch.c_str());
+ fprintf(jsonFile, " \"symbol-name\" : \"%s\",\n", binary.client.symbolName.c_str());
+
+ fprintf(jsonFile, " \"def-uuid\" : \"%s\",\n", binary.uuid.c_str());
+ fprintf(jsonFile, " \"def-project-name\" : \"%s\",\n", binary.projectName.c_str());
+ fprintf(jsonFile, " \"def-install-name\" : \"%s\",\n", binary.installName.c_str());
+ fprintf(jsonFile, " \"def-shared-cache-eligible\" : \"%s\",\n", defInSharedCache ? "yes" : "no");
+
+ fprintf(jsonFile, " \"use-uuid\" : \"%s\",\n", binary.client.uuid.c_str());
+ fprintf(jsonFile, " \"use-project-name\" : \"%s\",\n", binary.client.projectName.c_str());
+ fprintf(jsonFile, " \"use-path\" : \"%s\",\n", binary.client.path.c_str());
+ fprintf(jsonFile, " \"use-shared-cache-eligible\" : \"%s\"\n", useInSharedCache ? "yes" : "no");
+ fprintf(jsonFile, " }");
+ }
+ fprintf(jsonFile, "\n");
+
+ fprintf(jsonFile, " ],\n");
+ }
+
+ {
+ fprintf(jsonFile, " \"added-exports\" : [\n");
+
+ bool needsComma = false;
+ for ( const ExportsChangedBinary& binary : exportsChanged ) {
+ if ( !binary.wasAdded )
+ continue;
+
+ if ( needsComma )
+ fprintf(jsonFile, ",\n");
+ else
+ needsComma = true;
+
+ bool inSharedCache = Header::isSharedCacheEligiblePath(binary.installName.c_str());
+
+ fprintf(jsonFile, " {\n");
+
+ fprintf(jsonFile, " \"arch\" : \"%s\",\n", binary.arch.c_str());
+ fprintf(jsonFile, " \"symbol-name\" : \"%s\",\n", binary.symbolName.c_str());
+ fprintf(jsonFile, " \"uuid\" : \"%s\",\n", binary.uuid.c_str());
+ fprintf(jsonFile, " \"project-name\" : \"%s\",\n", binary.projectName.c_str());
+ fprintf(jsonFile, " \"install-name\" : \"%s\",\n", binary.installName.c_str());
+ fprintf(jsonFile, " \"shared-cache-eligible\" : \"%s\"\n", inSharedCache ? "yes" : "no");
+ fprintf(jsonFile, " }");
+ }
+ fprintf(jsonFile, "\n");
+
+ fprintf(jsonFile, " ],\n");
+ }
+
+ {
+ fprintf(jsonFile, " \"removed-exports\" : [\n");
+
+ bool needsComma = false;
+ for ( const ExportsChangedBinary& binary : exportsChanged ) {
+ if ( binary.wasAdded )
+ continue;
+
+ if ( needsComma )
+ fprintf(jsonFile, ",\n");
+ else
+ needsComma = true;
+
+ bool inSharedCache = Header::isSharedCacheEligiblePath(binary.installName.c_str());
+
+ fprintf(jsonFile, " {\n");
+
+ fprintf(jsonFile, " \"arch\" : \"%s\",\n", binary.arch.c_str());
+ fprintf(jsonFile, " \"symbol-name\" : \"%s\",\n", binary.symbolName.c_str());
+ fprintf(jsonFile, " \"uuid\" : \"%s\",\n", binary.uuid.c_str());
+ fprintf(jsonFile, " \"project-name\" : \"%s\",\n", binary.projectName.c_str());
+ fprintf(jsonFile, " \"install-name\" : \"%s\",\n", binary.installName.c_str());
+ fprintf(jsonFile, " \"shared-cache-eligible\" : \"%s\"\n", inSharedCache ? "yes" : "no");
+ fprintf(jsonFile, " }");
+ }
+ fprintf(jsonFile, "\n");
+
+ fprintf(jsonFile, " ]\n");
+ }
+
+ fprintf(jsonFile, "}\n");
+}