Add support for third-party build systems (#569)

Author: ileitch

CHANGELOG.md

@@ -7,6 +7,7 @@
 ##### Enhancements
 
 - Add CodeClimate output formatter available via the `--format codeclimate` option. 
+- Add support for third-party build systems, such as Bazel.
 
 ##### Bug Fixes
 

README.md

@@ -36,6 +36,7 @@
 - [Xcode Integration](#xcode-integration)
 - [Excluding Files](#excluding-files)
 - [Continuous Integration](#continuous-integration)
+- [Build Systems](#build-systems)
 - [Platforms](#platforms)
 - [Troubleshooting](#troubleshooting)
 - [Known Bugs](#known-bugs)
@@ -414,6 +415,31 @@ The index store generated by `xcodebuild` exists in DerivedData at a location de
 
 By default Periphery, looks for the index store at `.build/debug/index/store`. Therefore, if you intend to run Periphery directly after calling `swift test`, you can omit the `--index-store-path` option, and Periphery will use the index store created when the project was built for testing. However if this isn't the case, then you must provide Periphery the location of the index store with `--index-store-path`.
 
+## Build Systems
+
+Periphery can analyze projects using third-party build systems such as Bazel, though it cannot drive them automatically like SwiftPM and xcodebuild. Instead, you need to specify the index store location and provide a file-target mapping file.
+
+A file-target mapping file contains a simple mapping of source files to build targets. You will need to generate this file yourself using the appropriate tooling for your build system. The format is as follows:
+
+```
+{
+  "file_targets": {
+    "path/to/file_a.swift": ["TargetA"],
+    "path/to/file_b.swift": ["TargetB", "TargetC"]
+  }
+}
+```
+
+> Relative paths are assumed to be relative to the current directory.
+
+You can then invoke periphery as follows:
+
+```
+periphery scan --file-targets-path map.json --index-store-path index/store
+``` 
+
+> Both options support multiple paths.
+
 ## Platforms
 
 Periphery supports both macOS and Linux. macOS supports both Xcode and Swift Package Manager (SPM) projects, whereas only SPM projects are supported on Linux.

Sources/Frontend/Commands/ScanCommand.swift

@@ -1,5 +1,6 @@
 import Foundation
 import ArgumentParser
+import SystemPackage
 import Shared
 
 struct ScanCommand: FrontendCommand {
@@ -23,6 +24,9 @@ struct ScanCommand: FrontendCommand {
     @Option(help: "Path to your project's .xcodeproj - supply this option if your project doesn't have an .xcworkspace. Xcode projects only")
     var project: String?
 
+    @Option(parsing: .upToNextOption, help: "Path to file targets mapping. For use with third-party build systems. Multiple paths may be specified")
+    var fileTargetsPath: [FilePath] = defaultConfiguration.$fileTargetsPath.defaultValue
+
     @Option(help: "Comma-separated list of schemes that must be built in order to produce the targets passed to the --targets option. Xcode projects only", transform: split(by: ","))
     var schemes: [String] = defaultConfiguration.$schemes.defaultValue
 
@@ -41,10 +45,10 @@ struct ScanCommand: FrontendCommand {
     @Option(help: "Path glob of source files which should be included from the results. Note that this option is purely cosmetic, these files will still be indexed. Multiple globs may be delimited by a pipe", transform: split(by: "|"))
     var reportInclude: [String] = defaultConfiguration.$reportInclude.defaultValue
 
-    @Option(help: "Path to index store to use. Implies '--skip-build'")
-    var indexStorePath: String?
+    @Option(parsing: .upToNextOption, help: "Path to the index store. Multiple paths may be specified. Implies '--skip-build'")
+    var indexStorePath: [FilePath] = defaultConfiguration.$indexStorePath.defaultValue
 
-    @Flag(help: "Retain all public declarations - you'll likely want to enable this if you're scanning a framework/library project")
+    @Flag(help: "Retain all public declarations, recommended for framework/library projects")
     var retainPublic: Bool = defaultConfiguration.$retainPublic.defaultValue
 
     @Flag(help: "Disable identification of redundant public accessibility")
@@ -96,6 +100,7 @@ struct ScanCommand: FrontendCommand {
         configuration.guidedSetup = setup
         configuration.apply(\.$workspace, workspace)
         configuration.apply(\.$project, project)
+        configuration.apply(\.$fileTargetsPath, fileTargetsPath)
         configuration.apply(\.$schemes, schemes)
         configuration.apply(\.$targets, targets)
         configuration.apply(\.$indexExclude, indexExclude)
@@ -131,3 +136,9 @@ struct ScanCommand: FrontendCommand {
 }
 
 extension OutputFormat: ExpressibleByArgument {}
+
+extension FilePath: ExpressibleByArgument {
+    public init?(argument: String) {
+        self.init(argument)
+    }
+}

Sources/Frontend/Project.swift

@@ -13,6 +13,8 @@ final class Project {
 
         if configuration.workspace != nil || configuration.project != nil {
             return self.init(kind: .xcode)
+        } else if !configuration.fileTargetsPath.isEmpty {
+            return self.init(kind: .generic)
         } else if SPM.isSupported {
             return self.init(kind: .spm)
         }
@@ -59,6 +61,8 @@ final class Project {
             #endif
         case .spm:
             return try SPMProjectDriver.build()
+        case .generic:
+            return try GenericProjectDriver.build()
         }
     }
 }

Sources/Frontend/Scan.swift

@@ -12,7 +12,7 @@ final class Scan {
     }
 
     func perform(project: Project) throws -> [ScanResult] {
-        if configuration.indexStorePath != nil, !configuration.skipBuild {
+        if !configuration.indexStorePath.isEmpty, !configuration.skipBuild {
             logger.warn("The '--index-store-path' option implies '--skip-build', specify it to silence this warning")
             configuration.skipBuild = true
         }

Sources/PeripheryKit/Generic/GenericProjectDriver.swift

@@ -0,0 +1,62 @@
+import Foundation
+import SystemPackage
+import Shared
+
+public final class GenericProjectDriver {
+    public static func build() throws -> Self {
+        let configuration = Configuration.shared
+        let decoder = JSONDecoder()
+        decoder.keyDecodingStrategy = .convertFromSnakeCase
+
+        let sourceFiles = try configuration.fileTargetsPath
+            .reduce(into: [FilePath: Set<String>]()) { result, mapPath in
+                guard mapPath.exists else {
+                    throw PeripheryError.pathDoesNotExist(path: mapPath.string)
+                }
+
+                let data = try Data(contentsOf: mapPath.url)
+                let map = try decoder
+                    .decode(FileTargetMapContainer.self, from: data)
+                    .fileTargets
+                    .reduce(into: [FilePath: Set<String>](), { (result, tuple) in
+                        let (key, value) = tuple
+                        let path: FilePath
+                        if key.hasPrefix("/") {
+                            path = FilePath(key)
+                        } else {
+                            path = FilePath.current.appending(key)
+                        }
+
+                        if !path.exists {
+                            throw PeripheryError.pathDoesNotExist(path: path.string)
+                        }
+
+                        result[path] = value
+                    })
+                result.merge(map) { $0.union($1) }
+            }
+
+        return self.init(sourceFiles: sourceFiles, configuration: configuration)
+    }
+
+    private let sourceFiles: [FilePath: Set<String>]
+    private let configuration: Configuration
+
+    init(sourceFiles: [FilePath: Set<String>], configuration: Configuration) {
+        self.sourceFiles = sourceFiles
+        self.configuration = configuration
+    }
+}
+
+extension GenericProjectDriver: ProjectDriver {
+    public func build() throws {}
+
+    public func index(graph: SourceGraph) throws {
+        try SwiftIndexer(sourceFiles: sourceFiles, graph: graph, indexStorePaths: configuration.indexStorePath).perform()
+        graph.indexingComplete()
+    }
+}
+
+struct FileTargetMapContainer: Decodable {
+    let fileTargets: [String: Set<String>]
+}

Sources/PeripheryKit/Indexer/SwiftIndexer.swift

@@ -5,45 +5,48 @@ import SystemPackage
 import Shared
 
 public final class SwiftIndexer: Indexer {
-    private let sourceFiles: [FilePath: [String]]
+    private let sourceFiles: [FilePath: Set<String>]
     private let graph: SourceGraph
     private let logger: ContextualLogger
     private let configuration: Configuration
-    private let indexStore: IndexStore
-    private let indexStoreURL: URL
+    private let indexStorePaths: [FilePath]
 
     public required init(
-        sourceFiles: [FilePath: [String]],
+        sourceFiles: [FilePath: Set<String>],
         graph: SourceGraph,
-        indexStoreURL: URL,
+        indexStorePaths: [FilePath],
         logger: Logger = .init(),
         configuration: Configuration = .shared
-    ) throws {
+    ) {
         self.sourceFiles = sourceFiles
         self.graph = graph
-        self.indexStore = try .open(store: indexStoreURL, lib: .open())
-        self.indexStoreURL = indexStoreURL
+        self.indexStorePaths = indexStorePaths
         self.logger = logger.contextualized(with: "index:swift")
         self.configuration = configuration
         super.init(configuration: configuration)
     }
 
     public func perform() throws {
-        var unitsByFile: [FilePath: [IndexStoreUnit]] = [:]
+        var unitsByFile: [FilePath: [(IndexStore, IndexStoreUnit)]] = [:]
         let allSourceFiles = Set(sourceFiles.keys)
         let (includedFiles, excludedFiles) = filterIndexExcluded(from: allSourceFiles)
         excludedFiles.forEach { self.logger.debug("Excluding \($0.string)") }
 
-        try indexStore.forEachUnits(includeSystem: false) { unit -> Bool in
-            guard let filePath = try indexStore.mainFilePath(for: unit) else { return true }
+        for indexStorePath in indexStorePaths {
+            logger.debug("Reading \(indexStorePath)")
+            let indexStore = try IndexStore.open(store: URL(fileURLWithPath: indexStorePath.string), lib: .open())
 
-            let file = FilePath(filePath)
+            try indexStore.forEachUnits(includeSystem: false) { unit -> Bool in
+                guard let filePath = try indexStore.mainFilePath(for: unit) else { return true }
 
-            if includedFiles.contains(file) {
-                unitsByFile[file, default: []].append(unit)
-            }
+                let file = FilePath(filePath)
+
+                if includedFiles.contains(file) {
+                    unitsByFile[file, default: []].append((indexStore, unit))
+                }
 
-            return true
+                return true
+            }
         }
 
         let indexedFiles = Set(unitsByFile.keys)
@@ -52,15 +55,16 @@ public final class SwiftIndexer: Indexer {
         if !unindexedFiles.isEmpty {
             unindexedFiles.forEach { logger.debug("Source file not indexed: \($0)") }
             let targets: Set<String> = Set(unindexedFiles.flatMap { sourceFiles[$0] ?? [] })
-            throw PeripheryError.unindexedTargetsError(targets: targets, indexStorePath: indexStoreURL.path)
+            throw PeripheryError.unindexedTargetsError(targets: targets, indexStorePaths: indexStorePaths)
         }
 
         let jobs = try unitsByFile.map { (file, units) -> Job in
-            let modules = try units.reduce(into: Set<String>()) { (set, unit) in
+            let modules = try units.reduce(into: Set<String>()) { (set, tuple) in
+                let (indexStore, unit) = tuple
                 if let name = try indexStore.moduleName(for: unit) {
                     let (didInsert, _) = set.insert(name)
                     if !didInsert {
-                        let targets = try Set(units.compactMap { try indexStore.target(for: $0) })
+                        let targets = try Set(units.compactMap { try indexStore.target(for: $0.1) })
                         throw PeripheryError.conflictingIndexUnitsError(file: file, module: name, unitTargets: targets)
                     }
                 }
@@ -71,7 +75,6 @@ public final class SwiftIndexer: Indexer {
                 file: sourceFile,
                 units: units,
                 graph: graph,
-                indexStore: indexStore,
                 logger: logger,
                 configuration: configuration
             )
@@ -103,24 +106,21 @@ public final class SwiftIndexer: Indexer {
     private class Job {
         let file: SourceFile
 
-        private let units: [IndexStoreUnit]
+        private let units: [(IndexStore, IndexStoreUnit)]
         private let graph: SourceGraph
-        private let indexStore: IndexStore
         private let logger: ContextualLogger
         private let configuration: Configuration
 
         required init(
             file: SourceFile,
-            units: [IndexStoreUnit],
+            units: [(IndexStore, IndexStoreUnit)],
             graph: SourceGraph,
-            indexStore: IndexStore,
             logger: ContextualLogger,
             configuration: Configuration
         ) {
             self.file = file
             self.units = units
             self.graph = graph
-            self.indexStore = indexStore
             self.logger = logger
             self.configuration = configuration
         }
@@ -164,7 +164,7 @@ public final class SwiftIndexer: Indexer {
         func phaseOne() throws {
             var rawDeclsByKey: [RawDeclaration.Key: [(RawDeclaration, [RawRelation])]] = [:]
 
-            for unit in units {
+            for (indexStore, unit) in units {
                 try indexStore.forEachRecordDependencies(for: unit) { dependency in
                     guard case let .record(record) = dependency else { return true }
 
@@ -175,17 +175,17 @@ public final class SwiftIndexer: Indexer {
                               else { return true }
 
                         if !occurrence.roles.intersection([.definition, .declaration]).isEmpty {
-                            if let (decl, relations) = try parseRawDeclaration(occurrence, usr, location) {
+                            if let (decl, relations) = try parseRawDeclaration(occurrence, usr, location, indexStore) {
                                 rawDeclsByKey[decl.key, default: []].append((decl, relations))
                             }
                         }
 
                         if !occurrence.roles.intersection([.reference]).isEmpty {
-                            try parseReference(occurrence, usr, location)
+                            try parseReference(occurrence, usr, location, indexStore)
                         }
 
                         if !occurrence.roles.intersection([.implicit]).isEmpty {
-                            try parseImplicit(occurrence, usr, location)
+                            try parseImplicit(occurrence, usr, location, indexStore)
                         }
 
                         return true
@@ -438,7 +438,8 @@ public final class SwiftIndexer: Indexer {
         private func parseRawDeclaration(
             _ occurrence: IndexStoreOccurrence,
             _ usr: String,
-            _ location: SourceLocation
+            _ location: SourceLocation,
+            _ indexStore: IndexStore
         ) throws -> (RawDeclaration, [RawRelation])? {
             guard let kind = transformDeclarationKind(occurrence.symbol.kind, occurrence.symbol.subKind)
             else { return nil }
@@ -528,7 +529,8 @@ public final class SwiftIndexer: Indexer {
         private func parseImplicit(
             _ occurrence: IndexStoreOccurrence,
             _ occurrenceUsr: String,
-            _ location: SourceLocation
+            _ location: SourceLocation,
+            _ indexStore: IndexStore
         ) throws {
             var refs = [Reference]()
 
@@ -557,7 +559,8 @@ public final class SwiftIndexer: Indexer {
         private func parseReference(
             _ occurrence: IndexStoreOccurrence,
             _ occurrenceUsr: String,
-            _ location: SourceLocation
+            _ location: SourceLocation,
+            _ indexStore: IndexStore
         ) throws {
             guard let kind = transformReferenceKind(occurrence.symbol.kind, occurrence.symbol.subKind)
                   else { return }

Sources/PeripheryKit/SPM/SPMProjectDriver.swift

@@ -58,20 +58,20 @@ extension SPMProjectDriver: ProjectDriver {
     }
 
     public func index(graph: SourceGraph) throws {
-        let sourceFiles = targets.reduce(into: [FilePath: [String]]()) { result, target in
+        let sourceFiles = targets.reduce(into: [FilePath: Set<String>]()) { result, target in
             let targetPath = absolutePath(for: target)
-            target.sources.forEach { result[targetPath.appending($0), default: []].append(target.name) }
+            target.sources.forEach { result[targetPath.appending($0), default: []].insert(target.name) }
         }
 
-        let storePath: String
+        let storePaths: [FilePath]
 
-        if let path = configuration.indexStorePath {
-            storePath = path
+        if !configuration.indexStorePath.isEmpty {
+            storePaths = configuration.indexStorePath
         } else {
-            storePath = FilePath(package.path).appending(".build/debug/index/store").string
+            storePaths = [FilePath(package.path).appending(".build/debug/index/store")]
         }
 
-        try SwiftIndexer(sourceFiles: sourceFiles, graph: graph, indexStoreURL: URL(fileURLWithPath: storePath)).perform()
+        try SwiftIndexer(sourceFiles: sourceFiles, graph: graph, indexStorePaths: storePaths).perform()
 
         graph.indexingComplete()
     }