[Explicit Module Builds] Introduce an InterModuleDependencyOracle abstraction for tracking and sharing dependency scanning results

This PR refactors the handling of inter-module dependencies to centralize their handling in a new, reference-type, oracle.

`InterModuleDependencyOracle` will now be responsible for responding to queries about dependencies of a given module.
The intent behind it is to abstract the actual mechanism used for tracking/computing/caching inter-module dependencies. A single oracle reference is intended to be passed around to multiple Driver invocations, corresponding to different targets, by build-system clients (e.g. SwiftPM), replacing today's use of `ModuleInfoMap`-based API, to communicate already-computed dependency-scan results.

As implemented, it is a simple store of `ModuleInfo` value objects, which a `Driver` invocation populates by invoking a `-scan-dependencies` job; not unlike passing around an accumulating `ModuleInfoMap`.
In the future, the oracle's internal implementation may be replaced with a direct dependency-scanner library access, without having to break the clients' API.

Author: artemcm

Sources/SwiftDriver/CMakeLists.txt

@@ -9,12 +9,13 @@
 add_library(SwiftDriver
   "Explicit Module Builds/ClangModuleBuildJobCache.swift"
   "Explicit Module Builds/ClangVersionedDependencyResolution.swift"
-  "Explicit Module Builds/CommonDependencyGraphOperations.swift"
   "Explicit Module Builds/ExplicitDependencyBuildPlanner.swift"
-  "Explicit Module Builds/InterModuleDependencyGraph.swift"
   "Explicit Module Builds/ModuleDependencyScanning.swift"
   "Explicit Module Builds/PlaceholderDependencyResolution.swift"
   "Explicit Module Builds/SerializableModuleArtifacts.swift"
+  "Explicit Module Builds/Inter Module Dependencies/CommonDependencyOperations.swift"
+  "Explicit Module Builds/Inter Module Dependencies/InterModuleDependencyGraph.swift"
+  "Explicit Module Builds/Inter Module Dependencies/InterModuleDependencyOracle.swift"
 
   Driver/CompilerMode.swift
   Driver/DebugInfo.swift

Sources/SwiftDriver/Driver/Driver.swift

@@ -256,11 +256,21 @@ public struct Driver {
   /// dependencies in parallel builds.
   var forceEmitModuleBeforeCompile: Bool = false
 
+  // FIXME: We should soon be able to remove this from being in the Driver's state.
+  // Its only remaining use outside of actual dependency-build planning is in
+  // command-line option generation for the main module compile.
   /// Planner for constructing module build jobs using Explicit Module Builds.
   /// Constructed during the planning phase only when all modules will be prebuilt and treated
   /// as explicit by the various compilation jobs.
   @_spi(Testing) public var explicitDependencyBuildPlanner: ExplicitDependencyBuildPlanner? = nil
 
+  /// An oracle for querying inter-module dependencies
+  /// Can either be an argument to the driver in many-module contexts where dependency information
+  /// is shared across many targets; otherwise, a new instance is created by the driver itself.
+  @_spi(Testing) public let interModuleDependencyOracle: InterModuleDependencyOracle
+
+  // TODO: Once the clients have transitioned to using the InterModuleDependencyOracle API,
+  // this must convey information about the externally-prebuilt targets only
   /// All external artifacts a build system (e.g. SwiftPM) may pass in as input to the explicit
   /// build of the current module. Consists of a map of externally-built targets, and a map of all previously
   /// discovered/scanned modules and their infos.
@@ -311,7 +321,8 @@ public struct Driver {
     diagnosticsEngine: DiagnosticsEngine = DiagnosticsEngine(handlers: [Driver.stderrDiagnosticsHandler]),
     fileSystem: FileSystem = localFileSystem,
     executor: DriverExecutor,
-    externalBuildArtifacts: ExternalBuildArtifacts? = nil
+    externalBuildArtifacts: ExternalBuildArtifacts? = nil,
+    interModuleDependencyOracle: InterModuleDependencyOracle? = nil
   ) throws {
     self.env = env
     self.fileSystem = fileSystem
@@ -370,9 +381,18 @@ public struct Driver {
         guard let modTime = try? fileSystem
           .getFileInfo($0.file).modTime else { return nil }
         return ($0, modTime)
-      })
+    })
 
-    do {
+    // Create an instance of an inter-module dependency oracle, if the driver's
+    // client did not provide one. The clients are expected to provide an oracle
+    // when they wish to share module dependency information across targets.
+    if let dependencyOracle = interModuleDependencyOracle {
+      self.interModuleDependencyOracle = dependencyOracle
+    } else {
+      self.interModuleDependencyOracle = InterModuleDependencyOracle()
+    }
+    
+    do {    
       let outputFileMap: OutputFileMap?
       // Initialize an empty output file map, which will be populated when we start creating jobs.
       if let outputFileMapArg = parsedOptions.getLastArgument(.outputFileMap)?.asSingle {
@@ -382,14 +402,12 @@ public struct Driver {
         } catch {
           throw Error.unableToLoadOutputFileMap(outputFileMapArg)
         }
-      } else {
-        outputFileMap = nil
-      }
 
-      if let workingDirectory = self.workingDirectory {
-        self.outputFileMap = outputFileMap?.resolveRelativePaths(relativeTo: workingDirectory)
-      } else {
-        self.outputFileMap = outputFileMap
+        if let workingDirectory = self.workingDirectory {
+          self.outputFileMap = outputFileMap?.resolveRelativePaths(relativeTo: workingDirectory)
+        } else {
+          self.outputFileMap = outputFileMap
+        }
       }
     }
 

Sources/SwiftDriver/Explicit Module Builds/ClangVersionedDependencyResolution.swift

@@ -65,6 +65,9 @@ internal extension Driver {
   }
 }
 
+/// Resolution of versioned clang dependencies.
+/// FIXME: This code currently operates on instances of InterModuleDependencyGraph,
+/// It should be transitioned to operate on an instance of an InterModuleDependencyOracle.
 private extension InterModuleDependencyGraph {
   /// For each module scanned at multiple target versions, combine their dependencies across version-specific graphs.
   mutating func resolveVersionedClangModules(using versionedGraphMap: ModuleVersionedGraphMap)
@@ -108,24 +111,35 @@ private extension InterModuleDependencyGraph {
                pathPCMArtSet: Set<[String]>,
                pcmArgSetMap: inout [ModuleDependencyId : Set<[String]>])
     throws {
+      guard let moduleInfo = modules[moduleId] else {
+        throw Driver.Error.missingModuleDependency(moduleId.moduleName)
+      }
       switch moduleId {
         case .swift:
+          guard case .swift(let swiftModuleDetails) = moduleInfo.details else {
+            throw Driver.Error.malformedModuleDependency(moduleId.moduleName,
+                                                         "no Swift `details` object")
+          }
           // Add extraPCMArgs of the visited node to the current path set
           // and proceed to visit all direct dependencies
-          let modulePCMArgs = try swiftModulePCMArgs(of: moduleId)
+          let modulePCMArgs = swiftModuleDetails.extraPcmArgs
           var newPathPCMArgSet = pathPCMArtSet
           newPathPCMArgSet.insert(modulePCMArgs)
-          for dependencyId in try moduleInfo(of: moduleId).directDependencies! {
+          for dependencyId in moduleInfo.directDependencies! {
             try visit(dependencyId,
                       pathPCMArtSet: newPathPCMArgSet,
                       pcmArgSetMap: &pcmArgSetMap)
           }
         case .clang:
+          guard case .clang(let clangModuleDetails) = moduleInfo.details else {
+            throw Driver.Error.malformedModuleDependency(moduleId.moduleName,
+                                                         "no Clang `details` object")
+          }
           // The details of this module contain information on which sets of PCMArgs are already
           // captured in the described dependencies of this module. Only re-scan at PCMArgs not
           // already captured.
-          let moduleDetails = try clangModuleDetails(of: moduleId)
-          let alreadyCapturedPCMArgs = moduleDetails.dependenciesCapturedPCMArgs ?? Set<[String]>()
+          let alreadyCapturedPCMArgs =
+            clangModuleDetails.dependenciesCapturedPCMArgs ?? Set<[String]>()
           let newPCMArgSet = pathPCMArtSet.filter { !alreadyCapturedPCMArgs.contains($0) }
           // Add current path's PCMArgs to the SetMap and stop traversal
           if pcmArgSetMap[moduleId] != nil {
@@ -138,7 +152,7 @@ private extension InterModuleDependencyGraph {
           // We can rely on the fact that this pre-built module already has its
           // versioned-PCM dependencies satisfied, so we do not need to add additional
           // arguments. Proceed traversal to its dependencies.
-          for dependencyId in try moduleInfo(of: moduleId).directDependencies! {
+          for dependencyId in moduleInfo.directDependencies! {
             try visit(dependencyId,
                       pathPCMArtSet: pathPCMArtSet,
                       pcmArgSetMap: &pcmArgSetMap)
@@ -159,57 +173,19 @@ private extension InterModuleDependencyGraph {
                                                         [ModuleDependencyId : Set<[String]>]
   ) throws {
     for (moduleId, newPCMArgs) in pcmArgSetMap {
-      var moduleDetails = try clangModuleDetails(of: moduleId)
-      if moduleDetails.dependenciesCapturedPCMArgs == nil {
-        moduleDetails.dependenciesCapturedPCMArgs = Set<[String]>()
+      guard let moduleInfo = modules[moduleId] else {
+        throw Driver.Error.missingModuleDependency(moduleId.moduleName)
+      }
+      guard case .clang(var clangModuleDetails) = moduleInfo.details else {
+        throw Driver.Error.malformedModuleDependency(moduleId.moduleName,
+                                                     "no Clang `details` object")
       }
-      newPCMArgs.forEach { moduleDetails.dependenciesCapturedPCMArgs!.insert($0) }
-      modules[moduleId]!.details = .clang(moduleDetails)
+      if clangModuleDetails.dependenciesCapturedPCMArgs == nil {
+        clangModuleDetails.dependenciesCapturedPCMArgs = Set<[String]>()
+      }
+      newPCMArgs.forEach { clangModuleDetails.dependenciesCapturedPCMArgs!.insert($0) }
+      modules[moduleId]!.details = .clang(clangModuleDetails)
     }
   }
 }
 
-public extension InterModuleDependencyGraph {
-  /// Given two moduleInfos of clang modules, merge them by combining their directDependencies and
-  /// dependenciesCapturedPCMArgs and sourceFiles fields. These fields may differ across the same module
-  /// scanned at different PCMArgs (e.g. -target option).
-  static func mergeClangModuleInfoDependencies(_ firstInfo: ModuleInfo, _ secondInfo:ModuleInfo
-  ) -> ModuleInfo {
-    guard case .clang(let firstDetails) = firstInfo.details,
-          case .clang(let secondDetails) = secondInfo.details
-    else {
-      fatalError("mergeClangModules expected two valid ClangModuleDetails objects.")
-    }
-
-    // As far as their dependencies go, these module infos are identical
-    if firstInfo.directDependencies == secondInfo.directDependencies,
-       firstDetails.dependenciesCapturedPCMArgs == secondDetails.dependenciesCapturedPCMArgs,
-       firstInfo.sourceFiles == secondInfo.sourceFiles {
-      return firstInfo
-    }
-
-    // Create a new moduleInfo that represents this module with combined dependency information
-    let firstModuleSources = firstInfo.sourceFiles ?? []
-    let secondModuleSources = secondInfo.sourceFiles ?? []
-    let combinedSourceFiles = Array(Set(firstModuleSources + secondModuleSources))
-
-    let firstModuleDependencies = firstInfo.directDependencies ?? []
-    let secondModuleDependencies = secondInfo.directDependencies ?? []
-    let combinedDependencies = Array(Set(firstModuleDependencies + secondModuleDependencies))
-
-    let firstModuleCapturedPCMArgs = firstDetails.dependenciesCapturedPCMArgs ?? Set<[String]>()
-    let secondModuleCapturedPCMArgs = secondDetails.dependenciesCapturedPCMArgs ?? Set<[String]>()
-    let combinedCapturedPCMArgs = firstModuleCapturedPCMArgs.union(secondModuleCapturedPCMArgs)
-
-    let combinedModuleDetails =
-      ClangModuleDetails(moduleMapPath: firstDetails.moduleMapPath,
-                         dependenciesCapturedPCMArgs: combinedCapturedPCMArgs,
-                         contextHash: firstDetails.contextHash,
-                         commandLine: firstDetails.commandLine)
-
-    return ModuleInfo(modulePath: firstInfo.modulePath,
-                      sourceFiles: combinedSourceFiles,
-                      directDependencies: combinedDependencies,
-                      details: .clang(combinedModuleDetails))
-  }
-}