Add simple synthetic modules graph benchmarks (swiftlang#7465)

New `SyntheticModulesGraph` benchmark is introduced, which calls
`loadModulesGraph` function that we already run against test fixtures in
`ModulesGraphTests` and `BuildPlanTests`. The function under benchmark
almost immediately delegates to `ModulesGraph.load`, which is used in
real-world modules graph dependency resolution.

Benchmark parameters are controlled with
`SWIFTPM_BENCHMARK_MODULES_GRAPH_DEPTH` and
`SWIFTPM_BENCHMARK_MODULES_GRAPH_WIDTH` environment variables, so in the
future we should be able to plot graphs of benchmark metrics against
depth and width of modules graph over a given range.

Thresholds are now split into platform-specific directories so that
thresholds recorded on x86_64 don't interfere with thresholds for arm64,
same for the OS family.

Author: MaxDesiatov

Benchmarks/Benchmarks/PackageGraphBenchmarks/PackageGraphBenchmarks.swift

@@ -1,7 +1,13 @@
+@_spi(DontAdoptOutsideOfSwiftPMExposedForBenchmarksAndTestsOnly)
 import Basics
 import Benchmark
 import Foundation
 import PackageModel
+
+@_spi(DontAdoptOutsideOfSwiftPMExposedForBenchmarksAndTestsOnly)
+import func PackageGraph.loadModulesGraph
+
+import class TSCBasic.InMemoryFileSystem
 import Workspace
 
 let benchmarks = {
@@ -16,6 +22,32 @@ let benchmarks = {
         ]
     }
 
+    let modulesGraphDepth: Int
+    if let envVar = ProcessInfo.processInfo.environment["SWIFTPM_BENCHMARK_MODULES_GRAPH_DEPTH"],
+    let parsedValue = Int(envVar) {
+        modulesGraphDepth = parsedValue
+    } else {
+        modulesGraphDepth = 100
+    }
+
+    let modulesGraphWidth: Int
+    if let envVar = ProcessInfo.processInfo.environment["SWIFTPM_BENCHMARK_MODULES_GRAPH_WIDTH"],
+    let parsedValue = Int(envVar) {
+        modulesGraphWidth = parsedValue
+    } else {
+        modulesGraphWidth = 100
+    }
+
+    let packagesGraphDepth: Int
+    if let envVar = ProcessInfo.processInfo.environment["SWIFTPM_BENCHMARK_PACKAGES_GRAPH_DEPTH"],
+    let parsedValue = Int(envVar) {
+        packagesGraphDepth = parsedValue
+    } else {
+        packagesGraphDepth = 10
+    }
+
+    let noopObservability = ObservabilitySystem.NOOP
+
     // Benchmarks computation of a resolved graph of modules for a package using `Workspace` as an entry point. It runs PubGrub to get
     // resolved concrete versions of dependencies, assigning all modules and products to each other as corresponding dependencies
     // with their build triples, but with the build plan not yet constructed. In this benchmark specifically we're loading `Package.swift`
@@ -33,10 +65,57 @@ let benchmarks = {
     ) { benchmark in
         let path = try AbsolutePath(validating: #file).parentDirectory.parentDirectory.parentDirectory
         let workspace = try Workspace(fileSystem: localFileSystem, location: .init(forRootPackage: path, fileSystem: localFileSystem))
-        let system = ObservabilitySystem { _, _ in }
 
         for _ in benchmark.scaledIterations {
-            try workspace.loadPackageGraph(rootPath: path, observabilityScope: system.topScope)
+            try workspace.loadPackageGraph(rootPath: path, observabilityScope: noopObservability)
+        }
+    }
+
+
+    // Benchmarks computation of a resolved graph of modules for a synthesized package using `loadModulesGraph` as an
+    // entry point, which almost immediately delegates to `ModulesGraph.load` under the hood.
+    Benchmark(
+        "SyntheticModulesGraph",
+        configuration: .init(
+            metrics: defaultMetrics,
+            maxDuration: .seconds(10),
+            thresholds: [
+                .mallocCountTotal: .init(absolute: [.p90: 2500]),
+                .syscalls: .init(absolute: [.p90: 0]),
+            ]
+        )
+    ) { benchmark in
+        let targets = try (0..<modulesGraphWidth).map { i in
+            try TargetDescription(name: "Target\(i)", dependencies: (0..<min(i, modulesGraphDepth)).map {
+                .target(name: "Target\($0)")
+            })
+        }
+        let fileSystem = InMemoryFileSystem(
+            emptyFiles: targets.map { "/benchmark/Sources/\($0.name)/empty.swift" }
+        )
+        let rootPackagePath = try AbsolutePath(validating: "/benchmark")
+
+        let manifest = Manifest(
+            displayName: "benchmark",
+            path: rootPackagePath,
+            packageKind: .root(rootPackagePath),
+            packageLocation: rootPackagePath.pathString,
+            defaultLocalization: nil,
+            platforms: [],
+            version: nil,
+            revision: nil,
+            toolsVersion: .v5_10,
+            pkgConfig: nil,
+            providers: nil,
+            cLanguageStandard: nil,
+            cxxLanguageStandard: nil,
+            swiftLanguageVersions: nil
+        )
+
+        for _ in benchmark.scaledIterations {
+            try blackHole(
+                loadModulesGraph(fileSystem: fileSystem, manifests: [manifest], observabilityScope: noopObservability)
+            )
         }
     }
 }

Benchmarks/README.md

@@ -1,10 +1,13 @@
 # SwiftPM Benchmarks
 
-Benchmarks currently use [ordo-one/package-benchmark](https://github.com/ordo-one/package-benchmark) library for benchmarking.
+Benchmarks currently use [ordo-one/package-benchmark](https://github.com/ordo-one/package-benchmark) library for
+benchmarking.
 
 ## How to Run
 
-To run the benchmarks in their default configuration, run this commend in the `Benchmarks` subdirectory of the SwiftPM repository clone (the directory in which this `README.md` file is contained):
+To run the benchmarks in their default configuration, run this command in the `Benchmarks` subdirectory of the SwiftPM
+repository clone (the directory in which this `README.md` file is contained):
+
 ```
 swift package benchmark
 ```
@@ -17,13 +20,17 @@ SWIFTPM_BENCHMARK_ALL_METRICS=true swift package benchmark
 
 ## Benchmark Thresholds
 
-`Benchmarks/Thresholds` subdirectory contains recorded allocation and syscall counts for macOS on Apple Silicon when built with Swift 5.10. To record new thresholds, run the following command:
+`Benchmarks/Thresholds` subdirectory contains recorded allocation and syscall counts for macOS on Apple Silicon when
+built with Swift 5.10. To record new thresholds, run the following command:
 
 ```
-swift package --allow-writing-to-package-directory benchmark --format metricP90AbsoluteThresholds --path Thresholds/
+swift package --allow-writing-to-package-directory benchmark \
+  --format metricP90AbsoluteThresholds \
+  --path "Thresholds/$([[ $(uname) == Darwin ]] && echo macosx || echo linux)-$(uname -m)"
 ```
 
-To verify that recorded thresholds do not exceeded given relative or absolute values (passed as `thresholds` arguments to each benchmark's configuration), run this command:
+To verify that recorded thresholds do not exceeded given relative or absolute values (passed as `thresholds` arguments
+to each benchmark's configuration), run this command:
 
 ```
 swift package benchmark baseline check --check-absolute-path Thresholds/

Benchmarks/Thresholds/PackageGraphBenchmarks.PackageGraphLoading.p90.json

@@ -0,0 +1,4 @@
+{
+  "mallocCountTotal" : 10775,
+  "syscalls" : 1508
+}

Benchmarks/Thresholds/macosx-arm64/PackageGraphBenchmarks.SwiftPMWorkspaceModulesGraph.p90.json

@@ -0,0 +1,4 @@
+{
+  "mallocCountTotal" : 2427,
+  "syscalls" : 0
+}

Benchmarks/Thresholds/macosx-arm64/PackageGraphBenchmarks.SyntheticModulesGraph.p90.json

@@ -21,6 +21,7 @@ import class TSCBasic.FileLock
 import enum TSCBasic.FileMode
 import protocol TSCBasic.FileSystem
 import enum TSCBasic.FileSystemAttribute
+import class TSCBasic.InMemoryFileSystem
 import var TSCBasic.localFileSystem
 import protocol TSCBasic.WritableByteStream
 
@@ -632,3 +633,48 @@ extension FileLock {
         return try Self.prepareLock(fileToLock: fileToLock.underlying, at: lockFilesDirectory?.underlying)
     }
 }
+
+/// Convenience initializers for testing purposes.
+extension InMemoryFileSystem {
+    /// Create a new file system with the given files, provided as a map from
+    /// file path to contents.
+    public convenience init(files: [String: ByteString]) {
+        self.init()
+
+        for (path, contents) in files {
+            let path = try! AbsolutePath(validating: path)
+            try! createDirectory(path.parentDirectory, recursive: true)
+            try! writeFileContents(path, bytes: contents)
+        }
+    }
+
+    /// Create a new file system with an empty file at each provided path.
+    public convenience init(emptyFiles files: String...) {
+        self.init(emptyFiles: files)
+    }
+
+    /// Create a new file system with an empty file at each provided path.
+    public convenience init(emptyFiles files: [String]) {
+        self.init()
+        self.createEmptyFiles(at: .root, files: files)
+    }
+}
+
+extension FileSystem {
+    public func createEmptyFiles(at root: AbsolutePath, files: String...) {
+        self.createEmptyFiles(at: root, files: files)
+    }
+
+    public func createEmptyFiles(at root: AbsolutePath, files: [String]) {
+        do {
+            try createDirectory(root, recursive: true)
+            for path in files {
+                let path = try AbsolutePath(validating: String(path.dropFirst()), relativeTo: root)
+                try createDirectory(path.parentDirectory, recursive: true)
+                try writeFileContents(path, bytes: "")
+            }
+        } catch {
+            fatalError("Failed to create empty files: \(error)")
+        }
+    }
+}

Sources/Basics/FileSystem/FileSystem+Extensions.swift

@@ -13,16 +13,17 @@
 import struct DequeModule.Deque
 
 /// Directed graph that stores edges in [adjacency lists](https://en.wikipedia.org/wiki/Adjacency_list).
-struct DirectedGraph<Node> {
-    init(nodes: [Node]) {
+@_spi(DontAdoptOutsideOfSwiftPMExposedForBenchmarksAndTestsOnly)
+public struct DirectedGraph<Node> {
+    public init(nodes: [Node]) {
         self.nodes = nodes
         self.edges = .init(repeating: [], count: nodes.count)
     }
 
-    private var nodes: [Node]
+    public private(set) var nodes: [Node]
     private var edges: [[Int]]
 
-    mutating func addEdge(source: Int, destination: Int) {
+    public mutating func addEdge(source: Int, destination: Int) {
         self.edges[source].append(destination)
     }
 
@@ -31,7 +32,8 @@ struct DirectedGraph<Node> {
     ///   - source: `Index` of a node to start traversing edges from.
     ///   - destination: `Index` of a node to which a path could exist via edges from `source`.
     /// - Returns: `true` if a path from `source` to `destination` exists, `false` otherwise.
-    func areNodesConnected(source: Int, destination: Int) -> Bool {
+    @_spi(DontAdoptOutsideOfSwiftPMExposedForBenchmarksAndTestsOnly)
+    public func areNodesConnected(source: Int, destination: Int) -> Bool {
         var todo = Deque<Int>([source])
         var done = Set<Int>()
 

Sources/Basics/Graph/DirectedGraph.swift

@@ -13,16 +13,17 @@
 import struct DequeModule.Deque
 
 /// Undirected graph that stores edges in an [adjacency matrix](https://en.wikipedia.org/wiki/Adjacency_list).
-struct UndirectedGraph<Node> {
-    init(nodes: [Node]) {
+@_spi(DontAdoptOutsideOfSwiftPMExposedForBenchmarksAndTestsOnly)
+public struct UndirectedGraph<Node> {
+    public init(nodes: [Node]) {
         self.nodes = nodes
         self.edges = .init(rows: nodes.count, columns: nodes.count)
     }
 
     private var nodes: [Node]
     private var edges: AdjacencyMatrix
 
-    mutating func addEdge(source: Int, destination: Int) {
+    public mutating func addEdge(source: Int, destination: Int) {
         // Adjacency matrix is symmetrical for undirected graphs.
         self.edges[source, destination] = true
         self.edges[destination, source] = true
@@ -33,7 +34,7 @@ struct UndirectedGraph<Node> {
     ///   - source: `Index` of a node to start traversing edges from.
     ///   - destination: `Index` of a node to which a connection could exist via edges from `source`.
     /// - Returns: `true` if a path from `source` to `destination` exists, `false` otherwise.
-    func areNodesConnected(source: Int, destination: Int) -> Bool {
+    public func areNodesConnected(source: Int, destination: Int) -> Bool {
         var todo = Deque<Int>([source])
         var done = Set<Int>()
 

Sources/Basics/Graph/UndirectedGraph.swift

@@ -56,6 +56,10 @@ public class ObservabilitySystem {
             self.underlying(scope, diagnostic)
         }
     }
+
+    public static var NOOP: ObservabilityScope {
+        ObservabilitySystem { _, _ in }.topScope
+    }
 }
 
 public protocol ObservabilityHandlerProvider {

Sources/Basics/Observability.swift

@@ -10,6 +10,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+import protocol Basics.FileSystem
+import class Basics.ObservabilityScope
 import struct Basics.IdentifiableSet
 import OrderedCollections
 import PackageLoading
@@ -444,3 +446,50 @@ func topologicalSort<T: Identifiable>(
 
     return result.reversed()
 }
+
+@_spi(DontAdoptOutsideOfSwiftPMExposedForBenchmarksAndTestsOnly)
+public func loadModulesGraph(
+    identityResolver: IdentityResolver = DefaultIdentityResolver(),
+    fileSystem: FileSystem,
+    manifests: [Manifest],
+    binaryArtifacts: [PackageIdentity: [String: BinaryArtifact]] = [:],
+    explicitProduct: String? = .none,
+    shouldCreateMultipleTestProducts: Bool = false,
+    createREPLProduct: Bool = false,
+    useXCBuildFileRules: Bool = false,
+    customXCTestMinimumDeploymentTargets: [PackageModel.Platform: PlatformVersion]? = .none,
+    observabilityScope: ObservabilityScope
+) throws -> ModulesGraph {
+    let rootManifests = manifests.filter(\.packageKind.isRoot).spm_createDictionary { ($0.path, $0) }
+    let externalManifests = try manifests.filter { !$0.packageKind.isRoot }
+        .reduce(
+            into: OrderedCollections
+                .OrderedDictionary<PackageIdentity, (manifest: Manifest, fs: FileSystem)>()
+        ) { partial, item in
+            partial[try identityResolver.resolveIdentity(for: item.packageKind)] = (item, fileSystem)
+        }
+
+    let packages = Array(rootManifests.keys)
+    let input = PackageGraphRootInput(packages: packages)
+    let graphRoot = PackageGraphRoot(
+        input: input,
+        manifests: rootManifests,
+        explicitProduct: explicitProduct,
+        observabilityScope: observabilityScope
+    )
+
+    return try ModulesGraph.load(
+        root: graphRoot,
+        identityResolver: identityResolver,
+        additionalFileRules: useXCBuildFileRules ? FileRuleDescription.xcbuildFileTypes : FileRuleDescription
+            .swiftpmFileTypes,
+        externalManifests: externalManifests,
+        binaryArtifacts: binaryArtifacts,
+        shouldCreateMultipleTestProducts: shouldCreateMultipleTestProducts,
+        createREPLProduct: createREPLProduct,
+        customXCTestMinimumDeploymentTargets: customXCTestMinimumDeploymentTargets,
+        availableLibraries: [],
+        fileSystem: fileSystem,
+        observabilityScope: observabilityScope
+    )
+}

Sources/PackageGraph/ModulesGraph.swift

@@ -13,7 +13,12 @@
 import struct Basics.AbsolutePath
 import class Basics.ObservabilitySystem
 import class Basics.ObservabilityScope
+
 import struct PackageGraph.ModulesGraph
+
+@_spi(DontAdoptOutsideOfSwiftPMExposedForBenchmarksAndTestsOnly)
+import func PackageGraph.loadModulesGraph
+
 import class PackageModel.Manifest
 import struct PackageModel.ProductDescription
 import struct PackageModel.TargetDescription

Sources/SPMTestSupport/MockPackageGraphs.swift

@@ -24,10 +24,6 @@ extension ObservabilitySystem {
         let observabilitySystem = ObservabilitySystem(collector)
         return TestingObservability(collector: collector, topScope: observabilitySystem.topScope)
     }
-
-    package static var NOOP: ObservabilityScope {
-        ObservabilitySystem { _, _ in }.topScope
-    }
 }
 
 package struct TestingObservability {