Add HashEncoder and QueryEngineTests (swiftlang#7435)

Depends on swiftlang#7347.

Previously, `QueryEngine` couldn't cache anything due to a missing
hashing implementation that could be consistent across SwiftPM
relaunches. Here `HashEncoder` is implemented on top `Codable`. While it
probably has certain performance overhead when compared to an
implementation with macros, it should be good enough for prototyping.

New `QueryEngine` now works for trivial computations, as verified by
newly added `QueryEngineTests`.

Author: MaxDesiatov

Package.swift

@@ -359,7 +359,7 @@ let package = Package(
             /** Support for building using Xcode's build system */
             name: "XCBuildSupport",
             dependencies: ["DriverSupport", "SPMBuildCore", "PackageGraph"],
-            exclude: ["CMakeLists.txt", "CODEOWNERS"]
+            exclude: ["CMakeLists.txt"]
         ),
         .target(
             /** High level functionality */
@@ -473,6 +473,9 @@ let package = Package(
             dependencies: [
                 "Basics",
                 .product(name: "Crypto", package: "swift-crypto"),
+            ],
+            swiftSettings: [
+                .enableExperimentalFeature("StrictConcurrency=complete"),
             ]
         ),
 
@@ -533,7 +536,7 @@ let package = Package(
                 "Commands",
                 "SwiftSDKCommand",
                 "PackageCollectionsCommand",
-                "PackageRegistryCommand"
+                "PackageRegistryCommand",
             ],
             linkerSettings: swiftpmLinkSettings
         ),
@@ -672,6 +675,10 @@ let package = Package(
             name: "PackageSigningTests",
             dependencies: ["SPMTestSupport", "PackageSigning"]
         ),
+        .testTarget(
+            name: "QueryEngineTests",
+            dependencies: ["QueryEngine", "SPMTestSupport"]
+        ),
         .testTarget(
             name: "SourceControlTests",
             dependencies: ["SourceControl", "SPMTestSupport"],

Sources/Basics/SQLite.swift

@@ -19,20 +19,20 @@ import SPMSQLite3
 #endif
 
 /// A minimal SQLite wrapper.
-public final class SQLite {
+package final class SQLite {
     /// The location of the database.
-    public let location: Location
+    package let location: Location
 
     /// The configuration for the database.
-    public let configuration: Configuration
+    package let configuration: Configuration
 
     /// Pointer to the database.
     let db: OpaquePointer
 
     /// Create or open the database at the given path.
     ///
     /// The database is opened in serialized mode.
-    public init(location: Location, configuration: Configuration = Configuration()) throws {
+    package init(location: Location, configuration: Configuration = Configuration()) throws {
         self.location = location
         self.configuration = configuration
 
@@ -64,19 +64,19 @@ public final class SQLite {
     }
 
     @available(*, deprecated, message: "use init(location:configuration) instead")
-    public convenience init(dbPath: AbsolutePath) throws {
+    package convenience init(dbPath: AbsolutePath) throws {
         try self.init(location: .path(dbPath))
     }
 
     /// Prepare the given query.
-    public func prepare(query: String) throws -> PreparedStatement {
+    package func prepare(query: String) throws -> PreparedStatement {
         try PreparedStatement(db: self.db, query: query)
     }
 
     /// Directly execute the given query.
     ///
     /// Note: Use withCString for string arguments.
-    public func exec(query queryString: String, args: [CVarArg] = [], _ callback: SQLiteExecCallback? = nil) throws {
+    package func exec(query queryString: String, args: [CVarArg] = [], _ callback: SQLiteExecCallback? = nil) throws {
         let query = withVaList(args) { ptr in
             sqlite3_vmprintf(queryString, ptr)
         }
@@ -96,27 +96,27 @@ public final class SQLite {
         }
     }
 
-    public func close() throws {
+    package func close() throws {
         try Self.checkError { sqlite3_close(db) }
     }
 
-    public typealias SQLiteExecCallback = ([Column]) -> Void
+    package typealias SQLiteExecCallback = ([Column]) -> Void
 
-    public struct Configuration {
-        public var busyTimeoutMilliseconds: Int32
-        public var maxSizeInBytes: Int?
+    package struct Configuration {
+        package var busyTimeoutMilliseconds: Int32
+        package var maxSizeInBytes: Int?
 
         // https://www.sqlite.org/pgszchng2016.html
         private let defaultPageSizeInBytes = 1024
 
-        public init() {
+        package init() {
             self.busyTimeoutMilliseconds = 5000
             self.maxSizeInBytes = .none
         }
 
         // FIXME: deprecated 12/2020, remove once clients migrated over
         @available(*, deprecated, message: "use busyTimeout instead")
-        public var busyTimeoutSeconds: Int32 {
+        package var busyTimeoutSeconds: Int32 {
             get {
                 self._busyTimeoutSeconds
             } set {
@@ -133,7 +133,7 @@ public final class SQLite {
             }
         }
 
-        public var maxSizeInMegabytes: Int? {
+        package var maxSizeInMegabytes: Int? {
             get {
                 self.maxSizeInBytes.map { $0 / (1024 * 1024) }
             }
@@ -142,12 +142,12 @@ public final class SQLite {
             }
         }
 
-        public var maxPageCount: Int? {
+        package var maxPageCount: Int? {
             self.maxSizeInBytes.map { $0 / self.defaultPageSizeInBytes }
         }
     }
 
-    public enum Location {
+    package enum Location: Sendable {
         case path(AbsolutePath)
         case memory
         case temporary
@@ -165,59 +165,59 @@ public final class SQLite {
     }
 
     /// Represents an sqlite value.
-    public enum SQLiteValue {
+    package enum SQLiteValue {
         case null
         case string(String)
         case int(Int)
         case blob(Data)
     }
 
     /// Represents a row returned by called step() on a prepared statement.
-    public struct Row {
+    package struct Row {
         /// The pointer to the prepared statement.
         let stmt: OpaquePointer
 
         /// Get integer at the given column index.
-        public func int(at index: Int32) -> Int {
+        package func int(at index: Int32) -> Int {
             Int(sqlite3_column_int64(self.stmt, index))
         }
 
         /// Get blob data at the given column index.
-        public func blob(at index: Int32) -> Data {
+        package func blob(at index: Int32) -> Data {
             let bytes = sqlite3_column_blob(stmt, index)!
             let count = sqlite3_column_bytes(stmt, index)
             return Data(bytes: bytes, count: Int(count))
         }
 
         /// Get string at the given column index.
-        public func string(at index: Int32) -> String {
+        package func string(at index: Int32) -> String {
             String(cString: sqlite3_column_text(self.stmt, index))
         }
     }
 
-    public struct Column {
-        public var name: String
-        public var value: String
+    package struct Column {
+        package var name: String
+        package var value: String
     }
 
     /// Represents a prepared statement.
-    public struct PreparedStatement {
+    package struct PreparedStatement {
         typealias sqlite3_destructor_type = @convention(c) (UnsafeMutableRawPointer?) -> Void
         static let SQLITE_STATIC = unsafeBitCast(0, to: sqlite3_destructor_type.self)
         static let SQLITE_TRANSIENT = unsafeBitCast(-1, to: sqlite3_destructor_type.self)
 
         /// The pointer to the prepared statement.
         let stmt: OpaquePointer
 
-        public init(db: OpaquePointer, query: String) throws {
+        package init(db: OpaquePointer, query: String) throws {
             var stmt: OpaquePointer?
             try SQLite.checkError { sqlite3_prepare_v2(db, query, -1, &stmt, nil) }
             self.stmt = stmt!
         }
 
         /// Evaluate the prepared statement.
         @discardableResult
-        public func step() throws -> Row? {
+        package func step() throws -> Row? {
             let result = sqlite3_step(stmt)
 
             switch result {
@@ -231,7 +231,7 @@ public final class SQLite {
         }
 
         /// Bind the given arguments to the statement.
-        public func bind(_ arguments: [SQLiteValue]) throws {
+        package func bind(_ arguments: [SQLiteValue]) throws {
             for (idx, argument) in arguments.enumerated() {
                 let idx = Int32(idx) + 1
                 switch argument {
@@ -258,17 +258,17 @@ public final class SQLite {
         }
 
         /// Reset the prepared statement.
-        public func reset() throws {
+        package func reset() throws {
             try SQLite.checkError { sqlite3_reset(stmt) }
         }
 
         /// Clear bindings from the prepared statement.
-        public func clearBindings() throws {
+        package func clearBindings() throws {
             try SQLite.checkError { sqlite3_clear_bindings(stmt) }
         }
 
         /// Finalize the statement and free up resources.
-        public func finalize() throws {
+        package func finalize() throws {
             try SQLite.checkError { sqlite3_finalize(stmt) }
         }
     }
@@ -296,7 +296,7 @@ public final class SQLite {
         }
     }
 
-    public enum Errors: Error {
+    package enum Errors: Error {
         case databaseFull
     }
 }

Sources/Basics/SQLiteBackedCache.swift

@@ -17,13 +17,13 @@ import class TSCBasic.InMemoryFileSystem
 import var TSCBasic.localFileSystem
 
 /// SQLite backed persistent cache.
-public final class SQLiteBackedCache<Value: Codable>: Closable {
-    public typealias Key = String
+package final class SQLiteBackedCache<Value: Codable>: Closable {
+    package typealias Key = String
 
-    public let tableName: String
-    public let fileSystem: FileSystem
-    public let location: SQLite.Location
-    public let configuration: SQLiteBackedCacheConfiguration
+    package let tableName: String
+    package let fileSystem: FileSystem
+    package let location: SQLite.Location
+    package let configuration: SQLiteBackedCacheConfiguration
 
     private var state = State.idle
     private let stateLock = NSLock()
@@ -37,7 +37,7 @@ public final class SQLiteBackedCache<Value: Codable>: Closable {
     ///   - tableName: The SQLite table name. Must follow SQLite naming rules (e.g., no spaces).
     ///   - location: SQLite.Location
     ///   - configuration: Optional. Configuration for the cache.
-    public init(tableName: String, location: SQLite.Location, configuration: SQLiteBackedCacheConfiguration = .init()) {
+    package init(tableName: String, location: SQLite.Location, configuration: SQLiteBackedCacheConfiguration = .init()) {
         self.tableName = tableName
         self.location = location
         switch self.location {
@@ -57,7 +57,7 @@ public final class SQLiteBackedCache<Value: Codable>: Closable {
     ///   - tableName: The SQLite table name. Must follow SQLite naming rules (e.g., no spaces).
     ///   - path: The path of the SQLite database.
     ///   - configuration: Optional. Configuration for the cache.
-    public convenience init(
+    package convenience init(
         tableName: String,
         path: AbsolutePath,
         configuration: SQLiteBackedCacheConfiguration = .init()
@@ -75,7 +75,7 @@ public final class SQLiteBackedCache<Value: Codable>: Closable {
         }
     }
 
-    public func close() throws {
+    package func close() throws {
         try self.withStateLock {
             if case .connected(let db) = self.state {
                 try db.close()
@@ -122,7 +122,7 @@ public final class SQLiteBackedCache<Value: Codable>: Closable {
         }
     }
 
-    public func put(
+    package func put(
         blobKey key: some Sequence<UInt8>,
         value: Value,
         replace: Bool = false,
@@ -131,7 +131,7 @@ public final class SQLiteBackedCache<Value: Codable>: Closable {
         try self.put(rawKey: .blob(Data(key)), value: value, observabilityScope: observabilityScope)
     }
 
-    public func put(
+    package func put(
         key: Key,
         value: Value,
         replace: Bool = false,
@@ -140,7 +140,7 @@ public final class SQLiteBackedCache<Value: Codable>: Closable {
         try self.put(rawKey: .string(key), value: value, replace: replace, observabilityScope: observabilityScope)
     }
 
-    public func get(key: Key) throws -> Value? {
+    package func get(key: Key) throws -> Value? {
         let query = "SELECT value FROM \(self.tableName) WHERE key = ? LIMIT 1;"
         return try self.executeStatement(query) { statement -> Value? in
             try statement.bind([.string(key)])
@@ -151,7 +151,7 @@ public final class SQLiteBackedCache<Value: Codable>: Closable {
         }
     }
 
-    public func get(blobKey key: some Sequence<UInt8>) throws -> Value? {
+    package func get(blobKey key: some Sequence<UInt8>) throws -> Value? {
         let query = "SELECT value FROM \(self.tableName) WHERE key = ? LIMIT 1;"
         return try self.executeStatement(query) { statement -> Value? in
             try statement.bind([.blob(Data(key))])
@@ -162,7 +162,7 @@ public final class SQLiteBackedCache<Value: Codable>: Closable {
         }
     }
 
-    public func remove(key: Key) throws {
+    package func remove(key: Key) throws {
         let query = "DELETE FROM \(self.tableName) WHERE key = ?;"
         try self.executeStatement(query) { statement in
             try statement.bind([.string(key)])
@@ -254,20 +254,20 @@ public final class SQLiteBackedCache<Value: Codable>: Closable {
     }
 }
 
-public struct SQLiteBackedCacheConfiguration {
-    public var truncateWhenFull: Bool
+package struct SQLiteBackedCacheConfiguration {
+    package var truncateWhenFull: Bool
 
     fileprivate var underlying: SQLite.Configuration
 
-    public init() {
+    package init() {
         self.underlying = .init()
         self.truncateWhenFull = true
         self.maxSizeInMegabytes = 100
         // see https://www.sqlite.org/c3ref/busy_timeout.html
         self.busyTimeoutMilliseconds = 1000
     }
 
-    public var maxSizeInMegabytes: Int? {
+    package var maxSizeInMegabytes: Int? {
         get {
             self.underlying.maxSizeInMegabytes
         }
@@ -276,7 +276,7 @@ public struct SQLiteBackedCacheConfiguration {
         }
     }
 
-    public var maxSizeInBytes: Int? {
+    package var maxSizeInBytes: Int? {
         get {
             self.underlying.maxSizeInBytes
         }
@@ -285,7 +285,7 @@ public struct SQLiteBackedCacheConfiguration {
         }
     }
 
-    public var busyTimeoutMilliseconds: Int32 {
+    package var busyTimeoutMilliseconds: Int32 {
         get {
             self.underlying.busyTimeoutMilliseconds
         }