diff --git a/Package.swift b/Package.swift index b8f4923719d..ab065fca776 100644 --- a/Package.swift +++ b/Package.swift @@ -359,7 +359,7 @@ let package = Package( /** Support for building using Xcode's build system */ name: "XCBuildSupport", dependencies: ["DriverSupport", "SPMBuildCore", "PackageGraph"], - exclude: ["CMakeLists.txt", "CODEOWNERS"] + exclude: ["CMakeLists.txt"] ), .target( /** High level functionality */ @@ -473,6 +473,9 @@ let package = Package( dependencies: [ "Basics", .product(name: "Crypto", package: "swift-crypto"), + ], + swiftSettings: [ + .enableExperimentalFeature("StrictConcurrency=complete"), ] ), @@ -533,7 +536,7 @@ let package = Package( "Commands", "SwiftSDKCommand", "PackageCollectionsCommand", - "PackageRegistryCommand" + "PackageRegistryCommand", ], linkerSettings: swiftpmLinkSettings ), @@ -672,6 +675,10 @@ let package = Package( name: "PackageSigningTests", dependencies: ["SPMTestSupport", "PackageSigning"] ), + .testTarget( + name: "QueryEngineTests", + dependencies: ["QueryEngine", "SPMTestSupport"] + ), .testTarget( name: "SourceControlTests", dependencies: ["SourceControl", "SPMTestSupport"], diff --git a/Sources/Basics/SQLite.swift b/Sources/Basics/SQLite.swift index ab4e8ae34a5..d6cc6a108d2 100644 --- a/Sources/Basics/SQLite.swift +++ b/Sources/Basics/SQLite.swift @@ -19,12 +19,12 @@ import SPMSQLite3 #endif /// A minimal SQLite wrapper. -public final class SQLite { +package final class SQLite { /// The location of the database. - public let location: Location + package let location: Location /// The configuration for the database. - public let configuration: Configuration + package let configuration: Configuration /// Pointer to the database. let db: OpaquePointer @@ -32,7 +32,7 @@ public final class SQLite { /// Create or open the database at the given path. /// /// The database is opened in serialized mode. - public init(location: Location, configuration: Configuration = Configuration()) throws { + package init(location: Location, configuration: Configuration = Configuration()) throws { self.location = location self.configuration = configuration @@ -64,19 +64,19 @@ public final class SQLite { } @available(*, deprecated, message: "use init(location:configuration) instead") - public convenience init(dbPath: AbsolutePath) throws { + package convenience init(dbPath: AbsolutePath) throws { try self.init(location: .path(dbPath)) } /// Prepare the given query. - public func prepare(query: String) throws -> PreparedStatement { + package func prepare(query: String) throws -> PreparedStatement { try PreparedStatement(db: self.db, query: query) } /// Directly execute the given query. /// /// Note: Use withCString for string arguments. - public func exec(query queryString: String, args: [CVarArg] = [], _ callback: SQLiteExecCallback? = nil) throws { + package func exec(query queryString: String, args: [CVarArg] = [], _ callback: SQLiteExecCallback? = nil) throws { let query = withVaList(args) { ptr in sqlite3_vmprintf(queryString, ptr) } @@ -96,27 +96,27 @@ public final class SQLite { } } - public func close() throws { + package func close() throws { try Self.checkError { sqlite3_close(db) } } - public typealias SQLiteExecCallback = ([Column]) -> Void + package typealias SQLiteExecCallback = ([Column]) -> Void - public struct Configuration { - public var busyTimeoutMilliseconds: Int32 - public var maxSizeInBytes: Int? + package struct Configuration { + package var busyTimeoutMilliseconds: Int32 + package var maxSizeInBytes: Int? // https://www.sqlite.org/pgszchng2016.html private let defaultPageSizeInBytes = 1024 - public init() { + package init() { self.busyTimeoutMilliseconds = 5000 self.maxSizeInBytes = .none } // FIXME: deprecated 12/2020, remove once clients migrated over @available(*, deprecated, message: "use busyTimeout instead") - public var busyTimeoutSeconds: Int32 { + package var busyTimeoutSeconds: Int32 { get { self._busyTimeoutSeconds } set { @@ -133,7 +133,7 @@ public final class SQLite { } } - public var maxSizeInMegabytes: Int? { + package var maxSizeInMegabytes: Int? { get { self.maxSizeInBytes.map { $0 / (1024 * 1024) } } @@ -142,12 +142,12 @@ public final class SQLite { } } - public var maxPageCount: Int? { + package var maxPageCount: Int? { self.maxSizeInBytes.map { $0 / self.defaultPageSizeInBytes } } } - public enum Location { + package enum Location: Sendable { case path(AbsolutePath) case memory case temporary @@ -165,7 +165,7 @@ public final class SQLite { } /// Represents an sqlite value. - public enum SQLiteValue { + package enum SQLiteValue { case null case string(String) case int(Int) @@ -173,35 +173,35 @@ public final class SQLite { } /// Represents a row returned by called step() on a prepared statement. - public struct Row { + package struct Row { /// The pointer to the prepared statement. let stmt: OpaquePointer /// Get integer at the given column index. - public func int(at index: Int32) -> Int { + package func int(at index: Int32) -> Int { Int(sqlite3_column_int64(self.stmt, index)) } /// Get blob data at the given column index. - public func blob(at index: Int32) -> Data { + package func blob(at index: Int32) -> Data { let bytes = sqlite3_column_blob(stmt, index)! let count = sqlite3_column_bytes(stmt, index) return Data(bytes: bytes, count: Int(count)) } /// Get string at the given column index. - public func string(at index: Int32) -> String { + package func string(at index: Int32) -> String { String(cString: sqlite3_column_text(self.stmt, index)) } } - public struct Column { - public var name: String - public var value: String + package struct Column { + package var name: String + package var value: String } /// Represents a prepared statement. - public struct PreparedStatement { + package struct PreparedStatement { typealias sqlite3_destructor_type = @convention(c) (UnsafeMutableRawPointer?) -> Void static let SQLITE_STATIC = unsafeBitCast(0, to: sqlite3_destructor_type.self) static let SQLITE_TRANSIENT = unsafeBitCast(-1, to: sqlite3_destructor_type.self) @@ -209,7 +209,7 @@ public final class SQLite { /// The pointer to the prepared statement. let stmt: OpaquePointer - public init(db: OpaquePointer, query: String) throws { + package init(db: OpaquePointer, query: String) throws { var stmt: OpaquePointer? try SQLite.checkError { sqlite3_prepare_v2(db, query, -1, &stmt, nil) } self.stmt = stmt! @@ -217,7 +217,7 @@ public final class SQLite { /// Evaluate the prepared statement. @discardableResult - public func step() throws -> Row? { + package func step() throws -> Row? { let result = sqlite3_step(stmt) switch result { @@ -231,7 +231,7 @@ public final class SQLite { } /// Bind the given arguments to the statement. - public func bind(_ arguments: [SQLiteValue]) throws { + package func bind(_ arguments: [SQLiteValue]) throws { for (idx, argument) in arguments.enumerated() { let idx = Int32(idx) + 1 switch argument { @@ -258,17 +258,17 @@ public final class SQLite { } /// Reset the prepared statement. - public func reset() throws { + package func reset() throws { try SQLite.checkError { sqlite3_reset(stmt) } } /// Clear bindings from the prepared statement. - public func clearBindings() throws { + package func clearBindings() throws { try SQLite.checkError { sqlite3_clear_bindings(stmt) } } /// Finalize the statement and free up resources. - public func finalize() throws { + package func finalize() throws { try SQLite.checkError { sqlite3_finalize(stmt) } } } @@ -296,7 +296,7 @@ public final class SQLite { } } - public enum Errors: Error { + package enum Errors: Error { case databaseFull } } diff --git a/Sources/Basics/SQLiteBackedCache.swift b/Sources/Basics/SQLiteBackedCache.swift index abb269b453b..373755bd39b 100644 --- a/Sources/Basics/SQLiteBackedCache.swift +++ b/Sources/Basics/SQLiteBackedCache.swift @@ -17,13 +17,13 @@ import class TSCBasic.InMemoryFileSystem import var TSCBasic.localFileSystem /// SQLite backed persistent cache. -public final class SQLiteBackedCache: Closable { - public typealias Key = String +package final class SQLiteBackedCache: Closable { + package typealias Key = String - public let tableName: String - public let fileSystem: FileSystem - public let location: SQLite.Location - public let configuration: SQLiteBackedCacheConfiguration + package let tableName: String + package let fileSystem: FileSystem + package let location: SQLite.Location + package let configuration: SQLiteBackedCacheConfiguration private var state = State.idle private let stateLock = NSLock() @@ -37,7 +37,7 @@ public final class SQLiteBackedCache: Closable { /// - tableName: The SQLite table name. Must follow SQLite naming rules (e.g., no spaces). /// - location: SQLite.Location /// - configuration: Optional. Configuration for the cache. - public init(tableName: String, location: SQLite.Location, configuration: SQLiteBackedCacheConfiguration = .init()) { + package init(tableName: String, location: SQLite.Location, configuration: SQLiteBackedCacheConfiguration = .init()) { self.tableName = tableName self.location = location switch self.location { @@ -57,7 +57,7 @@ public final class SQLiteBackedCache: Closable { /// - tableName: The SQLite table name. Must follow SQLite naming rules (e.g., no spaces). /// - path: The path of the SQLite database. /// - configuration: Optional. Configuration for the cache. - public convenience init( + package convenience init( tableName: String, path: AbsolutePath, configuration: SQLiteBackedCacheConfiguration = .init() @@ -75,7 +75,7 @@ public final class SQLiteBackedCache: Closable { } } - public func close() throws { + package func close() throws { try self.withStateLock { if case .connected(let db) = self.state { try db.close() @@ -122,7 +122,7 @@ public final class SQLiteBackedCache: Closable { } } - public func put( + package func put( blobKey key: some Sequence, value: Value, replace: Bool = false, @@ -131,7 +131,7 @@ public final class SQLiteBackedCache: Closable { try self.put(rawKey: .blob(Data(key)), value: value, observabilityScope: observabilityScope) } - public func put( + package func put( key: Key, value: Value, replace: Bool = false, @@ -140,7 +140,7 @@ public final class SQLiteBackedCache: Closable { try self.put(rawKey: .string(key), value: value, replace: replace, observabilityScope: observabilityScope) } - public func get(key: Key) throws -> Value? { + package func get(key: Key) throws -> Value? { let query = "SELECT value FROM \(self.tableName) WHERE key = ? LIMIT 1;" return try self.executeStatement(query) { statement -> Value? in try statement.bind([.string(key)]) @@ -151,7 +151,7 @@ public final class SQLiteBackedCache: Closable { } } - public func get(blobKey key: some Sequence) throws -> Value? { + package func get(blobKey key: some Sequence) throws -> Value? { let query = "SELECT value FROM \(self.tableName) WHERE key = ? LIMIT 1;" return try self.executeStatement(query) { statement -> Value? in try statement.bind([.blob(Data(key))]) @@ -162,7 +162,7 @@ public final class SQLiteBackedCache: Closable { } } - public func remove(key: Key) throws { + package func remove(key: Key) throws { let query = "DELETE FROM \(self.tableName) WHERE key = ?;" try self.executeStatement(query) { statement in try statement.bind([.string(key)]) @@ -254,12 +254,12 @@ public final class SQLiteBackedCache: Closable { } } -public struct SQLiteBackedCacheConfiguration { - public var truncateWhenFull: Bool +package struct SQLiteBackedCacheConfiguration { + package var truncateWhenFull: Bool fileprivate var underlying: SQLite.Configuration - public init() { + package init() { self.underlying = .init() self.truncateWhenFull = true self.maxSizeInMegabytes = 100 @@ -267,7 +267,7 @@ public struct SQLiteBackedCacheConfiguration { self.busyTimeoutMilliseconds = 1000 } - public var maxSizeInMegabytes: Int? { + package var maxSizeInMegabytes: Int? { get { self.underlying.maxSizeInMegabytes } @@ -276,7 +276,7 @@ public struct SQLiteBackedCacheConfiguration { } } - public var maxSizeInBytes: Int? { + package var maxSizeInBytes: Int? { get { self.underlying.maxSizeInBytes } @@ -285,7 +285,7 @@ public struct SQLiteBackedCacheConfiguration { } } - public var busyTimeoutMilliseconds: Int32 { + package var busyTimeoutMilliseconds: Int32 { get { self.underlying.busyTimeoutMilliseconds } diff --git a/Sources/QueryEngine/CacheKey.swift b/Sources/QueryEngine/CacheKey.swift new file mode 100644 index 00000000000..2fb5fa174fa --- /dev/null +++ b/Sources/QueryEngine/CacheKey.swift @@ -0,0 +1,169 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +@_exported import protocol Crypto.HashFunction +import struct Foundation.URL +import struct SystemPackage.FilePath + +/// Indicates that values of a conforming type can be hashed with an arbitrary hashing function. Unlike `Hashable`, +/// this protocol doesn't utilize random seed values and produces consistent hash values across process launches. +package protocol CacheKey: Encodable { +} + +extension Bool: CacheKey { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + hashFunction.update(data: self ? [1] : [0]) + } +} + +extension Int: CacheKey { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + withUnsafeBytes(of: self) { + hashFunction.update(data: $0) + } + } +} + +extension Int8: CacheKey { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + withUnsafeBytes(of: self) { + hashFunction.update(data: $0) + } + } +} + +extension Int16: CacheKey { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + withUnsafeBytes(of: self) { + hashFunction.update(data: $0) + } + } +} + +extension Int32: CacheKey { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + withUnsafeBytes(of: self) { + hashFunction.update(data: $0) + } + } +} + +extension Int64: CacheKey { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + withUnsafeBytes(of: self) { + hashFunction.update(data: $0) + } + } +} + +extension UInt: CacheKey { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + withUnsafeBytes(of: self) { + hashFunction.update(data: $0) + } + } +} + +extension UInt8: CacheKey { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + withUnsafeBytes(of: self) { + hashFunction.update(data: $0) + } + } +} + +extension UInt16: CacheKey { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + withUnsafeBytes(of: self) { + hashFunction.update(data: $0) + } + } +} + +extension UInt32: CacheKey { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + withUnsafeBytes(of: self) { + hashFunction.update(data: $0) + } + } +} + +extension UInt64: CacheKey { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + withUnsafeBytes(of: self) { + hashFunction.update(data: $0) + } + } +} + +extension Float: CacheKey { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + withUnsafeBytes(of: self) { + hashFunction.update(data: $0) + } + } +} + +extension Double: CacheKey { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + withUnsafeBytes(of: self) { + hashFunction.update(data: $0) + } + } +} + +extension String: CacheKey { + func hash(with hashFunction: inout some HashFunction) { + var t = String(reflecting: Self.self) + t.withUTF8 { + hashFunction.update(data: $0) + } + var x = self + x.withUTF8 { + hashFunction.update(data: $0) + } + } +} + +extension FilePath: CacheKey { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + self.string.hash(with: &hashFunction) + } +} + +extension FilePath.Component: CacheKey { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + self.string.hash(with: &hashFunction) + } +} + +extension URL: CacheKey { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + self.description.hash(with: &hashFunction) + } +} diff --git a/Sources/QueryEngine/FileSystem/AsyncFileSystem.swift b/Sources/QueryEngine/FileSystem/AsyncFileSystem.swift index 9b461490260..02e82ca5593 100644 --- a/Sources/QueryEngine/FileSystem/AsyncFileSystem.swift +++ b/Sources/QueryEngine/FileSystem/AsyncFileSystem.swift @@ -15,9 +15,16 @@ import protocol Crypto.HashFunction import struct SystemPackage.Errno import struct SystemPackage.FilePath -public protocol AsyncFileSystem: Actor { - func withOpenReadableFile(_ path: FilePath, _ body: (OpenReadableFile) async throws -> T) async throws -> T - func withOpenWritableFile(_ path: FilePath, _ body: (OpenWritableFile) async throws -> T) async throws -> T +package protocol AsyncFileSystem: Actor { + func withOpenReadableFile( + _ path: FilePath, + _ body: @Sendable (OpenReadableFile) async throws -> T + ) async throws -> T + + func withOpenWritableFile( + _ path: FilePath, + _ body: @Sendable (OpenWritableFile) async throws -> T + ) async throws -> T } enum FileSystemError: Error { diff --git a/Sources/QueryEngine/FileSystem/FileCacheRecord.swift b/Sources/QueryEngine/FileSystem/FileCacheRecord.swift index 18f42c29f58..ed23fd78bb5 100644 --- a/Sources/QueryEngine/FileSystem/FileCacheRecord.swift +++ b/Sources/QueryEngine/FileSystem/FileCacheRecord.swift @@ -13,9 +13,9 @@ // FIXME: need a new swift-system tag to remove `@preconcurrency` @preconcurrency import struct SystemPackage.FilePath -public struct FileCacheRecord: Sendable { - public let path: FilePath - public let hash: String +package struct FileCacheRecord: Sendable { + package let path: FilePath + package let hash: String } extension FileCacheRecord: Codable { @@ -25,13 +25,13 @@ extension FileCacheRecord: Codable { } // FIXME: `Codable` on `FilePath` is broken, thus all `Codable` types with `FilePath` properties need a custom impl. - public init(from decoder: any Decoder) throws { + package init(from decoder: any Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) self.path = try FilePath(container.decode(String.self, forKey: .path)) self.hash = try container.decode(String.self, forKey: .hash) } - public func encode(to encoder: any Encoder) throws { + package func encode(to encoder: any Encoder) throws { var container = encoder.container(keyedBy: CodingKeys.self) try container.encode(self.path.string, forKey: .path) try container.encode(self.hash, forKey: .hash) diff --git a/Sources/QueryEngine/FileSystem/OpenReadableFile.swift b/Sources/QueryEngine/FileSystem/OpenReadableFile.swift index e4bdaa1fda8..dae8d37fd87 100644 --- a/Sources/QueryEngine/FileSystem/OpenReadableFile.swift +++ b/Sources/QueryEngine/FileSystem/OpenReadableFile.swift @@ -13,7 +13,7 @@ import protocol Crypto.HashFunction import struct SystemPackage.FileDescriptor -public struct OpenReadableFile { +package struct OpenReadableFile: Sendable { let readChunkSize: Int enum FileHandle { diff --git a/Sources/QueryEngine/FileSystem/OpenWritableFile.swift b/Sources/QueryEngine/FileSystem/OpenWritableFile.swift index a8f4da00416..be69f3cdc7c 100644 --- a/Sources/QueryEngine/FileSystem/OpenWritableFile.swift +++ b/Sources/QueryEngine/FileSystem/OpenWritableFile.swift @@ -13,8 +13,8 @@ import struct SystemPackage.FileDescriptor import struct SystemPackage.FilePath -public struct OpenWritableFile { - enum FileHandle { +package struct OpenWritableFile: Sendable { + enum FileHandle: Sendable { case local(FileDescriptor) case virtual(VirtualFileSystem.Storage, FilePath) } diff --git a/Sources/QueryEngine/FileSystem/ReadableFileStream.swift b/Sources/QueryEngine/FileSystem/ReadableFileStream.swift index 5b3628f5a89..f1fc7864ed5 100644 --- a/Sources/QueryEngine/FileSystem/ReadableFileStream.swift +++ b/Sources/QueryEngine/FileSystem/ReadableFileStream.swift @@ -13,17 +13,17 @@ import _Concurrency import SystemPackage -public enum ReadableFileStream: AsyncSequence { - public typealias Element = [UInt8] +package enum ReadableFileStream: AsyncSequence { + package typealias Element = [UInt8] case local(LocalReadableFileStream) case virtual(VirtualReadableFileStream) - public enum Iterator: AsyncIteratorProtocol { + package enum Iterator: AsyncIteratorProtocol { case local(LocalReadableFileStream.Iterator) case virtual(VirtualReadableFileStream.Iterator) - public func next() async throws -> [UInt8]? { + package func next() async throws -> [UInt8]? { switch self { case .local(let local): try await local.next() @@ -33,7 +33,7 @@ public enum ReadableFileStream: AsyncSequence { } } - public func makeAsyncIterator() -> Iterator { + package func makeAsyncIterator() -> Iterator { switch self { case .local(let local): .local(local.makeAsyncIterator()) @@ -43,13 +43,13 @@ public enum ReadableFileStream: AsyncSequence { } } -public struct LocalReadableFileStream: AsyncSequence { - public typealias Element = [UInt8] +package struct LocalReadableFileStream: AsyncSequence { + package typealias Element = [UInt8] let fileDescriptor: FileDescriptor let readChunkSize: Int - public final class Iterator: AsyncIteratorProtocol { + package final class Iterator: AsyncIteratorProtocol { init(_ fileDescriptor: FileDescriptor, readChunkSize: Int) { self.fileDescriptor = fileDescriptor self.readChunkSize = readChunkSize @@ -58,7 +58,7 @@ public struct LocalReadableFileStream: AsyncSequence { private let fileDescriptor: FileDescriptor private let readChunkSize: Int - public func next() async throws -> [UInt8]? { + package func next() async throws -> [UInt8]? { var buffer = [UInt8](repeating: 0, count: readChunkSize) let bytesRead = try buffer.withUnsafeMutableBytes { @@ -74,22 +74,22 @@ public struct LocalReadableFileStream: AsyncSequence { } } - public func makeAsyncIterator() -> Iterator { + package func makeAsyncIterator() -> Iterator { Iterator(self.fileDescriptor, readChunkSize: self.readChunkSize) } } -public struct VirtualReadableFileStream: AsyncSequence { - public typealias Element = [UInt8] +package struct VirtualReadableFileStream: AsyncSequence { + package typealias Element = [UInt8] - public final class Iterator: AsyncIteratorProtocol { + package final class Iterator: AsyncIteratorProtocol { init(bytes: [UInt8]? = nil) { self.bytes = bytes } var bytes: [UInt8]? - public func next() async throws -> [UInt8]? { + package func next() async throws -> [UInt8]? { defer { bytes = nil } return self.bytes @@ -98,7 +98,7 @@ public struct VirtualReadableFileStream: AsyncSequence { let bytes: [UInt8] - public func makeAsyncIterator() -> Iterator { + package func makeAsyncIterator() -> Iterator { Iterator(bytes: self.bytes) } } diff --git a/Sources/QueryEngine/FileSystem/VirtualFileSystem.swift b/Sources/QueryEngine/FileSystem/VirtualFileSystem.swift index 581e0776aa9..db8af267bbf 100644 --- a/Sources/QueryEngine/FileSystem/VirtualFileSystem.swift +++ b/Sources/QueryEngine/FileSystem/VirtualFileSystem.swift @@ -13,7 +13,7 @@ import struct SystemPackage.FilePath actor VirtualFileSystem: AsyncFileSystem { - public static let defaultChunkSize = 512 * 1024 + package static let defaultChunkSize = 512 * 1024 let readChunkSize: Int @@ -32,14 +32,20 @@ actor VirtualFileSystem: AsyncFileSystem { self.readChunkSize = readChunkSize } - func withOpenReadableFile(_ path: FilePath, _ body: (OpenReadableFile) async throws -> T) async throws -> T { + func withOpenReadableFile( + _ path: FilePath, + _ body: (OpenReadableFile) async throws -> T + ) async throws -> T { guard let bytes = storage.content[path] else { throw FileSystemError.fileDoesNotExist(path) } return try await body(.init(readChunkSize: self.readChunkSize, fileHandle: .virtual(bytes))) } - func withOpenWritableFile(_ path: FilePath, _ body: (OpenWritableFile) async throws -> T) async throws -> T { + func withOpenWritableFile( + _ path: FilePath, + _ body: (OpenWritableFile) async throws -> T + ) async throws -> T { try await body(.init(fileHandle: .virtual(self.storage, path))) } } diff --git a/Sources/QueryEngine/Query.swift b/Sources/QueryEngine/Query.swift index 536db5d22a6..e04fc7d6879 100644 --- a/Sources/QueryEngine/Query.swift +++ b/Sources/QueryEngine/Query.swift @@ -12,12 +12,255 @@ import Crypto import struct SystemPackage.FilePath -public protocol Query: Encodable { +package protocol Query: CacheKey, Sendable { func run(engine: QueryEngine) async throws -> FilePath } -extension Query { - func hash(with hashFunction: inout some HashFunction) { - fatalError("\(#function) not implemented") +// SwiftPM has to be built with Swift 5.8 on CI and also needs to support CMake for bootstrapping on Windows. +// This means we can't implement persistable hashing with macros (unavailable in Swift 5.8 and additional effort to +// set up with CMake when Swift 5.9 is available for all CI jobs) and have to stick to `Encodable` for now. +final class HashEncoder: Encoder { + enum Error: Swift.Error { + case noCacheKeyConformance(Encodable.Type) + } + + var codingPath: [any CodingKey] + + var userInfo: [CodingUserInfoKey : Any] + + func container(keyedBy type: Key.Type) -> KeyedEncodingContainer where Key : CodingKey { + String(reflecting: Key.self).hash(with: &self.hashFunction) + return .init(KeyedContainer(encoder: self)) + } + + func unkeyedContainer() -> any UnkeyedEncodingContainer { + self + } + + func singleValueContainer() -> any SingleValueEncodingContainer { + self + } + + init() { + self.hashFunction = Hash() + self.codingPath = [] + self.userInfo = [:] + } + + fileprivate var hashFunction = Hash() + + func finalize() -> Hash.Digest { + hashFunction.finalize() + } +} + +extension HashEncoder: SingleValueEncodingContainer { + func encodeNil() throws { + // FIXME: this doesn't encode the name of the underlying optional type, + // but `Encoder` protocol is limited and can't provide this for us. + var str = "nil" + str.withUTF8 { + self.hashFunction.update(data: $0) + } + } + + func encode(_ value: Bool) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: String) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: Double) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: Float) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: Int) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: Int8) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: Int16) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: Int32) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: Int64) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: UInt) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: UInt8) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: UInt16) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: UInt32) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: UInt64) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: T) throws where T : Encodable { + guard value is CacheKey else { + throw Error.noCacheKeyConformance(T.self) + } + + try value.encode(to: self) + } +} + +extension HashEncoder: UnkeyedEncodingContainer { + var count: Int { + 0 + } + + func nestedContainer(keyedBy keyType: NestedKey.Type) -> KeyedEncodingContainer where NestedKey : CodingKey { + KeyedEncodingContainer(KeyedContainer(encoder: self)) + } + + func nestedUnkeyedContainer() -> any UnkeyedEncodingContainer { + self + } + + func superEncoder() -> any Encoder { + fatalError() + } +} + +extension HashEncoder { + struct KeyedContainer: KeyedEncodingContainerProtocol { + var encoder: HashEncoder + var codingPath: [any CodingKey] { self.encoder.codingPath } + + mutating func encodeNil(forKey key: K) throws { + // FIXME: this doesn't encode the name of the underlying optional type, + // but `Encoder` protocol is limited and can't provide this for us. + var str = "nil" + str.withUTF8 { + self.encoder.hashFunction.update(data: $0) + } + } + + mutating func encode(_ value: Bool, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: String, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: Double, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: Float, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: Int, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: Int8, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: Int16, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: Int32, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: Int64, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: UInt, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: UInt8, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: UInt16, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: UInt32, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: UInt64, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: T, forKey key: K) throws where T : Encodable { + guard value is CacheKey else { + throw Error.noCacheKeyConformance(T.self) + } + + key.stringValue.hash(with: &self.encoder.hashFunction) + try value.encode(to: self.encoder) + } + + mutating func nestedContainer( + keyedBy keyType: NestedKey.Type, + forKey key: K + ) -> KeyedEncodingContainer where NestedKey : CodingKey { + key.stringValue.hash(with: &self.encoder.hashFunction) + return self.encoder.nestedContainer(keyedBy: keyType) + } + + mutating func nestedUnkeyedContainer(forKey key: K) -> any UnkeyedEncodingContainer { + key.stringValue.hash(with: &self.encoder.hashFunction) + return self.encoder + } + + mutating func superEncoder() -> any Encoder { + fatalError() + } + + mutating func superEncoder(forKey key: K) -> any Encoder { + fatalError() + } + + typealias Key = K } } diff --git a/Sources/QueryEngine/QueryEngine.swift b/Sources/QueryEngine/QueryEngine.swift index a6536ac9892..dfeb3997bd2 100644 --- a/Sources/QueryEngine/QueryEngine.swift +++ b/Sources/QueryEngine/QueryEngine.swift @@ -12,8 +12,9 @@ import Basics import Crypto +@preconcurrency import SystemPackage -public func withQueryEngine( +package func withQueryEngine( _ fileSystem: any AsyncFileSystem, _ observabilityScope: ObservabilityScope, cacheLocation: SQLite.Location, @@ -34,13 +35,13 @@ public func withQueryEngine( /// Cacheable computations engine. Currently the engine makes an assumption that computations produce same results for /// the same query values and write results to a single file path. -public actor QueryEngine { +package actor QueryEngine { private(set) var cacheHits = 0 private(set) var cacheMisses = 0 - public let fileSystem: any AsyncFileSystem - public let httpClient = HTTPClient() - public let observabilityScope: ObservabilityScope + package let fileSystem: any AsyncFileSystem + package let httpClient = HTTPClient() + package let observabilityScope: ObservabilityScope private let resultsCache: SQLiteBackedCache private var isShutDown = false @@ -60,7 +61,7 @@ public actor QueryEngine { self.resultsCache = SQLiteBackedCache(tableName: "cache_table", location: cacheLocation) } - public func shutDown() async throws { + package func shutDown() async throws { precondition(!self.isShutDown, "`QueryEngine/shutDown` should be called only once") try self.resultsCache.close() @@ -78,18 +79,19 @@ public actor QueryEngine { /// Executes a given query if no cached result of it is available. Otherwise fetches the result from engine's cache. /// - Parameter query: A query value to execute. /// - Returns: A file path to query's result recorded in a file. - public subscript(_ query: some Query) -> FileCacheRecord { + package subscript(_ query: some Query) -> FileCacheRecord { get async throws { - var hashFunction = SHA512() - query.hash(with: &hashFunction) - let key = hashFunction.finalize() + let hashEncoder = HashEncoder() + try query.encode(to: hashEncoder) + let key = hashEncoder.finalize() if let fileRecord = try resultsCache.get(blobKey: key) { - hashFunction = SHA512() - try await self.fileSystem.withOpenReadableFile(fileRecord.path) { + + let fileHash = try await self.fileSystem.withOpenReadableFile(fileRecord.path) { + var hashFunction = SHA512() try await $0.hash(with: &hashFunction) + return hashFunction.finalize().description } - let fileHash = hashFunction.finalize().description if fileHash == fileRecord.hash { self.cacheHits += 1 @@ -99,13 +101,13 @@ public actor QueryEngine { self.cacheMisses += 1 let resultPath = try await query.run(engine: self) - hashFunction = SHA512() - try await self.fileSystem.withOpenReadableFile(resultPath) { + let resultHash = try await self.fileSystem.withOpenReadableFile(resultPath) { + var hashFunction = SHA512() try await $0.hash(with: &hashFunction) + return hashFunction.finalize().description } - let resultHash = hashFunction.finalize() - let result = FileCacheRecord(path: resultPath, hash: resultHash.description) + let result = FileCacheRecord(path: resultPath, hash: resultHash) // FIXME: update `SQLiteBackedCache` to store `resultHash` directly instead of relying on string conversions try self.resultsCache.put(blobKey: key, value: result) diff --git a/Tests/QueryEngineTests/QueryEngineTests.swift b/Tests/QueryEngineTests/QueryEngineTests.swift new file mode 100644 index 00000000000..39c92af40d2 --- /dev/null +++ b/Tests/QueryEngineTests/QueryEngineTests.swift @@ -0,0 +1,153 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import class Basics.ObservabilitySystem +import struct Foundation.Data +@testable import QueryEngine +import struct SystemPackage.FilePath +import SPMTestSupport +import XCTest + +private let encoder = JSONEncoder() +private let decoder = JSONDecoder() + +private extension AsyncFileSystem { + func read(_ path: FilePath, bufferLimit: Int = 10 * 1024 * 1024, as: V.Type) async throws -> V { + let data = try await self.withOpenReadableFile(path) { + var data = Data() + for try await chunk in try await $0.read() { + data.append(contentsOf: chunk) + + guard data.count < bufferLimit else { + throw FileSystemError.bufferLimitExceeded(path) + } + } + return data + } + + return try decoder.decode(V.self, from: data) + } + + func write(_ path: FilePath, _ value: some Encodable) async throws { + let data = try encoder.encode(value) + try await self.withOpenWritableFile(path) { fileHandle in + try await fileHandle.write(data) + } + } +} + +private struct Const: Query { + let x: Int + + func run(engine: QueryEngine) async throws -> FilePath { + let resultPath = FilePath("/Const-\(x)") + try await engine.fileSystem.write(resultPath, self.x) + return resultPath + } +} + +private struct MultiplyByTwo: Query { + let x: Int + + func run(engine: QueryEngine) async throws -> FilePath { + let constPath = try await engine[Const(x: self.x)].path + let constResult = try await engine.fileSystem.read(constPath, as: Int.self) + + let resultPath = FilePath("/MultiplyByTwo-\(constResult)") + try await engine.fileSystem.write(resultPath, constResult * 2) + return resultPath + } +} + +private struct AddThirty: Query { + let x: Int + + func run(engine: QueryEngine) async throws -> FilePath { + let constPath = try await engine[Const(x: self.x)].path + let constResult = try await engine.fileSystem.read(constPath, as: Int.self) + + let resultPath = FilePath("/AddThirty-\(constResult)") + try await engine.fileSystem.write(resultPath, constResult + 30) + return resultPath + } +} + +private struct Expression: Query { + let x: Int + let y: Int + + func run(engine: QueryEngine) async throws -> FilePath { + let multiplyPath = try await engine[MultiplyByTwo(x: self.x)].path + let addThirtyPath = try await engine[AddThirty(x: self.y)].path + + let multiplyResult = try await engine.fileSystem.read(multiplyPath, as: Int.self) + let addThirtyResult = try await engine.fileSystem.read(addThirtyPath, as: Int.self) + + let resultPath = FilePath("/Expression-\(multiplyResult)-\(addThirtyResult)") + try await engine.fileSystem.write(resultPath, multiplyResult + addThirtyResult) + return resultPath + } +} + +final class QueryEngineTests: XCTestCase { + func testSimpleCaching() async throws { + let observabilitySystem = ObservabilitySystem.makeForTesting() + let engine = QueryEngine( + VirtualFileSystem(), + observabilitySystem.topScope, + cacheLocation: .memory + ) + + var resultPath = try await engine[Expression(x: 1, y: 2)].path + var result = try await engine.fileSystem.read(resultPath, as: Int.self) + + XCTAssertEqual(result, 34) + + var cacheMisses = await engine.cacheMisses + XCTAssertEqual(cacheMisses, 5) + + var cacheHits = await engine.cacheHits + XCTAssertEqual(cacheHits, 0) + + resultPath = try await engine[Expression(x: 1, y: 2)].path + result = try await engine.fileSystem.read(resultPath, as: Int.self) + XCTAssertEqual(result, 34) + + cacheMisses = await engine.cacheMisses + XCTAssertEqual(cacheMisses, 5) + + cacheHits = await engine.cacheHits + XCTAssertEqual(cacheHits, 1) + + resultPath = try await engine[Expression(x: 2, y: 1)].path + result = try await engine.fileSystem.read(resultPath, as: Int.self) + XCTAssertEqual(result, 35) + + cacheMisses = await engine.cacheMisses + XCTAssertEqual(cacheMisses, 8) + + cacheHits = await engine.cacheHits + XCTAssertEqual(cacheHits, 3) + + resultPath = try await engine[Expression(x: 2, y: 1)].path + result = try await engine.fileSystem.read(resultPath, as: Int.self) + XCTAssertEqual(result, 35) + + cacheMisses = await engine.cacheMisses + XCTAssertEqual(cacheMisses, 8) + + cacheHits = await engine.cacheHits + XCTAssertEqual(cacheHits, 4) + + try await engine.shutDown() + } +}