Skip to content

Commit addc8d2

Browse files
MaxDesiatovMaxDesiatov
authored
Basics: add support for .tar.gz archives (#6368)
`swift experimental-destination install` subcommand fails when pointed to a destination bundle compressed with `.tar.gz` extension, compressed with `tar` and `gzip`. This archival format is vastly superior to `zip` in the context of cross-compilation destination bundles. In a typical scenario we see that `.tar.gz` archives are at least 4x better than `.zip`, since with `.zip` files are compressed individually, while with `.tar.gz` compression is applied to the whole archive at once. On macOS `tar` is always available, and we can also assume it's also available on Linux Docker images, since `tar` is used to unpack `.tar.gz` distributions of Swift itself. Since `ZipArchiver` uses `.tar.exe` on Windows, we make the same assumption that command is available, as we did for `ZipArchiver`. Added new `TarArchiver` class, `TarArchiverTests`, and corresponding test files. This is technically NFC, since `TarArchiver` is not called from anywhere yet.
1 parent 0c22d23 commit addc8d2

File tree

9 files changed

+335
-10
lines changed

9 files changed

+335
-10
lines changed

‎Package.swift

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -554,8 +554,10 @@ let package = Package(
554554
name: "BasicsTests",
555555
dependencies: ["Basics", "SPMTestSupport", "tsan_utils"],
556556
exclude: [
557-
"Inputs/archive.zip",
558-
"Inputs/invalid_archive.zip",
557+
"Archiver/Inputs/archive.tar.gz",
558+
"Archiver/Inputs/archive.zip",
559+
"Archiver/Inputs/invalid_archive.tar.gz",
560+
"Archiver/Inputs/invalid_archive.zip",
559561
]
560562
),
561563
.testTarget(

‎Sources/Basics/Archiver+Tar.swift

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// This source file is part of the Swift open source project
4+
//
5+
// Copyright (c) 2014-2023 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See http://swift.org/LICENSE.txt for license information
9+
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
import class Dispatch.DispatchQueue
14+
import struct Dispatch.DispatchTime
15+
import struct TSCBasic.AbsolutePath
16+
import protocol TSCBasic.FileSystem
17+
import struct TSCBasic.FileSystemError
18+
import class TSCBasic.Process
19+
20+
/// An `Archiver` that handles Tar archives using the command-line `tar` tool.
21+
public struct TarArchiver: Archiver {
22+
public let supportedExtensions: Set<String> = ["tar", "tar.gz"]
23+
24+
/// The file-system implementation used for various file-system operations and checks.
25+
private let fileSystem: FileSystem
26+
27+
/// Helper for cancelling in-flight requests
28+
private let cancellator: Cancellator
29+
30+
/// The underlying command
31+
private let tarCommand: String
32+
33+
/// Creates a `TarArchiver`.
34+
///
35+
/// - Parameters:
36+
/// - fileSystem: The file system to used by the `TarArchiver`.
37+
/// - cancellator: Cancellation handler
38+
public init(fileSystem: FileSystem, cancellator: Cancellator? = .none) {
39+
self.fileSystem = fileSystem
40+
self.cancellator = cancellator ?? Cancellator(observabilityScope: .none)
41+
42+
#if os(Windows)
43+
self.tarCommand = "tar.exe"
44+
#else
45+
self.tarCommand = "tar"
46+
#endif
47+
}
48+
49+
public func extract(
50+
from archivePath: AbsolutePath,
51+
to destinationPath: AbsolutePath,
52+
completion: @escaping (Result<Void, Error>) -> Void
53+
) {
54+
do {
55+
guard self.fileSystem.exists(archivePath) else {
56+
throw FileSystemError(.noEntry, archivePath)
57+
}
58+
59+
guard self.fileSystem.isDirectory(destinationPath) else {
60+
throw FileSystemError(.notDirectory, destinationPath)
61+
}
62+
63+
let process = TSCBasic.Process(
64+
arguments: [self.tarCommand, "zxf", archivePath.pathString, "-C", destinationPath.pathString]
65+
)
66+
67+
guard let registrationKey = self.cancellator.register(process) else {
68+
throw CancellationError.failedToRegisterProcess(process)
69+
}
70+
71+
DispatchQueue.sharedConcurrent.async {
72+
defer { self.cancellator.deregister(registrationKey) }
73+
completion(.init(catching: {
74+
try process.launch()
75+
let processResult = try process.waitUntilExit()
76+
guard processResult.exitStatus == .terminated(code: 0) else {
77+
throw try StringError(processResult.utf8stderrOutput())
78+
}
79+
}))
80+
}
81+
} catch {
82+
return completion(.failure(error))
83+
}
84+
}
85+
86+
public func compress(
87+
directory: AbsolutePath,
88+
to destinationPath: AbsolutePath,
89+
completion: @escaping (Result<Void, Error>) -> Void
90+
) {
91+
do {
92+
guard self.fileSystem.isDirectory(directory) else {
93+
throw FileSystemError(.notDirectory, directory)
94+
}
95+
96+
let process = TSCBasic.Process(
97+
arguments: [self.tarCommand, "acf", destinationPath.pathString, directory.basename],
98+
workingDirectory: directory.parentDirectory
99+
)
100+
101+
guard let registrationKey = self.cancellator.register(process) else {
102+
throw CancellationError.failedToRegisterProcess(process)
103+
}
104+
105+
DispatchQueue.sharedConcurrent.async {
106+
defer { self.cancellator.deregister(registrationKey) }
107+
completion(.init(catching: {
108+
try process.launch()
109+
let processResult = try process.waitUntilExit()
110+
guard processResult.exitStatus == .terminated(code: 0) else {
111+
throw try StringError(processResult.utf8stderrOutput())
112+
}
113+
}))
114+
}
115+
} catch {
116+
return completion(.failure(error))
117+
}
118+
}
119+
120+
public func validate(path: AbsolutePath, completion: @escaping (Result<Bool, Error>) -> Void) {
121+
do {
122+
guard self.fileSystem.exists(path) else {
123+
throw FileSystemError(.noEntry, path)
124+
}
125+
126+
let process = TSCBasic.Process(arguments: [self.tarCommand, "tf", path.pathString])
127+
guard let registrationKey = self.cancellator.register(process) else {
128+
throw CancellationError.failedToRegisterProcess(process)
129+
}
130+
131+
DispatchQueue.sharedConcurrent.async {
132+
defer { self.cancellator.deregister(registrationKey) }
133+
completion(.init(catching: {
134+
try process.launch()
135+
let processResult = try process.waitUntilExit()
136+
return processResult.exitStatus == .terminated(code: 0)
137+
}))
138+
}
139+
} catch {
140+
return completion(.failure(error))
141+
}
142+
}
143+
144+
public func cancel(deadline: DispatchTime) throws {
145+
try self.cancellator.cancel(deadline: deadline)
146+
}
147+
}

‎Sources/Basics/Archiver+Zip.swift

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ public struct ZipArchiver: Archiver, Cancellable {
5353
let process = TSCBasic.Process(arguments: ["unzip", archivePath.pathString, "-d", destinationPath.pathString])
5454
#endif
5555
guard let registrationKey = self.cancellator.register(process) else {
56-
throw StringError("cancellation")
56+
throw CancellationError.failedToRegisterProcess(process)
5757
}
5858

5959
DispatchQueue.sharedConcurrent.async {
@@ -95,7 +95,7 @@ public struct ZipArchiver: Archiver, Cancellable {
9595
#endif
9696

9797
guard let registrationKey = self.cancellator.register(process) else {
98-
throw StringError("Failed to register cancellation for Archiver")
98+
throw CancellationError.failedToRegisterProcess(process)
9999
}
100100

101101
DispatchQueue.sharedConcurrent.async {
@@ -125,7 +125,7 @@ public struct ZipArchiver: Archiver, Cancellable {
125125
let process = TSCBasic.Process(arguments: ["unzip", "-t", path.pathString])
126126
#endif
127127
guard let registrationKey = self.cancellator.register(process) else {
128-
throw StringError("cancellation")
128+
throw CancellationError.failedToRegisterProcess(process)
129129
}
130130

131131
DispatchQueue.sharedConcurrent.async {

‎Sources/Basics/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
add_library(Basics
1010
Archiver.swift
11+
Archiver+Tar.swift
1112
Archiver+Zip.swift
1213
AuthorizationProvider.swift
1314
ByteString+Extensions.swift

‎Sources/Basics/Cancellator.swift

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,9 +112,24 @@ public protocol Cancellable {
112112
}
113113

114114
public struct CancellationError: Error, CustomStringConvertible {
115-
public let description = "Operation cancelled"
115+
public let description: String
116116

117-
public init() {}
117+
public init() {
118+
self.init(description: "Operation cancelled")
119+
}
120+
121+
private init(description: String) {
122+
self.description = description
123+
}
124+
125+
static func failedToRegisterProcess(_ process: TSCBasic.Process) -> Self {
126+
Self(description: """
127+
failed to register a cancellation handler for this process invocation `\(
128+
process.arguments.joined(separator: " ")
129+
)`
130+
"""
131+
)
132+
}
118133
}
119134

120135
extension TSCBasic.Process {

‎Tests/BasicsTests/Inputs/archive.tar.gz

346 Bytes
Binary file not shown.

‎Tests/BasicsTests/Inputs/invalid_archive.tar.gz

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
not an archive

‎Tests/BasicsTests/TarArchiverTests.swift

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// This source file is part of the Swift open source project
4+
//
5+
// Copyright (c) 2014-2023 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See http://swift.org/LICENSE.txt for license information
9+
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
import Basics
14+
import TSCBasic
15+
import TSCclibc // for SPM_posix_spawn_file_actions_addchdir_np_supported
16+
import TSCTestSupport
17+
import XCTest
18+
19+
final class TarArchiverTests: XCTestCase {
20+
func testSuccess() throws {
21+
try testWithTemporaryDirectory { tmpdir in
22+
let archiver = TarArchiver(fileSystem: localFileSystem)
23+
let inputArchivePath = AbsolutePath(path: #file).parentDirectory
24+
.appending(components: "Inputs", "archive.tar.gz")
25+
try archiver.extract(from: inputArchivePath, to: tmpdir)
26+
let content = tmpdir.appending("file")
27+
XCTAssert(localFileSystem.exists(content))
28+
XCTAssertEqual((try? localFileSystem.readFileContents(content))?.cString, "Hello World!")
29+
}
30+
}
31+
32+
func testArchiveDoesntExist() {
33+
let fileSystem = InMemoryFileSystem()
34+
let archiver = TarArchiver(fileSystem: fileSystem)
35+
let archive = AbsolutePath("/archive.tar.gz")
36+
XCTAssertThrowsError(try archiver.extract(from: archive, to: "/")) { error in
37+
XCTAssertEqual(error as? FileSystemError, FileSystemError(.noEntry, archive))
38+
}
39+
}
40+
41+
func testDestinationDoesntExist() throws {
42+
let fileSystem = InMemoryFileSystem(emptyFiles: "/archive.tar.gz")
43+
let archiver = TarArchiver(fileSystem: fileSystem)
44+
let destination = AbsolutePath("/destination")
45+
XCTAssertThrowsError(try archiver.extract(from: "/archive.tar.gz", to: destination)) { error in
46+
XCTAssertEqual(error as? FileSystemError, FileSystemError(.notDirectory, destination))
47+
}
48+
}
49+
50+
func testDestinationIsFile() throws {
51+
let fileSystem = InMemoryFileSystem(emptyFiles: "/archive.tar.gz", "/destination")
52+
let archiver = TarArchiver(fileSystem: fileSystem)
53+
let destination = AbsolutePath("/destination")
54+
XCTAssertThrowsError(try archiver.extract(from: "/archive.tar.gz", to: destination)) { error in
55+
XCTAssertEqual(error as? FileSystemError, FileSystemError(.notDirectory, destination))
56+
}
57+
}
58+
59+
func testInvalidArchive() throws {
60+
try testWithTemporaryDirectory { tmpdir in
61+
let archiver = TarArchiver(fileSystem: localFileSystem)
62+
let inputArchivePath = AbsolutePath(path: #file).parentDirectory
63+
.appending(components: "Inputs", "invalid_archive.tar.gz")
64+
XCTAssertThrowsError(try archiver.extract(from: inputArchivePath, to: tmpdir)) { error in
65+
#if os(Linux)
66+
XCTAssertMatch((error as? StringError)?.description, .contains("not in gzip format"))
67+
#else
68+
XCTAssertMatch((error as? StringError)?.description, .contains("Unrecognized archive format"))
69+
#endif
70+
}
71+
}
72+
}
73+
74+
func testValidation() throws {
75+
// valid
76+
try testWithTemporaryDirectory { _ in
77+
let archiver = TarArchiver(fileSystem: localFileSystem)
78+
let path = AbsolutePath(path: #file).parentDirectory
79+
.appending(components: "Inputs", "archive.tar.gz")
80+
XCTAssertTrue(try archiver.validate(path: path))
81+
}
82+
// invalid
83+
try testWithTemporaryDirectory { _ in
84+
let archiver = TarArchiver(fileSystem: localFileSystem)
85+
let path = AbsolutePath(path: #file).parentDirectory
86+
.appending(components: "Inputs", "invalid_archive.tar.gz")
87+
XCTAssertFalse(try archiver.validate(path: path))
88+
}
89+
// error
90+
try testWithTemporaryDirectory { _ in
91+
let archiver = TarArchiver(fileSystem: localFileSystem)
92+
let path = AbsolutePath.root.appending("does_not_exist.tar.gz")
93+
XCTAssertThrowsError(try archiver.validate(path: path)) { error in
94+
XCTAssertEqual(error as? FileSystemError, FileSystemError(.noEntry, path))
95+
}
96+
}
97+
}
98+
99+
func testCompress() throws {
100+
#if os(Linux)
101+
guard SPM_posix_spawn_file_actions_addchdir_np_supported() else {
102+
throw XCTSkip("working directory not supported on this platform")
103+
}
104+
#endif
105+
106+
try testWithTemporaryDirectory { tmpdir in
107+
let archiver = TarArchiver(fileSystem: localFileSystem)
108+
109+
let rootDir = tmpdir.appending(component: UUID().uuidString)
110+
try localFileSystem.createDirectory(rootDir)
111+
try localFileSystem.writeFileContents(rootDir.appending("file1.txt"), string: "Hello World!")
112+
113+
let dir1 = rootDir.appending("dir1")
114+
try localFileSystem.createDirectory(dir1)
115+
try localFileSystem.writeFileContents(dir1.appending("file2.txt"), string: "Hello World 2!")
116+
117+
let dir2 = dir1.appending("dir2")
118+
try localFileSystem.createDirectory(dir2)
119+
try localFileSystem.writeFileContents(dir2.appending("file3.txt"), string: "Hello World 3!")
120+
try localFileSystem.writeFileContents(dir2.appending("file4.txt"), string: "Hello World 4!")
121+
122+
let archivePath = tmpdir.appending(component: UUID().uuidString + ".tar.gz")
123+
try archiver.compress(directory: rootDir, to: archivePath)
124+
XCTAssertFileExists(archivePath)
125+
126+
let extractRootDir = tmpdir.appending(component: UUID().uuidString)
127+
try localFileSystem.createDirectory(extractRootDir)
128+
try archiver.extract(from: archivePath, to: extractRootDir)
129+
try localFileSystem.stripFirstLevel(of: extractRootDir)
130+
131+
XCTAssertFileExists(extractRootDir.appending("file1.txt"))
132+
XCTAssertEqual(
133+
try? localFileSystem.readFileContents(extractRootDir.appending("file1.txt")),
134+
"Hello World!"
135+
)
136+
137+
let extractedDir1 = extractRootDir.appending("dir1")
138+
XCTAssertDirectoryExists(extractedDir1)
139+
XCTAssertFileExists(extractedDir1.appending("file2.txt"))
140+
XCTAssertEqual(
141+
try? localFileSystem.readFileContents(extractedDir1.appending("file2.txt")),
142+
"Hello World 2!"
143+
)
144+
145+
let extractedDir2 = extractedDir1.appending("dir2")
146+
XCTAssertDirectoryExists(extractedDir2)
147+
XCTAssertFileExists(extractedDir2.appending("file3.txt"))
148+
XCTAssertEqual(
149+
try? localFileSystem.readFileContents(extractedDir2.appending("file3.txt")),
150+
"Hello World 3!"
151+
)
152+
XCTAssertFileExists(extractedDir2.appending("file4.txt"))
153+
XCTAssertEqual(
154+
try? localFileSystem.readFileContents(extractedDir2.appending("file4.txt")),
155+
"Hello World 4!"
156+
)
157+
}
158+
}
159+
}

0 commit comments

Comments
 (0)