Tests: Exercise output and diagnostic messages from command plugins

Fixtures/Miscellaneous/Plugins/CommandPluginDiagnosticsStub/Package.swift

@@ -0,0 +1,19 @@
+// swift-tools-version: 5.9
+
+import PackageDescription
+
+let package = Package(
+    name: "CommandPluginDiagnostics",
+    targets: [
+        .plugin(
+            name: "diagnostics-stub",
+            capability: .command(intent: .custom(
+                verb: "print-diagnostics",
+                description: "Writes diagnostic messages for testing"
+            ))
+        ),
+        .executableTarget(
+            name: "placeholder"
+        ),
+    ]
+)

Fixtures/Miscellaneous/Plugins/CommandPluginDiagnosticsStub/Plugins/diagnostics_stub.swift

@@ -0,0 +1,28 @@
+import Foundation
+import PackagePlugin
+
+@main
+struct diagnostics_stub: CommandPlugin {
+    // This is a helper for testing plugin diagnostics.  It sends different messages to SwiftPM depending on its arguments.
+    func performCommand(context: PluginContext, arguments: [String]) async throws {
+        // Anything a plugin writes to standard output appears on standard output.
+        // Printing to stderr will also go to standard output because SwiftPM combines
+        // stdout and stderr before launching the plugin.
+        if arguments.contains("print") {
+           print("command plugin: print")
+        }
+
+        // Diagnostics are collected by SwiftPM and printed to standard error, depending on the current log verbosity level.
+        if arguments.contains("remark") {
+           Diagnostics.remark("command plugin: Diagnostics.remark")     // prefixed with 'info:' when printed
+        }
+
+        if arguments.contains("warning") {
+           Diagnostics.warning("command plugin: Diagnostics.warning")   // prefixed with 'warning:' when printed
+        }
+
+        if arguments.contains("error") {
+           Diagnostics.error("command plugin: Diagnostics.error")       // prefixed with 'error:' when printed
+        }
+    }
+}

Fixtures/Miscellaneous/Plugins/CommandPluginDiagnosticsStub/Sources/main.swift

@@ -0,0 +1,4 @@
+// The Swift Programming Language
+// https://docs.swift.org/swift-book
+
+print("Hello, world from executable target!")

Tests/CommandsTests/PackageToolTests.swift

@@ -1874,6 +1874,115 @@ final class PackageToolTests: CommandsTestCase {
         }
     }
 
+    // Test reporting of plugin diagnostic messages at different verbosity levels
+    func testCommandPluginDiagnostics() throws {
+        // Only run the test if the environment in which we're running actually supports Swift concurrency (which the plugin APIs require).
+        try XCTSkipIf(!UserToolchain.default.supportsSwiftConcurrency(), "skipping because test environment doesn't support concurrency")
+
+        // Match patterns for expected messages
+        let isEmpty = StringPattern.equal("")
+        let isOnlyPrint = StringPattern.equal("command plugin: print\n")
+        let containsRemark = StringPattern.contains("command plugin: Diagnostics.remark")
+        let containsWarning = StringPattern.contains("command plugin: Diagnostics.warning")
+        let containsError = StringPattern.contains("command plugin: Diagnostics.error")
+
+        try fixture(name: "Miscellaneous/Plugins/CommandPluginDiagnosticsStub") { fixturePath in
+            func runPlugin(flags: [String], diagnostics: [String], completion: (String, String) -> Void) throws {
+                let (stdout, stderr) = try SwiftPM.Package.execute(flags + ["print-diagnostics"] + diagnostics, packagePath: fixturePath)
+                completion(stdout, stderr)
+            }
+
+            // Diagnostics.error causes SwiftPM to return a non-zero exit code, but we still need to check stdout and stderr
+            func runPluginWithError(flags: [String], diagnostics: [String], completion: (String, String) -> Void) throws {
+                XCTAssertThrowsError(try SwiftPM.Package.execute(flags + ["print-diagnostics"] + diagnostics, packagePath: fixturePath)) { error in
+                    guard case SwiftPMError.executionFailure(_, let stdout, let stderr) = error else {
+                        return XCTFail("invalid error \(error)")
+                    }
+                    completion(stdout, stderr)
+                }
+            }
+
+            // Default verbosity
+            //   - stdout is always printed
+            //   - Diagnostics below 'warning' are suppressed
+
+            try runPlugin(flags: [], diagnostics: ["print"]) { stdout, stderr in
+                XCTAssertMatch(stdout, isOnlyPrint)
+                XCTAssertMatch(stderr, isEmpty)
+            }
+
+            try runPlugin(flags: [], diagnostics: ["print", "remark"]) { stdout, stderr in
+            	XCTAssertMatch(stdout, isOnlyPrint)
+                XCTAssertMatch(stderr, isEmpty)
+            }
+
+            try runPlugin(flags: [], diagnostics: ["print", "remark", "warning"]) { stdout, stderr in
+            	XCTAssertMatch(stdout, isOnlyPrint)
+                XCTAssertMatch(stderr, containsWarning)
+            }
+
+         	try runPluginWithError(flags: [], diagnostics: ["print", "remark", "warning", "error"]) { stdout, stderr in
+                XCTAssertMatch(stdout, isOnlyPrint)
+                XCTAssertMatch(stderr, containsWarning)
+                XCTAssertMatch(stderr, containsError)
+            }
+
+            // Quiet Mode
+            //   - stdout is always printed
+            //   - Diagnostics below 'error' are suppressed
+
+            try runPlugin(flags: ["-q"], diagnostics: ["print"]) { stdout, stderr in
+                XCTAssertMatch(stdout, isOnlyPrint)
+                XCTAssertMatch(stderr, isEmpty)
+            }
+
+            try runPlugin(flags: ["-q"], diagnostics: ["print", "remark"]) { stdout, stderr in
+            	XCTAssertMatch(stdout, isOnlyPrint)
+                XCTAssertMatch(stderr, isEmpty)
+            }
+
+            try runPlugin(flags: ["-q"], diagnostics: ["print", "remark", "warning"]) { stdout, stderr in
+            	XCTAssertMatch(stdout, isOnlyPrint)
+                XCTAssertMatch(stderr, isEmpty)
+            }
+
+            try runPluginWithError(flags: ["-q"], diagnostics: ["print", "remark", "warning", "error"]) { stdout, stderr in
+            	XCTAssertMatch(stdout, isOnlyPrint)
+            	XCTAssertNoMatch(stderr, containsRemark)
+                XCTAssertNoMatch(stderr, containsWarning)
+                XCTAssertMatch(stderr, containsError)
+            }
+
+            // Verbose Mode
+            //   - stdout is always printed
+            //   - All diagnostics are printed
+            //   - Substantial amounts of additional compiler output are also printed
+
+            try runPlugin(flags: ["-v"], diagnostics: ["print"]) { stdout, stderr in
+                XCTAssertMatch(stdout, isOnlyPrint)
+                // At this level stderr contains extra compiler output even if the plugin does not print diagnostics
+            }
+
+            try runPlugin(flags: ["-v"], diagnostics: ["print", "remark"]) { stdout, stderr in
+            	XCTAssertMatch(stdout, isOnlyPrint)
+                XCTAssertMatch(stderr, containsRemark)
+            }
+
+            try runPlugin(flags: ["-v"], diagnostics: ["print", "remark", "warning"]) { stdout, stderr in
+            	XCTAssertMatch(stdout, isOnlyPrint)
+                XCTAssertMatch(stderr, containsRemark)
+                XCTAssertMatch(stderr, containsWarning)
+            }
+
+            try runPluginWithError(flags: ["-v"], diagnostics: ["print", "remark", "warning", "error"]) { stdout, stderr in
+            	XCTAssertMatch(stdout, isOnlyPrint)
+            	XCTAssertMatch(stderr, containsRemark)
+                XCTAssertMatch(stderr, containsWarning)
+                XCTAssertMatch(stderr, containsError)
+            }
+        }
+    }
+
     func testCommandPluginNetworkingPermissions(permissionsManifestFragment: String, permissionError: String, reason: String, remedy: [String]) throws {
         // Only run the test if the environment in which we're running actually supports Swift concurrency (which the plugin APIs require).
         try XCTSkipIf(!UserToolchain.default.supportsSwiftConcurrency(), "skipping because test environment doesn't support concurrency")