Test command output can be written asynchronously

Author: plemarquand

Sources/_InternalTestSupport/SwiftTesting+Helpers.swift

@@ -10,6 +10,8 @@
 
 import Basics
 import Testing
+import Foundation
+import class TSCBasic.BufferedOutputByteStream
 
 public func expectFileExists(
     at path: AbsolutePath,
@@ -116,3 +118,34 @@ public func expectAsyncThrowsError<T>(
         errorHandler(error)
     }
 }
+
+/// Checks if an output stream contains a specific string, with retry logic for asynchronous writes.
+/// - Parameters:
+///   - outputStream: The output stream to check
+///   - needle: The string to search for in the output stream
+///   - timeout: Maximum time to wait for the string to appear (default: 3 seconds)
+///   - retryInterval: Time to wait between checks (default: 50 milliseconds)
+/// - Returns: True if the string was found within the timeout period
+public func waitForOutputStreamToContain(
+    _ outputStream: BufferedOutputByteStream,
+    _ needle: String,
+    timeout: TimeInterval = 3.0,
+    retryInterval: TimeInterval = 0.05
+) async throws -> Bool {
+    let description = outputStream.bytes.description
+    if description.contains(needle) {
+        return true
+    }
+
+    let startTime = Date()
+    while Date().timeIntervalSince(startTime) < timeout {
+        let description = outputStream.bytes.description
+        if description.contains(needle) {
+            return true
+        }
+
+        try await Task.sleep(nanoseconds: UInt64(retryInterval * 1_000_000_000))
+    }
+
+    return outputStream.bytes.description.contains(needle)
+}

Tests/CommandsTests/PackageCommandTests.swift

@@ -44,8 +44,8 @@ fileprivate func execute(
         try localFileSystem.writeFileContents(packagePath.appending("Package.swift"), string: manifest)
     }
 
-    environment["SWIFTPM_TESTS_LLDB"] = "1"
-
+    // don't ignore local packages when caching
+    environment["SWIFTPM_TESTS_PACKAGECACHE"] = "1"
     return try await executeSwiftPackage(
         packagePath,
         configuration: configuration,

Tests/CommandsTests/TestCommandTests.swift

@@ -1194,10 +1194,12 @@ struct TestCommandTests {
 
             #expect(error == ExitCode.failure, "Expected ExitCode.failure, got \(String(describing: error))")
 
-            let errorDescription = outputStream.bytes.description
+            // The output stream is written to asynchronously on a DispatchQueue and can
+            // receive output after the command has thrown.
+            let found = try await waitForOutputStreamToContain(outputStream, "--debugger cannot be used with --parallel")
             #expect(
-                errorDescription.contains("--debugger cannot be used with --parallel"),
-                "Expected error about incompatible flags, got: \(errorDescription)"
+                found,
+                "Expected error about incompatible flags, got: \(outputStream.bytes.description)"
             )
         }
 
@@ -1213,11 +1215,11 @@ struct TestCommandTests {
 
             #expect(error == ExitCode.failure, "Expected ExitCode.failure, got \(String(describing: error))")
 
-            let errorDescription = outputStream.bytes.description
             // Should hit the --parallel error first since validation is done in order
+            let found = try await waitForOutputStreamToContain(outputStream, "--debugger cannot be used with --parallel")
             #expect(
-                errorDescription.contains("--debugger cannot be used with --parallel"),
-                "Expected error about incompatible flags, got: \(errorDescription)"
+                found,
+                "Expected error about incompatible flags, got: \(outputStream.bytes.description)"
             )
         }
 
@@ -1233,10 +1235,10 @@ struct TestCommandTests {
 
             #expect(error == ExitCode.failure, "Expected ExitCode.failure, got \(String(describing: error))")
 
-            let errorDescription = outputStream.bytes.description
+            let found = try await waitForOutputStreamToContain(outputStream, "--debugger cannot be used with --num-workers")
             #expect(
-                errorDescription.contains("--debugger cannot be used with --num-workers"),
-                "Expected error about incompatible flags, got: \(errorDescription)"
+                found,
+                "Expected error about incompatible flags, got: \(outputStream.bytes.description)"
             )
         }
 
@@ -1252,10 +1254,10 @@ struct TestCommandTests {
 
             #expect(error == ExitCode.failure, "Expected ExitCode.failure, got \(String(describing: error))")
 
-            let errorDescription = outputStream.bytes.description
+            let found = try await waitForOutputStreamToContain(outputStream, "--debugger cannot be used with --list-tests")
             #expect(
-                errorDescription.contains("--debugger cannot be used with --list-tests"),
-                "Expected error about incompatible flags, got: \(errorDescription)"
+                found,
+                "Expected error about incompatible flags, got: \(outputStream.bytes.description)"
             )
         }
 
@@ -1271,10 +1273,10 @@ struct TestCommandTests {
 
             #expect(error == ExitCode.failure, "Expected ExitCode.failure, got \(String(describing: error))")
 
-            let errorDescription = outputStream.bytes.description
+            let found = try await waitForOutputStreamToContain(outputStream, "--debugger cannot be used with --show-codecov-path")
             #expect(
-                errorDescription.contains("--debugger cannot be used with --show-codecov-path"),
-                "Expected error about incompatible flags, got: \(errorDescription)"
+                found,
+                "Expected error about incompatible flags, got: \(outputStream.bytes.description)"
             )
         }
 
@@ -1290,10 +1292,10 @@ struct TestCommandTests {
 
             #expect(error == ExitCode.failure, "Expected ExitCode.failure, got \(String(describing: error))")
 
-            let errorDescription = outputStream.bytes.description
+            let found = try await waitForOutputStreamToContain(outputStream, "--debugger cannot be used with release configuration")
             #expect(
-                errorDescription.contains("--debugger cannot be used with release configuration"),
-                "Expected error about incompatible flags, got: \(errorDescription)"
+                found,
+                "Expected error about incompatible flags, got: \(outputStream.bytes.description)"
             )
         }