Fix lit / taef integration (#7870)

* TAEF writes everything to stdout, so don't worry about separate
out/err streams
* Use a single regex match for the Summary string
* TAEF's exit code doesn't reliably indicate skipped tests, so look for
these explicitly in the summary
* Always return the output with the result - so that `--show-all` will
work correctly and we can see the output from tests regardless of their
final result.

Author: damyanp

utils/lit/lit/formats/taef.py

@@ -1,28 +1,34 @@
 from __future__ import absolute_import
 import os
-import sys
 import signal
 import subprocess
+import re
 
 import lit.Test
-import lit.TestRunner
 import lit.util
 from .base import TestFormat
 
 # TAEF must be run with custom command line string and shell=True
 # because of the way it manually processes quoted arguments in a
 # non-standard way.
+
+
 def executeCommandForTaef(command, cwd=None, env=None):
-    p = subprocess.Popen(command, cwd=cwd,
-                        shell=True,
-                        stdin=subprocess.PIPE,
-                        stdout=subprocess.PIPE,
-                        stderr=subprocess.PIPE,
-                        env=env,
-                        # Close extra file handles on UNIX (on Windows this cannot be done while
-                        # also redirecting input). Taef only run on windows.
-                        close_fds=False)
-    out,err = p.communicate()
+    p = subprocess.Popen(
+        command,
+        cwd=cwd,
+        shell=True,
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        # TAEF doesn't seem to really use stderr, so we just
+        # deal with stdout
+        stderr=subprocess.STDOUT,
+        env=env,
+        # Close extra file handles on UNIX (on Windows this cannot be done while
+        # also redirecting input). Taef only run on windows.
+        close_fds=False,
+    )
+    out, _ = p.communicate()
     exitCode = p.wait()
 
     # Detect Ctrl-C in subprocess.
@@ -31,9 +37,9 @@ def executeCommandForTaef(command, cwd=None, env=None):
 
     # Ensure the resulting output is always of string type.
     out = lit.util.convert_string(out)
-    err = lit.util.convert_string(err)
 
-    return out, err, exitCode
+    return out, exitCode
+
 
 class TaefTest(TestFormat):
     def __init__(self, te_path, test_dll, test_path, select_filter, extra_params):
@@ -63,21 +69,27 @@ def getTaefTests(self, dll_path, litConfig, localConfig):
 
         if litConfig.debug:
             litConfig.note('searching taef test in %r' % dll_path)
-        
+
         cmd = [self.te, dll_path, "/list", "/select:", self.select_filter]
 
         try:
-            lines,err,exitCode = executeCommandForTaef(cmd)
+            lines, exitCode = executeCommandForTaef(cmd)
             # this is for windows
             lines = lines.replace('\r', '')
             lines = lines.split('\n')
 
         except:
-            litConfig.error("unable to discover taef tests in %r, using %s. exeption encountered." % (dll_path, self.te))
+            litConfig.error(
+                "unable to discover taef tests in %r, using %s. exception encountered."
+                % (dll_path, self.te)
+            )
             raise StopIteration
 
         if exitCode:
-            litConfig.error("unable to discover taef tests in %r, using %s. error: %s." % (dll_path, self.te, err))
+            litConfig.error(
+                "unable to discover taef tests in %r, using %s. error: %s."
+                % (dll_path, self.te, lines)
+            )
             raise StopIteration
 
         for ln in lines:
@@ -118,7 +130,7 @@ def getTestsInDirectory(self, testSuite, path_in_suite,
     def execute(self, test, litConfig):
         test_dll = test.getFilePath()
 
-        testPath,testName = os.path.split(test.getSourcePath())
+        testPath, testName = os.path.split(test.getSourcePath())
 
         select_filter = str.format("@Name='{}'", testName)
 
@@ -141,30 +153,48 @@ def execute(self, test, litConfig):
         if litConfig.noExecute:
             return lit.Test.PASS, ''
 
-        out, err, exitCode = executeCommandForTaef(
-            cmd, env = test.config.environment)
+        out, exitCode = executeCommandForTaef(cmd, env=test.config.environment)
 
-        if exitCode:
-            skipped = 'Failed=0, Blocked=0, Not Run=0, Skipped=1'
-            if skipped in out:
-                return lit.Test.UNSUPPORTED, ''
+        return getTestResult(out, exitCode), out
+
+
+def getTestResult(out, exitCode):
+    unselected = "The selection criteria did not match any tests."
+    if unselected in out:
+        return lit.Test.UNSUPPORTED
+
+    # TAEF's exit code cannot reliably indicate if a test was skipped, failed or
+    # passed. The summary string instead is used to determine this.
+
+    # Example summary string:
+    # Summary: Total=1, Passed=1, Failed=0, Blocked=0, Not Run=0, Skipped=0
+    regex = r"Summary: Total=\d+, Passed=(\d+), Failed=(\d+), Blocked=(\d+), Not Run=(\d+), Skipped=(\d+)"
+
+    # Get the last occurance of the summary in the output
+    match = None
+    for m in re.finditer(regex, out):
+        match = m
+
+    if not match:
+        return lit.Test.UNRESOLVED
 
-            unselected = 'The selection criteria did not match any tests.'
-            if unselected in out:
-                return lit.Test.UNSUPPORTED, ''
+    passed = int(match.group(1))
+    failed = int(match.group(2))
+    blocked = int(match.group(3))
+    not_run = int(match.group(4))
+    skipped = int(match.group(5))
 
-            return lit.Test.FAIL, out + err
+    # TAEF docs claim that exitCode should be non-zero if a test is skipped,
+    # but that doesn't seem to be the case with at least v10.88k. So we
+    # explicitly look for skipped in the summary - and any other non-zero exit
+    # codes after this are some kind of failure.
+    if skipped > 0 or blocked > 0 or not_run > 0:
+        return lit.Test.UNSUPPORTED
 
-        summary = 'Summary: Total='
-        if summary not in out:
-            msg = ('Unable to find %r in taef output:\n\n%s%s' %
-                   (summary, out, err))
-            return lit.Test.UNRESOLVED, msg
-        no_fail = 'Failed=0, Blocked=0, Not Run=0, Skipped=0'
-        if no_fail not in out == -1:
-            msg = ('Unable to find %r in taef output:\n\n%s%s' %
-                   (no_fail, out, err))
-            return lit.Test.UNRESOLVED, msg
+    if exitCode or failed > 0:
+        return lit.Test.FAIL
 
-        return lit.Test.PASS,''
+    if passed > 0:
+        return lit.Test.PASS
 
+    return lit.Test.UNRESOLVED