From 3b46553f47a202842ea6172c016a2df64aa3d34e Mon Sep 17 00:00:00 2001
From: Paul Fenwick <pjf@perltraining.com.au>
Date: Fri, 24 Nov 2017 03:50:52 -0500
Subject: [PATCH] Improve TAP compliance in test suite

This allows tools like `prove` to operate on our test-suite, allowing us
to do things like run recently-failing tests first, randomising test
order, and fancier reporting options.

Changes made:

- TAP output always goes to STDOUT, never to STDERR
- Skipped tests signaled as `ok 1 # skip` as per standard
- Expected failures signaled as `not ok 1 # TODO` as per standard
- `.prove` added to .gitignore
---
 .gitignore                 |  1 +
 test/simpletap/__init__.py | 12 ++++++++----
 test/test.cpp              |  3 ++-
 3 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/.gitignore b/.gitignore
index 3ba2ade24..03300d5c8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,3 +19,4 @@ CPackSourceConfig.cmake
 patches
 *.exe
 tutorials
+.prove
diff --git a/test/simpletap/__init__.py b/test/simpletap/__init__.py
index 357f63f20..6ab4c7fa4 100644
--- a/test/simpletap/__init__.py
+++ b/test/simpletap/__init__.py
@@ -155,12 +155,12 @@ class TAPTestResult(unittest.result.TestResult):
         if status:
 
             if status == "SKIP":
-                self.stream.writeln("{0} {1} - {2}: {3}".format(
-                    color("skip", "yellow"), self.testsRun, filename, desc)
+                self.stream.writeln("{0} {1} - {2}: {3} # skip".format(
+                    color("ok", "yellow"), self.testsRun, filename, desc)
                 )
             elif status == "EXPECTED_FAILURE":
-                self.stream.writeln("{0} {1} - {2}: {3}".format(
-                    color("skip", "yellow"), self.testsRun, filename, desc)
+                self.stream.writeln("{0} {1} - {2}: {3} # TODO".format(
+                    color("not ok", "yellow"), self.testsRun, filename, desc)
                 )
             else:
                 self.stream.writeln("{0} {1} - {2}: {3}".format(
@@ -227,6 +227,10 @@ class TAPTestRunner(unittest.runner.TextTestRunner):
         unittest.signals.registerResult(result)
         result.failfast = self.failfast
 
+        # TAP requires output is on STDOUT.
+        # TODO: Define this at __init__ time
+        result.stream = unittest.runner._WritelnDecorator(sys.stdout)
+
         with warnings.catch_warnings():
             if getattr(self, "warnings", None):
                 # if self.warnings is set, use it to filter all the warnings
diff --git a/test/test.cpp b/test/test.cpp
index 9dc7552ca..ecc212e8a 100644
--- a/test/test.cpp
+++ b/test/test.cpp
@@ -452,11 +452,12 @@ void UnitTest::skip (const std::string& text)
 {
   ++_counter;
   ++_skipped;
-  std::cout << yellow ("skip")
+  std::cout << yellow ("ok")
             << " "
             << _counter
             << " - "
             << text
+            << " # skip"
             << "\n";
 }