Store failing stdout, stderr, and diffs, rather than deleting them.

Reading diffs is not always practical, and this will allow later
diffing with other diff tools.

1 files changed, 34 insertions, 20 deletions

diff --git a/cmdtest b/cmdtest
index 4d9f183..5c61242 100755
--- a/cmdtest
+++ b/cmdtest
@@ -117,25 +117,41 @@ class CommandTester(cliapp.Application):
                 argv.extend(self.expand(self.lines(test.args)))
 
         stdin = self.cat(test.stdin or '/dev/null')
-        exit, out, err = self.runcmd_unchecked(argv, 
-                                               env=self.add_to_env(),
-                                               stdin=stdin)
-        
-        expected_exit = int(self.cat(test.exit or '/dev/null').strip() or '0')
-        expected_stdout = self.cat(test.stdout or '/dev/null')
-        expected_stderr = self.cat(test.stderr or '/dev/null')
-        
+        stdout_name = test.path_prefix + '.stdout-actual'
+        stderr_name = test.path_prefix + '.stderr-actual'
+        with open(stdout_name, 'wb') as stdout:
+            with open(stderr_name, 'wb') as stderr:
+                exit, out, err = self.runcmd_unchecked(argv, 
+                                                       env=self.add_to_env(),
+                                                       stdin=stdin,
+                                                       stdout=stdout,
+                                                       stderr=stderr)
+
         errors = []
-        if out != expected_stdout:
-            diff = self.diff(expected_stdout, out)
-            errors.append(TestFailure(test, 'stdout diff:\n%s' % diff))
-        if err != expected_stderr:
-            diff = self.diff(expected_stderr, err)
-            errors.append(TestFailure(test, 'stderr diff:\n%s' % diff))
+
+        stdout_diff_name = test.path_prefix + '.stdout-diff'
+        stdout_diff = self.diff(test.stdout or '/dev/null', stdout_name,
+                                stdout_diff_name)
+        if stdout_diff:
+            errors.append(TestFailure(test, 'stdout diff:\n%s' % stdout_diff))
+
+        stderr_diff_name = test.path_prefix + '.stderr-diff'
+        stderr_diff = self.diff(test.stderr or '/dev/null', stderr_name,
+                                stderr_diff_name)
+        if stderr_diff:
+            errors.append(TestFailure(test, 'stderr diff:\n%s' % stderr_diff))
+
+        expected_exit = int(self.cat(test.exit or '/dev/null').strip() or '0')
         if exit != expected_exit:
             errors.append(TestFailure(test, 
                                       'got exit code %s, expected %s' %
                                         (exit, expected_exit)))
+
+        if not errors:
+            os.remove(stdout_name)
+            os.remove(stderr_name)
+            os.remove(stdout_diff_name)
+            os.remove(stderr_diff_name)
         
         return errors
 
@@ -155,12 +171,10 @@ class CommandTester(cliapp.Application):
         }
         return [s % variables for s in strings]
 
-    def diff(self, expected, actual):
-        e = os.path.join(self.tempdir, 'expected')
-        a = os.path.join(self.tempdir, 'actual')
-        self.write_file(e, expected)
-        self.write_file(a, actual)
-        exit, out, err = self.runcmd_unchecked(['diff', '-u', e, a])
+    def diff(self, expected_name, actual_name, diff_name):
+        exit, out, err = self.runcmd_unchecked(['diff', '-u', 
+                                                 expected_name, actual_name])
+        self.write_file(diff_name, out)
         return out
 
     def write_file(self, filename, content):