Fix cmdtest script to run test cases as they should be run.

1 files changed, 69 insertions, 47 deletions

diff --git a/cmdtest b/cmdtest
index 66c7ba9..c7e616e 100755
--- a/cmdtest
+++ b/cmdtest
@@ -26,10 +26,7 @@ import tempfile
 import ttystatus
 import unittest
 
-
-class TestCase(object):
-
-    pass
+import cmdtest
 
 
 class TestFailure(Exception):
@@ -48,26 +45,31 @@ class CommandTester(cliapp.Application):
                              'test COMMAND (executable name/path)', 
                              metavar='COMMAND')
 
-    def process_args(self, args):
+    def process_args(self, dirnames):
         self.settings.require('command')
         self.setup_ttystatus()
         self.setup_tempdir()
-        tests = self.load_tests(args)
-        self.ts['tests'] = tests
+
+        td = self.load_tests(dirnames)
+        self.ts['tests'] = td.tests
+
         errors = 0
-        for test in tests:
+        self.run_script(td.setup_once)
+        for test in td.tests:
             self.ts['test'] = test
-            try:
-                self.run_test(test)
-            except TestFailure, e:
+            self.run_script(td.setup)
+            for e in self.run_test(test):
                 logging.error(str(e))
-                self.ts.notify(str(e))
+                self.ts.clear()
+                self.output.write('%s\n' % str(e))
                 errors += 1
+            self.run_script(td.teardown)
+        self.run_script(td.teardown_once)
                 
-        ok = len(tests) - errors
+        ok = len(td.tests) - errors
         self.ts.finish()
         self.output.write('%d/%d tests OK, %d failures\n' % 
-                            (ok, len(tests), errors))
+                            (ok, len(td.tests), errors))
         if errors:
             sys.exit(1)
 
@@ -77,27 +79,61 @@ class CommandTester(cliapp.Application):
         self.ts.add(ttystatus.Index('test', 'tests'))
         
     def load_tests(self, dirnames):
-        tests = []
+        td = cmdtest.TestDir()
         for dirname in dirnames:
-            tests.append(self.load_test(dirname))
-        return tests
-
-    def load_test(self, dirname):
-        t = TestCase()
-        t.name = os.path.basename(dirname)
-        t.setup_cmds = self.lines(os.path.join(dirname, 'setup'))
-        t.command = self.settings['command']
-        t.args = self.lines(os.path.join(dirname, 'args'))
-        t.stdout = self.cat(os.path.join(dirname, 'stdout'))
-        return t
+            td.scan(dirname)
+        return td
 
     def setup_tempdir(self):
         self.tempdir = tempfile.mkdtemp()
         logging.info('Temporary directory %s' % self.tempdir)
+        self.datadir = os.path.join(self.tempdir, 'data')
+        os.mkdir(self.datadir)
         
     def cleanup_tempdir(self):
+        logging.info('Removing temporary directory %s' % self.tempdir)
         shutil.rmtree(self.tempdir)
-        logging.info('Removed temporary directory %s' % self.tempdir)
+
+    def run_script(self, script_name):
+        if script_name:
+            self.runcmd([script_name], env=self.add_to_env())
+
+    def add_to_env(self):
+        env = dict(os.environ)
+        env['DATADIR'] = self.datadir
+        return env
+
+    def run_test(self, test):
+        logging.info('Test case: %s' % test.name)
+
+        self.run_script(test.setup)
+
+        argv = [self.settings['command']]
+        if test.args:
+            argv.extend(self.expand(self.lines(test.args)))
+
+        stdin = self.cat(test.stdin or '/dev/null')
+        exit, out, err = self.runcmd_unchecked(argv, 
+                                               env=self.add_to_env(),
+                                               stdin=stdin)
+        
+        expected_exit = test.exit or 0
+        expected_stdout = self.cat(test.stdout or '/dev/null')
+        expected_stderr = self.cat(test.stderr or '/dev/null')
+
+        errors = []
+        if out != expected_stdout:
+            diff = self.diff(expected_stdout, out)
+            errors.append(TestFailure(test, 'stdout diff:\n%s' % diff))
+        if err != expected_stderr:
+            diff = self.diff(expected_stderr, err)
+            errors.append(TestFailure(test, 'stderr diff:\n%s' % diff))
+        if exit != expected_exit:
+            errors.append(TestFailure(test, 
+                                      'got exit code %s, expected %s' %
+                                        (exit, expected_exit)))
+        
+        return errors
 
     def cat(self, filename):
         if os.path.exists(filename):
@@ -109,32 +145,18 @@ class CommandTester(cliapp.Application):
     def lines(self, filename):
         return self.cat(filename).splitlines()
 
-    def run_test(self, test):
-        logging.info('Test case: %s' % test.name)
-
-        logging.debug('Running setup commands')
-        for setup_cmd in test.setup_cmds:
-            expanded = self.expand([setup_cmd])[0]
-            self.runcmd([expanded], shell=True)
-
-        logging.debug('Running tested command')
-        argv = [self.settings['command']] + self.expand(test.args)
-        out = self.runcmd(argv)
-
-        if out != test.stdout:
-            actual = self.write_file('actual_stdout', out)
-            expected = self.write_file('expected_stdout', test.stdout)
-            diff_argv = ['diff', '-u', expected, actual]
-            exit, diff, err = self.runcmd_unchecked(diff_argv)
-            raise TestFailure(test, 'stdout difference:\n%s' % diff)
-        logging.info('Test %s passed' % test.name)
-
     def expand(self, strings):
         variables = {
             'tempdir': self.tempdir,
         }
         return [s % variables for s in strings]
 
+    def diff(self, expected, actual):
+        e = os.path.join(self.tempdir, 'expected')
+        a = os.path.join(self.tempdir, 'actual')
+        exit, out, err = self.runcmd_unchecked(['diff', '-u', e, a])
+        return out
+
     def write_file(self, basename, content):
         filename = os.path.join(self.tempdir, basename)
         with open(filename, 'w') as f: