From 8dcb783e5bcd9b5df28fafc2e9b29ec4b03735d2 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 15 Aug 2011 20:25:16 +0100 Subject: Fix cmdtest script to run test cases as they should be run. --- cmdtest | 116 ++++++++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 69 insertions(+), 47 deletions(-) (limited to 'cmdtest') diff --git a/cmdtest b/cmdtest index 66c7ba9..c7e616e 100755 --- a/cmdtest +++ b/cmdtest @@ -26,10 +26,7 @@ import tempfile import ttystatus import unittest - -class TestCase(object): - - pass +import cmdtest class TestFailure(Exception): @@ -48,26 +45,31 @@ class CommandTester(cliapp.Application): 'test COMMAND (executable name/path)', metavar='COMMAND') - def process_args(self, args): + def process_args(self, dirnames): self.settings.require('command') self.setup_ttystatus() self.setup_tempdir() - tests = self.load_tests(args) - self.ts['tests'] = tests + + td = self.load_tests(dirnames) + self.ts['tests'] = td.tests + errors = 0 - for test in tests: + self.run_script(td.setup_once) + for test in td.tests: self.ts['test'] = test - try: - self.run_test(test) - except TestFailure, e: + self.run_script(td.setup) + for e in self.run_test(test): logging.error(str(e)) - self.ts.notify(str(e)) + self.ts.clear() + self.output.write('%s\n' % str(e)) errors += 1 + self.run_script(td.teardown) + self.run_script(td.teardown_once) - ok = len(tests) - errors + ok = len(td.tests) - errors self.ts.finish() self.output.write('%d/%d tests OK, %d failures\n' % - (ok, len(tests), errors)) + (ok, len(td.tests), errors)) if errors: sys.exit(1) @@ -77,27 +79,61 @@ class CommandTester(cliapp.Application): self.ts.add(ttystatus.Index('test', 'tests')) def load_tests(self, dirnames): - tests = [] + td = cmdtest.TestDir() for dirname in dirnames: - tests.append(self.load_test(dirname)) - return tests - - def load_test(self, dirname): - t = TestCase() - t.name = os.path.basename(dirname) - t.setup_cmds = self.lines(os.path.join(dirname, 'setup')) - t.command = self.settings['command'] - t.args = self.lines(os.path.join(dirname, 'args')) - t.stdout = self.cat(os.path.join(dirname, 'stdout')) - return t + td.scan(dirname) + return td def setup_tempdir(self): self.tempdir = tempfile.mkdtemp() logging.info('Temporary directory %s' % self.tempdir) + self.datadir = os.path.join(self.tempdir, 'data') + os.mkdir(self.datadir) def cleanup_tempdir(self): + logging.info('Removing temporary directory %s' % self.tempdir) shutil.rmtree(self.tempdir) - logging.info('Removed temporary directory %s' % self.tempdir) + + def run_script(self, script_name): + if script_name: + self.runcmd([script_name], env=self.add_to_env()) + + def add_to_env(self): + env = dict(os.environ) + env['DATADIR'] = self.datadir + return env + + def run_test(self, test): + logging.info('Test case: %s' % test.name) + + self.run_script(test.setup) + + argv = [self.settings['command']] + if test.args: + argv.extend(self.expand(self.lines(test.args))) + + stdin = self.cat(test.stdin or '/dev/null') + exit, out, err = self.runcmd_unchecked(argv, + env=self.add_to_env(), + stdin=stdin) + + expected_exit = test.exit or 0 + expected_stdout = self.cat(test.stdout or '/dev/null') + expected_stderr = self.cat(test.stderr or '/dev/null') + + errors = [] + if out != expected_stdout: + diff = self.diff(expected_stdout, out) + errors.append(TestFailure(test, 'stdout diff:\n%s' % diff)) + if err != expected_stderr: + diff = self.diff(expected_stderr, err) + errors.append(TestFailure(test, 'stderr diff:\n%s' % diff)) + if exit != expected_exit: + errors.append(TestFailure(test, + 'got exit code %s, expected %s' % + (exit, expected_exit))) + + return errors def cat(self, filename): if os.path.exists(filename): @@ -109,32 +145,18 @@ class CommandTester(cliapp.Application): def lines(self, filename): return self.cat(filename).splitlines() - def run_test(self, test): - logging.info('Test case: %s' % test.name) - - logging.debug('Running setup commands') - for setup_cmd in test.setup_cmds: - expanded = self.expand([setup_cmd])[0] - self.runcmd([expanded], shell=True) - - logging.debug('Running tested command') - argv = [self.settings['command']] + self.expand(test.args) - out = self.runcmd(argv) - - if out != test.stdout: - actual = self.write_file('actual_stdout', out) - expected = self.write_file('expected_stdout', test.stdout) - diff_argv = ['diff', '-u', expected, actual] - exit, diff, err = self.runcmd_unchecked(diff_argv) - raise TestFailure(test, 'stdout difference:\n%s' % diff) - logging.info('Test %s passed' % test.name) - def expand(self, strings): variables = { 'tempdir': self.tempdir, } return [s % variables for s in strings] + def diff(self, expected, actual): + e = os.path.join(self.tempdir, 'expected') + a = os.path.join(self.tempdir, 'actual') + exit, out, err = self.runcmd_unchecked(['diff', '-u', e, a]) + return out + def write_file(self, basename, content): filename = os.path.join(self.tempdir, basename) with open(filename, 'w') as f: -- cgit v1.2.1