From 6414e1b3d53f01fc903876a66497de1f253f819c Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sat, 25 Sep 2021 10:23:15 +0300 Subject: feat: add --run-all (-k) option to Python test runner This allows the user to run all scenarios even if one fails. Sponsored-by: author --- share/python/template/main.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) (limited to 'share') diff --git a/share/python/template/main.py b/share/python/template/main.py index fa42814..b4c32bb 100644 --- a/share/python/template/main.py +++ b/share/python/template/main.py @@ -3,6 +3,7 @@ import logging import os import random import shutil +import sys import tarfile import tempfile @@ -74,6 +75,7 @@ def parse_command_line(): p = argparse.ArgumentParser() p.add_argument("--log") p.add_argument("--env", action="append", default=[]) + p.add_argument("--run-all", "-k", action="store_true") p.add_argument("--save-on-failure") p.add_argument("patterns", nargs="*") return p.parse_args() @@ -128,18 +130,30 @@ def main(scenarios): (name, value) = env.split("=", 1) extra_env[name] = value - try: - for scen in todo: + errors = [] + for scen in todo: + try: scen.run(_datadir, extra_env) - except Exception as e: - logging.error(str(e), exc_info=True) - if args.save_on_failure: - print(args.save_on_failure) - filename = os.path.abspath(os.path.join(srcdir, args.save_on_failure)) - print(filename) - save_directory(_datadir, filename) - raise + except Exception as e: + logging.error(str(e), exc_info=True) + errors.append((scen, e)) + if args.save_on_failure: + print(args.save_on_failure) + filename = os.path.abspath(os.path.join(srcdir, args.save_on_failure)) + print(filename) + save_directory(_datadir, filename) + if not args.run_all: + raise shutil.rmtree(_datadir) + + if errors: + sys.stderr.write(f"ERROR: {len(errors)} scenarios failed\n") + for (scean, e) in errors: + sys.stderr.write(f" - Scenario {scen.get_title()} failed:\n {e}\n") + if args.log: + sys.stderr.write(f"Log file in {args.log}\n") + sys.exit(1) + print("OK, all scenarios finished successfully") logging.info("OK, all scenarios finished successfully") -- cgit v1.2.1