feat: add --run-all (-k) option to Python test runner

This allows the user to run all scenarios even if one fails.

Sponsored-by: author

1 files changed, 24 insertions, 10 deletions

diff --git a/share/python/template/main.py b/share/python/template/main.py
index fa42814..b4c32bb 100644
--- a/share/python/template/main.py
+++ b/share/python/template/main.py
@@ -3,6 +3,7 @@ import logging
 import os
 import random
 import shutil
+import sys
 import tarfile
 import tempfile
 
@@ -74,6 +75,7 @@ def parse_command_line():
     p = argparse.ArgumentParser()
     p.add_argument("--log")
     p.add_argument("--env", action="append", default=[])
+    p.add_argument("--run-all", "-k", action="store_true")
     p.add_argument("--save-on-failure")
     p.add_argument("patterns", nargs="*")
     return p.parse_args()
@@ -128,18 +130,30 @@ def main(scenarios):
         (name, value) = env.split("=", 1)
         extra_env[name] = value
 
-    try:
-        for scen in todo:
+    errors = []
+    for scen in todo:
+        try:
             scen.run(_datadir, extra_env)
-    except Exception as e:
-        logging.error(str(e), exc_info=True)
-        if args.save_on_failure:
-            print(args.save_on_failure)
-            filename = os.path.abspath(os.path.join(srcdir, args.save_on_failure))
-            print(filename)
-            save_directory(_datadir, filename)
-        raise
+        except Exception as e:
+            logging.error(str(e), exc_info=True)
+            errors.append((scen, e))
+            if args.save_on_failure:
+                print(args.save_on_failure)
+                filename = os.path.abspath(os.path.join(srcdir, args.save_on_failure))
+                print(filename)
+                save_directory(_datadir, filename)
+            if not args.run_all:
+                raise
 
     shutil.rmtree(_datadir)
+
+    if errors:
+        sys.stderr.write(f"ERROR: {len(errors)} scenarios failed\n")
+        for (scean, e) in errors:
+            sys.stderr.write(f" - Scenario {scen.get_title()} failed:\n  {e}\n")
+        if args.log:
+            sys.stderr.write(f"Log file in {args.log}\n")
+        sys.exit(1)
+
     print("OK, all scenarios finished successfully")
     logging.info("OK, all scenarios finished successfully")