From eb9c82812e31439b35f29bb06aef2ff7afe9147b Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sat, 13 Jan 2024 16:52:18 +0200 Subject: fix: adapt to changes in Subplot the subplot/daemon.{py,yaml} files are copies from Subplot, to be removed when Subplot is fixed. Signed-off-by: Lars Wirzenius Sponsored-by: author --- ewww.subplot | 4 +- subplot/daemon.py | 329 ++++++++++++++++++++++++++++++++++++++++++++++++++++ subplot/daemon.yaml | 165 ++++++++++++++++++++++++++ subplot/ewww.yaml | 21 ++++ 4 files changed, 517 insertions(+), 2 deletions(-) create mode 100644 subplot/daemon.py create mode 100644 subplot/daemon.yaml diff --git a/ewww.subplot b/ewww.subplot index d8f3958..4344948 100644 --- a/ewww.subplot +++ b/ewww.subplot @@ -5,14 +5,14 @@ markdowns: - ewww.md bindings: - subplot/ewww.yaml - - python/lib/daemon.yaml + - subplot/daemon.yaml - lib/files.yaml - lib/runcmd.yaml impls: python: - subplot/ewww.py - subplot/http.py - - python/lib/daemon.py + - subplot/daemon.py - lib/files.py - lib/runcmd.py classes: diff --git a/subplot/daemon.py b/subplot/daemon.py new file mode 100644 index 0000000..9f58af0 --- /dev/null +++ b/subplot/daemon.py @@ -0,0 +1,329 @@ +import json +import logging +import os +import signal +import socket +import subprocess +import time + + +# A helper function for testing lib/daemon itself. +def _daemon_shell_script(ctx, filename=None): + get_file = globals()["get_file"] + data = get_file(filename) + with open(filename, "wb") as f: + f.write(data) + os.chmod(filename, 0o755) + + +# Start a daemon that will open a port on localhost. +def daemon_start_on_port(ctx, path=None, args=None, name=None, port=None, env=None): + _daemon_start(ctx, path=path, args=args, name=name, env=env) + daemon_wait_for_port("localhost", port) + + +# Start a daemon after a little wait. This is used only for testing the +# port-waiting code. +def _daemon_start_soonish(ctx, path=None, args=None, name=None, port=None, env=None): + _daemon_start(ctx, path=path, args=args, name=name, env=env) + daemon = ctx.declare("_daemon") + + # Store the PID of the process we just started so that _daemon_stop_soonish + # can kill it during the cleanup phase. This works around the Subplot + # Python template not giving the step captures to cleanup functions. Note + # that this code assume at most one _soonish function is called. + daemon["_soonish"] = daemon[name]["pid"] + + try: + daemon_wait_for_port("localhost", port) + except Exception as e: + daemon["_start_error"] = repr(e) + + logging.info("pgrep: %r", _daemon_pgrep(path)) + + +def _daemon_stop_soonish(ctx, path=None, args=None, name=None, port=None, env=None): + ns = ctx.declare("_daemon") + pid = ns["_soonish"] + logging.debug(f"Stopping soonishly-started daemon, {pid}") + signo = signal.SIGKILL + try: + os.kill(pid, signo) + except ProcessLookupError: + logging.warning("Process did not actually exist (anymore?)") + + +# Find a binary akin to the `which` or `whereis` command +def _daemon_whereis(path): + if "/" in path: + logging.debug(f"Not using PATH for daemon {path}") + return path + for prefix in os.environ["PATH"].split(":"): + absolute = os.path.join(prefix, path) + logging.debug(f"Checking for {absolute}") + if os.access(absolute, os.X_OK) and os.path.isfile(absolute): + return absolute + return path + + +# Start a daeamon, get its PID. Don't wait for a port or anything. This is +# meant for background processes that don't have port. Useful for testing the +# lib/daemon library of Subplot, but not much else. +def _daemon_start(ctx, path=None, args=None, name=None, env=None): + runcmd_run = globals()["runcmd_run"] + runcmd_exit_code_is = globals()["runcmd_exit_code_is"] + runcmd_get_exit_code = globals()["runcmd_get_exit_code"] + runcmd_get_stderr = globals()["runcmd_get_stderr"] + runcmd_prepend_to_path = globals()["runcmd_prepend_to_path"] + + path = _daemon_whereis(path) + path = os.path.abspath(path) + argv = [path] + args.split() + + env = json.loads(env or "{}") + env_vars = [] + if env: + for (key, value) in env.items(): + env_vars.extend(["-E", f"{key}={value}"]) + + logging.debug(f"Starting daemon {name}") + logging.debug(f" ctx={ctx.as_dict()}") + logging.debug(f" name={name}") + logging.debug(f" path={path}") + logging.debug(f" args={args}") + logging.debug(f" argv={argv}") + logging.debug(f" env={env}") + + ns = ctx.declare("_daemon") + + this = ns[name] = { + "pid-file": f"{name}.pid", + "stderr": f"{name}.stderr", + "stdout": f"{name}.stdout", + } + + # Debian installs `daemonize` to /usr/sbin, which isn't part of the minimal + # environment that Subplot sets up. So we add /usr/sbin to the PATH. + runcmd_prepend_to_path(ctx, "/usr/sbin") + runcmd_run( + ctx, + [ + "daemonize", + "-c", + os.getcwd(), + "-p", + this["pid-file"], + "-e", + this["stderr"], + "-o", + this["stdout"], + ] + + env_vars + + argv, + ) + + # Check that daemonize has exited OK. If it hasn't, it didn't start the + # background process at all. If so, log the stderr in case there was + # something useful there for debugging. + exit = runcmd_get_exit_code(ctx) + if exit != 0: + stderr = runcmd_get_stderr(ctx) + logging.error(f"daemon {name} stderr: {stderr}") + runcmd_exit_code_is(ctx, 0) + + # Get the pid of the background process, from the pid file created by + # daemonize. We don't need to wait for it, since we know daemonize already + # exited. If it isn't there now, it's won't appear later. + if not os.path.exists(this["pid-file"]): + raise Exception("daemonize didn't create a PID file") + + this["pid"] = _daemon_wait_for_pid(this["pid-file"], 10.0) + + logging.debug(f"Started daemon {name}") + logging.debug(f" pid={this['pid']}") + logging.debug(f" ctx={ctx.as_dict()}") + + +def _daemon_wait_for_pid(filename, timeout): + start = time.time() + while time.time() < start + timeout: + with open(filename) as f: + data = f.read().strip() + if data: + return int(data) + raise Exception("daemonize created a PID file without a PID") + + +def daemon_wait_for_port(host, port, timeout=5.0): + addr = (host, port) + until = time.time() + timeout + while True: + try: + s = socket.create_connection(addr, timeout=timeout) + s.close() + return + except socket.timeout: + logging.error( + f"daemon did not respond at port {port} within {timeout} seconds" + ) + raise + except socket.error as e: + logging.info(f"could not connect to daemon at {port}: {e}") + pass + if time.time() >= until: + logging.error( + f"could not connect to daemon at {port} within {timeout} seconds" + ) + raise ConnectionRefusedError() + # Sleep a bit to avoid consuming too much CPU while busy-waiting. + time.sleep(0.1) + + +# Stop a daemon. +def daemon_stop(ctx, path=None, args=None, name=None): + logging.debug(f"Stopping daemon {name}") + + ns = ctx.declare("_daemon") + logging.debug(f" ns={ns}") + pid = ns[name]["pid"] + signo = signal.SIGTERM + + this = ns[name] + + _daemon_log_long(f"{name} stdout, before:", open(this["stdout"]).read()) + _daemon_log_long(f"{name} stderr, before:", open(this["stderr"]).read()) + + logging.debug(f"Terminating process {pid} with signal {signo}") + try: + os.kill(pid, signo) + except ProcessLookupError: + logging.warning("Process did not actually exist (anymore?)") + + while True: + try: + os.kill(pid, 0) + logging.debug(f"Daemon {name}, pid {pid} still exists") + time.sleep(1) + except ProcessLookupError: + break + logging.debug(f"Daemon {name} is gone") + + _daemon_log_long(f"{name} stdout, after:", open(this["stdout"]).read()) + _daemon_log_long(f"{name} stderr, after:", open(this["stderr"]).read()) + + +def _daemon_log_long(prefix, data): + logging.debug(prefix) + for line in data.splitlines(): + logging.debug(f" {line}") + + +def daemon_no_such_process(ctx, args=None): + assert not _daemon_pgrep(args) + + +def daemon_process_exists(ctx, args=None): + assert _daemon_pgrep(args) + + +def _daemon_pgrep(pattern): + logging.info(f"checking if process exists: pattern={pattern}") + exit = subprocess.call( + ["pgrep", "-laf", pattern], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + logging.info(f"exit code: {exit}") + return exit == 0 + + +def daemon_start_fails_with(ctx, message=None): + daemon = ctx.declare("_daemon") + error = daemon["_start_error"] + logging.debug(f"daemon_start_fails_with: error={error!r}") + logging.debug(f"daemon_start_fails_with: message={message!r}") + assert message.lower() in error.lower() + + +def daemon_start_succeeds(ctx): + daemon = ctx.declare("_daemon") + logging.debug(f"daemon_start_succeeds: {daemon}") + for name in daemon.keys(): + if isinstance(daemon[name], dict): + logging.debug(f"name={name}") + stderr = daemon_get_stderr(ctx, name) + logging.debug(f"daemon_start_succeeds: {name}:\n{stderr}") + assert "_start_error" not in daemon + + +def daemon_get_stdout(ctx, name): + return _daemon_get_output(ctx, name, "stdout") + + +def daemon_get_stderr(ctx, name): + return _daemon_get_output(ctx, name, "stderr") + + +def _daemon_get_output(ctx, name, which): + ns = ctx.declare("_daemon") + this = ns[name] + filename = this[which] + data = open(filename).read() + logging.debug(f"Read {which} of daemon {name} from {filename}: {data!r}") + return data + + +def daemon_has_produced_output(ctx, name=None): + started = time.time() + timeout = 5.0 + while time.time() < started + timeout: + stdout = daemon_get_stdout(ctx, name) + stderr = daemon_get_stderr(ctx, name) + if stdout and stderr: + break + time.sleep(0.1) + + +def daemon_stdout_is(ctx, name=None, text=None): + daemon_get_stdout = globals()["daemon_get_stdout"] + _daemon_output_is(ctx, name, text, daemon_get_stdout) + + +def daemon_stdout_contains(ctx, name=None, text=None): + daemon_get_stdout = globals()["daemon_get_stdout"] + assert_eq = globals()["assert_eq"] + result = _daemon_output_contains(ctx, name, text, daemon_get_stdout) + assert_eq(result, True) + + +def daemon_stdout_doesnt_contain(ctx, name=None, text=None): + daemon_get_stdout = globals()["daemon_get_stdout"] + assert_eq = globals()["assert_eq"] + result = _daemon_output_contains(ctx, name, text, daemon_get_stdout) + assert_eq(result, False) + + +def daemon_stderr_is(ctx, name=None, text=None): + daemon_get_stderr = globals()["daemon_get_stderr"] + _daemon_output_is(ctx, name, text, daemon_get_stderr) + + +def _daemon_output_is(ctx, name, text, getter): + assert_eq = globals()["assert_eq"] + text = bytes(text, "UTF-8").decode("unicode_escape") + output = getter(ctx, name) + assert_eq(text, output) + + +def _daemon_output_contains(ctx, name, text, getter): + log_lines = globals()["log_lines"] + indent = " " * 4 + + text = bytes(text, "UTF-8").decode("unicode_escape") + output = getter(ctx, name) + + logging.debug(" output:") + log_lines(indent, output) + + logging.debug(" text:") + log_lines(indent, text) + + return text in output diff --git a/subplot/daemon.yaml b/subplot/daemon.yaml new file mode 100644 index 0000000..e385880 --- /dev/null +++ b/subplot/daemon.yaml @@ -0,0 +1,165 @@ +- given: there is no "{args:text}" process + impl: + python: + function: daemon_no_such_process + doc: | + Ensure a given process is not running. + +- given: a daemon helper shell script {filename} + impl: + python: + function: _daemon_shell_script + types: + filename: file + doc: | + Install a helper script from an embedded file. + +- when: I start "{path}{args:text}" as a background process as {name}, on port {port} + impl: + python: + function: daemon_start_on_port + doc: | + Start a process in the background (as a daemon) and wait until it + listens on its assigned port. + +- when: I start "(?P[^ "]+)(?P[^"]*)" as a background process as (?P[^,]+), on port (?P\d+), with environment (?P.*) + regex: true + types: + args: text + path: path + name: text + port: uint + env: text + impl: + python: + function: daemon_start_on_port + doc: | + Start a process in the background (as a daemon) and wait until it + listens on its assigned port. Remember the process under the given + name. + +- when: I try to start "{path}{args:text}" as {name}, on port {port} + impl: + python: + function: _daemon_start_soonish + cleanup: _daemon_stop_soonish + doc: | + Try to start a background process (as a daemon), but don't fail if + starting it fails. + +- when: I try to start "(?P[^ "]+)(?P[^"]*)" as (?P[^,]+), on port (?P\d+), with environment (?P.*) + regex: true + types: + path: path + args: text + name: text + port: uint + env: text + impl: + python: + function: _daemon_start_soonish + cleanup: _daemon_stop_soonish + doc: | + Start a process in the background (as a daemon) and wait until it + listens on its assigned port. Remember the process under the given + name. Don't fail if this fails. + +- when: I start "{path}{args:text}" as a background process as {name} + impl: + python: + function: _daemon_start + doc: | + Start a process in the background (as a daemon). Remember the + process under the given name. Don't fail if this fails. + +- when: I start "(?P[^ "]+)(?P[^"]*)" as a background process as (?P[^,]+), with environment (?P.*) + regex: true + types: + path: path + args: text + name: text + env: text + impl: + python: + function: _daemon_start + doc: | + Start a process in the background (as a daemon), with specific + environment variables set. Remember the process under the given + name. Don't fail if this fails. + +- when: I stop background process {name} + impl: + python: + function: daemon_stop + doc: | + Stop a background process that was started earlier with the given + name. + +- when: daemon {name} has produced output + impl: + python: + function: daemon_has_produced_output + doc: | + Wait until the named daemon has produced output to its stdout or + stderr. + +- then: a process "{args:text}" is running + impl: + python: + function: daemon_process_exists + doc: | + Check that a given process is running. + +- then: there is no "{args:text}" process + impl: + python: + function: daemon_no_such_process + doc: | + Check that a given process is not running. + +- then: starting daemon fails with "{message:text}" + impl: + python: + function: daemon_start_fails_with + doc: | + Check that starting a daemon previously failed, and the error + message contains the given text. + +- then: starting the daemon succeeds + impl: + python: + function: daemon_start_succeeds + doc: | + Check that staring a daemon previous succeeded. + +- then: daemon {name} stdout is "{text:text}" + impl: + python: + function: daemon_stdout_is + doc: | + Check that the named daemon has written exactly the given text to + its stdout. + +- then: daemon {name} stdout contains "{text:text}" + impl: + python: + function: daemon_stdout_contains + doc: | + Check that the named daemon has written the given text to its + stdout, possibly among other text. + +- then: daemon {name} stdout doesn't contain "{text:text}" + impl: + python: + function: daemon_stdout_doesnt_contain + doc: | + Check that the named daemon has not written the given text to its + stdout. + +- then: daemon {name} stderr is "{text:text}" + impl: + python: + function: daemon_stderr_is + doc: | + Check that the named daemon has written exactly the given text to + its stderr. diff --git a/subplot/ewww.yaml b/subplot/ewww.yaml index 421bfad..f758e20 100644 --- a/subplot/ewww.yaml +++ b/subplot/ewww.yaml @@ -1,4 +1,7 @@ - given: a self-signed certificate as {cert}, using key {key} + types: + cert: word + key: word impl: python: function: copy_test_certificate @@ -12,43 +15,61 @@ filename: file - given: directory {dirname} + types: + dirname: path impl: python: function: create_directory - then: I am redirected to {location} + types: + location: word impl: python: function: fixme - then: I can do at least {number} requests per second + types: + number: uint impl: python: function: fixme - then: I get status code {code} + types: + code: uint impl: python: function: http_status_code_is - then: 'header (?P
\S+) is "(?P.+)"' regex: true + types: + header: word + value: text impl: python: function: http_header_is - then: 'body is "(?P.*)"' regex: true + types: + body: text impl: python: function: http_body_is - when: I request {method} {url} + types: + url: word impl: python: function: request - when: I request files under {url} in random order {count} times + types: + url: word + count: uint impl: python: function: fixme -- cgit v1.2.1 From c52df697788004830709af50d49509b4278277bf Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sat, 13 Jan 2024 17:20:51 +0200 Subject: tests(check): add --offline option Signed-off-by: Lars Wirzenius Sponsored-by: author --- check | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/check b/check index c143a4a..5dfe653 100755 --- a/check +++ b/check @@ -3,6 +3,7 @@ set -eu verbose=false +offline= moar=true while [ "$#" -gt 0 ] && $moar; do case "$1" in @@ -10,6 +11,10 @@ while [ "$#" -gt 0 ] && $moar; do verbose=true shift 1 ;; + --offline) + offline=--offline + shift 1 + ;; esac done @@ -35,14 +40,14 @@ docgen() { subplot docgen "$1" --output "$2" } -$hideok cargo build --all-targets +$hideok cargo build --all-targets $offline if cargo --list | awk '{ print $1 }' | grep 'clippy$' >/dev/null; then # shellcheck disable=SC2086 - cargo clippy $quiet + cargo clippy $quiet $offline fi # shellcheck disable=SC2086 -$hideok cargo test $quiet +$hideok cargo test $quiet $offline if cargo fmt --help >/dev/null 2>/dev/null; then $hideok cargo fmt -- --check -- cgit v1.2.1 From 9dffa9f611693f3c6fea9f540772406e7e19a9d8 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sat, 13 Jan 2024 17:39:16 +0200 Subject: fix(debian/control): drop unnecessary build-dependencies Signed-off-by: Lars Wirzenius Sponsored-by: author --- debian/control | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/debian/control b/debian/control index 2bfdc2d..c1b687b 100644 --- a/debian/control +++ b/debian/control @@ -13,10 +13,7 @@ Build-Depends: python3, python3-requests, python3-yaml, - subplot, - texlive-fonts-recommended, - texlive-latex-base, - texlive-latex-recommended + subplot Homepage: https://ewww.liw.fi/ Package: ewww -- cgit v1.2.1 From 1a6d669a9ade98ccd21c0896b3fb0b122473ab73 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sat, 13 Jan 2024 17:39:31 +0200 Subject: fix(debian/rules): install in offline mode Signed-off-by: Lars Wirzenius Sponsored-by: author --- debian/rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/rules b/debian/rules index e3d7fd8..5a9e628 100755 --- a/debian/rules +++ b/debian/rules @@ -7,7 +7,7 @@ override_dh_auto_build: true override_dh_auto_install: - cargo install --path=. --root=debian/ewww + cargo install --path=. --root=debian/ewww --offline install -d debian/ewww/lib/systemd/system install -m 0644 ewww.service debian/ewww/lib/systemd/system/ewww.service rm -f debian/ewww/.crates.toml -- cgit v1.2.1