From 8c4729010511d7bd491675522378507c2af0b583 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 31 Dec 2020 11:02:32 +0200 Subject: chore: update vendored subplot librarires: runcmd, daemon --- obnam.md | 2 +- subplot/daemon.py | 92 ------------------------- subplot/server.py | 37 ++++------ subplot/vendored/daemon.md | 38 +++++++++++ subplot/vendored/daemon.py | 139 +++++++++++++++++++++++++++++++++++++ subplot/vendored/daemon.yaml | 17 +++++ subplot/vendored/files.md | 82 ++++++++++++++++++++++ subplot/vendored/files.py | 158 +++++++++++++++++++++++++++++++++++++++++++ subplot/vendored/files.yaml | 62 +++++++++++++++++ subplot/vendored/runcmd.md | 1 + 10 files changed, 510 insertions(+), 118 deletions(-) delete mode 100644 subplot/daemon.py create mode 100644 subplot/vendored/daemon.md create mode 100644 subplot/vendored/daemon.py create mode 100644 subplot/vendored/daemon.yaml create mode 100644 subplot/vendored/files.md create mode 100644 subplot/vendored/files.py create mode 100644 subplot/vendored/files.yaml diff --git a/obnam.md b/obnam.md index c3b03a8..e877e4c 100644 --- a/obnam.md +++ b/obnam.md @@ -939,7 +939,7 @@ functions: - subplot/server.py - subplot/client.py - subplot/data.py - - subplot/daemon.py + - subplot/vendored/daemon.py - subplot/vendored/runcmd.py classes: - json diff --git a/subplot/daemon.py b/subplot/daemon.py deleted file mode 100644 index 4d60fd0..0000000 --- a/subplot/daemon.py +++ /dev/null @@ -1,92 +0,0 @@ -############################################################################# -# Start and stop daemons, or background processes. - - -import logging -import os -import signal -import time - - -# Start a process in the background. -def start_daemon(ctx, name, argv): - runcmd_run = globals()["runcmd_run"] - runcmd_exit_code_is = globals()["runcmd_exit_code_is"] - runcmd_get_exit_code = globals()["runcmd_get_exit_code"] - runcmd_get_stderr = globals()["runcmd_get_stderr"] - runcmd_prepend_to_path = globals()["runcmd_prepend_to_path"] - - logging.debug(f"Starting daemon {name}") - logging.debug(f" ctx={ctx.as_dict()}") - logging.debug(f" name={name}") - logging.debug(f" argv={argv}") - - if "daemon" not in ctx.as_dict(): - ctx["daemon"] = {} - assert name not in ctx["daemon"] - this = ctx["daemon"][name] = { - "pid-file": f"{name}.pid", - "stderr": f"{name}.stderr", - "stdout": f"{name}.stdout", - } - # Debian up to 10 installs `daemonize` to /usr/sbin, which isn't part of - # the minimal environment that Subplot sets up. - runcmd_prepend_to_path(ctx, "/usr/sbin") - runcmd_run( - ctx, - [ - "daemonize", - "-c", - os.getcwd(), - "-p", - this["pid-file"], - "-e", - this["stderr"], - "-o", - this["stdout"], - ] - + argv, - ) - - # Wait for a bit for daemon to start and maybe find a problem and die. - time.sleep(3) - exit = runcmd_get_exit_code(ctx) - stderr = runcmd_get_stderr(ctx) - if exit != 0: - logging.error(f"obnam-server stderr: {stderr}") - - runcmd_exit_code_is(ctx, 0) - this["pid"] = int(open(this["pid-file"]).read().strip()) - assert process_exists(this["pid"]) - - logging.debug(f"Started daemon {name}") - logging.debug(f" ctx={ctx.as_dict()}") - - -# Stop a daemon. -def stop_daemon(ctx, name): - logging.debug(f"Stopping daemon {name}") - logging.debug(f" ctx={ctx.as_dict()}") - logging.debug(f" ctx['daemon']={ctx.as_dict()['daemon']}") - - this = ctx["daemon"][name] - terminate_process(this["pid"], signal.SIGKILL) - - -# Does a process exist? -def process_exists(pid): - try: - os.kill(pid, 0) - except ProcessLookupError: - return False - return True - - -# Terminate process. -def terminate_process(pid, signalno): - logging.debug(f"Terminating process {pid} with signal {signalno}") - try: - os.kill(pid, signalno) - except ProcessLookupError: - logging.debug("Process did not actually exist (anymore?)") - pass diff --git a/subplot/server.py b/subplot/server.py index b07e944..dca7f10 100644 --- a/subplot/server.py +++ b/subplot/server.py @@ -15,7 +15,7 @@ urllib3.disable_warnings() def start_chunk_server(ctx): - start_daemon = globals()["start_daemon"] + daemon_start_on_port = globals()["daemon_start_on_port"] srcdir = globals()["srcdir"] logging.debug(f"Starting obnam-server") @@ -34,27 +34,27 @@ def start_chunk_server(ctx): "address": f"localhost:{port}", } + server_binary = os.path.abspath(os.path.join(srcdir, "target", "debug", "obnam-server")) + filename = "config.yaml" yaml.safe_dump(config, stream=open(filename, "w")) logging.debug(f"Picked randomly port for obnam-server: {config['address']}") ctx["server_url"] = f"https://{config['address']}" - start_daemon( + daemon_start_on_port( ctx, - "obnam-server", - [os.path.join(srcdir, "target", "debug", "obnam-server"), filename], + name="obnam-server", + path=server_binary, + args=filename, + port=port, ) - if not port_open("localhost", port, 5.0): - stderr = open(ctx["daemon"]["obnam-server"]["stderr"]).read() - logging.debug(f"Stderr from daemon: {stderr!r}") - def stop_chunk_server(ctx): logging.debug("Stopping obnam-server") - stop_daemon = globals()["stop_daemon"] - stop_daemon(ctx, "obnam-server") + daemon_stop = globals()["daemon_stop"] + daemon_stop(ctx, name="obnam-server") def post_file(ctx, filename=None, path=None, header=None, json=None): @@ -127,20 +127,6 @@ def json_body_matches(ctx, wanted=None): assert_eq(body.get(key, "not.there"), wanted[key]) -# Wait for a port to be open -def port_open(host, port, timeout): - logging.debug(f"Waiting for port localhost:{port} to be available") - started = time.time() - while time.time() < started + timeout: - try: - socket.create_connection((host, port), timeout=timeout) - return True - except socket.error: - pass - logging.error(f"Port localhost:{port} is not open") - return False - - # Make an HTTP request. def _request(ctx, method, url, headers=None, data=None): r = method(url, headers=headers, data=data, verify=False) @@ -159,7 +145,8 @@ def _request(ctx, method, url, headers=None, data=None): logging.debug(f" json: {ctx['http.json']!r}") logging.debug(f" text: {r.content!r}") if not r.ok: - stderr = open(ctx["daemon"]["obnam-server"]["stderr"], "rb").read() + daemon = ctx.declare("_daemon") + stderr = open(daemon["obnam-server"]["stderr"], "rb").read() logging.debug(f" server stderr: {stderr!r}") diff --git a/subplot/vendored/daemon.md b/subplot/vendored/daemon.md new file mode 100644 index 0000000..131dcb1 --- /dev/null +++ b/subplot/vendored/daemon.md @@ -0,0 +1,38 @@ +# Introduction + +The [Subplot][] library `daemon` for Python provides scenario steps +and their implementations for running a background process and +terminating at the end of the scenario. + +[Subplot]: https://subplot.liw.fi/ + +This document explains the acceptance criteria for the library and how +they're verified. It uses the steps and functions from the +`lib/daemon` library. The scenarios all have the same structure: run a +command, then examine the exit code, verify the process is running. + +# Daemon is started and terminated + +This scenario starts a background process, verifies it's started, and +verifies it's terminated after the scenario ends. + +~~~scenario +given there is no "/bin/sleep 12765" process +when I start "/bin/sleep 12765" as a background process as sleepyhead +then a process "/bin/sleep 12765" is running +when I stop background process sleepyhead +then there is no "/bin/sleep 12765" process +~~~ + + + +--- +title: Acceptance criteria for the lib/daemon Subplot library +author: The Subplot project +bindings: +- daemon.yaml +template: python +functions: +- daemon.py +- runcmd.py +... diff --git a/subplot/vendored/daemon.py b/subplot/vendored/daemon.py new file mode 100644 index 0000000..febf392 --- /dev/null +++ b/subplot/vendored/daemon.py @@ -0,0 +1,139 @@ +import logging +import os +import signal +import socket +import subprocess +import time + + +# Start a daemon that will open a port on localhost. +def daemon_start_on_port(ctx, path=None, args=None, name=None, port=None): + daemon_start(ctx, path=path, args=args, name=name) + daemon_wait_for_port("localhost", port) + + +# Start a daeamon, get its PID. Don't wait for a port or anything. This is +# meant for background processes that don't have port. Useful for testing the +# lib/daemon library of Subplot, but not much else. +def daemon_start(ctx, path=None, args=None, name=None): + runcmd_run = globals()["runcmd_run"] + runcmd_exit_code_is = globals()["runcmd_exit_code_is"] + runcmd_get_exit_code = globals()["runcmd_get_exit_code"] + runcmd_get_stderr = globals()["runcmd_get_stderr"] + runcmd_prepend_to_path = globals()["runcmd_prepend_to_path"] + + argv = [path] + args.split() + + logging.debug(f"Starting daemon {name}") + logging.debug(f" ctx={ctx.as_dict()}") + logging.debug(f" name={name}") + logging.debug(f" path={path}") + logging.debug(f" args={args}") + logging.debug(f" argv={argv}") + + ns = ctx.declare("_daemon") + + this = ns[name] = { + "pid-file": f"{name}.pid", + "stderr": f"{name}.stderr", + "stdout": f"{name}.stdout", + } + + # Debian installs `daemonize` to /usr/sbin, which isn't part of the minimal + # environment that Subplot sets up. So we add /usr/sbin to the PATH. + runcmd_prepend_to_path(ctx, "/usr/sbin") + runcmd_run( + ctx, + [ + "daemonize", + "-c", + os.getcwd(), + "-p", + this["pid-file"], + "-e", + this["stderr"], + "-o", + this["stdout"], + ] + + argv, + ) + + # Check that daemonize has exited OK. If it hasn't, it didn't start the + # background process at all. If so, log the stderr in case there was + # something useful there for debugging. + exit = runcmd_get_exit_code(ctx) + if exit != 0: + stderr = runcmd_get_stderr(ctx) + logging.error(f"daemon {name} stderr: {stderr}") + runcmd_exit_code_is(ctx, 0) + + # Get the pid of the background process, from the pid file created by + # daemonize. We don't need to wait for it, since we know daemonize already + # exited. If it isn't there now, it's won't appear later. + if not os.path.exists(this["pid-file"]): + raise Exception("daemonize didn't create a PID file") + + this["pid"] = _daemon_wait_for_pid(this["pid-file"], 10.0) + + logging.debug(f"Started daemon {name}") + logging.debug(f" pid={this['pid']}") + logging.debug(f" ctx={ctx.as_dict()}") + + +def _daemon_wait_for_pid(filename, timeout): + start = time.time() + while time.time() < start + timeout: + with open(filename) as f: + data = f.read().strip() + if data: + return int(data) + raise Exception("daemonize created a PID file without a PID") + + +def daemon_wait_for_port(host, port, timeout=3.0): + addr = (host, port) + until = time.time() + timeout + while True: + try: + s = socket.create_connection(addr, timeout=timeout) + s.close() + return + except socket.timeout: + logging.error(f"daemon did not respond at port {port} within {timeout} seconds") + raise + except socket.error as e: + logging.info(f"could not connect to daemon at {port}: {e}") + pass + if time.time() >= until: + logging.error(f"could not connect to daemon at {port} within {timeout} seconds") + raise ConnectionRefusedError() + + +# Stop a daemon. +def daemon_stop(ctx, name=None): + logging.debug(f"Stopping daemon {name}") + ns = ctx.declare("_daemon") + logging.debug(f" ns={ns}") + pid = ns[name]["pid"] + signo = signal.SIGKILL + + logging.debug(f"Terminating process {pid} with signal {signo}") + try: + os.kill(pid, signo) + except ProcessLookupError: + logging.warning("Process did not actually exist (anymore?)") + + +def daemon_no_such_process(ctx, args=None): + assert not _daemon_pgrep(args) + + +def daemon_process_exists(ctx, args=None): + assert _daemon_pgrep(args) + + +def _daemon_pgrep(pattern): + logging.info(f"checking if process exists: pattern={pattern}") + exit = subprocess.call(["pgrep", "-laf", pattern]) + logging.info(f"exit code: {exit}") + return exit == 0 diff --git a/subplot/vendored/daemon.yaml b/subplot/vendored/daemon.yaml new file mode 100644 index 0000000..6165c62 --- /dev/null +++ b/subplot/vendored/daemon.yaml @@ -0,0 +1,17 @@ +- given: there is no "{args:text}" process + function: daemon_no_such_process + +- when: I start "{path}{args:text}" as a background process as {name}, on port {port} + function: daemon_start_on_port + +- when: I start "{path}{args:text}" as a background process as {name} + function: daemon_start + +- when: I stop background process {name} + function: daemon_stop + +- then: a process "{args:text}" is running + function: daemon_process_exists + +- then: there is no "{args:text}" process + function: daemon_no_such_process diff --git a/subplot/vendored/files.md b/subplot/vendored/files.md new file mode 100644 index 0000000..68ef1ac --- /dev/null +++ b/subplot/vendored/files.md @@ -0,0 +1,82 @@ +# Introduction + +The [Subplot][] library `files` provides scenario steps and their +implementations for managing files on the file system during tests. +The library consists of a bindings file `lib/files.yaml` and +implementations in Python in `lib/files.py`. + +[Subplot]: https://subplot.liw.fi/ + +This document explains the acceptance criteria for the library and how +they're verified. It uses the steps and functions from the `files` +library. + +# Create on-disk files from embedded files + +Subplot allows the source document to embed test files, and the +`files` library provides steps to create real, on-disk files from +the embedded files. + +~~~scenario +given file hello.txt +then file hello.txt exists +and file hello.txt contains "hello, world" +and file other.txt does not exist +given file other.txt from hello.txt +then file other.txt exists +and files hello.txt and other.txt match +and only files hello.txt, other.txt exist +~~~ + +~~~{#hello.txt .file .numberLines} +hello, world +~~~ + + +# File metadata + +These steps create files and manage their metadata. + +~~~scenario +given file hello.txt +when I remember metadata for file hello.txt +then file hello.txt has same metadata as before + +when I write "yo" to file hello.txt +then file hello.txt has different metadata from before +~~~ + +# File modification time + +These steps manipulate and test file modification times. + +~~~scenario +given file foo.dat has modification time 1970-01-02 03:04:05 +then file foo.dat has a very old modification time + +when I touch file foo.dat +then file foo.dat has a very recent modification time +~~~ + + +# File contents + +These steps verify contents of files. + +~~~scenario +given file hello.txt +then file hello.txt contains "hello, world" +and file hello.txt matches regex "hello, .*" +and file hello.txt matches regex /hello, .*/ +~~~ + + +--- +title: Acceptance criteria for the files Subplot library +author: The Subplot project +template: python +bindings: +- files.yaml +functions: +- files.py +... diff --git a/subplot/vendored/files.py b/subplot/vendored/files.py new file mode 100644 index 0000000..ec37b9d --- /dev/null +++ b/subplot/vendored/files.py @@ -0,0 +1,158 @@ +import logging +import os +import re +import time + + +def files_create_from_embedded(ctx, filename=None): + files_create_from_embedded_with_other_name( + ctx, filename_on_disk=filename, embedded_filename=filename + ) + + +def files_create_from_embedded_with_other_name( + ctx, filename_on_disk=None, embedded_filename=None +): + get_file = globals()["get_file"] + with open(filename_on_disk, "wb") as f: + f.write(get_file(embedded_filename)) + + +def files_create_from_text(ctx, filename=None, text=None): + with open(filename, "w") as f: + f.write(text) + + +def files_file_exists(ctx, filename=None): + assert_eq = globals()["assert_eq"] + assert_eq(os.path.exists(filename), True) + + +def files_file_does_not_exist(ctx, filename=None): + assert_eq = globals()["assert_eq"] + assert_eq(os.path.exists(filename), False) + + +def files_only_these_exist(ctx, filenames=None): + assert_eq = globals()["assert_eq"] + filenames = filenames.replace(",", "").split() + assert_eq(set(os.listdir(".")), set(filenames)) + + +def files_file_contains(ctx, filename=None, data=None): + assert_eq = globals()["assert_eq"] + with open(filename, "rb") as f: + actual = f.read() + actual = actual.decode("UTF-8") + assert_eq(data in actual, True) + + +def files_file_matches_regex(ctx, filename=None, regex=None): + assert_eq = globals()["assert_eq"] + with open(filename) as f: + content = f.read() + m = re.search(regex, content) + if m is None: + logging.debug(f"files_file_matches_regex: no match") + logging.debug(f" filenamed: {filename}") + logging.debug(f" regex: {regex}") + logging.debug(f" content: {regex}") + logging.debug(f" match: {m}") + assert_eq(bool(m), True) + + +def files_match(ctx, filename1=None, filename2=None): + assert_eq = globals()["assert_eq"] + with open(filename1, "rb") as f: + data1 = f.read() + with open(filename2, "rb") as f: + data2 = f.read() + assert_eq(data1, data2) + + +def files_touch_with_timestamp( + ctx, + filename=None, + year=None, + month=None, + day=None, + hour=None, + minute=None, + second=None, +): + t = ( + int(year), + int(month), + int(day), + int(hour), + int(minute), + int(second), + -1, + -1, + -1, + ) + ts = time.mktime(t) + _files_touch(filename, ts) + + +def files_touch(ctx, filename=None): + _files_touch(filename, None) + + +def _files_touch(filename, ts): + if not os.path.exists(filename): + open(filename, "w").close() + times = None + if ts is not None: + times = (ts, ts) + os.utime(filename, times=times) + + +def files_mtime_is_recent(ctx, filename=None): + st = os.stat(filename) + age = abs(st.st_mtime - time.time()) + assert age < 1.0 + + +def files_mtime_is_ancient(ctx, filename=None): + st = os.stat(filename) + age = abs(st.st_mtime - time.time()) + year = 365 * 24 * 60 * 60 + required = 39 * year + logging.debug(f"ancient? mtime={st.st_mtime} age={age} required={required}") + assert age > required + + +def files_remember_metadata(ctx, filename=None): + meta = _files_remembered(ctx) + meta[filename] = _files_get_metadata(filename) + logging.debug("files_remember_metadata:") + logging.debug(f" meta: {meta}") + logging.debug(f" ctx: {ctx}") + + +# Check that current metadata of a file is as stored in the context. +def files_has_remembered_metadata(ctx, filename=None): + assert_eq = globals()["assert_eq"] + meta = _files_remembered(ctx) + logging.debug("files_has_remembered_metadata:") + logging.debug(f" meta: {meta}") + logging.debug(f" ctx: {ctx}") + assert_eq(meta[filename], _files_get_metadata(filename)) + + +def files_has_different_metadata(ctx, filename=None): + assert_ne = globals()["assert_ne"] + meta = _files_remembered(ctx) + assert_ne(meta[filename], _files_get_metadata(filename)) + + +def _files_remembered(ctx): + ns = ctx.declare("_files") + return ns.get("remembered-metadata", {}) + + +def _files_get_metadata(filename): + st = os.lstat(filename) + keys = ["st_dev", "st_gid", "st_ino", "st_mode", "st_mtime", "st_size", "st_uid"] + return {key: getattr(st, key) for key in keys} diff --git a/subplot/vendored/files.yaml b/subplot/vendored/files.yaml new file mode 100644 index 0000000..be69920 --- /dev/null +++ b/subplot/vendored/files.yaml @@ -0,0 +1,62 @@ +- given: file {filename} + function: files_create_from_embedded + types: + filename: file + +- given: file {filename_on_disk} from {embedded_filename} + function: files_create_from_embedded_with_other_name + types: + embedded_filename: file + +- given: file {filename} has modification time {year}-{month}-{day} {hour}:{minute}:{second} + function: files_touch_with_timestamp + +- when: I write "(?P.*)" to file (?P\S+) + regex: true + function: files_create_from_text + +- when: I remember metadata for file {filename} + function: files_remember_metadata + +- when: I touch file {filename} + function: files_touch + +- then: file {filename} exists + function: files_file_exists + +- then: file {filename} does not exist + function: files_file_does_not_exist + +- then: only files (?P.+) exist + function: files_only_these_exist + regex: true + +- then: file (?P\S+) contains "(?P.*)" + regex: true + function: files_file_contains + +- then: file (?P\S+) matches regex /(?P.*)/ + regex: true + function: files_file_matches_regex + +- then: file (?P\S+) matches regex "(?P.*)" + regex: true + function: files_file_matches_regex + +- then: files {filename1} and {filename2} match + function: files_match + +- then: file {filename} has same metadata as before + function: files_has_remembered_metadata + +- then: file {filename} has different metadata from before + function: files_has_different_metadata + +- then: file {filename} has changed from before + function: files_has_different_metadata + +- then: file {filename} has a very recent modification time + function: files_mtime_is_recent + +- then: file {filename} has a very old modification time + function: files_mtime_is_ancient diff --git a/subplot/vendored/runcmd.md b/subplot/vendored/runcmd.md index bb42005..a9d4ed4 100644 --- a/subplot/vendored/runcmd.md +++ b/subplot/vendored/runcmd.md @@ -162,6 +162,7 @@ then stderr doesn't match regex world$ --- title: Acceptance criteria for the lib/runcmd Subplot library author: The Subplot project +template: python bindings: - runcmd.yaml functions: -- cgit v1.2.1