refactor: move ancillary subplot files to subplot/

This way, the root of the source tree is less cluttered.

I'm leaving the subplot.md file in the root, though, since it's meant
to be more visible, more "in your face".

4 files changed, 325 insertions, 0 deletions

diff --git a/subplot/daemon.py b/subplot/daemon.py
new file mode 100644
index 0000000..e223505
--- /dev/null
+++ b/subplot/daemon.py
@@ -0,0 +1,84 @@
+#############################################################################
+# Start and stop daemons, or background processes.
+
+
+import logging
+import os
+import signal
+import time
+
+
+# Start a process in the background.
+def start_daemon(ctx, name, argv):
+    runcmd = globals()["runcmd"]
+    exit_code_is = globals()["exit_code_is"]
+
+    logging.debug(f"Starting daemon {name}")
+    logging.debug(f"  ctx={ctx.as_dict()}")
+    logging.debug(f"  name={name}")
+    logging.debug(f"  argv={argv}")
+
+    if "daemon" not in ctx.as_dict():
+        ctx["daemon"] = {}
+    assert name not in ctx["daemon"]
+    this = ctx["daemon"][name] = {
+        "pid-file": f"{name}.pid",
+        "stderr": f"{name}.stderr",
+        "stdout": f"{name}.stdout",
+    }
+    runcmd(
+        ctx,
+        [
+            "/usr/sbin/daemonize",
+            "-c",
+            os.getcwd(),
+            "-p",
+            this["pid-file"],
+            "-e",
+            this["stderr"],
+            "-o",
+            this["stdout"],
+        ]
+        + argv,
+    )
+
+    # Wait for a bit for daemon to start and maybe find a problem and die.
+    time.sleep(3)
+    if ctx["exit"] != 0:
+        logging.error(f"obnam-server stderr: {ctx['stderr']}")
+
+    exit_code_is(ctx, 0)
+    this["pid"] = int(open(this["pid-file"]).read().strip())
+    assert process_exists(this["pid"])
+
+    logging.debug(f"Started daemon {name}")
+    logging.debug(f"  ctx={ctx.as_dict()}")
+
+
+# Stop a daemon.
+def stop_daemon(ctx, name):
+    logging.debug(f"Stopping daemon {name}")
+    logging.debug(f"  ctx={ctx.as_dict()}")
+    logging.debug(f"  ctx['daemon']={ctx.as_dict()['daemon']}")
+
+    this = ctx["daemon"][name]
+    terminate_process(this["pid"], signal.SIGKILL)
+
+
+# Does a process exist?
+def process_exists(pid):
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    return True
+
+
+# Terminate process.
+def terminate_process(pid, signalno):
+    logging.debug(f"Terminating process {pid} with signal {signalno}")
+    try:
+        os.kill(pid, signalno)
+    except ProcessLookupError:
+        logging.debug("Process did not actually exist (anymore?)")
+        pass
diff --git a/subplot/obnam.py b/subplot/obnam.py
new file mode 100644
index 0000000..c827180
--- /dev/null
+++ b/subplot/obnam.py
@@ -0,0 +1,134 @@
+import logging
+import os
+import random
+import requests
+import shutil
+import socket
+import time
+import urllib3
+import yaml
+
+
+urllib3.disable_warnings()
+
+
+def start_chunk_server(ctx):
+    start_daemon = globals()["start_daemon"]
+    srcdir = globals()["srcdir"]
+
+    logging.debug(f"Starting obnam-server")
+
+    for x in ["test.pem", "test.key"]:
+        shutil.copy(os.path.join(srcdir, x), x)
+
+    chunks = "chunks"
+    os.mkdir(chunks)
+
+    config = {"chunks": chunks, "tls_key": "test.key", "tls_cert": "test.pem"}
+    port = config["port"] = random.randint(2000, 30000)
+    filename = "config.yaml"
+    yaml.safe_dump(config, stream=open(filename, "w"))
+    logging.debug(f"Picked randomly port for obnam-server: {config['port']}")
+    ctx["config"] = config
+
+    ctx["url"] = f"https://localhost:{port}"
+
+    start_daemon(ctx, "obnam-server", [_binary("obnam-server"), filename])
+
+    if not port_open("localhost", port, 5.0):
+        stderr = open(ctx["daemon"]["obnam-server"]["stderr"]).read()
+        logging.debug(f"Stderr from daemon: {stderr!r}")
+
+
+def stop_chunk_server(ctx):
+    logging.debug("Stopping obnam-server")
+    stop_daemon = globals()["stop_daemon"]
+    stop_daemon(ctx, "obnam-server")
+
+
+def create_file_with_random_data(ctx, filename=None):
+    N = 128
+    data = "".join(chr(random.randint(0, 255)) for i in range(N)).encode("UTF-8")
+    with open(filename, "wb") as f:
+        f.write(data)
+
+
+def post_file(ctx, filename=None, path=None, header=None, json=None):
+    url = f"{ctx['url']}/chunks"
+    headers = {header: json}
+    data = open(filename, "rb").read()
+    _request(ctx, requests.post, url, headers=headers, data=data)
+
+
+def get_chunk(ctx, var=None):
+    chunk_id = ctx["vars"][var]
+    url = f"{ctx['url']}/chunks/{chunk_id}"
+    _request(ctx, requests.get, url)
+
+
+def status_code_is(ctx, status=None):
+    assert_eq = globals()["assert_eq"]
+    assert_eq(ctx["http.status"], int(status))
+
+
+def header_is(ctx, header=None, value=None):
+    assert_eq = globals()["assert_eq"]
+    assert_eq(ctx["http.headers"][header], value)
+
+
+def remember_json_field(ctx, field=None, var=None):
+    v = ctx.get("vars", {})
+    v[var] = ctx["http.json"][field]
+    ctx["vars"] = v
+
+
+def body_matches_file(ctx, filename=None):
+    assert_eq = globals()["assert_eq"]
+    content = open(filename, "rb").read()
+    logging.debug(f"body_matches_file:")
+    logging.debug(f"  filename: {filename}")
+    logging.debug(f"  content: {content!r}")
+    logging.debug(f"  body: {ctx['http.raw']!r}")
+    assert_eq(ctx["http.raw"], content)
+
+
+# Name of Rust binary, debug-build.
+def _binary(name):
+    srcdir = globals()["srcdir"]
+    return os.path.abspath(os.path.join(srcdir, "target", "debug", name))
+
+
+# Wait for a port to be open
+def port_open(host, port, timeout):
+    logging.debug(f"Waiting for port localhost:{port} to be available")
+    started = time.time()
+    while time.time() < started + timeout:
+        try:
+            socket.create_connection((host, port), timeout=timeout)
+            return True
+        except socket.error:
+            pass
+    logging.error(f"Port localhost:{port} is not open")
+    return False
+
+
+# Make an HTTP request.
+def _request(ctx, method, url, headers=None, data=None):
+    r = method(url, headers=headers, data=data, verify=False)
+    ctx["http.status"] = r.status_code
+    ctx["http.headers"] = dict(r.headers)
+    try:
+        ctx["http.json"] = dict(r.json())
+    except ValueError:
+        ctx["http.json"] = None
+    ctx["http.raw"] = r.content
+    logging.debug("HTTP request:")
+    logging.debug(f"  url: {url}")
+    logging.debug(f"  header: {headers!r}")
+    logging.debug("HTTP response:")
+    logging.debug(f"  status: {r.status_code}")
+    logging.debug(f"  json: {ctx['http.json']!r}")
+    logging.debug(f"  text: {r.content!r}")
+    if not r.ok:
+        stderr = open(ctx["daemon"]["obnam-server"]["stderr"], "rb").read()
+        logging.debug(f"  server stderr: {stderr!r}")
diff --git a/subplot/obnam.yaml b/subplot/obnam.yaml
new file mode 100644
index 0000000..7acf581
--- /dev/null
+++ b/subplot/obnam.yaml
@@ -0,0 +1,30 @@
+- given: "a chunk server"
+  function: start_chunk_server
+  cleanup: stop_chunk_server
+
+- given: >
+    a file (?P<filename>\\S+) containing "(?P<data>.*)"
+  regex: true
+  function: create_file_with_given_data
+
+- given: "a file {filename} containing some random data"
+  function: create_file_with_random_data
+
+- when: "I POST (?P<filename>\\S+) to (?P<path>\\S+), with (?P<header>\\S+): (?P<json>.*)"
+  regex: true
+  function: post_file
+
+- when: "I GET /chunks/<{var}>"
+  function: get_chunk
+
+- then: "HTTP status code is {status}"
+  function: status_code_is
+
+- then: "{header} is {value}"
+  function: header_is
+
+- then: "the JSON body has a field {field}, henceforth {var}"
+  function: remember_json_field
+
+- then: "the body matches file {filename}"
+  function: body_matches_file
diff --git a/subplot/runcmd.py b/subplot/runcmd.py
new file mode 100644
index 0000000..7193c15
--- /dev/null
+++ b/subplot/runcmd.py
@@ -0,0 +1,77 @@
+# Some step implementations for running commands and capturing the result.
+
+import subprocess
+
+
+# Run a command, capture its stdout, stderr, and exit code in context.
+def runcmd(ctx, argv, **kwargs):
+    p = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
+    stdout, stderr = p.communicate("")
+    ctx["argv"] = argv
+    ctx["stdout"] = stdout.decode("utf-8")
+    ctx["stderr"] = stderr.decode("utf-8")
+    ctx["exit"] = p.returncode
+
+
+# Check that latest exit code captured by runcmd was a specific one.
+def exit_code_is(ctx, wanted):
+    if ctx.get("exit") != wanted:
+        print("context:", ctx.as_dict())
+    assert_eq(ctx.get("exit"), wanted)
+
+
+# Check that latest exit code captured by runcmd was not a specific one.
+def exit_code_is_not(ctx, unwanted):
+    if ctx.get("exit") == unwanted:
+        print("context:", ctx.as_dict())
+    assert_ne(ctx.get("exit"), unwanted)
+
+
+# Check that latest exit code captured by runcmd was zero.
+def exit_code_zero(ctx):
+    exit_code_is(ctx, 0)
+
+
+# Check that latest exit code captured by runcmd was not zero.
+def exit_code_nonzero(ctx):
+    exit_code_is_not(ctx, 0)
+
+
+# Check that stdout of latest runcmd contains a specific string.
+def stdout_contains(ctx, pattern=None):
+    stdout = ctx.get("stdout", "")
+    if pattern not in stdout:
+        print("pattern:", repr(pattern))
+        print("stdout:", repr(stdout))
+        print("ctx:", ctx.as_dict())
+    assert_eq(pattern in stdout, True)
+
+
+# Check that stdout of latest runcmd does not contain a specific string.
+def stdout_does_not_contain(ctx, pattern=None):
+    stdout = ctx.get("stdout", "")
+    if pattern in stdout:
+        print("pattern:", repr(pattern))
+        print("stdout:", repr(stdout))
+        print("ctx:", ctx.as_dict())
+    assert_eq(pattern not in stdout, True)
+
+
+# Check that stderr of latest runcmd does contains a specific string.
+def stderr_contains(ctx, pattern=None):
+    stderr = ctx.get("stderr", "")
+    if pattern not in stderr:
+        print("pattern:", repr(pattern))
+        print("stderr:", repr(stderr))
+        print("ctx:", ctx.as_dict())
+    assert_eq(pattern in stderr, True)
+
+
+# Check that stderr of latest runcmd does not contain a specific string.
+def stderr_does_not_contain(ctx, pattern=None):
+    stderr = ctx.get("stderr", "")
+    if pattern not in stderr:
+        print("pattern:", repr(pattern))
+        print("stderr:", repr(stderr))
+        print("ctx:", ctx.as_dict())
+    assert_eq(pattern not in stderr, True)