summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLars Wirzenius <liw@liw.fi>2020-12-31 11:02:32 +0200
committerLars Wirzenius <liw@liw.fi>2020-12-31 11:02:32 +0200
commit8c4729010511d7bd491675522378507c2af0b583 (patch)
tree2cbff9a84d0c401828ebd7c9129068600b3cb9ea
parent89c2c522682ea0490f3512e43bf855b254cd924f (diff)
downloadobnam2-8c4729010511d7bd491675522378507c2af0b583.tar.gz
chore: update vendored subplot librarires: runcmd, daemon
-rw-r--r--obnam.md2
-rw-r--r--subplot/daemon.py92
-rw-r--r--subplot/server.py37
-rw-r--r--subplot/vendored/daemon.md38
-rw-r--r--subplot/vendored/daemon.py139
-rw-r--r--subplot/vendored/daemon.yaml17
-rw-r--r--subplot/vendored/files.md82
-rw-r--r--subplot/vendored/files.py158
-rw-r--r--subplot/vendored/files.yaml62
-rw-r--r--subplot/vendored/runcmd.md1
10 files changed, 510 insertions, 118 deletions
diff --git a/obnam.md b/obnam.md
index c3b03a8..e877e4c 100644
--- a/obnam.md
+++ b/obnam.md
@@ -939,7 +939,7 @@ functions:
- subplot/server.py
- subplot/client.py
- subplot/data.py
- - subplot/daemon.py
+ - subplot/vendored/daemon.py
- subplot/vendored/runcmd.py
classes:
- json
diff --git a/subplot/daemon.py b/subplot/daemon.py
deleted file mode 100644
index 4d60fd0..0000000
--- a/subplot/daemon.py
+++ /dev/null
@@ -1,92 +0,0 @@
-#############################################################################
-# Start and stop daemons, or background processes.
-
-
-import logging
-import os
-import signal
-import time
-
-
-# Start a process in the background.
-def start_daemon(ctx, name, argv):
- runcmd_run = globals()["runcmd_run"]
- runcmd_exit_code_is = globals()["runcmd_exit_code_is"]
- runcmd_get_exit_code = globals()["runcmd_get_exit_code"]
- runcmd_get_stderr = globals()["runcmd_get_stderr"]
- runcmd_prepend_to_path = globals()["runcmd_prepend_to_path"]
-
- logging.debug(f"Starting daemon {name}")
- logging.debug(f" ctx={ctx.as_dict()}")
- logging.debug(f" name={name}")
- logging.debug(f" argv={argv}")
-
- if "daemon" not in ctx.as_dict():
- ctx["daemon"] = {}
- assert name not in ctx["daemon"]
- this = ctx["daemon"][name] = {
- "pid-file": f"{name}.pid",
- "stderr": f"{name}.stderr",
- "stdout": f"{name}.stdout",
- }
- # Debian up to 10 installs `daemonize` to /usr/sbin, which isn't part of
- # the minimal environment that Subplot sets up.
- runcmd_prepend_to_path(ctx, "/usr/sbin")
- runcmd_run(
- ctx,
- [
- "daemonize",
- "-c",
- os.getcwd(),
- "-p",
- this["pid-file"],
- "-e",
- this["stderr"],
- "-o",
- this["stdout"],
- ]
- + argv,
- )
-
- # Wait for a bit for daemon to start and maybe find a problem and die.
- time.sleep(3)
- exit = runcmd_get_exit_code(ctx)
- stderr = runcmd_get_stderr(ctx)
- if exit != 0:
- logging.error(f"obnam-server stderr: {stderr}")
-
- runcmd_exit_code_is(ctx, 0)
- this["pid"] = int(open(this["pid-file"]).read().strip())
- assert process_exists(this["pid"])
-
- logging.debug(f"Started daemon {name}")
- logging.debug(f" ctx={ctx.as_dict()}")
-
-
-# Stop a daemon.
-def stop_daemon(ctx, name):
- logging.debug(f"Stopping daemon {name}")
- logging.debug(f" ctx={ctx.as_dict()}")
- logging.debug(f" ctx['daemon']={ctx.as_dict()['daemon']}")
-
- this = ctx["daemon"][name]
- terminate_process(this["pid"], signal.SIGKILL)
-
-
-# Does a process exist?
-def process_exists(pid):
- try:
- os.kill(pid, 0)
- except ProcessLookupError:
- return False
- return True
-
-
-# Terminate process.
-def terminate_process(pid, signalno):
- logging.debug(f"Terminating process {pid} with signal {signalno}")
- try:
- os.kill(pid, signalno)
- except ProcessLookupError:
- logging.debug("Process did not actually exist (anymore?)")
- pass
diff --git a/subplot/server.py b/subplot/server.py
index b07e944..dca7f10 100644
--- a/subplot/server.py
+++ b/subplot/server.py
@@ -15,7 +15,7 @@ urllib3.disable_warnings()
def start_chunk_server(ctx):
- start_daemon = globals()["start_daemon"]
+ daemon_start_on_port = globals()["daemon_start_on_port"]
srcdir = globals()["srcdir"]
logging.debug(f"Starting obnam-server")
@@ -34,27 +34,27 @@ def start_chunk_server(ctx):
"address": f"localhost:{port}",
}
+ server_binary = os.path.abspath(os.path.join(srcdir, "target", "debug", "obnam-server"))
+
filename = "config.yaml"
yaml.safe_dump(config, stream=open(filename, "w"))
logging.debug(f"Picked randomly port for obnam-server: {config['address']}")
ctx["server_url"] = f"https://{config['address']}"
- start_daemon(
+ daemon_start_on_port(
ctx,
- "obnam-server",
- [os.path.join(srcdir, "target", "debug", "obnam-server"), filename],
+ name="obnam-server",
+ path=server_binary,
+ args=filename,
+ port=port,
)
- if not port_open("localhost", port, 5.0):
- stderr = open(ctx["daemon"]["obnam-server"]["stderr"]).read()
- logging.debug(f"Stderr from daemon: {stderr!r}")
-
def stop_chunk_server(ctx):
logging.debug("Stopping obnam-server")
- stop_daemon = globals()["stop_daemon"]
- stop_daemon(ctx, "obnam-server")
+ daemon_stop = globals()["daemon_stop"]
+ daemon_stop(ctx, name="obnam-server")
def post_file(ctx, filename=None, path=None, header=None, json=None):
@@ -127,20 +127,6 @@ def json_body_matches(ctx, wanted=None):
assert_eq(body.get(key, "not.there"), wanted[key])
-# Wait for a port to be open
-def port_open(host, port, timeout):
- logging.debug(f"Waiting for port localhost:{port} to be available")
- started = time.time()
- while time.time() < started + timeout:
- try:
- socket.create_connection((host, port), timeout=timeout)
- return True
- except socket.error:
- pass
- logging.error(f"Port localhost:{port} is not open")
- return False
-
-
# Make an HTTP request.
def _request(ctx, method, url, headers=None, data=None):
r = method(url, headers=headers, data=data, verify=False)
@@ -159,7 +145,8 @@ def _request(ctx, method, url, headers=None, data=None):
logging.debug(f" json: {ctx['http.json']!r}")
logging.debug(f" text: {r.content!r}")
if not r.ok:
- stderr = open(ctx["daemon"]["obnam-server"]["stderr"], "rb").read()
+ daemon = ctx.declare("_daemon")
+ stderr = open(daemon["obnam-server"]["stderr"], "rb").read()
logging.debug(f" server stderr: {stderr!r}")
diff --git a/subplot/vendored/daemon.md b/subplot/vendored/daemon.md
new file mode 100644
index 0000000..131dcb1
--- /dev/null
+++ b/subplot/vendored/daemon.md
@@ -0,0 +1,38 @@
+# Introduction
+
+The [Subplot][] library `daemon` for Python provides scenario steps
+and their implementations for running a background process and
+terminating at the end of the scenario.
+
+[Subplot]: https://subplot.liw.fi/
+
+This document explains the acceptance criteria for the library and how
+they're verified. It uses the steps and functions from the
+`lib/daemon` library. The scenarios all have the same structure: run a
+command, then examine the exit code, verify the process is running.
+
+# Daemon is started and terminated
+
+This scenario starts a background process, verifies it's started, and
+verifies it's terminated after the scenario ends.
+
+~~~scenario
+given there is no "/bin/sleep 12765" process
+when I start "/bin/sleep 12765" as a background process as sleepyhead
+then a process "/bin/sleep 12765" is running
+when I stop background process sleepyhead
+then there is no "/bin/sleep 12765" process
+~~~
+
+
+
+---
+title: Acceptance criteria for the lib/daemon Subplot library
+author: The Subplot project
+bindings:
+- daemon.yaml
+template: python
+functions:
+- daemon.py
+- runcmd.py
+...
diff --git a/subplot/vendored/daemon.py b/subplot/vendored/daemon.py
new file mode 100644
index 0000000..febf392
--- /dev/null
+++ b/subplot/vendored/daemon.py
@@ -0,0 +1,139 @@
+import logging
+import os
+import signal
+import socket
+import subprocess
+import time
+
+
+# Start a daemon that will open a port on localhost.
+def daemon_start_on_port(ctx, path=None, args=None, name=None, port=None):
+ daemon_start(ctx, path=path, args=args, name=name)
+ daemon_wait_for_port("localhost", port)
+
+
+# Start a daeamon, get its PID. Don't wait for a port or anything. This is
+# meant for background processes that don't have port. Useful for testing the
+# lib/daemon library of Subplot, but not much else.
+def daemon_start(ctx, path=None, args=None, name=None):
+ runcmd_run = globals()["runcmd_run"]
+ runcmd_exit_code_is = globals()["runcmd_exit_code_is"]
+ runcmd_get_exit_code = globals()["runcmd_get_exit_code"]
+ runcmd_get_stderr = globals()["runcmd_get_stderr"]
+ runcmd_prepend_to_path = globals()["runcmd_prepend_to_path"]
+
+ argv = [path] + args.split()
+
+ logging.debug(f"Starting daemon {name}")
+ logging.debug(f" ctx={ctx.as_dict()}")
+ logging.debug(f" name={name}")
+ logging.debug(f" path={path}")
+ logging.debug(f" args={args}")
+ logging.debug(f" argv={argv}")
+
+ ns = ctx.declare("_daemon")
+
+ this = ns[name] = {
+ "pid-file": f"{name}.pid",
+ "stderr": f"{name}.stderr",
+ "stdout": f"{name}.stdout",
+ }
+
+ # Debian installs `daemonize` to /usr/sbin, which isn't part of the minimal
+ # environment that Subplot sets up. So we add /usr/sbin to the PATH.
+ runcmd_prepend_to_path(ctx, "/usr/sbin")
+ runcmd_run(
+ ctx,
+ [
+ "daemonize",
+ "-c",
+ os.getcwd(),
+ "-p",
+ this["pid-file"],
+ "-e",
+ this["stderr"],
+ "-o",
+ this["stdout"],
+ ]
+ + argv,
+ )
+
+ # Check that daemonize has exited OK. If it hasn't, it didn't start the
+ # background process at all. If so, log the stderr in case there was
+ # something useful there for debugging.
+ exit = runcmd_get_exit_code(ctx)
+ if exit != 0:
+ stderr = runcmd_get_stderr(ctx)
+ logging.error(f"daemon {name} stderr: {stderr}")
+ runcmd_exit_code_is(ctx, 0)
+
+ # Get the pid of the background process, from the pid file created by
+ # daemonize. We don't need to wait for it, since we know daemonize already
+ # exited. If it isn't there now, it's won't appear later.
+ if not os.path.exists(this["pid-file"]):
+ raise Exception("daemonize didn't create a PID file")
+
+ this["pid"] = _daemon_wait_for_pid(this["pid-file"], 10.0)
+
+ logging.debug(f"Started daemon {name}")
+ logging.debug(f" pid={this['pid']}")
+ logging.debug(f" ctx={ctx.as_dict()}")
+
+
+def _daemon_wait_for_pid(filename, timeout):
+ start = time.time()
+ while time.time() < start + timeout:
+ with open(filename) as f:
+ data = f.read().strip()
+ if data:
+ return int(data)
+ raise Exception("daemonize created a PID file without a PID")
+
+
+def daemon_wait_for_port(host, port, timeout=3.0):
+ addr = (host, port)
+ until = time.time() + timeout
+ while True:
+ try:
+ s = socket.create_connection(addr, timeout=timeout)
+ s.close()
+ return
+ except socket.timeout:
+ logging.error(f"daemon did not respond at port {port} within {timeout} seconds")
+ raise
+ except socket.error as e:
+ logging.info(f"could not connect to daemon at {port}: {e}")
+ pass
+ if time.time() >= until:
+ logging.error(f"could not connect to daemon at {port} within {timeout} seconds")
+ raise ConnectionRefusedError()
+
+
+# Stop a daemon.
+def daemon_stop(ctx, name=None):
+ logging.debug(f"Stopping daemon {name}")
+ ns = ctx.declare("_daemon")
+ logging.debug(f" ns={ns}")
+ pid = ns[name]["pid"]
+ signo = signal.SIGKILL
+
+ logging.debug(f"Terminating process {pid} with signal {signo}")
+ try:
+ os.kill(pid, signo)
+ except ProcessLookupError:
+ logging.warning("Process did not actually exist (anymore?)")
+
+
+def daemon_no_such_process(ctx, args=None):
+ assert not _daemon_pgrep(args)
+
+
+def daemon_process_exists(ctx, args=None):
+ assert _daemon_pgrep(args)
+
+
+def _daemon_pgrep(pattern):
+ logging.info(f"checking if process exists: pattern={pattern}")
+ exit = subprocess.call(["pgrep", "-laf", pattern])
+ logging.info(f"exit code: {exit}")
+ return exit == 0
diff --git a/subplot/vendored/daemon.yaml b/subplot/vendored/daemon.yaml
new file mode 100644
index 0000000..6165c62
--- /dev/null
+++ b/subplot/vendored/daemon.yaml
@@ -0,0 +1,17 @@
+- given: there is no "{args:text}" process
+ function: daemon_no_such_process
+
+- when: I start "{path}{args:text}" as a background process as {name}, on port {port}
+ function: daemon_start_on_port
+
+- when: I start "{path}{args:text}" as a background process as {name}
+ function: daemon_start
+
+- when: I stop background process {name}
+ function: daemon_stop
+
+- then: a process "{args:text}" is running
+ function: daemon_process_exists
+
+- then: there is no "{args:text}" process
+ function: daemon_no_such_process
diff --git a/subplot/vendored/files.md b/subplot/vendored/files.md
new file mode 100644
index 0000000..68ef1ac
--- /dev/null
+++ b/subplot/vendored/files.md
@@ -0,0 +1,82 @@
+# Introduction
+
+The [Subplot][] library `files` provides scenario steps and their
+implementations for managing files on the file system during tests.
+The library consists of a bindings file `lib/files.yaml` and
+implementations in Python in `lib/files.py`.
+
+[Subplot]: https://subplot.liw.fi/
+
+This document explains the acceptance criteria for the library and how
+they're verified. It uses the steps and functions from the `files`
+library.
+
+# Create on-disk files from embedded files
+
+Subplot allows the source document to embed test files, and the
+`files` library provides steps to create real, on-disk files from
+the embedded files.
+
+~~~scenario
+given file hello.txt
+then file hello.txt exists
+and file hello.txt contains "hello, world"
+and file other.txt does not exist
+given file other.txt from hello.txt
+then file other.txt exists
+and files hello.txt and other.txt match
+and only files hello.txt, other.txt exist
+~~~
+
+~~~{#hello.txt .file .numberLines}
+hello, world
+~~~
+
+
+# File metadata
+
+These steps create files and manage their metadata.
+
+~~~scenario
+given file hello.txt
+when I remember metadata for file hello.txt
+then file hello.txt has same metadata as before
+
+when I write "yo" to file hello.txt
+then file hello.txt has different metadata from before
+~~~
+
+# File modification time
+
+These steps manipulate and test file modification times.
+
+~~~scenario
+given file foo.dat has modification time 1970-01-02 03:04:05
+then file foo.dat has a very old modification time
+
+when I touch file foo.dat
+then file foo.dat has a very recent modification time
+~~~
+
+
+# File contents
+
+These steps verify contents of files.
+
+~~~scenario
+given file hello.txt
+then file hello.txt contains "hello, world"
+and file hello.txt matches regex "hello, .*"
+and file hello.txt matches regex /hello, .*/
+~~~
+
+
+---
+title: Acceptance criteria for the files Subplot library
+author: The Subplot project
+template: python
+bindings:
+- files.yaml
+functions:
+- files.py
+...
diff --git a/subplot/vendored/files.py b/subplot/vendored/files.py
new file mode 100644
index 0000000..ec37b9d
--- /dev/null
+++ b/subplot/vendored/files.py
@@ -0,0 +1,158 @@
+import logging
+import os
+import re
+import time
+
+
+def files_create_from_embedded(ctx, filename=None):
+ files_create_from_embedded_with_other_name(
+ ctx, filename_on_disk=filename, embedded_filename=filename
+ )
+
+
+def files_create_from_embedded_with_other_name(
+ ctx, filename_on_disk=None, embedded_filename=None
+):
+ get_file = globals()["get_file"]
+ with open(filename_on_disk, "wb") as f:
+ f.write(get_file(embedded_filename))
+
+
+def files_create_from_text(ctx, filename=None, text=None):
+ with open(filename, "w") as f:
+ f.write(text)
+
+
+def files_file_exists(ctx, filename=None):
+ assert_eq = globals()["assert_eq"]
+ assert_eq(os.path.exists(filename), True)
+
+
+def files_file_does_not_exist(ctx, filename=None):
+ assert_eq = globals()["assert_eq"]
+ assert_eq(os.path.exists(filename), False)
+
+
+def files_only_these_exist(ctx, filenames=None):
+ assert_eq = globals()["assert_eq"]
+ filenames = filenames.replace(",", "").split()
+ assert_eq(set(os.listdir(".")), set(filenames))
+
+
+def files_file_contains(ctx, filename=None, data=None):
+ assert_eq = globals()["assert_eq"]
+ with open(filename, "rb") as f:
+ actual = f.read()
+ actual = actual.decode("UTF-8")
+ assert_eq(data in actual, True)
+
+
+def files_file_matches_regex(ctx, filename=None, regex=None):
+ assert_eq = globals()["assert_eq"]
+ with open(filename) as f:
+ content = f.read()
+ m = re.search(regex, content)
+ if m is None:
+ logging.debug(f"files_file_matches_regex: no match")
+ logging.debug(f" filenamed: {filename}")
+ logging.debug(f" regex: {regex}")
+ logging.debug(f" content: {regex}")
+ logging.debug(f" match: {m}")
+ assert_eq(bool(m), True)
+
+
+def files_match(ctx, filename1=None, filename2=None):
+ assert_eq = globals()["assert_eq"]
+ with open(filename1, "rb") as f:
+ data1 = f.read()
+ with open(filename2, "rb") as f:
+ data2 = f.read()
+ assert_eq(data1, data2)
+
+
+def files_touch_with_timestamp(
+ ctx,
+ filename=None,
+ year=None,
+ month=None,
+ day=None,
+ hour=None,
+ minute=None,
+ second=None,
+):
+ t = (
+ int(year),
+ int(month),
+ int(day),
+ int(hour),
+ int(minute),
+ int(second),
+ -1,
+ -1,
+ -1,
+ )
+ ts = time.mktime(t)
+ _files_touch(filename, ts)
+
+
+def files_touch(ctx, filename=None):
+ _files_touch(filename, None)
+
+
+def _files_touch(filename, ts):
+ if not os.path.exists(filename):
+ open(filename, "w").close()
+ times = None
+ if ts is not None:
+ times = (ts, ts)
+ os.utime(filename, times=times)
+
+
+def files_mtime_is_recent(ctx, filename=None):
+ st = os.stat(filename)
+ age = abs(st.st_mtime - time.time())
+ assert age < 1.0
+
+
+def files_mtime_is_ancient(ctx, filename=None):
+ st = os.stat(filename)
+ age = abs(st.st_mtime - time.time())
+ year = 365 * 24 * 60 * 60
+ required = 39 * year
+ logging.debug(f"ancient? mtime={st.st_mtime} age={age} required={required}")
+ assert age > required
+
+
+def files_remember_metadata(ctx, filename=None):
+ meta = _files_remembered(ctx)
+ meta[filename] = _files_get_metadata(filename)
+ logging.debug("files_remember_metadata:")
+ logging.debug(f" meta: {meta}")
+ logging.debug(f" ctx: {ctx}")
+
+
+# Check that current metadata of a file is as stored in the context.
+def files_has_remembered_metadata(ctx, filename=None):
+ assert_eq = globals()["assert_eq"]
+ meta = _files_remembered(ctx)
+ logging.debug("files_has_remembered_metadata:")
+ logging.debug(f" meta: {meta}")
+ logging.debug(f" ctx: {ctx}")
+ assert_eq(meta[filename], _files_get_metadata(filename))
+
+
+def files_has_different_metadata(ctx, filename=None):
+ assert_ne = globals()["assert_ne"]
+ meta = _files_remembered(ctx)
+ assert_ne(meta[filename], _files_get_metadata(filename))
+
+
+def _files_remembered(ctx):
+ ns = ctx.declare("_files")
+ return ns.get("remembered-metadata", {})
+
+
+def _files_get_metadata(filename):
+ st = os.lstat(filename)
+ keys = ["st_dev", "st_gid", "st_ino", "st_mode", "st_mtime", "st_size", "st_uid"]
+ return {key: getattr(st, key) for key in keys}
diff --git a/subplot/vendored/files.yaml b/subplot/vendored/files.yaml
new file mode 100644
index 0000000..be69920
--- /dev/null
+++ b/subplot/vendored/files.yaml
@@ -0,0 +1,62 @@
+- given: file {filename}
+ function: files_create_from_embedded
+ types:
+ filename: file
+
+- given: file {filename_on_disk} from {embedded_filename}
+ function: files_create_from_embedded_with_other_name
+ types:
+ embedded_filename: file
+
+- given: file {filename} has modification time {year}-{month}-{day} {hour}:{minute}:{second}
+ function: files_touch_with_timestamp
+
+- when: I write "(?P<text>.*)" to file (?P<filename>\S+)
+ regex: true
+ function: files_create_from_text
+
+- when: I remember metadata for file {filename}
+ function: files_remember_metadata
+
+- when: I touch file {filename}
+ function: files_touch
+
+- then: file {filename} exists
+ function: files_file_exists
+
+- then: file {filename} does not exist
+ function: files_file_does_not_exist
+
+- then: only files (?P<filenames>.+) exist
+ function: files_only_these_exist
+ regex: true
+
+- then: file (?P<filename>\S+) contains "(?P<data>.*)"
+ regex: true
+ function: files_file_contains
+
+- then: file (?P<filename>\S+) matches regex /(?P<regex>.*)/
+ regex: true
+ function: files_file_matches_regex
+
+- then: file (?P<filename>\S+) matches regex "(?P<regex>.*)"
+ regex: true
+ function: files_file_matches_regex
+
+- then: files {filename1} and {filename2} match
+ function: files_match
+
+- then: file {filename} has same metadata as before
+ function: files_has_remembered_metadata
+
+- then: file {filename} has different metadata from before
+ function: files_has_different_metadata
+
+- then: file {filename} has changed from before
+ function: files_has_different_metadata
+
+- then: file {filename} has a very recent modification time
+ function: files_mtime_is_recent
+
+- then: file {filename} has a very old modification time
+ function: files_mtime_is_ancient
diff --git a/subplot/vendored/runcmd.md b/subplot/vendored/runcmd.md
index bb42005..a9d4ed4 100644
--- a/subplot/vendored/runcmd.md
+++ b/subplot/vendored/runcmd.md
@@ -162,6 +162,7 @@ then stderr doesn't match regex world$
---
title: Acceptance criteria for the lib/runcmd Subplot library
author: The Subplot project
+template: python
bindings:
- runcmd.yaml
functions: