Merge branch 'subplot-refresh' into 'main'

refactor: update the vendored Subplot libraries

See merge request larswirzenius/obnam!128

7 files changed, 334 insertions, 9 deletions

diff --git a/subplot/vendored/daemon.md b/subplot/vendored/daemon.md
index 131dcb1..9484926 100644
--- a/subplot/vendored/daemon.md
+++ b/subplot/vendored/daemon.md
@@ -25,6 +25,84 @@ then there is no "/bin/sleep 12765" process
 ~~~
 
 
+# Daemon takes a while to open its port
+
+[netcat]: https://en.wikipedia.org/wiki/Netcat
+
+This scenario verifies that if the background process never starts
+listening on its port, the daemon library handles that correctly. We
+do this by using [netcat][] to start a dummy daemon, after a short
+delay. The lib/daemon code will wait for netcat to open its port, by
+connecting to the port. It then closes the port, which causes netcat
+to terminate.
+
+~~~scenario
+given a daemon helper shell script slow-start-daemon.sh
+given there is no "slow-start-daemon.sh" process
+when I try to start "./slow-start-daemon.sh" as slow-daemon, on port 8888
+when I stop background process slow-daemon
+then there is no "slow-start-daemon.sh" process
+~~~
+
+~~~{#slow-start-daemon.sh .file .sh .numberLines}
+#!/bin/bash
+
+set -euo pipefail
+
+sleep 2
+netcat -l 8888 > /dev/null
+echo OK
+~~~
+
+# Daemon never opens the intended port
+
+This scenario verifies that if the background process never starts
+listening on its port, the daemon library handles that correctly.
+
+~~~scenario
+given there is no "/bin/sleep 12765" process
+when I try to start "/bin/sleep 12765" as sleepyhead, on port 8888
+then starting daemon fails with "ConnectionRefusedError"
+then a process "/bin/sleep 12765" is running
+when I stop background process sleepyhead
+then there is no "/bin/sleep 12765" process
+~~~
+
+
+# Daemon stdout and stderr are retrievable
+
+Sometimes it's useful for the step functions to be able to retrieve
+the stdout or stderr of of the daemon, after it's started, or even
+after it's terminated. This scenario verifies that `lib/daemon` can do
+that.
+
+~~~scenario
+given a daemon helper shell script chatty-daemon.sh
+given there is no "chatty-daemon" process
+when I start "./chatty-daemon.sh" as a background process as chatty-daemon
+when daemon chatty-daemon has produced output
+when I stop background process chatty-daemon
+then there is no "chatty-daemon" process
+then daemon chatty-daemon stdout is "hi there\n"
+then daemon chatty-daemon stderr is "hola\n"
+~~~
+
+We make for the daemon to exit, to work around a race condition: if
+the test program retrieves the daemon's output too fast, it may not
+have had time to produce it yet.
+
+
+~~~{#chatty-daemon.sh .file .sh .numberLines}
+#!/bin/bash
+
+set -euo pipefail
+
+trap 'exit 0' TERM
+
+echo hola 1>&2
+echo hi there
+~~~
+
 
 ---
 title: Acceptance criteria for the lib/daemon Subplot library
diff --git a/subplot/vendored/daemon.py b/subplot/vendored/daemon.py
index febf392..11f65bf 100644
--- a/subplot/vendored/daemon.py
+++ b/subplot/vendored/daemon.py
@@ -6,22 +6,63 @@ import subprocess
 import time
 
 
+# A helper function for testing lib/daemon itself.
+def _daemon_shell_script(ctx, filename=None):
+    get_file = globals()["get_file"]
+    data = get_file(filename)
+    with open(filename, "wb") as f:
+        f.write(data)
+    os.chmod(filename, 0o755)
+
+
 # Start a daemon that will open a port on localhost.
 def daemon_start_on_port(ctx, path=None, args=None, name=None, port=None):
-    daemon_start(ctx, path=path, args=args, name=name)
+    _daemon_start(ctx, path=path, args=args, name=name)
     daemon_wait_for_port("localhost", port)
 
 
+# Start a daemon after a little wait. This is used only for testing the
+# port-waiting code.
+def _daemon_start_soonish(ctx, path=None, args=None, name=None, port=None):
+    _daemon_start(ctx, path=os.path.abspath(path), args=args, name=name)
+    daemon = ctx.declare("_daemon")
+
+    # Store the PID of the process we just started so that _daemon_stop_soonish
+    # can kill it during the cleanup phase. This works around the Subplot
+    # Python template not giving the step captures to cleanup functions. Note
+    # that this code assume at most one _soonish function is called.
+    daemon["_soonish"] = daemon[name]["pid"]
+
+    try:
+        daemon_wait_for_port("localhost", port)
+    except Exception as e:
+        daemon["_start_error"] = repr(e)
+
+    logging.info("pgrep: %r", _daemon_pgrep(path))
+
+
+def _daemon_stop_soonish(ctx, path=None, args=None, name=None, port=None):
+    ns = ctx.declare("_daemon")
+    pid = ns["_soonish"]
+    logging.debug(f"Stopping soonishly-started daemon, {pid}")
+    signo = signal.SIGKILL
+    try:
+        os.kill(pid, signo)
+    except ProcessLookupError:
+        logging.warning("Process did not actually exist (anymore?)")
+
+
 # Start a daeamon, get its PID. Don't wait for a port or anything. This is
 # meant for background processes that don't have port. Useful for testing the
 # lib/daemon library of Subplot, but not much else.
-def daemon_start(ctx, path=None, args=None, name=None):
+def _daemon_start(ctx, path=None, args=None, name=None):
     runcmd_run = globals()["runcmd_run"]
     runcmd_exit_code_is = globals()["runcmd_exit_code_is"]
     runcmd_get_exit_code = globals()["runcmd_get_exit_code"]
     runcmd_get_stderr = globals()["runcmd_get_stderr"]
     runcmd_prepend_to_path = globals()["runcmd_prepend_to_path"]
 
+    path = os.path.abspath(path)
     argv = [path] + args.split()
 
     logging.debug(f"Starting daemon {name}")
@@ -90,7 +131,7 @@ def _daemon_wait_for_pid(filename, timeout):
     raise Exception("daemonize created a PID file without a PID")
 
 
-def daemon_wait_for_port(host, port, timeout=3.0):
+def daemon_wait_for_port(host, port, timeout=5.0):
     addr = (host, port)
     until = time.time() + timeout
     while True:
@@ -99,23 +140,36 @@ def daemon_wait_for_port(host, port, timeout=3.0):
             s.close()
             return
         except socket.timeout:
-            logging.error(f"daemon did not respond at port {port} within {timeout} seconds")
+            logging.error(
+                f"daemon did not respond at port {port} within {timeout} seconds"
+            )
             raise
         except socket.error as e:
             logging.info(f"could not connect to daemon at {port}: {e}")
             pass
         if time.time() >= until:
-            logging.error(f"could not connect to daemon at {port} within {timeout} seconds")
+            logging.error(
+                f"could not connect to daemon at {port} within {timeout} seconds"
+            )
             raise ConnectionRefusedError()
+        # Sleep a bit to avoid consuming too much CPU while busy-waiting.
+        time.sleep(0.1)
 
 
 # Stop a daemon.
-def daemon_stop(ctx, name=None):
+def daemon_stop(ctx, path=None, args=None, name=None):
     logging.debug(f"Stopping daemon {name}")
+
     ns = ctx.declare("_daemon")
     logging.debug(f"  ns={ns}")
     pid = ns[name]["pid"]
-    signo = signal.SIGKILL
+    signo = signal.SIGTERM
+
+    this = ns[name]
+    data = open(this["stdout"]).read()
+    logging.debug(f"{name} stdout, before: {data!r}")
+    data = open(this["stderr"]).read()
+    logging.debug(f"{name} stderr, before: {data!r}")
 
     logging.debug(f"Terminating process {pid} with signal {signo}")
     try:
@@ -123,6 +177,20 @@ def daemon_stop(ctx, name=None):
     except ProcessLookupError:
         logging.warning("Process did not actually exist (anymore?)")
 
+    while True:
+        try:
+            os.kill(pid, 0)
+            logging.debug(f"Daemon {name}, pid {pid} still exists")
+            time.sleep(1)
+        except ProcessLookupError:
+            break
+    logging.debug(f"Daemon {name} is gone")
+
+    data = open(this["stdout"]).read()
+    logging.debug(f"{name} stdout, after: {data!r}")
+    data = open(this["stderr"]).read()
+    logging.debug(f"{name} stderr, after: {data!r}")
+
 
 def daemon_no_such_process(ctx, args=None):
     assert not _daemon_pgrep(args)
@@ -134,6 +202,61 @@ def daemon_process_exists(ctx, args=None):
 
 def _daemon_pgrep(pattern):
     logging.info(f"checking if process exists: pattern={pattern}")
-    exit = subprocess.call(["pgrep", "-laf", pattern])
+    exit = subprocess.call(
+        ["pgrep", "-laf", pattern], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+    )
     logging.info(f"exit code: {exit}")
     return exit == 0
+
+
+def daemon_start_fails_with(ctx, message=None):
+    daemon = ctx.declare("_daemon")
+    error = daemon["_start_error"]
+    logging.debug(f"daemon_start_fails_with: error={error!r}")
+    logging.debug(f"daemon_start_fails_with: message={message!r}")
+    assert message.lower() in error.lower()
+
+
+def daemon_get_stdout(ctx, name):
+    return _daemon_get_output(ctx, name, "stdout")
+
+
+def daemon_get_stderr(ctx, name):
+    return _daemon_get_output(ctx, name, "stderr")
+
+
+def _daemon_get_output(ctx, name, which):
+    ns = ctx.declare("_daemon")
+    this = ns[name]
+    filename = this[which]
+    data = open(filename).read()
+    logging.debug(f"Read {which} of daemon {name} from {filename}: {data!r}")
+    return data
+
+
+def daemon_has_produced_output(ctx, name=None):
+    started = time.time()
+    timeout = 5.0
+    while time.time() < started + timeout:
+        stdout = daemon_get_stdout(ctx, name)
+        stderr = daemon_get_stderr(ctx, name)
+        if stdout and stderr:
+            break
+        time.sleep(0.1)
+
+
+def daemon_stdout_is(ctx, name=None, text=None):
+    daemon_get_stdout = globals()["daemon_get_stdout"]
+    _daemon_output_is(ctx, name, text, daemon_get_stdout)
+
+
+def daemon_stderr_is(ctx, name=None, text=None):
+    daemon_get_stderr = globals()["daemon_get_stderr"]
+    _daemon_output_is(ctx, name, text, daemon_get_stderr)
+
+
+def _daemon_output_is(ctx, name, text, getter):
+    assert_eq = globals()["assert_eq"]
+    text = bytes(text, "UTF-8").decode("unicode_escape")
+    output = getter(ctx, name)
+    assert_eq(text, output)
diff --git a/subplot/vendored/daemon.yaml b/subplot/vendored/daemon.yaml
index 6165c62..4fab1f6 100644
--- a/subplot/vendored/daemon.yaml
+++ b/subplot/vendored/daemon.yaml
@@ -1,17 +1,37 @@
 - given: there is no "{args:text}" process
   function: daemon_no_such_process
 
+- given: a daemon helper shell script {filename}
+  function: _daemon_shell_script
+
 - when: I start "{path}{args:text}" as a background process as {name}, on port {port}
   function: daemon_start_on_port
 
+- when: I try to start "{path}{args:text}" as {name}, on port {port}
+  function: _daemon_start_soonish
+  cleanup: _daemon_stop_soonish
+
 - when: I start "{path}{args:text}" as a background process as {name}
-  function: daemon_start
+  function: _daemon_start
 
 - when: I stop background process {name}
   function: daemon_stop
 
+- when: daemon {name} has produced output
+  function: daemon_has_produced_output
+
 - then: a process "{args:text}" is running
   function: daemon_process_exists
 
 - then: there is no "{args:text}" process
   function: daemon_no_such_process
+
+- then: starting daemon fails with "{message:text}"
+  function: daemon_start_fails_with
+
+- then: daemon {name} stdout is "{text:text}"
+  function: daemon_stdout_is
+
+- then: daemon {name} stderr is "{text:text}"
+  function: daemon_stderr_is
+  
diff --git a/subplot/vendored/files.md b/subplot/vendored/files.md
index 68ef1ac..823c760 100644
--- a/subplot/vendored/files.md
+++ b/subplot/vendored/files.md
@@ -70,6 +70,29 @@ and file hello.txt matches regex "hello, .*"
 and file hello.txt matches regex /hello, .*/
 ~~~
 
+# Directories
+
+There are also a large number of directory based steps and some directory
+based behaviour available in creating files which are available in the files
+library.
+
+```scenario
+given a directory first
+then directory first exists
+and directory first is empty
+and directory second does not exist
+when I remove directory first
+then directory first does not exist
+when I create directory second
+then directory second exists
+and directory second is empty
+given file second/third/hello.txt from hello.txt
+then directory second is not empty
+and directory second/third exists
+and directory second/third is not empty
+when I remove directory second
+then directory second does not exist
+```
 
 ---
 title: Acceptance criteria for the files Subplot library
diff --git a/subplot/vendored/files.py b/subplot/vendored/files.py
index ec37b9d..dd5b9f8 100644
--- a/subplot/vendored/files.py
+++ b/subplot/vendored/files.py
@@ -1,10 +1,12 @@
 import logging
 import os
 import re
+import shutil
 import time
 
 
 def files_create_from_embedded(ctx, filename=None):
+    files_make_directory(ctx, path=os.path.dirname(filename) or ".")
     files_create_from_embedded_with_other_name(
         ctx, filename_on_disk=filename, embedded_filename=filename
     )
@@ -14,15 +16,29 @@ def files_create_from_embedded_with_other_name(
     ctx, filename_on_disk=None, embedded_filename=None
 ):
     get_file = globals()["get_file"]
+
+    files_make_directory(ctx, path=os.path.dirname(filename_on_disk) or ".")
     with open(filename_on_disk, "wb") as f:
         f.write(get_file(embedded_filename))
 
 
 def files_create_from_text(ctx, filename=None, text=None):
+    files_make_directory(ctx, path=os.path.dirname(filename) or ".")
     with open(filename, "w") as f:
         f.write(text)
 
 
+def files_make_directory(ctx, path=None):
+    path = "./" + path
+    if not os.path.exists(path):
+        os.makedirs(path)
+
+
+def files_remove_directory(ctx, path=None):
+    path = "./" + path
+    shutil.rmtree(path)
+
+
 def files_file_exists(ctx, filename=None):
     assert_eq = globals()["assert_eq"]
     assert_eq(os.path.exists(filename), True)
@@ -33,6 +49,26 @@ def files_file_does_not_exist(ctx, filename=None):
     assert_eq(os.path.exists(filename), False)
 
 
+def files_directory_exists(ctx, path=None):
+    assert_eq = globals()["assert_eq"]
+    assert_eq(os.path.isdir(path), True)
+
+
+def files_directory_does_not_exist(ctx, path=None):
+    assert_eq = globals()["assert_eq"]
+    assert_eq(os.path.isdir(path), False)
+
+
+def files_directory_is_empty(ctx, path=None):
+    assert_eq = globals()["assert_eq"]
+    assert_eq(os.listdir(path), [])
+
+
+def files_directory_is_not_empty(ctx, path=None):
+    assert_ne = globals()["assert_ne"]
+    assert_ne(os.listdir(path), False)
+
+
 def files_only_these_exist(ctx, filenames=None):
     assert_eq = globals()["assert_eq"]
     filenames = filenames.replace(",", "").split()
diff --git a/subplot/vendored/files.yaml b/subplot/vendored/files.yaml
index be69920..f18b8cd 100644
--- a/subplot/vendored/files.yaml
+++ b/subplot/vendored/files.yaml
@@ -60,3 +60,24 @@
 
 - then: file {filename} has a very old modification time
   function: files_mtime_is_ancient
+
+- given: a directory {path}
+  function: files_make_directory
+
+- when: I create directory {path}
+  function: files_make_directory
+
+- when: I remove directory {path}
+  function: files_remove_directory
+
+- then: directory {path} exists
+  function: files_directory_exists
+
+- then: directory {path} does not exist
+  function: files_directory_does_not_exist
+
+- then: directory {path} is empty
+  function: files_directory_is_empty
+
+- then: directory {path} is not empty
+  function: files_directory_is_not_empty
diff --git a/subplot/vendored/runcmd.md b/subplot/vendored/runcmd.md
index a9d4ed4..4615f69 100644
--- a/subplot/vendored/runcmd.md
+++ b/subplot/vendored/runcmd.md
@@ -45,6 +45,27 @@ then exit code is not 0
 and command fails
 ~~~
 
+# Check we can prepend to $PATH
+
+This scenario verifies that we can add a directory to the beginning of
+the PATH environment variable, so that we can have `runcmd` invoke a
+binary from our build tree rather than from system directories. This
+is especially useful for testing new versions of software that's
+already installed on the system.
+
+~~~scenario
+given executable script ls from ls.sh
+when I prepend . to PATH
+when I run ls
+then command is successful
+then stdout contains "custom ls, not system ls"
+~~~
+
+~~~{#ls.sh .file .sh .numberLines}
+#!/bin/sh
+echo "custom ls, not system ls"
+~~~
+
 # Check output has what we want
 
 These scenarios verify that stdout or stderr do have something we want
@@ -165,6 +186,9 @@ author: The Subplot project
 template: python
 bindings:
 - runcmd.yaml
+- runcmd_test.yaml
 functions:
 - runcmd.py
+- runcmd_test.py
+- files.py
 ...