test: change backup/restore smoke test to verify metadata

5 files changed, 543 insertions, 0 deletions

diff --git a/subplot/client.py b/subplot/client.py
index acdc48a..75baa8f 100644
--- a/subplot/client.py
+++ b/subplot/client.py
@@ -47,6 +47,35 @@ def capture_generation_id(ctx, varname=None):
     ctx["vars"] = v
 
 
+def create_manifest_of_live(ctx, dirname=None, manifest=None):
+    _create_manifest_of_directory(ctx, dirname=dirname, manifest=manifest)
+
+
+def create_manifest_of_restored(ctx, dirname=None, restored=None, manifest=None):
+    _create_manifest_of_directory(
+        ctx, dirname=os.path.join(restored, "./" + dirname), manifest=manifest
+    )
+
+
+def _create_manifest_of_directory(ctx, dirname=None, manifest=None):
+    runcmd_run = globals()["runcmd_run"]
+    runcmd_get_exit_code = globals()["runcmd_get_exit_code"]
+    runcmd_get_stdout = globals()["runcmd_get_stdout"]
+
+    runcmd_run(ctx, ["summain", dirname])
+    assert runcmd_get_exit_code(ctx) == 0
+    stdout = runcmd_get_stdout(ctx)
+    open(manifest, "w").write(stdout)
+
+
+def files_match(ctx, first=None, second=None):
+    assert_eq = globals()["assert_eq"]
+
+    f = open(first).read()
+    s = open(first).read()
+    assert_eq(f, s)
+
+
 def live_and_restored_data_match(ctx, live=None, restore=None):
     subprocess.check_call(["diff", "-rq", f"{live}/.", f"{restore}/{live}/."])
 
diff --git a/subplot/client.yaml b/subplot/client.yaml
index 63033c3..6894025 100644
--- a/subplot/client.yaml
+++ b/subplot/client.yaml
@@ -4,6 +4,12 @@
 - given: "a client config based on {filename}"
   function: configure_client
 
+- given: a manifest of the directory {dirname} in {manifest}
+  function: create_manifest_of_live
+
+- given: a manifest of the directory {dirname} restored in {restored} in {manifest}
+  function: create_manifest_of_restored
+
 - when: "I invoke obnam restore {filename} <{genid}> {todir}"
   function: run_obnam_restore
 
@@ -12,3 +18,6 @@
 
 - then: "generation list contains <{gen_id}>"
   function: generation_list_contains
+
+- then: files {first} and {second} match
+  function: files_match
diff --git a/subplot/runcmd.md b/subplot/runcmd.md
new file mode 100644
index 0000000..a9d4ed4
--- /dev/null
+++ b/subplot/runcmd.md
@@ -0,0 +1,170 @@
+# Introduction
+
+The [Subplot][] library `runcmd` for Python provides scenario steps
+and their implementations for running Unix commands and examining the
+results. The library consists of a bindings file `lib/runcmd.yaml` and
+implementations in Python in `lib/runcmd.py`. There is no Bash
+version.
+
+[Subplot]: https://subplot.liw.fi/
+
+This document explains the acceptance criteria for the library and how
+they're verified. It uses the steps and functions from the
+`lib/runcmd` library. The scenarios all have the same structure: run a
+command, then examine the exit code, standard output (stdout for
+short), or standard error output (stderr) of the command.
+
+The scenarios use the Unix commands `/bin/true` and `/bin/false` to
+generate exit codes, and `/bin/echo` to produce stdout. To generate
+stderr, they use the little helper script below.
+
+~~~{#err.sh .file .sh .numberLines}
+#!/bin/sh
+echo "$@" 1>&2
+~~~
+
+# Check exit code
+
+These scenarios verify the exit code. To make it easier to write
+scenarios in language that flows more naturally, there are a couple of
+variations.
+
+## Successful execution
+
+~~~scenario
+when I run /bin/true
+then exit code is 0
+and command is successful
+~~~
+
+## Failed execution
+
+~~~scenario
+when I try to run /bin/false
+then exit code is not 0
+and command fails
+~~~
+
+# Check output has what we want
+
+These scenarios verify that stdout or stderr do have something we want
+to have.
+
+## Check stdout is exactly as wanted
+
+Note that the string is surrounded by double quotes to make it clear
+to the reader what's inside. Also, C-style string escapes are
+understood.
+
+~~~scenario
+when I run /bin/echo hello, world
+then stdout is exactly "hello, world\n"
+~~~
+
+## Check stderr is exactly as wanted
+
+~~~scenario
+given helper script err.sh for runcmd
+when I run sh err.sh hello, world
+then stderr is exactly "hello, world\n"
+~~~
+
+## Check stdout using sub-string search
+
+Exact string comparisons are not always enough, so we can verify a
+sub-string is in output.
+
+~~~scenario
+when I run /bin/echo hello, world
+then stdout contains "world\n"
+and exit code is 0
+~~~
+
+## Check stderr using sub-string search
+
+~~~scenario
+given helper script err.sh for runcmd
+when I run sh err.sh hello, world
+then stderr contains "world\n"
+~~~
+
+## Check stdout using regular expressions
+
+Fixed strings are not always enough, so we can verify output matches a
+regular expression. Note that the regular expression is not delimited
+and does not get any C-style string escaped decoded.
+
+~~~scenario
+when I run /bin/echo hello, world
+then stdout matches regex world$
+~~~
+
+## Check stderr using regular expressions
+
+~~~scenario
+given helper script err.sh for runcmd
+when I run sh err.sh hello, world
+then stderr matches regex world$
+~~~
+
+# Check output doesn't have what we want to avoid
+
+These scenarios verify that the stdout or stderr do not
+have something we want to avoid.
+
+## Check stdout is not exactly something
+
+~~~scenario
+when I run /bin/echo hi
+then stdout isn't exactly "hello, world\n"
+~~~
+
+## Check stderr is not exactly something
+
+~~~scenario
+given helper script err.sh for runcmd
+when I run sh err.sh hi
+then stderr isn't exactly "hello, world\n"
+~~~
+
+## Check stdout doesn't contain sub-string
+
+~~~scenario
+when I run /bin/echo hi
+then stdout doesn't contain "world"
+~~~
+
+## Check stderr doesn't contain sub-string
+
+~~~scenario
+given helper script err.sh for runcmd
+when I run sh err.sh hi
+then stderr doesn't contain "world"
+~~~
+
+## Check stdout doesn't match regular expression
+
+~~~scenario
+when I run /bin/echo hi
+then stdout doesn't match regex world$
+
+~~~
+
+## Check stderr doesn't match regular expressions
+
+~~~scenario
+given helper script err.sh for runcmd
+when I run sh err.sh hi
+then stderr doesn't match regex world$
+~~~
+
+
+---
+title: Acceptance criteria for the lib/runcmd Subplot library
+author: The Subplot project
+template: python
+bindings:
+- runcmd.yaml
+functions:
+- runcmd.py
+...
diff --git a/subplot/runcmd.py b/subplot/runcmd.py
new file mode 100644
index 0000000..a2564c6
--- /dev/null
+++ b/subplot/runcmd.py
@@ -0,0 +1,252 @@
+import logging
+import os
+import re
+import shlex
+import subprocess
+
+
+#
+# Helper functions.
+#
+
+# Get exit code or other stored data about the latest command run by
+# runcmd_run.
+
+
+def _runcmd_get(ctx, name):
+    ns = ctx.declare("_runcmd")
+    return ns[name]
+
+
+def runcmd_get_exit_code(ctx):
+    return _runcmd_get(ctx, "exit")
+
+
+def runcmd_get_stdout(ctx):
+    return _runcmd_get(ctx, "stdout")
+
+
+def runcmd_get_stdout_raw(ctx):
+    return _runcmd_get(ctx, "stdout.raw")
+
+
+def runcmd_get_stderr(ctx):
+    return _runcmd_get(ctx, "stderr")
+
+
+def runcmd_get_stderr_raw(ctx):
+    return _runcmd_get(ctx, "stderr.raw")
+
+
+def runcmd_get_argv(ctx):
+    return _runcmd_get(ctx, "argv")
+
+
+# Run a command, given an argv and other arguments for subprocess.Popen.
+#
+# This is meant to be a helper function, not bound directly to a step. The
+# stdout, stderr, and exit code are stored in the "_runcmd" namespace in the
+# ctx context.
+def runcmd_run(ctx, argv, **kwargs):
+    ns = ctx.declare("_runcmd")
+
+    # The Subplot Python template empties os.environ at startup, modulo a small
+    # number of variables with carefully chosen values. Here, we don't need to
+    # care about what those variables are, but we do need to not overwrite
+    # them, so we just add anything in the env keyword argument, if any, to
+    # os.environ.
+    env = dict(os.environ)
+    for key, arg in kwargs.pop("env", {}).items():
+        env[key] = arg
+
+    pp = ns.get("path-prefix")
+    if pp:
+        env["PATH"] = pp + ":" + env["PATH"]
+
+    logging.debug(f"runcmd_run")
+    logging.debug(f"  argv: {argv}")
+    logging.debug(f"  env: {env}")
+    p = subprocess.Popen(
+        argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, **kwargs
+    )
+    stdout, stderr = p.communicate("")
+    ns["argv"] = argv
+    ns["stdout.raw"] = stdout
+    ns["stderr.raw"] = stderr
+    ns["stdout"] = stdout.decode("utf-8")
+    ns["stderr"] = stderr.decode("utf-8")
+    ns["exit"] = p.returncode
+    logging.debug(f"  ctx: {ctx}")
+    logging.debug(f"  ns: {ns}")
+
+
+# Step: prepend srcdir to PATH whenever runcmd runs a command.
+def runcmd_helper_srcdir_path(ctx):
+    srcdir = globals()["srcdir"]
+    runcmd_prepend_to_path(ctx, srcdir)
+
+
+# Step: This creates a helper script.
+def runcmd_helper_script(ctx, filename=None):
+    get_file = globals()["get_file"]
+    with open(filename, "wb") as f:
+        f.write(get_file(filename))
+
+
+#
+# Step functions for running commands.
+#
+
+
+def runcmd_prepend_to_path(ctx, dirname=None):
+    ns = ctx.declare("_runcmd")
+    pp = ns.get("path-prefix", "")
+    if pp:
+        pp = f"{pp}:{dirname}"
+    else:
+        pp = dirname
+    ns["path-prefix"] = pp
+
+
+def runcmd_step(ctx, argv0=None, args=None):
+    runcmd_try_to_run(ctx, argv0=argv0, args=args)
+    runcmd_exit_code_is_zero(ctx)
+
+
+def runcmd_try_to_run(ctx, argv0=None, args=None):
+    argv = [shlex.quote(argv0)] + shlex.split(args)
+    runcmd_run(ctx, argv)
+
+
+#
+# Step functions for examining exit codes.
+#
+
+
+def runcmd_exit_code_is_zero(ctx):
+    runcmd_exit_code_is(ctx, exit=0)
+
+
+def runcmd_exit_code_is(ctx, exit=None):
+    assert_eq = globals()["assert_eq"]
+    assert_eq(runcmd_get_exit_code(ctx), int(exit))
+
+
+def runcmd_exit_code_is_nonzero(ctx):
+    runcmd_exit_code_is_not(ctx, exit=0)
+
+
+def runcmd_exit_code_is_not(ctx, exit=None):
+    assert_ne = globals()["assert_ne"]
+    assert_ne(runcmd_get_exit_code(ctx), int(exit))
+
+
+#
+# Step functions and helpers for examining output in various ways.
+#
+
+
+def runcmd_stdout_is(ctx, text=None):
+    _runcmd_output_is(runcmd_get_stdout(ctx), text)
+
+
+def runcmd_stdout_isnt(ctx, text=None):
+    _runcmd_output_isnt(runcmd_get_stdout(ctx), text)
+
+
+def runcmd_stderr_is(ctx, text=None):
+    _runcmd_output_is(runcmd_get_stderr(ctx), text)
+
+
+def runcmd_stderr_isnt(ctx, text=None):
+    _runcmd_output_isnt(runcmd_get_stderr(ctx), text)
+
+
+def _runcmd_output_is(actual, wanted):
+    assert_eq = globals()["assert_eq"]
+    wanted = bytes(wanted, "utf8").decode("unicode_escape")
+    logging.debug("_runcmd_output_is:")
+    logging.debug(f"  actual: {actual!r}")
+    logging.debug(f"  wanted: {wanted!r}")
+    assert_eq(actual, wanted)
+
+
+def _runcmd_output_isnt(actual, wanted):
+    assert_ne = globals()["assert_ne"]
+    wanted = bytes(wanted, "utf8").decode("unicode_escape")
+    logging.debug("_runcmd_output_isnt:")
+    logging.debug(f"  actual: {actual!r}")
+    logging.debug(f"  wanted: {wanted!r}")
+    assert_ne(actual, wanted)
+
+
+def runcmd_stdout_contains(ctx, text=None):
+    _runcmd_output_contains(runcmd_get_stdout(ctx), text)
+
+
+def runcmd_stdout_doesnt_contain(ctx, text=None):
+    _runcmd_output_doesnt_contain(runcmd_get_stdout(ctx), text)
+
+
+def runcmd_stderr_contains(ctx, text=None):
+    _runcmd_output_contains(runcmd_get_stderr(ctx), text)
+
+
+def runcmd_stderr_doesnt_contain(ctx, text=None):
+    _runcmd_output_doesnt_contain(runcmd_get_stderr(ctx), text)
+
+
+def _runcmd_output_contains(actual, wanted):
+    assert_eq = globals()["assert_eq"]
+    wanted = bytes(wanted, "utf8").decode("unicode_escape")
+    logging.debug("_runcmd_output_contains:")
+    logging.debug(f"  actual: {actual!r}")
+    logging.debug(f"  wanted: {wanted!r}")
+    assert_eq(wanted in actual, True)
+
+
+def _runcmd_output_doesnt_contain(actual, wanted):
+    assert_ne = globals()["assert_ne"]
+    wanted = bytes(wanted, "utf8").decode("unicode_escape")
+    logging.debug("_runcmd_output_doesnt_contain:")
+    logging.debug(f"  actual: {actual!r}")
+    logging.debug(f"  wanted: {wanted!r}")
+    assert_ne(wanted in actual, True)
+
+
+def runcmd_stdout_matches_regex(ctx, regex=None):
+    _runcmd_output_matches_regex(runcmd_get_stdout(ctx), regex)
+
+
+def runcmd_stdout_doesnt_match_regex(ctx, regex=None):
+    _runcmd_output_doesnt_match_regex(runcmd_get_stdout(ctx), regex)
+
+
+def runcmd_stderr_matches_regex(ctx, regex=None):
+    _runcmd_output_matches_regex(runcmd_get_stderr(ctx), regex)
+
+
+def runcmd_stderr_doesnt_match_regex(ctx, regex=None):
+    _runcmd_output_doesnt_match_regex(runcmd_get_stderr(ctx), regex)
+
+
+def _runcmd_output_matches_regex(actual, regex):
+    assert_ne = globals()["assert_ne"]
+    r = re.compile(regex)
+    m = r.search(actual)
+    logging.debug("_runcmd_output_matches_regex:")
+    logging.debug(f"  actual: {actual!r}")
+    logging.debug(f"  regex: {regex!r}")
+    logging.debug(f"  match: {m}")
+    assert_ne(m, None)
+
+
+def _runcmd_output_doesnt_match_regex(actual, regex):
+    assert_eq = globals()["assert_eq"]
+    r = re.compile(regex)
+    m = r.search(actual)
+    logging.debug("_runcmd_output_doesnt_match_regex:")
+    logging.debug(f"  actual: {actual!r}")
+    logging.debug(f"  regex: {regex!r}")
+    logging.debug(f"  match: {m}")
+    assert_eq(m, None)
diff --git a/subplot/runcmd.yaml b/subplot/runcmd.yaml
new file mode 100644
index 0000000..48dde90
--- /dev/null
+++ b/subplot/runcmd.yaml
@@ -0,0 +1,83 @@
+# Steps to run commands.
+
+- given: helper script {filename} for runcmd
+  function: runcmd_helper_script
+
+- given: srcdir is in the PATH
+  function: runcmd_helper_srcdir_path
+
+- when: I run (?P<argv0>\S+)(?P<args>.*)
+  regex: true
+  function: runcmd_step
+
+- when: I try to run (?P<argv0>\S+)(?P<args>.*)
+  regex: true
+  function: runcmd_try_to_run
+
+# Steps to examine exit code of latest command.
+
+- then: exit code is {exit}
+  function: runcmd_exit_code_is
+
+- then: exit code is not {exit}
+  function: runcmd_exit_code_is_not
+
+- then: command is successful
+  function: runcmd_exit_code_is_zero
+
+- then: command fails
+  function: runcmd_exit_code_is_nonzero
+
+# Steps to examine stdout/stderr for exact content.
+
+- then: stdout is exactly "(?P<text>.*)"
+  regex: true
+  function: runcmd_stdout_is
+
+- then: "stdout isn't exactly \"(?P<text>.*)\""
+  regex: true
+  function: runcmd_stdout_isnt
+
+- then: stderr is exactly "(?P<text>.*)"
+  regex: true
+  function: runcmd_stderr_is
+
+- then: "stderr isn't exactly \"(?P<text>.*)\""
+  regex: true
+  function: runcmd_stderr_isnt
+
+# Steps to examine stdout/stderr for sub-strings.
+
+- then: stdout contains "(?P<text>.*)"
+  regex: true
+  function: runcmd_stdout_contains
+
+- then: "stdout doesn't contain \"(?P<text>.*)\""
+  regex: true
+  function: runcmd_stdout_doesnt_contain
+
+- then: stderr contains "(?P<text>.*)"
+  regex: true
+  function: runcmd_stderr_contains
+
+- then: "stderr doesn't contain \"(?P<text>.*)\""
+  regex: true
+  function: runcmd_stderr_doesnt_contain
+
+# Steps to match stdout/stderr against regular expressions.
+
+- then: stdout matches regex (?P<regex>.*)
+  regex: true
+  function: runcmd_stdout_matches_regex
+
+- then: stdout doesn't match regex (?P<regex>.*)
+  regex: true
+  function: runcmd_stdout_doesnt_match_regex
+
+- then: stderr matches regex (?P<regex>.*)
+  regex: true
+  function: runcmd_stderr_matches_regex
+
+- then: stderr doesn't match regex (?P<regex>.*)
+  regex: true
+  function: runcmd_stderr_doesnt_match_regex