feat: verify checksum of chunks downloaded from server

7 files changed, 34 insertions, 505 deletions

diff --git a/subplot/client.py b/subplot/client.py
index c1f5159..53c7f6e 100644
--- a/subplot/client.py
+++ b/subplot/client.py
@@ -50,6 +50,13 @@ def run_obnam_restore_with_genref(ctx, filename=None, genref=None, todir=None):
     )
 
 
+def run_obnam_get_chunk(ctx, filename=None, gen_id=None, todir=None):
+    runcmd_run = globals()["runcmd_run"]
+    gen_id = ctx["vars"][gen_id]
+    logging.debug(f"run_obnam_get_chunk: gen_id={gen_id}")
+    runcmd_run(ctx, ["obnam", "--config", filename, "get-chunk", gen_id])
+
+
 def capture_generation_id(ctx, varname=None):
     runcmd_get_stdout = globals()["runcmd_get_stdout"]
 
@@ -95,3 +102,11 @@ def get_backup_reason(ctx, filename):
     lines = [line for line in lines if filename in line]
     line = lines[0]
     return line.split()[-1]
+
+
+def stdout_matches_file(ctx, filename=None):
+    runcmd_get_stdout = globals()["runcmd_get_stdout"]
+    assert_eq = globals()["assert_eq"]
+    stdout = runcmd_get_stdout(ctx)
+    data = open(filename).read()
+    assert_eq(stdout, data)
diff --git a/subplot/client.yaml b/subplot/client.yaml
index db55679..b1f9b19 100644
--- a/subplot/client.yaml
+++ b/subplot/client.yaml
@@ -10,6 +10,9 @@
 - when: "I invoke obnam --config {filename} restore latest {todir}"
   function: run_obnam_restore_latest
 
+- when: "I invoke obnam --config {filename} get-chunk <{gen_id}>"
+  function: run_obnam_get_chunk
+
 - then: "backup generation is {varname}"
   function: capture_generation_id
 
@@ -24,3 +27,6 @@
 
 - then: "file {filename} was not backed up because it was unchanged"
   function: file_was_unchanged
+
+- then: "stdout matches file {filename}"
+  function: stdout_matches_file
diff --git a/subplot/runcmd.md b/subplot/runcmd.md
deleted file mode 100644
index a9d4ed4..0000000
--- a/subplot/runcmd.md
+++ /dev/null
@@ -1,170 +0,0 @@
-# Introduction
-
-The [Subplot][] library `runcmd` for Python provides scenario steps
-and their implementations for running Unix commands and examining the
-results. The library consists of a bindings file `lib/runcmd.yaml` and
-implementations in Python in `lib/runcmd.py`. There is no Bash
-version.
-
-[Subplot]: https://subplot.liw.fi/
-
-This document explains the acceptance criteria for the library and how
-they're verified. It uses the steps and functions from the
-`lib/runcmd` library. The scenarios all have the same structure: run a
-command, then examine the exit code, standard output (stdout for
-short), or standard error output (stderr) of the command.
-
-The scenarios use the Unix commands `/bin/true` and `/bin/false` to
-generate exit codes, and `/bin/echo` to produce stdout. To generate
-stderr, they use the little helper script below.
-
-~~~{#err.sh .file .sh .numberLines}
-#!/bin/sh
-echo "$@" 1>&2
-~~~
-
-# Check exit code
-
-These scenarios verify the exit code. To make it easier to write
-scenarios in language that flows more naturally, there are a couple of
-variations.
-
-## Successful execution
-
-~~~scenario
-when I run /bin/true
-then exit code is 0
-and command is successful
-~~~
-
-## Failed execution
-
-~~~scenario
-when I try to run /bin/false
-then exit code is not 0
-and command fails
-~~~
-
-# Check output has what we want
-
-These scenarios verify that stdout or stderr do have something we want
-to have.
-
-## Check stdout is exactly as wanted
-
-Note that the string is surrounded by double quotes to make it clear
-to the reader what's inside. Also, C-style string escapes are
-understood.
-
-~~~scenario
-when I run /bin/echo hello, world
-then stdout is exactly "hello, world\n"
-~~~
-
-## Check stderr is exactly as wanted
-
-~~~scenario
-given helper script err.sh for runcmd
-when I run sh err.sh hello, world
-then stderr is exactly "hello, world\n"
-~~~
-
-## Check stdout using sub-string search
-
-Exact string comparisons are not always enough, so we can verify a
-sub-string is in output.
-
-~~~scenario
-when I run /bin/echo hello, world
-then stdout contains "world\n"
-and exit code is 0
-~~~
-
-## Check stderr using sub-string search
-
-~~~scenario
-given helper script err.sh for runcmd
-when I run sh err.sh hello, world
-then stderr contains "world\n"
-~~~
-
-## Check stdout using regular expressions
-
-Fixed strings are not always enough, so we can verify output matches a
-regular expression. Note that the regular expression is not delimited
-and does not get any C-style string escaped decoded.
-
-~~~scenario
-when I run /bin/echo hello, world
-then stdout matches regex world$
-~~~
-
-## Check stderr using regular expressions
-
-~~~scenario
-given helper script err.sh for runcmd
-when I run sh err.sh hello, world
-then stderr matches regex world$
-~~~
-
-# Check output doesn't have what we want to avoid
-
-These scenarios verify that the stdout or stderr do not
-have something we want to avoid.
-
-## Check stdout is not exactly something
-
-~~~scenario
-when I run /bin/echo hi
-then stdout isn't exactly "hello, world\n"
-~~~
-
-## Check stderr is not exactly something
-
-~~~scenario
-given helper script err.sh for runcmd
-when I run sh err.sh hi
-then stderr isn't exactly "hello, world\n"
-~~~
-
-## Check stdout doesn't contain sub-string
-
-~~~scenario
-when I run /bin/echo hi
-then stdout doesn't contain "world"
-~~~
-
-## Check stderr doesn't contain sub-string
-
-~~~scenario
-given helper script err.sh for runcmd
-when I run sh err.sh hi
-then stderr doesn't contain "world"
-~~~
-
-## Check stdout doesn't match regular expression
-
-~~~scenario
-when I run /bin/echo hi
-then stdout doesn't match regex world$
-
-~~~
-
-## Check stderr doesn't match regular expressions
-
-~~~scenario
-given helper script err.sh for runcmd
-when I run sh err.sh hi
-then stderr doesn't match regex world$
-~~~
-
-
----
-title: Acceptance criteria for the lib/runcmd Subplot library
-author: The Subplot project
-template: python
-bindings:
-- runcmd.yaml
-functions:
-- runcmd.py
-...
diff --git a/subplot/runcmd.py b/subplot/runcmd.py
deleted file mode 100644
index a2564c6..0000000
--- a/subplot/runcmd.py
+++ /dev/null
@@ -1,252 +0,0 @@
-import logging
-import os
-import re
-import shlex
-import subprocess
-
-
-#
-# Helper functions.
-#
-
-# Get exit code or other stored data about the latest command run by
-# runcmd_run.
-
-
-def _runcmd_get(ctx, name):
-    ns = ctx.declare("_runcmd")
-    return ns[name]
-
-
-def runcmd_get_exit_code(ctx):
-    return _runcmd_get(ctx, "exit")
-
-
-def runcmd_get_stdout(ctx):
-    return _runcmd_get(ctx, "stdout")
-
-
-def runcmd_get_stdout_raw(ctx):
-    return _runcmd_get(ctx, "stdout.raw")
-
-
-def runcmd_get_stderr(ctx):
-    return _runcmd_get(ctx, "stderr")
-
-
-def runcmd_get_stderr_raw(ctx):
-    return _runcmd_get(ctx, "stderr.raw")
-
-
-def runcmd_get_argv(ctx):
-    return _runcmd_get(ctx, "argv")
-
-
-# Run a command, given an argv and other arguments for subprocess.Popen.
-#
-# This is meant to be a helper function, not bound directly to a step. The
-# stdout, stderr, and exit code are stored in the "_runcmd" namespace in the
-# ctx context.
-def runcmd_run(ctx, argv, **kwargs):
-    ns = ctx.declare("_runcmd")
-
-    # The Subplot Python template empties os.environ at startup, modulo a small
-    # number of variables with carefully chosen values. Here, we don't need to
-    # care about what those variables are, but we do need to not overwrite
-    # them, so we just add anything in the env keyword argument, if any, to
-    # os.environ.
-    env = dict(os.environ)
-    for key, arg in kwargs.pop("env", {}).items():
-        env[key] = arg
-
-    pp = ns.get("path-prefix")
-    if pp:
-        env["PATH"] = pp + ":" + env["PATH"]
-
-    logging.debug(f"runcmd_run")
-    logging.debug(f"  argv: {argv}")
-    logging.debug(f"  env: {env}")
-    p = subprocess.Popen(
-        argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, **kwargs
-    )
-    stdout, stderr = p.communicate("")
-    ns["argv"] = argv
-    ns["stdout.raw"] = stdout
-    ns["stderr.raw"] = stderr
-    ns["stdout"] = stdout.decode("utf-8")
-    ns["stderr"] = stderr.decode("utf-8")
-    ns["exit"] = p.returncode
-    logging.debug(f"  ctx: {ctx}")
-    logging.debug(f"  ns: {ns}")
-
-
-# Step: prepend srcdir to PATH whenever runcmd runs a command.
-def runcmd_helper_srcdir_path(ctx):
-    srcdir = globals()["srcdir"]
-    runcmd_prepend_to_path(ctx, srcdir)
-
-
-# Step: This creates a helper script.
-def runcmd_helper_script(ctx, filename=None):
-    get_file = globals()["get_file"]
-    with open(filename, "wb") as f:
-        f.write(get_file(filename))
-
-
-#
-# Step functions for running commands.
-#
-
-
-def runcmd_prepend_to_path(ctx, dirname=None):
-    ns = ctx.declare("_runcmd")
-    pp = ns.get("path-prefix", "")
-    if pp:
-        pp = f"{pp}:{dirname}"
-    else:
-        pp = dirname
-    ns["path-prefix"] = pp
-
-
-def runcmd_step(ctx, argv0=None, args=None):
-    runcmd_try_to_run(ctx, argv0=argv0, args=args)
-    runcmd_exit_code_is_zero(ctx)
-
-
-def runcmd_try_to_run(ctx, argv0=None, args=None):
-    argv = [shlex.quote(argv0)] + shlex.split(args)
-    runcmd_run(ctx, argv)
-
-
-#
-# Step functions for examining exit codes.
-#
-
-
-def runcmd_exit_code_is_zero(ctx):
-    runcmd_exit_code_is(ctx, exit=0)
-
-
-def runcmd_exit_code_is(ctx, exit=None):
-    assert_eq = globals()["assert_eq"]
-    assert_eq(runcmd_get_exit_code(ctx), int(exit))
-
-
-def runcmd_exit_code_is_nonzero(ctx):
-    runcmd_exit_code_is_not(ctx, exit=0)
-
-
-def runcmd_exit_code_is_not(ctx, exit=None):
-    assert_ne = globals()["assert_ne"]
-    assert_ne(runcmd_get_exit_code(ctx), int(exit))
-
-
-#
-# Step functions and helpers for examining output in various ways.
-#
-
-
-def runcmd_stdout_is(ctx, text=None):
-    _runcmd_output_is(runcmd_get_stdout(ctx), text)
-
-
-def runcmd_stdout_isnt(ctx, text=None):
-    _runcmd_output_isnt(runcmd_get_stdout(ctx), text)
-
-
-def runcmd_stderr_is(ctx, text=None):
-    _runcmd_output_is(runcmd_get_stderr(ctx), text)
-
-
-def runcmd_stderr_isnt(ctx, text=None):
-    _runcmd_output_isnt(runcmd_get_stderr(ctx), text)
-
-
-def _runcmd_output_is(actual, wanted):
-    assert_eq = globals()["assert_eq"]
-    wanted = bytes(wanted, "utf8").decode("unicode_escape")
-    logging.debug("_runcmd_output_is:")
-    logging.debug(f"  actual: {actual!r}")
-    logging.debug(f"  wanted: {wanted!r}")
-    assert_eq(actual, wanted)
-
-
-def _runcmd_output_isnt(actual, wanted):
-    assert_ne = globals()["assert_ne"]
-    wanted = bytes(wanted, "utf8").decode("unicode_escape")
-    logging.debug("_runcmd_output_isnt:")
-    logging.debug(f"  actual: {actual!r}")
-    logging.debug(f"  wanted: {wanted!r}")
-    assert_ne(actual, wanted)
-
-
-def runcmd_stdout_contains(ctx, text=None):
-    _runcmd_output_contains(runcmd_get_stdout(ctx), text)
-
-
-def runcmd_stdout_doesnt_contain(ctx, text=None):
-    _runcmd_output_doesnt_contain(runcmd_get_stdout(ctx), text)
-
-
-def runcmd_stderr_contains(ctx, text=None):
-    _runcmd_output_contains(runcmd_get_stderr(ctx), text)
-
-
-def runcmd_stderr_doesnt_contain(ctx, text=None):
-    _runcmd_output_doesnt_contain(runcmd_get_stderr(ctx), text)
-
-
-def _runcmd_output_contains(actual, wanted):
-    assert_eq = globals()["assert_eq"]
-    wanted = bytes(wanted, "utf8").decode("unicode_escape")
-    logging.debug("_runcmd_output_contains:")
-    logging.debug(f"  actual: {actual!r}")
-    logging.debug(f"  wanted: {wanted!r}")
-    assert_eq(wanted in actual, True)
-
-
-def _runcmd_output_doesnt_contain(actual, wanted):
-    assert_ne = globals()["assert_ne"]
-    wanted = bytes(wanted, "utf8").decode("unicode_escape")
-    logging.debug("_runcmd_output_doesnt_contain:")
-    logging.debug(f"  actual: {actual!r}")
-    logging.debug(f"  wanted: {wanted!r}")
-    assert_ne(wanted in actual, True)
-
-
-def runcmd_stdout_matches_regex(ctx, regex=None):
-    _runcmd_output_matches_regex(runcmd_get_stdout(ctx), regex)
-
-
-def runcmd_stdout_doesnt_match_regex(ctx, regex=None):
-    _runcmd_output_doesnt_match_regex(runcmd_get_stdout(ctx), regex)
-
-
-def runcmd_stderr_matches_regex(ctx, regex=None):
-    _runcmd_output_matches_regex(runcmd_get_stderr(ctx), regex)
-
-
-def runcmd_stderr_doesnt_match_regex(ctx, regex=None):
-    _runcmd_output_doesnt_match_regex(runcmd_get_stderr(ctx), regex)
-
-
-def _runcmd_output_matches_regex(actual, regex):
-    assert_ne = globals()["assert_ne"]
-    r = re.compile(regex)
-    m = r.search(actual)
-    logging.debug("_runcmd_output_matches_regex:")
-    logging.debug(f"  actual: {actual!r}")
-    logging.debug(f"  regex: {regex!r}")
-    logging.debug(f"  match: {m}")
-    assert_ne(m, None)
-
-
-def _runcmd_output_doesnt_match_regex(actual, regex):
-    assert_eq = globals()["assert_eq"]
-    r = re.compile(regex)
-    m = r.search(actual)
-    logging.debug("_runcmd_output_doesnt_match_regex:")
-    logging.debug(f"  actual: {actual!r}")
-    logging.debug(f"  regex: {regex!r}")
-    logging.debug(f"  match: {m}")
-    assert_eq(m, None)
diff --git a/subplot/runcmd.yaml b/subplot/runcmd.yaml
deleted file mode 100644
index 48dde90..0000000
--- a/subplot/runcmd.yaml
+++ /dev/null
@@ -1,83 +0,0 @@
-# Steps to run commands.
-
-- given: helper script {filename} for runcmd
-  function: runcmd_helper_script
-
-- given: srcdir is in the PATH
-  function: runcmd_helper_srcdir_path
-
-- when: I run (?P<argv0>\S+)(?P<args>.*)
-  regex: true
-  function: runcmd_step
-
-- when: I try to run (?P<argv0>\S+)(?P<args>.*)
-  regex: true
-  function: runcmd_try_to_run
-
-# Steps to examine exit code of latest command.
-
-- then: exit code is {exit}
-  function: runcmd_exit_code_is
-
-- then: exit code is not {exit}
-  function: runcmd_exit_code_is_not
-
-- then: command is successful
-  function: runcmd_exit_code_is_zero
-
-- then: command fails
-  function: runcmd_exit_code_is_nonzero
-
-# Steps to examine stdout/stderr for exact content.
-
-- then: stdout is exactly "(?P<text>.*)"
-  regex: true
-  function: runcmd_stdout_is
-
-- then: "stdout isn't exactly \"(?P<text>.*)\""
-  regex: true
-  function: runcmd_stdout_isnt
-
-- then: stderr is exactly "(?P<text>.*)"
-  regex: true
-  function: runcmd_stderr_is
-
-- then: "stderr isn't exactly \"(?P<text>.*)\""
-  regex: true
-  function: runcmd_stderr_isnt
-
-# Steps to examine stdout/stderr for sub-strings.
-
-- then: stdout contains "(?P<text>.*)"
-  regex: true
-  function: runcmd_stdout_contains
-
-- then: "stdout doesn't contain \"(?P<text>.*)\""
-  regex: true
-  function: runcmd_stdout_doesnt_contain
-
-- then: stderr contains "(?P<text>.*)"
-  regex: true
-  function: runcmd_stderr_contains
-
-- then: "stderr doesn't contain \"(?P<text>.*)\""
-  regex: true
-  function: runcmd_stderr_doesnt_contain
-
-# Steps to match stdout/stderr against regular expressions.
-
-- then: stdout matches regex (?P<regex>.*)
-  regex: true
-  function: runcmd_stdout_matches_regex
-
-- then: stdout doesn't match regex (?P<regex>.*)
-  regex: true
-  function: runcmd_stdout_doesnt_match_regex
-
-- then: stderr matches regex (?P<regex>.*)
-  regex: true
-  function: runcmd_stderr_matches_regex
-
-- then: stderr doesn't match regex (?P<regex>.*)
-  regex: true
-  function: runcmd_stderr_doesnt_match_regex
diff --git a/subplot/server.py b/subplot/server.py
index 5cc9d9b..289e181 100644
--- a/subplot/server.py
+++ b/subplot/server.py
@@ -90,6 +90,16 @@ def delete_chunk_by_id(ctx, chunk_id=None):
     _request(ctx, requests.delete, url)
 
 
+def make_chunk_file_be_empty(ctx, chunk_id=None):
+    chunk_id = ctx["vars"][chunk_id]
+    chunks = ctx["config"]["chunks"]
+    for (dirname, _, _) in os.walk(chunks):
+        filename = os.path.join(dirname, chunk_id + ".data")
+        if os.path.exists(filename):
+            logging.debug(f"emptying chunk file {filename}")
+            open(filename, "w").close()
+
+
 def status_code_is(ctx, status=None):
     assert_eq = globals()["assert_eq"]
     assert_eq(ctx["http.status"], int(status))
diff --git a/subplot/server.yaml b/subplot/server.yaml
index 2cc2b5f..68f8f0c 100644
--- a/subplot/server.yaml
+++ b/subplot/server.yaml
@@ -25,6 +25,9 @@
 - when: "I try to DELETE /chunks/{chunk_id}"
   function: delete_chunk_by_id
 
+- when: "chunk <{chunk_id}> on chunk server is replaced by an empty file"
+  function: make_chunk_file_be_empty
+
 - then: "HTTP status code is {status}"
   function: status_code_is