From 650b7cee5700eae9ab6c300fbdb816dead6f01f5 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 4 Jan 2021 20:01:14 +0200 Subject: Revert "test: make sure Cargo.toml and debian/changelog versions match" This reverts commit 4a779c234c3dbf2685ad81be92a96688266fec58. CI munges the Debian version, making the version check always fail. Back to the drawing board for this. --- check | 9 --------- debian/changelog | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/check b/check index 73a02b7..65514cc 100755 --- a/check +++ b/check @@ -21,15 +21,6 @@ got_cargo_cmd() cargo --list | grep " $1 " > /dev/null } -# Check version numbers in Cargo.toml vs debian/changelog -v1="$(awk '/^version/ { print $NF }' Cargo.toml | tr -d '"')" -v2="$(dpkg-parsechangelog -SVersion | sed 's/-.*$//')" -if [ "$v1" != "$v2" ] -then - echo "Version from Carog.toml ($v1) and debian/changelog ($v2) don't match" 1>&2 - exit 1 -fi - cargo build --all-targets $quiet got_cargo_cmd clippy && cargo clippy $quiet got_cargo_cmd fmt && cargo fmt -- --check diff --git a/debian/changelog b/debian/changelog index 8b8d16a..3a8701c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -obnam (0.1.0-1) unstable; urgency=low +obnam (0.1-1) unstable; urgency=low * Initial packaging. This is not intended to be uploaded to Debian, so no closing of an ITP bug. -- cgit v1.2.1 From 8efabc60f87e5462b01f4832d575f68382929624 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 4 Jan 2021 18:57:49 +0200 Subject: feat: verify checksum of chunks downloaded from server --- obnam.md | 20 ++++ src/bin/obnam.rs | 7 +- src/chunkid.rs | 8 +- src/client.rs | 19 +++- src/cmd/get_chunk.rs | 15 +++ src/cmd/mod.rs | 3 + src/error.rs | 6 ++ subplot/client.py | 15 +++ subplot/client.yaml | 6 ++ subplot/runcmd.md | 170 ---------------------------------- subplot/runcmd.py | 252 --------------------------------------------------- subplot/runcmd.yaml | 83 ----------------- subplot/server.py | 10 ++ subplot/server.yaml | 3 + 14 files changed, 108 insertions(+), 509 deletions(-) create mode 100644 src/cmd/get_chunk.rs delete mode 100644 subplot/runcmd.md delete mode 100644 subplot/runcmd.py delete mode 100644 subplot/runcmd.yaml diff --git a/obnam.md b/obnam.md index 0c4ab50..accfca2 100644 --- a/obnam.md +++ b/obnam.md @@ -976,6 +976,26 @@ when I run obnam --config smoke.yaml list-files then file live/data.dat was backed up because it was changed ~~~ +## Checksum verification + +Each chunk has metadata with the checksum of the chunk contents. This +scenario verifies that the client checks the contents hasn't been +modified. + +~~~scenario +given an installed obnam +and a running chunk server +and a client config based on smoke.yaml +and a file live/data.dat containing some random data +when I run obnam --config smoke.yaml backup +then backup generation is GEN +when I invoke obnam --config smoke.yaml get-chunk +then exit code is 0 +when chunk on chunk server is replaced by an empty file +when I invoke obnam --config smoke.yaml get-chunk +then command fails +~~~ + ## Tricky filenames Obnam needs to handle all filenames the underlying operating and file diff --git a/src/bin/obnam.rs b/src/bin/obnam.rs index f31884b..e5703ed 100644 --- a/src/bin/obnam.rs +++ b/src/bin/obnam.rs @@ -2,7 +2,7 @@ use log::{debug, error, info, LevelFilter}; use log4rs::append::file::FileAppender; use log4rs::config::{Appender, Config, Logger, Root}; use obnam::client::ClientConfig; -use obnam::cmd::{backup, list, list_files, restore}; +use obnam::cmd::{backup, get_chunk, list, list_files, restore}; use std::path::{Path, PathBuf}; use structopt::StructOpt; @@ -23,6 +23,7 @@ fn main() -> anyhow::Result<()> { Command::List => list(&config), Command::ListFiles { gen_id } => list_files(&config, &gen_id), Command::Restore { gen_id, to } => restore(&config, &gen_id, &to), + Command::GetChunk { chunk_id } => get_chunk(&config, &chunk_id), }; if let Err(ref e) = result { @@ -60,6 +61,10 @@ enum Command { #[structopt(parse(from_os_str))] to: PathBuf, }, + GetChunk { + #[structopt()] + chunk_id: String, + }, } fn setup_logging(filename: &Path) -> anyhow::Result<()> { diff --git a/src/chunkid.rs b/src/chunkid.rs index 9eec41f..3933d4b 100644 --- a/src/chunkid.rs +++ b/src/chunkid.rs @@ -37,6 +37,10 @@ impl ChunkId { } } + pub fn from_str(s: &str) -> Self { + ChunkId { id: s.to_string() } + } + pub fn as_bytes(&self) -> &[u8] { self.id.as_bytes() } @@ -81,7 +85,7 @@ impl FromStr for ChunkId { type Err = (); fn from_str(s: &str) -> Result { - Ok(ChunkId { id: s.to_string() }) + Ok(ChunkId::from_str(s)) } } @@ -113,6 +117,6 @@ mod test { fn survives_round_trip() { let id = ChunkId::new(); let id_str = id.to_string(); - assert_eq!(id, id_str.parse().unwrap()); + assert_eq!(id, ChunkId::from_str(&id_str)) } } diff --git a/src/client.rs b/src/client.rs index 616ceef..6b1bb80 100644 --- a/src/client.rs +++ b/src/client.rs @@ -4,6 +4,7 @@ use crate::chunk::GenerationChunk; use crate::chunker::Chunker; use crate::chunkid::ChunkId; use crate::chunkmeta::ChunkMeta; +use crate::error::ObnamError; use crate::fsentry::{FilesystemEntry, FilesystemKind}; use crate::generation::{FinishedGeneration, LocalGeneration}; use crate::genlist::GenerationList; @@ -212,8 +213,24 @@ impl BackupClient { return Err(ClientError::ChunkNotFound(chunk_id.to_string()).into()); } + let headers = res.headers(); + let meta = headers.get("chunk-meta"); + if meta.is_none() { + return Err(ObnamError::NoChunkMeta(chunk_id.to_string()).into()); + } + let meta = meta.unwrap().to_str()?; + let meta: ChunkMeta = serde_json::from_str(meta)?; + let body = res.bytes()?; - Ok(DataChunk::new(body.to_vec())) + let body = body.to_vec(); + let actual = sha256(&body); + if actual != meta.sha256() { + return Err(ObnamError::WrongChecksum(chunk_id.to_string()).into()); + } + + let chunk: DataChunk = DataChunk::new(body); + + Ok(chunk) } fn fetch_generation_chunk(&self, gen_id: &str) -> anyhow::Result { diff --git a/src/cmd/get_chunk.rs b/src/cmd/get_chunk.rs new file mode 100644 index 0000000..bf653ff --- /dev/null +++ b/src/cmd/get_chunk.rs @@ -0,0 +1,15 @@ +use crate::chunkid::ChunkId; +use crate::client::BackupClient; +use crate::client::ClientConfig; +use std::io::{stdout, Write}; + +pub fn get_chunk(config: &ClientConfig, chunk_id: &str) -> anyhow::Result<()> { + let client = BackupClient::new(&config.server_url)?; + let chunk_id: ChunkId = chunk_id.parse().unwrap(); + let chunk = client.fetch_chunk(&chunk_id)?; + + let stdout = stdout(); + let mut handle = stdout.lock(); + handle.write_all(chunk.data())?; + Ok(()) +} diff --git a/src/cmd/mod.rs b/src/cmd/mod.rs index 2919d88..c3cebe2 100644 --- a/src/cmd/mod.rs +++ b/src/cmd/mod.rs @@ -9,3 +9,6 @@ pub use list_files::list_files; pub mod restore; pub use restore::restore; + +pub mod get_chunk; +pub use get_chunk::get_chunk; diff --git a/src/error.rs b/src/error.rs index 3b3f573..360e62d 100644 --- a/src/error.rs +++ b/src/error.rs @@ -9,4 +9,10 @@ pub enum ObnamError { #[error("Generation has more than one file with the name {0}")] TooManyFiles(PathBuf), + + #[error("Server response did not have a 'chunk-meta' header for chunk {0}")] + NoChunkMeta(String), + + #[error("Wrong checksum for chunk {0}")] + WrongChecksum(String), } diff --git a/subplot/client.py b/subplot/client.py index c1f5159..53c7f6e 100644 --- a/subplot/client.py +++ b/subplot/client.py @@ -50,6 +50,13 @@ def run_obnam_restore_with_genref(ctx, filename=None, genref=None, todir=None): ) +def run_obnam_get_chunk(ctx, filename=None, gen_id=None, todir=None): + runcmd_run = globals()["runcmd_run"] + gen_id = ctx["vars"][gen_id] + logging.debug(f"run_obnam_get_chunk: gen_id={gen_id}") + runcmd_run(ctx, ["obnam", "--config", filename, "get-chunk", gen_id]) + + def capture_generation_id(ctx, varname=None): runcmd_get_stdout = globals()["runcmd_get_stdout"] @@ -95,3 +102,11 @@ def get_backup_reason(ctx, filename): lines = [line for line in lines if filename in line] line = lines[0] return line.split()[-1] + + +def stdout_matches_file(ctx, filename=None): + runcmd_get_stdout = globals()["runcmd_get_stdout"] + assert_eq = globals()["assert_eq"] + stdout = runcmd_get_stdout(ctx) + data = open(filename).read() + assert_eq(stdout, data) diff --git a/subplot/client.yaml b/subplot/client.yaml index db55679..b1f9b19 100644 --- a/subplot/client.yaml +++ b/subplot/client.yaml @@ -10,6 +10,9 @@ - when: "I invoke obnam --config {filename} restore latest {todir}" function: run_obnam_restore_latest +- when: "I invoke obnam --config {filename} get-chunk <{gen_id}>" + function: run_obnam_get_chunk + - then: "backup generation is {varname}" function: capture_generation_id @@ -24,3 +27,6 @@ - then: "file {filename} was not backed up because it was unchanged" function: file_was_unchanged + +- then: "stdout matches file {filename}" + function: stdout_matches_file diff --git a/subplot/runcmd.md b/subplot/runcmd.md deleted file mode 100644 index a9d4ed4..0000000 --- a/subplot/runcmd.md +++ /dev/null @@ -1,170 +0,0 @@ -# Introduction - -The [Subplot][] library `runcmd` for Python provides scenario steps -and their implementations for running Unix commands and examining the -results. The library consists of a bindings file `lib/runcmd.yaml` and -implementations in Python in `lib/runcmd.py`. There is no Bash -version. - -[Subplot]: https://subplot.liw.fi/ - -This document explains the acceptance criteria for the library and how -they're verified. It uses the steps and functions from the -`lib/runcmd` library. The scenarios all have the same structure: run a -command, then examine the exit code, standard output (stdout for -short), or standard error output (stderr) of the command. - -The scenarios use the Unix commands `/bin/true` and `/bin/false` to -generate exit codes, and `/bin/echo` to produce stdout. To generate -stderr, they use the little helper script below. - -~~~{#err.sh .file .sh .numberLines} -#!/bin/sh -echo "$@" 1>&2 -~~~ - -# Check exit code - -These scenarios verify the exit code. To make it easier to write -scenarios in language that flows more naturally, there are a couple of -variations. - -## Successful execution - -~~~scenario -when I run /bin/true -then exit code is 0 -and command is successful -~~~ - -## Failed execution - -~~~scenario -when I try to run /bin/false -then exit code is not 0 -and command fails -~~~ - -# Check output has what we want - -These scenarios verify that stdout or stderr do have something we want -to have. - -## Check stdout is exactly as wanted - -Note that the string is surrounded by double quotes to make it clear -to the reader what's inside. Also, C-style string escapes are -understood. - -~~~scenario -when I run /bin/echo hello, world -then stdout is exactly "hello, world\n" -~~~ - -## Check stderr is exactly as wanted - -~~~scenario -given helper script err.sh for runcmd -when I run sh err.sh hello, world -then stderr is exactly "hello, world\n" -~~~ - -## Check stdout using sub-string search - -Exact string comparisons are not always enough, so we can verify a -sub-string is in output. - -~~~scenario -when I run /bin/echo hello, world -then stdout contains "world\n" -and exit code is 0 -~~~ - -## Check stderr using sub-string search - -~~~scenario -given helper script err.sh for runcmd -when I run sh err.sh hello, world -then stderr contains "world\n" -~~~ - -## Check stdout using regular expressions - -Fixed strings are not always enough, so we can verify output matches a -regular expression. Note that the regular expression is not delimited -and does not get any C-style string escaped decoded. - -~~~scenario -when I run /bin/echo hello, world -then stdout matches regex world$ -~~~ - -## Check stderr using regular expressions - -~~~scenario -given helper script err.sh for runcmd -when I run sh err.sh hello, world -then stderr matches regex world$ -~~~ - -# Check output doesn't have what we want to avoid - -These scenarios verify that the stdout or stderr do not -have something we want to avoid. - -## Check stdout is not exactly something - -~~~scenario -when I run /bin/echo hi -then stdout isn't exactly "hello, world\n" -~~~ - -## Check stderr is not exactly something - -~~~scenario -given helper script err.sh for runcmd -when I run sh err.sh hi -then stderr isn't exactly "hello, world\n" -~~~ - -## Check stdout doesn't contain sub-string - -~~~scenario -when I run /bin/echo hi -then stdout doesn't contain "world" -~~~ - -## Check stderr doesn't contain sub-string - -~~~scenario -given helper script err.sh for runcmd -when I run sh err.sh hi -then stderr doesn't contain "world" -~~~ - -## Check stdout doesn't match regular expression - -~~~scenario -when I run /bin/echo hi -then stdout doesn't match regex world$ - -~~~ - -## Check stderr doesn't match regular expressions - -~~~scenario -given helper script err.sh for runcmd -when I run sh err.sh hi -then stderr doesn't match regex world$ -~~~ - - ---- -title: Acceptance criteria for the lib/runcmd Subplot library -author: The Subplot project -template: python -bindings: -- runcmd.yaml -functions: -- runcmd.py -... diff --git a/subplot/runcmd.py b/subplot/runcmd.py deleted file mode 100644 index a2564c6..0000000 --- a/subplot/runcmd.py +++ /dev/null @@ -1,252 +0,0 @@ -import logging -import os -import re -import shlex -import subprocess - - -# -# Helper functions. -# - -# Get exit code or other stored data about the latest command run by -# runcmd_run. - - -def _runcmd_get(ctx, name): - ns = ctx.declare("_runcmd") - return ns[name] - - -def runcmd_get_exit_code(ctx): - return _runcmd_get(ctx, "exit") - - -def runcmd_get_stdout(ctx): - return _runcmd_get(ctx, "stdout") - - -def runcmd_get_stdout_raw(ctx): - return _runcmd_get(ctx, "stdout.raw") - - -def runcmd_get_stderr(ctx): - return _runcmd_get(ctx, "stderr") - - -def runcmd_get_stderr_raw(ctx): - return _runcmd_get(ctx, "stderr.raw") - - -def runcmd_get_argv(ctx): - return _runcmd_get(ctx, "argv") - - -# Run a command, given an argv and other arguments for subprocess.Popen. -# -# This is meant to be a helper function, not bound directly to a step. The -# stdout, stderr, and exit code are stored in the "_runcmd" namespace in the -# ctx context. -def runcmd_run(ctx, argv, **kwargs): - ns = ctx.declare("_runcmd") - - # The Subplot Python template empties os.environ at startup, modulo a small - # number of variables with carefully chosen values. Here, we don't need to - # care about what those variables are, but we do need to not overwrite - # them, so we just add anything in the env keyword argument, if any, to - # os.environ. - env = dict(os.environ) - for key, arg in kwargs.pop("env", {}).items(): - env[key] = arg - - pp = ns.get("path-prefix") - if pp: - env["PATH"] = pp + ":" + env["PATH"] - - logging.debug(f"runcmd_run") - logging.debug(f" argv: {argv}") - logging.debug(f" env: {env}") - p = subprocess.Popen( - argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, **kwargs - ) - stdout, stderr = p.communicate("") - ns["argv"] = argv - ns["stdout.raw"] = stdout - ns["stderr.raw"] = stderr - ns["stdout"] = stdout.decode("utf-8") - ns["stderr"] = stderr.decode("utf-8") - ns["exit"] = p.returncode - logging.debug(f" ctx: {ctx}") - logging.debug(f" ns: {ns}") - - -# Step: prepend srcdir to PATH whenever runcmd runs a command. -def runcmd_helper_srcdir_path(ctx): - srcdir = globals()["srcdir"] - runcmd_prepend_to_path(ctx, srcdir) - - -# Step: This creates a helper script. -def runcmd_helper_script(ctx, filename=None): - get_file = globals()["get_file"] - with open(filename, "wb") as f: - f.write(get_file(filename)) - - -# -# Step functions for running commands. -# - - -def runcmd_prepend_to_path(ctx, dirname=None): - ns = ctx.declare("_runcmd") - pp = ns.get("path-prefix", "") - if pp: - pp = f"{pp}:{dirname}" - else: - pp = dirname - ns["path-prefix"] = pp - - -def runcmd_step(ctx, argv0=None, args=None): - runcmd_try_to_run(ctx, argv0=argv0, args=args) - runcmd_exit_code_is_zero(ctx) - - -def runcmd_try_to_run(ctx, argv0=None, args=None): - argv = [shlex.quote(argv0)] + shlex.split(args) - runcmd_run(ctx, argv) - - -# -# Step functions for examining exit codes. -# - - -def runcmd_exit_code_is_zero(ctx): - runcmd_exit_code_is(ctx, exit=0) - - -def runcmd_exit_code_is(ctx, exit=None): - assert_eq = globals()["assert_eq"] - assert_eq(runcmd_get_exit_code(ctx), int(exit)) - - -def runcmd_exit_code_is_nonzero(ctx): - runcmd_exit_code_is_not(ctx, exit=0) - - -def runcmd_exit_code_is_not(ctx, exit=None): - assert_ne = globals()["assert_ne"] - assert_ne(runcmd_get_exit_code(ctx), int(exit)) - - -# -# Step functions and helpers for examining output in various ways. -# - - -def runcmd_stdout_is(ctx, text=None): - _runcmd_output_is(runcmd_get_stdout(ctx), text) - - -def runcmd_stdout_isnt(ctx, text=None): - _runcmd_output_isnt(runcmd_get_stdout(ctx), text) - - -def runcmd_stderr_is(ctx, text=None): - _runcmd_output_is(runcmd_get_stderr(ctx), text) - - -def runcmd_stderr_isnt(ctx, text=None): - _runcmd_output_isnt(runcmd_get_stderr(ctx), text) - - -def _runcmd_output_is(actual, wanted): - assert_eq = globals()["assert_eq"] - wanted = bytes(wanted, "utf8").decode("unicode_escape") - logging.debug("_runcmd_output_is:") - logging.debug(f" actual: {actual!r}") - logging.debug(f" wanted: {wanted!r}") - assert_eq(actual, wanted) - - -def _runcmd_output_isnt(actual, wanted): - assert_ne = globals()["assert_ne"] - wanted = bytes(wanted, "utf8").decode("unicode_escape") - logging.debug("_runcmd_output_isnt:") - logging.debug(f" actual: {actual!r}") - logging.debug(f" wanted: {wanted!r}") - assert_ne(actual, wanted) - - -def runcmd_stdout_contains(ctx, text=None): - _runcmd_output_contains(runcmd_get_stdout(ctx), text) - - -def runcmd_stdout_doesnt_contain(ctx, text=None): - _runcmd_output_doesnt_contain(runcmd_get_stdout(ctx), text) - - -def runcmd_stderr_contains(ctx, text=None): - _runcmd_output_contains(runcmd_get_stderr(ctx), text) - - -def runcmd_stderr_doesnt_contain(ctx, text=None): - _runcmd_output_doesnt_contain(runcmd_get_stderr(ctx), text) - - -def _runcmd_output_contains(actual, wanted): - assert_eq = globals()["assert_eq"] - wanted = bytes(wanted, "utf8").decode("unicode_escape") - logging.debug("_runcmd_output_contains:") - logging.debug(f" actual: {actual!r}") - logging.debug(f" wanted: {wanted!r}") - assert_eq(wanted in actual, True) - - -def _runcmd_output_doesnt_contain(actual, wanted): - assert_ne = globals()["assert_ne"] - wanted = bytes(wanted, "utf8").decode("unicode_escape") - logging.debug("_runcmd_output_doesnt_contain:") - logging.debug(f" actual: {actual!r}") - logging.debug(f" wanted: {wanted!r}") - assert_ne(wanted in actual, True) - - -def runcmd_stdout_matches_regex(ctx, regex=None): - _runcmd_output_matches_regex(runcmd_get_stdout(ctx), regex) - - -def runcmd_stdout_doesnt_match_regex(ctx, regex=None): - _runcmd_output_doesnt_match_regex(runcmd_get_stdout(ctx), regex) - - -def runcmd_stderr_matches_regex(ctx, regex=None): - _runcmd_output_matches_regex(runcmd_get_stderr(ctx), regex) - - -def runcmd_stderr_doesnt_match_regex(ctx, regex=None): - _runcmd_output_doesnt_match_regex(runcmd_get_stderr(ctx), regex) - - -def _runcmd_output_matches_regex(actual, regex): - assert_ne = globals()["assert_ne"] - r = re.compile(regex) - m = r.search(actual) - logging.debug("_runcmd_output_matches_regex:") - logging.debug(f" actual: {actual!r}") - logging.debug(f" regex: {regex!r}") - logging.debug(f" match: {m}") - assert_ne(m, None) - - -def _runcmd_output_doesnt_match_regex(actual, regex): - assert_eq = globals()["assert_eq"] - r = re.compile(regex) - m = r.search(actual) - logging.debug("_runcmd_output_doesnt_match_regex:") - logging.debug(f" actual: {actual!r}") - logging.debug(f" regex: {regex!r}") - logging.debug(f" match: {m}") - assert_eq(m, None) diff --git a/subplot/runcmd.yaml b/subplot/runcmd.yaml deleted file mode 100644 index 48dde90..0000000 --- a/subplot/runcmd.yaml +++ /dev/null @@ -1,83 +0,0 @@ -# Steps to run commands. - -- given: helper script {filename} for runcmd - function: runcmd_helper_script - -- given: srcdir is in the PATH - function: runcmd_helper_srcdir_path - -- when: I run (?P\S+)(?P.*) - regex: true - function: runcmd_step - -- when: I try to run (?P\S+)(?P.*) - regex: true - function: runcmd_try_to_run - -# Steps to examine exit code of latest command. - -- then: exit code is {exit} - function: runcmd_exit_code_is - -- then: exit code is not {exit} - function: runcmd_exit_code_is_not - -- then: command is successful - function: runcmd_exit_code_is_zero - -- then: command fails - function: runcmd_exit_code_is_nonzero - -# Steps to examine stdout/stderr for exact content. - -- then: stdout is exactly "(?P.*)" - regex: true - function: runcmd_stdout_is - -- then: "stdout isn't exactly \"(?P.*)\"" - regex: true - function: runcmd_stdout_isnt - -- then: stderr is exactly "(?P.*)" - regex: true - function: runcmd_stderr_is - -- then: "stderr isn't exactly \"(?P.*)\"" - regex: true - function: runcmd_stderr_isnt - -# Steps to examine stdout/stderr for sub-strings. - -- then: stdout contains "(?P.*)" - regex: true - function: runcmd_stdout_contains - -- then: "stdout doesn't contain \"(?P.*)\"" - regex: true - function: runcmd_stdout_doesnt_contain - -- then: stderr contains "(?P.*)" - regex: true - function: runcmd_stderr_contains - -- then: "stderr doesn't contain \"(?P.*)\"" - regex: true - function: runcmd_stderr_doesnt_contain - -# Steps to match stdout/stderr against regular expressions. - -- then: stdout matches regex (?P.*) - regex: true - function: runcmd_stdout_matches_regex - -- then: stdout doesn't match regex (?P.*) - regex: true - function: runcmd_stdout_doesnt_match_regex - -- then: stderr matches regex (?P.*) - regex: true - function: runcmd_stderr_matches_regex - -- then: stderr doesn't match regex (?P.*) - regex: true - function: runcmd_stderr_doesnt_match_regex diff --git a/subplot/server.py b/subplot/server.py index 5cc9d9b..289e181 100644 --- a/subplot/server.py +++ b/subplot/server.py @@ -90,6 +90,16 @@ def delete_chunk_by_id(ctx, chunk_id=None): _request(ctx, requests.delete, url) +def make_chunk_file_be_empty(ctx, chunk_id=None): + chunk_id = ctx["vars"][chunk_id] + chunks = ctx["config"]["chunks"] + for (dirname, _, _) in os.walk(chunks): + filename = os.path.join(dirname, chunk_id + ".data") + if os.path.exists(filename): + logging.debug(f"emptying chunk file {filename}") + open(filename, "w").close() + + def status_code_is(ctx, status=None): assert_eq = globals()["assert_eq"] assert_eq(ctx["http.status"], int(status)) diff --git a/subplot/server.yaml b/subplot/server.yaml index 2cc2b5f..68f8f0c 100644 --- a/subplot/server.yaml +++ b/subplot/server.yaml @@ -25,6 +25,9 @@ - when: "I try to DELETE /chunks/{chunk_id}" function: delete_chunk_by_id +- when: "chunk <{chunk_id}> on chunk server is replaced by an empty file" + function: make_chunk_file_be_empty + - then: "HTTP status code is {status}" function: status_code_is -- cgit v1.2.1 From 34a9fc7cf754552b02b194f7e1186adc9414a623 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 4 Jan 2021 19:57:54 +0200 Subject: refactor: use fetch_chunk to fetch generation Also, add GenerationChunk::from_data_chunk function. --- src/chunk.rs | 6 ++++++ src/client.rs | 16 +++------------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/chunk.rs b/src/chunk.rs index 7fdeccb..4917b60 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -35,6 +35,12 @@ impl GenerationChunk { Self { chunk_ids } } + pub fn from_data_chunk(chunk: &DataChunk) -> anyhow::Result { + let data = chunk.data(); + let data = std::str::from_utf8(data)?; + Ok(serde_json::from_str(data)?) + } + pub fn is_empty(&self) -> bool { self.chunk_ids.is_empty() } diff --git a/src/client.rs b/src/client.rs index 6b1bb80..1b507d3 100644 --- a/src/client.rs +++ b/src/client.rs @@ -234,19 +234,9 @@ impl BackupClient { } fn fetch_generation_chunk(&self, gen_id: &str) -> anyhow::Result { - let url = format!("{}/{}", &self.chunks_url(), gen_id); - trace!("fetch_generation_chunk: url={:?}", url); - let req = self.client.get(&url).build()?; - let res = self.client.execute(req)?; - debug!("fetch_generation_chunk: status={}", res.status()); - if res.status() != 200 { - return Err(ClientError::GenerationNotFound(gen_id.to_string()).into()); - } - - let text = res.text()?; - debug!("fetch_generation_chunk: text={:?}", text); - let gen: GenerationChunk = serde_json::from_str(&text)?; - debug!("fetch_generation_chunk: {:?}", gen); + let chunk_id = ChunkId::from_str(gen_id); + let chunk = self.fetch_chunk(&chunk_id)?; + let gen = GenerationChunk::from_data_chunk(&chunk)?; Ok(gen) } -- cgit v1.2.1