diff options
| -rw-r--r-- | Cargo.toml | 5 | ||||
| -rw-r--r-- | client.yaml | 2 | ||||
| -rw-r--r-- | hello.txt | 1 | ||||
| -rw-r--r-- | obnam.md | 7 | ||||
| -rw-r--r-- | src/bin/obnam-backup.rs | 151 | ||||
| -rw-r--r-- | src/bin/obnam-server.rs | 13 | ||||
| -rw-r--r-- | subplot/obnam.py | 32 | ||||
| -rw-r--r-- | subplot/obnam.yaml | 6 |
8 files changed, 213 insertions, 4 deletions
@@ -7,9 +7,14 @@ edition = "2018" [dependencies] anyhow = "1" bytes = "0.5" +indicatif = "0.15" +log = "0.4" +pretty_env_logger = "0.4" +reqwest = { version = "0.10", features = ["blocking", "json"]} serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.8" +sha2 = "0.9" structopt = "0.3" thiserror = "1" tokio = { version = "0.2", features = ["macros"] } diff --git a/client.yaml b/client.yaml new file mode 100644 index 0000000..3405a80 --- /dev/null +++ b/client.yaml @@ -0,0 +1,2 @@ +server_name: localhost +server_port: 8888 diff --git a/hello.txt b/hello.txt new file mode 100644 index 0000000..ce01362 --- /dev/null +++ b/hello.txt @@ -0,0 +1 @@ +hello @@ -329,6 +329,13 @@ one directory can be backed up and restored, and the restored files and their metadata are identical to the original. This is the simplest possible, but still useful requirement for a backup system. +~~~scenario +given a chunk server +and a file live/data.dat containing some random data +when I back up live with obnam-backup +then backup command is successful +~~~ + ## Backups and restores These scenarios verify that every kind of file system object can be diff --git a/src/bin/obnam-backup.rs b/src/bin/obnam-backup.rs new file mode 100644 index 0000000..2b767af --- /dev/null +++ b/src/bin/obnam-backup.rs @@ -0,0 +1,151 @@ +// Read stdin, split into chunks, upload new chunks to chunk server. + +use indicatif::{ProgressBar, ProgressStyle}; +use obnam::chunk::Chunk; +use obnam::chunkmeta::ChunkMeta; +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::io::prelude::*; +use std::io::BufReader; +use std::path::{Path, PathBuf}; +use structopt::StructOpt; + +const BUFFER_SIZE: usize = 1024 * 1024; + +#[derive(Debug, StructOpt)] +#[structopt(name = "obnam-backup", about = "Simplistic backup client")] +struct Opt { + #[structopt(parse(from_os_str))] + config: PathBuf, +} + +fn main() -> anyhow::Result<()> { + let opt = Opt::from_args(); + let config = Config::read_config(&opt.config).unwrap(); + let pb = ProgressBar::new_spinner(); + pb.set_style( + ProgressStyle::default_bar() + .template("backing up:\n{bytes} ({bytes_per_sec}) {elapsed} {msg} {spinner}"), + ); + + println!("config: {:?}", config); + + let client = reqwest::blocking::Client::builder() + .danger_accept_invalid_certs(true) + .build()?; + + let stdin = std::io::stdin(); + let mut stdin = BufReader::new(stdin); + let mut dup = 0; + loop { + match read_chunk(&mut stdin)? { + None => break, + Some(chunk) => { + let n = chunk.data().len() as u64; + if !has_chunk(&client, &config, &chunk.meta())? { + pb.inc(n); + upload_chunk(&client, &config, chunk)?; + } else { + dup += n; + } + } + } + } + pb.finish(); + println!( + "read total {} bytes from stdin ({} dup)", + pb.position(), + dup + ); + Ok(()) +} + +#[derive(Debug, Deserialize, Clone)] +pub struct Config { + pub server_name: String, + pub server_port: u16, +} + +impl Config { + pub fn read_config(filename: &Path) -> anyhow::Result<Config> { + let config = std::fs::read_to_string(filename)?; + let config: Config = serde_yaml::from_str(&config)?; + Ok(config) + } +} + +fn read_chunk<H>(handle: &mut H) -> anyhow::Result<Option<Chunk>> +where + H: Read + BufRead, +{ + let mut buffer = [0; BUFFER_SIZE]; + let mut used = 0; + + loop { + let n = handle.read(&mut buffer[used..])?; + used += n; + if n == 0 || used == BUFFER_SIZE { + break; + } + } + + if used == 0 { + return Ok(None); + } + + let buffer = &buffer[..used]; + let mut hasher = Sha256::new(); + hasher.update(buffer); + let hash = hasher.finalize(); + let hash = format!("{:x}", hash); + let meta = ChunkMeta::new(&hash); + + let chunk = Chunk::new(meta, buffer.to_vec()); + Ok(Some(chunk)) +} + +fn upload_chunk( + client: &reqwest::blocking::Client, + config: &Config, + chunk: Chunk, +) -> anyhow::Result<()> { + let url = format!( + "http://{}:{}/chunks", + config.server_name, config.server_port + ); + + client + .post(&url) + .header("chunk-meta", chunk.meta().to_json()) + .body(chunk.data().to_vec()) + .send()?; + Ok(()) +} + +fn has_chunk( + client: &reqwest::blocking::Client, + config: &Config, + meta: &ChunkMeta, +) -> anyhow::Result<bool> { + let url = format!( + "http://{}:{}/chunks", + config.server_name, config.server_port, + ); + + let req = client + .get(&url) + .query(&[("sha256", meta.sha256())]) + .build()?; + + let res = client.execute(req)?; + let has = if res.status() != 200 { + false + } else { + let text = res.text()?; + let hits: HashMap<String, ChunkMeta> = serde_json::from_str(&text)?; + !hits.is_empty() + }; + + Ok(has) +} diff --git a/src/bin/obnam-server.rs b/src/bin/obnam-server.rs index 8ad792e..5723b23 100644 --- a/src/bin/obnam-server.rs +++ b/src/bin/obnam-server.rs @@ -1,4 +1,5 @@ use bytes::Bytes; +use log::{debug, error, info}; use obnam::{chunk::Chunk, chunkid::ChunkId, chunkmeta::ChunkMeta, index::Index, store::Store}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -19,6 +20,8 @@ struct Opt { #[tokio::main] async fn main() -> anyhow::Result<()> { + pretty_env_logger::init(); + let opt = Opt::from_args(); let config = Config::read_config(&opt.config).unwrap(); let config_bare = config.clone(); @@ -28,6 +31,9 @@ async fn main() -> anyhow::Result<()> { let index = Arc::new(Mutex::new(Index::default())); let index = warp::any().map(move || Arc::clone(&index)); + info!("Obnam server starting up"); + debug!("Configuration: {:?}", config_bare); + let create = warp::post() .and(warp::path("chunks")) .and(config.clone()) @@ -58,9 +64,9 @@ async fn main() -> anyhow::Result<()> { let webroot = create.or(fetch).or(search).or(delete); warp::serve(webroot) - .tls() - .key_path(config_bare.tls_key) - .cert_path(config_bare.tls_cert) + // .tls() + // .key_path(config_bare.tls_key) + // .cert_path(config_bare.tls_cert) .run(([127, 0, 0, 1], config_bare.port)) .await; Ok(()) @@ -148,6 +154,7 @@ pub async fn create_chunk( index.insert_generation(id.clone()); } + info!("created chunk {}: {:?}", id, meta); Ok(ChunkResult::Created(id)) } diff --git a/subplot/obnam.py b/subplot/obnam.py index ccfdc67..5947f1f 100644 --- a/subplot/obnam.py +++ b/subplot/obnam.py @@ -6,6 +6,7 @@ import re import requests import shutil import socket +import tarfile import time import urllib3 import yaml @@ -33,7 +34,7 @@ def start_chunk_server(ctx): logging.debug(f"Picked randomly port for obnam-server: {config['port']}") ctx["config"] = config - ctx["url"] = f"https://localhost:{port}" + ctx["url"] = f"http://localhost:{port}" start_daemon(ctx, "obnam-server", [_binary("obnam-server"), filename]) @@ -51,6 +52,9 @@ def stop_chunk_server(ctx): def create_file_with_random_data(ctx, filename=None): N = 128 data = "".join(chr(random.randint(0, 255)) for i in range(N)).encode("UTF-8") + dirname = os.path.dirname(filename) or "." + logging.debug(f"create_file_with_random_data: dirname={dirname}") + os.makedirs(dirname, exist_ok=True) with open(filename, "wb") as f: f.write(data) @@ -125,6 +129,32 @@ def json_body_matches(ctx, wanted=None): assert_eq(body.get(key, "not.there"), wanted[key]) +def back_up_directory(ctx, dirname=None): + runcmd = globals()["runcmd"] + + runcmd(ctx, ["pgrep", "-laf", "obnam"]) + + config = {"server_name": "localhost", "server_port": ctx["config"]["port"]} + config = yaml.safe_dump(config) + logging.debug(f"back_up_directory: {config}") + filename = "client.yaml" + with open(filename, "w") as f: + f.write(config) + + tarball = f"{dirname}.tar" + t = tarfile.open(name=tarball, mode="w") + t.add(dirname, arcname=".") + t.close() + + with open(tarball, "rb") as f: + runcmd(ctx, [_binary("obnam-backup"), filename], stdin=f) + + +def command_is_successful(ctx): + exit_code_zero = globals()["exit_code_zero"] + exit_code_zero(ctx) + + # Name of Rust binary, debug-build. def _binary(name): srcdir = globals()["srcdir"] diff --git a/subplot/obnam.yaml b/subplot/obnam.yaml index 065cb01..c2a3608 100644 --- a/subplot/obnam.yaml +++ b/subplot/obnam.yaml @@ -30,6 +30,9 @@ - when: "I try to DELETE /chunks/{chunk_id}" function: delete_chunk_by_id +- when: "I back up {dirname} with obnam-backup" + function: back_up_directory + - then: "HTTP status code is {status}" function: status_code_is @@ -45,3 +48,6 @@ - then: "the body matches file {filename}" function: body_matches_file + +- then: "backup command is successful" + function: command_is_successful |