From 38b51382e710dda515e6d0df8fdc4f0b253d9cd7 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 13 Nov 2020 09:40:39 +0200 Subject: refactor: put all client functionality into one program --- obnam.md | 7 ++- src/bin/obnam-backup.rs | 37 -------------- src/bin/obnam-list.rs | 27 ----------- src/bin/obnam-restore.rs | 122 ----------------------------------------------- src/bin/obnam.rs | 52 ++++++++++++++++++++ src/cmd/backup.rs | 21 ++++++++ src/cmd/list.rs | 13 +++++ src/cmd/mod.rs | 8 ++++ src/cmd/restore.rs | 115 ++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + subplot/obnam.py | 18 ++++++- subplot/obnam.yaml | 10 +++- 12 files changed, 239 insertions(+), 192 deletions(-) delete mode 100644 src/bin/obnam-backup.rs delete mode 100644 src/bin/obnam-list.rs delete mode 100644 src/bin/obnam-restore.rs create mode 100644 src/bin/obnam.rs create mode 100644 src/cmd/backup.rs create mode 100644 src/cmd/list.rs create mode 100644 src/cmd/mod.rs create mode 100644 src/cmd/restore.rs diff --git a/obnam.md b/obnam.md index 26a4cfc..920f037 100644 --- a/obnam.md +++ b/obnam.md @@ -310,10 +310,13 @@ given an installed obnam and a chunk server and a client config based on smoke.yaml and a file live/data.dat containing some random data -when I invoke obnam-backup smoke.yaml +when I invoke obnam backup smoke.yaml then backup command is successful and backup generation is GEN -when I invoke obnam-restore smoke.yaml restore.db rest +when I invoke obnam list smoke.yaml +then backup command is successful +and generation list contains +when I invoke obnam restore smoke.yaml restore.db rest then data in live and rest match ~~~ diff --git a/src/bin/obnam-backup.rs b/src/bin/obnam-backup.rs deleted file mode 100644 index 7d7e07e..0000000 --- a/src/bin/obnam-backup.rs +++ /dev/null @@ -1,37 +0,0 @@ -use log::{debug, info}; -use obnam::client::{BackupClient, ClientConfig}; -use obnam::fsiter::FsIterator; -use obnam::generation::Generation; -use std::path::PathBuf; -use structopt::StructOpt; - -const BUFFER_SIZE: usize = 1024 * 1024; - -fn main() -> anyhow::Result<()> { - pretty_env_logger::init(); - - let opt = Opt::from_args(); - info!("obnam-backup starts"); - debug!("opt: {:?}", opt); - let config = ClientConfig::read_config(&opt.config)?; - let client = BackupClient::new(&config.server_name, config.server_port)?; - - { - let mut gen = Generation::create(&config.dbname)?; - gen.insert_iter(FsIterator::new(&config.root).map(|entry| match entry { - Err(err) => Err(err), - Ok(entry) => client.upload_filesystem_entry(entry, BUFFER_SIZE), - }))?; - } - let gen_id = client.upload_generation(&config.dbname, BUFFER_SIZE)?; - println!("gen id: {}", gen_id); - - Ok(()) -} - -#[derive(Debug, StructOpt)] -#[structopt(name = "obnam-backup", about = "Simplistic backup client")] -struct Opt { - #[structopt(parse(from_os_str))] - config: PathBuf, -} diff --git a/src/bin/obnam-list.rs b/src/bin/obnam-list.rs deleted file mode 100644 index 18c49f1..0000000 --- a/src/bin/obnam-list.rs +++ /dev/null @@ -1,27 +0,0 @@ -use log::{debug, info}; -use obnam::client::{BackupClient, ClientConfig}; -use std::path::PathBuf; -use structopt::StructOpt; - -fn main() -> anyhow::Result<()> { - pretty_env_logger::init(); - - let opt = Opt::from_args(); - info!("obnam-list starts"); - debug!("opt: {:?}", opt); - let config = ClientConfig::read_config(&opt.config)?; - let client = BackupClient::new(&config.server_name, config.server_port)?; - - for gen_id in client.list_generations()? { - println!("{}", gen_id); - } - - Ok(()) -} - -#[derive(Debug, StructOpt)] -#[structopt(name = "obnam-backup", about = "Simplistic backup client")] -struct Opt { - #[structopt(parse(from_os_str))] - config: PathBuf, -} diff --git a/src/bin/obnam-restore.rs b/src/bin/obnam-restore.rs deleted file mode 100644 index 442104a..0000000 --- a/src/bin/obnam-restore.rs +++ /dev/null @@ -1,122 +0,0 @@ -use log::{debug, info}; -use obnam::client::BackupClient; -use obnam::fsentry::{FilesystemEntry, FilesystemKind}; -use obnam::generation::Generation; -//use obnam::chunkmeta::ChunkMeta; -use serde::Deserialize; -use std::fs::File; -use std::io::prelude::*; -use std::path::{Path, PathBuf}; -use structopt::StructOpt; - -fn main() -> anyhow::Result<()> { - pretty_env_logger::init(); - - let opt = Opt::from_args(); - let config = Config::read_config(&opt.config).unwrap(); - - info!("obnam-restore starts up"); - info!("opt: {:?}", opt); - info!("config: {:?}", config); - - let client = BackupClient::new(&config.server_name, config.server_port)?; - let gen_chunk = client.fetch_generation(&opt.gen_id)?; - debug!("gen: {:?}", gen_chunk); - { - let mut dbfile = File::create(&opt.dbname)?; - for id in gen_chunk.chunk_ids() { - let chunk = client.fetch_chunk(id)?; - dbfile.write_all(chunk.data())?; - } - } - info!("downloaded generation to {}", opt.dbname.display()); - - let gen = Generation::open(&opt.dbname)?; - for (fileid, entry) in gen.files()? { - restore(&client, &gen, fileid, entry, &opt.to)?; - } - - Ok(()) -} - -#[derive(Debug, StructOpt)] -#[structopt(name = "obnam-backup", about = "Simplistic backup client")] -struct Opt { - #[structopt(parse(from_os_str))] - config: PathBuf, - - #[structopt()] - gen_id: String, - - #[structopt(parse(from_os_str))] - dbname: PathBuf, - - #[structopt(parse(from_os_str))] - to: PathBuf, -} - -#[derive(Debug, Deserialize, Clone)] -pub struct Config { - pub server_name: String, - pub server_port: u16, -} - -impl Config { - pub fn read_config(filename: &Path) -> anyhow::Result { - let config = std::fs::read_to_string(filename)?; - let config: Config = serde_yaml::from_str(&config)?; - Ok(config) - } -} - -fn restore( - client: &BackupClient, - gen: &Generation, - fileid: u64, - entry: FilesystemEntry, - to: &Path, -) -> anyhow::Result<()> { - println!("restoring {}:{}", fileid, entry.path().display()); - - let path = if entry.path().is_absolute() { - entry.path().strip_prefix("/")? - } else { - entry.path() - }; - let to = to.join(path); - debug!(" to: {}", to.display()); - - match entry.kind() { - FilesystemKind::Regular => restore_regular(client, &gen, &to, fileid, &entry)?, - FilesystemKind::Directory => restore_directory(&to)?, - } - Ok(()) -} - -fn restore_directory(path: &Path) -> anyhow::Result<()> { - debug!("restoring directory {}", path.display()); - std::fs::create_dir_all(path)?; - Ok(()) -} - -fn restore_regular( - client: &BackupClient, - gen: &Generation, - path: &Path, - fileid: u64, - _entry: &FilesystemEntry, -) -> anyhow::Result<()> { - debug!("restoring regular {}", path.display()); - let parent = path.parent().unwrap(); - debug!(" mkdir {}", parent.display()); - std::fs::create_dir_all(parent)?; - { - let mut file = std::fs::File::create(path)?; - for chunkid in gen.chunkids(fileid)? { - let chunk = client.fetch_chunk(&chunkid)?; - file.write_all(chunk.data())?; - } - } - debug!("restored regular {}", path.display()); - Ok(()) -} diff --git a/src/bin/obnam.rs b/src/bin/obnam.rs new file mode 100644 index 0000000..d0cf271 --- /dev/null +++ b/src/bin/obnam.rs @@ -0,0 +1,52 @@ +use log::{debug, info}; +use obnam::cmd::{backup, list, restore}; +use std::path::PathBuf; +use structopt::StructOpt; + +const BUFFER_SIZE: usize = 1024 * 1024; + +fn main() -> anyhow::Result<()> { + pretty_env_logger::init(); + + let opt = Opt::from_args(); + info!("obnam starts"); + debug!("opt: {:?}", opt); + + match opt { + Opt::Backup { config } => backup(&config, BUFFER_SIZE)?, + Opt::List { config } => list(&config)?, + Opt::Restore { + config, + gen_id, + dbname, + to, + } => restore(&config, &gen_id, &dbname, &to)?, + } + Ok(()) +} + +#[derive(Debug, StructOpt)] +#[structopt(name = "obnam-backup", about = "Simplistic backup client")] +enum Opt { + Backup { + #[structopt(parse(from_os_str))] + config: PathBuf, + }, + List { + #[structopt(parse(from_os_str))] + config: PathBuf, + }, + Restore { + #[structopt(parse(from_os_str))] + config: PathBuf, + + #[structopt()] + gen_id: String, + + #[structopt(parse(from_os_str))] + dbname: PathBuf, + + #[structopt(parse(from_os_str))] + to: PathBuf, + }, +} diff --git a/src/cmd/backup.rs b/src/cmd/backup.rs new file mode 100644 index 0000000..308eafb --- /dev/null +++ b/src/cmd/backup.rs @@ -0,0 +1,21 @@ +use crate::client::{BackupClient, ClientConfig}; +use crate::fsiter::FsIterator; +use crate::generation::Generation; +use std::path::Path; + +pub fn backup(config: &Path, buffer_size: usize) -> anyhow::Result<()> { + let config = ClientConfig::read_config(config)?; + let client = BackupClient::new(&config.server_name, config.server_port)?; + + { + let mut gen = Generation::create(&config.dbname)?; + gen.insert_iter(FsIterator::new(&config.root).map(|entry| match entry { + Err(err) => Err(err), + Ok(entry) => client.upload_filesystem_entry(entry, buffer_size), + }))?; + } + let gen_id = client.upload_generation(&config.dbname, buffer_size)?; + println!("gen id: {}", gen_id); + + Ok(()) +} diff --git a/src/cmd/list.rs b/src/cmd/list.rs new file mode 100644 index 0000000..1972144 --- /dev/null +++ b/src/cmd/list.rs @@ -0,0 +1,13 @@ +use crate::client::{BackupClient, ClientConfig}; +use std::path::Path; + +pub fn list(config: &Path) -> anyhow::Result<()> { + let config = ClientConfig::read_config(&config)?; + let client = BackupClient::new(&config.server_name, config.server_port)?; + + for gen_id in client.list_generations()? { + println!("{}", gen_id); + } + + Ok(()) +} diff --git a/src/cmd/mod.rs b/src/cmd/mod.rs new file mode 100644 index 0000000..ca5ff42 --- /dev/null +++ b/src/cmd/mod.rs @@ -0,0 +1,8 @@ +mod backup; +pub use backup::backup; + +mod list; +pub use list::list; + +pub mod restore; +pub use restore::restore; diff --git a/src/cmd/restore.rs b/src/cmd/restore.rs new file mode 100644 index 0000000..6e2690c --- /dev/null +++ b/src/cmd/restore.rs @@ -0,0 +1,115 @@ +use crate::client::BackupClient; +use crate::fsentry::{FilesystemEntry, FilesystemKind}; +use crate::generation::Generation; +use log::{debug, info}; +//use obnam::chunkmeta::ChunkMeta; +use serde::Deserialize; +use std::fs::File; +use std::io::prelude::*; +use std::path::{Path, PathBuf}; +use structopt::StructOpt; + +pub fn restore(config: &Path, gen_id: &str, dbname: &Path, to: &Path) -> anyhow::Result<()> { + let config = Config::read_config(&config).unwrap(); + + let client = BackupClient::new(&config.server_name, config.server_port)?; + let gen_chunk = client.fetch_generation(&gen_id)?; + debug!("gen: {:?}", gen_chunk); + { + let mut dbfile = File::create(&dbname)?; + for id in gen_chunk.chunk_ids() { + let chunk = client.fetch_chunk(id)?; + dbfile.write_all(chunk.data())?; + } + } + info!("downloaded generation to {}", dbname.display()); + + let gen = Generation::open(&dbname)?; + for (fileid, entry) in gen.files()? { + restore_generation(&client, &gen, fileid, entry, &to)?; + } + + Ok(()) +} + +#[derive(Debug, StructOpt)] +#[structopt(name = "obnam-backup", about = "Simplistic backup client")] +struct Opt { + #[structopt(parse(from_os_str))] + config: PathBuf, + + #[structopt()] + gen_id: String, + + #[structopt(parse(from_os_str))] + dbname: PathBuf, + + #[structopt(parse(from_os_str))] + to: PathBuf, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct Config { + pub server_name: String, + pub server_port: u16, +} + +impl Config { + pub fn read_config(filename: &Path) -> anyhow::Result { + let config = std::fs::read_to_string(filename)?; + let config: Config = serde_yaml::from_str(&config)?; + Ok(config) + } +} + +fn restore_generation( + client: &BackupClient, + gen: &Generation, + fileid: u64, + entry: FilesystemEntry, + to: &Path, +) -> anyhow::Result<()> { + println!("restoring {}:{}", fileid, entry.path().display()); + + let path = if entry.path().is_absolute() { + entry.path().strip_prefix("/")? + } else { + entry.path() + }; + let to = to.join(path); + debug!(" to: {}", to.display()); + + match entry.kind() { + FilesystemKind::Regular => restore_regular(client, &gen, &to, fileid, &entry)?, + FilesystemKind::Directory => restore_directory(&to)?, + } + Ok(()) +} + +fn restore_directory(path: &Path) -> anyhow::Result<()> { + debug!("restoring directory {}", path.display()); + std::fs::create_dir_all(path)?; + Ok(()) +} + +fn restore_regular( + client: &BackupClient, + gen: &Generation, + path: &Path, + fileid: u64, + _entry: &FilesystemEntry, +) -> anyhow::Result<()> { + debug!("restoring regular {}", path.display()); + let parent = path.parent().unwrap(); + debug!(" mkdir {}", parent.display()); + std::fs::create_dir_all(parent)?; + { + let mut file = std::fs::File::create(path)?; + for chunkid in gen.chunkids(fileid)? { + let chunk = client.fetch_chunk(&chunkid)?; + file.write_all(chunk.data())?; + } + } + debug!("restored regular {}", path.display()); + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index f72ea56..3d20797 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ pub mod chunker; pub mod chunkid; pub mod chunkmeta; pub mod client; +pub mod cmd; pub mod fsentry; pub mod fsiter; pub mod generation; diff --git a/subplot/obnam.py b/subplot/obnam.py index ee48985..7df283a 100644 --- a/subplot/obnam.py +++ b/subplot/obnam.py @@ -237,7 +237,14 @@ def run_obnam_backup(ctx, filename=None): runcmd_run = globals()["runcmd_run"] _write_obnam_client_config(ctx, filename) - runcmd_run(ctx, ["env", "RUST_LOG=obnam", "obnam-backup", filename]) + runcmd_run(ctx, ["env", "RUST_LOG=obnam", "obnam", "backup", filename]) + + +def run_obnam_list(ctx, filename=None): + runcmd_run = globals()["runcmd_run"] + + _write_obnam_client_config(ctx, filename) + runcmd_run(ctx, ["env", "RUST_LOG=obnam", "obnam", "list", filename]) def _write_obnam_client_config(ctx, filename): @@ -254,7 +261,8 @@ def run_obnam_restore(ctx, filename=None, genid=None, dbname=None, todir=None): genid = ctx["vars"][genid] _write_obnam_client_config(ctx, filename) runcmd_run( - ctx, ["env", "RUST_LOG=obnam", "obnam-restore", filename, genid, dbname, todir] + ctx, + ["env", "RUST_LOG=obnam", "obnam", "restore", filename, genid, dbname, todir], ) @@ -274,3 +282,9 @@ def capture_generation_id(ctx, varname=None): def live_and_restored_data_match(ctx, live=None, restore=None): subprocess.check_call(["diff", "-rq", f"{live}/.", f"{restore}/{live}/."]) + + +def generation_list_contains(ctx, gen_id=None): + runcmd_stdout_contains = globals()["runcmd_stdout_contains"] + gen_id = ctx["vars"][gen_id] + runcmd_stdout_contains(ctx, text=gen_id) diff --git a/subplot/obnam.yaml b/subplot/obnam.yaml index 7d2d85b..8bde009 100644 --- a/subplot/obnam.yaml +++ b/subplot/obnam.yaml @@ -39,10 +39,13 @@ - when: "I back up {dirname} with obnam-backup" function: back_up_directory -- when: "I invoke obnam-backup {filename}" +- when: "I invoke obnam backup {filename}" function: run_obnam_backup -- when: "I invoke obnam-restore {filename} <{genid}> {dbname} {todir}" +- when: "I invoke obnam list {filename}" + function: run_obnam_list + +- when: "I invoke obnam restore {filename} <{genid}> {dbname} {todir}" function: run_obnam_restore - then: "HTTP status code is {status}" @@ -69,3 +72,6 @@ - then: "data in {live} and {restore} match" function: live_and_restored_data_match + +- then: "generation list contains <{gen_id}>" + function: generation_list_contains -- cgit v1.2.1