From 7cfa4142bc1859f9084a35e7e7fd5f67d3a655a3 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sat, 6 Feb 2021 19:22:10 +0200 Subject: feat! back up multiple roots This changes the client configuration file "root" field (with a single string) to "roots" (a list of strings). --- client.yaml | 3 ++- obnam.md | 42 ++++++++++++++++++++++++++++----- src/client.rs | 10 ++++---- src/cmd/backup.rs | 70 +++++++++++++++++++++++++++++++++++++------------------ 4 files changed, 90 insertions(+), 35 deletions(-) diff --git a/client.yaml b/client.yaml index f985fae..99feee4 100644 --- a/client.yaml +++ b/client.yaml @@ -1,4 +1,5 @@ server_url: https://localhost:8888 verify_tls_cert: false -root: /home/liw/tmp/Foton +roots: + - /home/liw/tmp/Foton log: obnam.log diff --git a/obnam.md b/obnam.md index 34535a5..204d6d9 100644 --- a/obnam.md +++ b/obnam.md @@ -1001,7 +1001,7 @@ then stdout, as JSON, matches file config.json ~~~ ~~~{#config.yaml .file .yaml .numberLines} -root: live +roots: [live] server_url: https://backup.example.com verify_tls_cert: true ~~~ @@ -1022,7 +1022,7 @@ then stderr contains "https:" ~~~ ~~~{#http.yaml .file .yaml .numberLines} -root: live +roots: [live] server_url: http://backup.example.com verify_tls_cert: true ~~~ @@ -1045,7 +1045,7 @@ then stderr contains "self signed certificate" ~~~{#ca-required.yaml .file .yaml .numberLines} verify_tls_cert: true -root: live +roots: [live] ~~~ @@ -1078,7 +1078,7 @@ then files live.yaml and rest.yaml match ~~~{#smoke.yaml .file .yaml .numberLines} verify_tls_cert: false -root: live +roots: [live] ~~~ @@ -1093,7 +1093,7 @@ All these scenarios use the following configuration file. ~~~{#metadata.yaml .file .yaml .numberLines} verify_tls_cert: false -root: live +roots: [live] ~~~ ### Modification time @@ -1167,7 +1167,7 @@ then server has 3 file chunks ~~~{#tiny-chunk-size.yaml .file .yaml .numberLines} verify_tls_cert: false -root: live +roots: [live] chunk_size: 1 ~~~ @@ -1292,6 +1292,36 @@ given a manifest of the directory live restored in rest in rest.yaml then files second.yaml and rest.yaml match ~~~ +## Back up multiple directories + +This scenario verifies that Obnam can back up more than one directory +at a time. + + +~~~scenario +given an installed obnam +and a running chunk server +and a client config based on roots.yaml +and a file live/one/data.dat containing some random data +and a file live/two/data.dat containing some random data +and a manifest of the directory live/one in one.yaml +and a manifest of the directory live/two in two.yaml +when I run obnam --config roots.yaml backup +then backup generation is GEN +when I invoke obnam --config roots.yaml restore rest +given a manifest of the directory live/one restored in rest in rest-one.yaml +given a manifest of the directory live/two restored in rest in rest-two.yaml +then files one.yaml and rest-one.yaml match +then files two.yaml and rest-two.yaml match +~~~ + +~~~{#roots.yaml .file .yaml .numberLines} +roots: +- live/one +- live/two +~~~ + + # Acceptance criteria for backup encryption This chapter outlines scenarios, to be implemented later, for diff --git a/src/client.rs b/src/client.rs index 7a4ce21..e4d9be8 100644 --- a/src/client.rs +++ b/src/client.rs @@ -25,7 +25,7 @@ struct TentativeClientConfig { server_url: String, verify_tls_cert: Option, chunk_size: Option, - root: PathBuf, + roots: Vec, log: Option, } @@ -34,7 +34,7 @@ pub struct ClientConfig { pub server_url: String, pub verify_tls_cert: bool, pub chunk_size: usize, - pub root: PathBuf, + pub roots: Vec, pub log: PathBuf, } @@ -43,7 +43,7 @@ pub enum ClientConfigError { #[error("server_url is empty")] ServerUrlIsEmpty, - #[error("backup root is unset or empty")] + #[error("No backup roots in config; at least one is needed")] NoBackupRoot, #[error("server URL doesn't use https: {0}")] @@ -66,7 +66,7 @@ impl ClientConfig { let config = ClientConfig { server_url: tentative.server_url, - root: tentative.root, + roots: tentative.roots, verify_tls_cert: tentative.verify_tls_cert.or(Some(false)).unwrap(), chunk_size: tentative.chunk_size.or(Some(DEFAULT_CHUNK_SIZE)).unwrap(), log: tentative.log.or(Some(PathBuf::from(DEVNULL))).unwrap(), @@ -83,7 +83,7 @@ impl ClientConfig { if !self.server_url.starts_with("https://") { return Err(ClientConfigError::NotHttps(self.server_url.to_string())); } - if self.root.to_string_lossy().is_empty() { + if self.roots.is_empty() { return Err(ClientConfigError::NoBackupRoot); } Ok(()) diff --git a/src/cmd/backup.rs b/src/cmd/backup.rs index fd1d876..cb2e9af 100644 --- a/src/cmd/backup.rs +++ b/src/cmd/backup.rs @@ -1,9 +1,11 @@ use crate::backup_run::BackupRun; +use crate::chunkid::ChunkId; use crate::client::ClientConfig; use crate::error::ObnamError; use crate::fsiter::FsIterator; use crate::generation::NascentGeneration; use log::info; +use std::path::{Path, PathBuf}; use std::time::SystemTime; use tempfile::NamedTempFile; @@ -31,40 +33,62 @@ pub fn backup(config: &ClientConfig) -> Result<(), ObnamError> { }; let genlist = run.client().list_generations()?; - let file_count = { - let iter = FsIterator::new(&config.root); - let mut new = NascentGeneration::create(&newname)?; - - match genlist.resolve("latest") { - Err(_) => { - info!("fresh backup without a previous generation"); - new.insert_iter(iter.map(|entry| run.backup_file_initially(entry)))?; - } - Ok(old) => { - info!("incremental backup based on {}", old); - let old = run.client().fetch_generation(&old, &oldname)?; - run.progress() - .files_in_previous_generation(old.file_count()? as u64); - new.insert_iter(iter.map(|entry| run.backup_file_incrementally(entry, &old)))?; - } - } - run.progress().finish(); - new.file_count() + let file_count = match genlist.resolve("latest") { + Err(_) => initial_backup(&config.roots, &newname, &run)?, + Ok(old) => incremental_backup(&old, &config.roots, &newname, &oldname, &run)?, }; + run.progress().finish(); // Upload the SQLite file, i.e., the named temporary file, which // still exists, since we persisted it above. let gen_id = run .client() .upload_generation(&newname, SQLITE_CHUNK_SIZE)?; - println!("status: OK"); - println!("duration: {}", runtime.elapsed()?.as_secs()); - println!("file-count: {}", file_count); - println!("generation-id: {}", gen_id); // Delete the temporary file.q std::fs::remove_file(&newname)?; std::fs::remove_file(&oldname)?; + report_stats(&runtime, file_count, &gen_id)?; + Ok(()) } + +fn report_stats(runtime: &SystemTime, file_count: i64, gen_id: &ChunkId) -> Result<(), ObnamError> { + println!("status: OK"); + println!("duration: {}", runtime.elapsed()?.as_secs()); + println!("file-count: {}", file_count); + println!("generation-id: {}", gen_id); + Ok(()) +} + +fn initial_backup(roots: &[PathBuf], newname: &Path, run: &BackupRun) -> Result { + info!("fresh backup without a previous generation"); + + let mut new = NascentGeneration::create(&newname)?; + for root in roots { + let iter = FsIterator::new(root); + new.insert_iter(iter.map(|entry| run.backup_file_initially(entry)))?; + } + Ok(new.file_count()) +} + +fn incremental_backup( + old: &str, + roots: &[PathBuf], + newname: &Path, + oldname: &Path, + run: &BackupRun, +) -> Result { + info!("incremental backup based on {}", old); + + let old = run.client().fetch_generation(&old, &oldname)?; + let mut new = NascentGeneration::create(&newname)?; + for root in roots { + let iter = FsIterator::new(root); + run.progress() + .files_in_previous_generation(old.file_count()? as u64); + new.insert_iter(iter.map(|entry| run.backup_file_incrementally(entry, &old)))?; + } + Ok(new.file_count()) +} -- cgit v1.2.1