From 48139725676fcce89a70897546969623f2474693 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 21 Mar 2022 09:12:49 +0200 Subject: feat! store list of generations in a "client trust root" chunk Backups made with this version can't be restored with old clients, and vice version. Sponsored-by: author --- obnam.md | 7 ++-- src/backup_run.rs | 3 +- src/chunk.rs | 95 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/client.rs | 52 ++++++++++++++++++++++------ src/cmd/backup.rs | 17 +++++++-- src/cmd/gen_info.rs | 9 ++++- src/cmd/inspect.rs | 9 ++++- src/cmd/list.rs | 8 ++++- src/cmd/list_files.rs | 8 ++++- src/cmd/resolve.rs | 8 ++++- src/cmd/restore.rs | 8 ++++- src/cmd/show_gen.rs | 8 ++++- src/error.rs | 5 +++ 13 files changed, 214 insertions(+), 23 deletions(-) diff --git a/obnam.md b/obnam.md index 12ac087..da641a5 100644 --- a/obnam.md +++ b/obnam.md @@ -1538,14 +1538,15 @@ data. The backup uses a chunk size of one byte, and backs up a file with three bytes. This results in three chunks for the file data, plus one for the generation SQLite file (not split into chunks of one byte), -plus a chunk for the generation itself. A total of five chunks. +plus a chunk for the generation itself. Additionally, the "trust root" +chunk exists. A total of six chunks. ~~~scenario given a working Obnam system given a client config based on tiny-chunk-size.yaml given a file live/data.dat containing "abc" when I run obnam backup -then server has 5 chunks +then server has 6 chunks ~~~ ~~~{#tiny-chunk-size.yaml .file .yaml .numberLines} @@ -1846,10 +1847,10 @@ then exit code is 1 and stdout contains "live/CACHEDIR.TAG" when I run obnam list-files then exit code is 0 +~~~ then file live/CACHEDIR.TAG was backed up because it was new and stdout doesn't contain "live/data1.dat" and stdout doesn't contain "live/data2.dat" -~~~ ### Ignore CACHEDIR.TAGs if `exclude_cache_tag_directories` is disabled diff --git a/src/backup_run.rs b/src/backup_run.rs index b03a7ee..5ceaed3 100644 --- a/src/backup_run.rs +++ b/src/backup_run.rs @@ -406,7 +406,8 @@ impl<'a> BackupRun<'a> { } } -fn current_timestamp() -> String { +/// Current timestamp as an ISO 8601 string. +pub fn current_timestamp() -> String { let now: DateTime = Local::now(); format!("{}", now.format("%Y-%m-%d %H:%M:%S.%f %z")) } diff --git a/src/chunk.rs b/src/chunk.rs index a37aa57..238bd01 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -102,3 +102,98 @@ impl GenerationChunk { Ok(DataChunk::new(bytes, meta)) } } + +/// A client trust root chunk. +/// +/// This chunk contains all per-client backup information. As long as +/// this chunk can be trusted, everything it links to can also be +/// trusted, thanks to cryptographic signatures. +#[derive(Debug, Serialize, Deserialize)] +pub struct ClientTrust { + client_name: String, + previous_version: Option, + timestamp: String, + backups: Vec, +} + +/// All the errors that may be returned for `ClientTrust` operations. +#[derive(Debug, thiserror::Error)] +pub enum ClientTrustError { + /// Error converting text from UTF8. + #[error(transparent)] + Utf8Error(#[from] std::str::Utf8Error), + + /// Error parsing JSON as chunk metadata. + #[error("failed to parse JSON: {0}")] + JsonParse(serde_json::Error), + + /// Error generating JSON from chunk metadata. + #[error("failed to serialize to JSON: {0}")] + JsonGenerate(serde_json::Error), +} + +impl ClientTrust { + /// Create a new ClientTrust object. + pub fn new( + name: &str, + previous_version: Option, + timestamp: String, + backups: Vec, + ) -> Self { + Self { + client_name: name.to_string(), + previous_version, + timestamp, + backups, + } + } + + /// Return client name. + pub fn client_name(&self) -> &str { + &self.client_name + } + + /// Return id of previous version, if any. + pub fn previous_version(&self) -> Option { + self.previous_version.clone() + } + + /// Return timestamp. + pub fn timestamp(&self) -> &str { + &self.timestamp + } + + /// Return list of all backup generations known. + pub fn backups(&self) -> &[ChunkId] { + &self.backups + } + + /// Append a backup generation to the list. + pub fn append_backup(&mut self, id: &ChunkId) { + self.backups.push(id.clone()); + } + + /// Update for new upload. + /// + /// This needs to happen every time the chunk is updated so that + /// the timestamp gets updated. + pub fn finalize(&mut self, timestamp: String) { + self.timestamp = timestamp; + } + + /// Convert generation chunk to a data chunk. + pub fn to_data_chunk(&self) -> Result { + let json: String = serde_json::to_string(self).map_err(ClientTrustError::JsonGenerate)?; + let bytes = json.as_bytes().to_vec(); + let checksum = Checksum::sha256_from_str_unchecked("client-trust"); + let meta = ChunkMeta::new_generation(&checksum, ""); + Ok(DataChunk::new(bytes, meta)) + } + + /// Create a new ClientTrust from a data chunk. + pub fn from_data_chunk(chunk: &DataChunk) -> Result { + let data = chunk.data(); + let data = std::str::from_utf8(data)?; + serde_json::from_str(data).map_err(ClientTrustError::JsonParse) + } +} diff --git a/src/client.rs b/src/client.rs index c4fbfec..5b13cb7 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,6 +1,8 @@ //! Client to the Obnam server HTTP API. -use crate::chunk::{DataChunk, GenerationChunk, GenerationChunkError}; +use crate::chunk::{ + ClientTrust, ClientTrustError, DataChunk, GenerationChunk, GenerationChunkError, +}; use crate::chunkid::ChunkId; use crate::chunkmeta::ChunkMeta; use crate::cipher::{CipherEngine, CipherError}; @@ -54,6 +56,10 @@ pub enum ClientError { #[error(transparent)] GenerationChunkError(#[from] GenerationChunkError), + /// An error regarding client trust. + #[error(transparent)] + ClientTrust(#[from] ClientTrustError), + /// An error using a backup's local metadata. #[error(transparent)] LocalGenerationError(#[from] LocalGenerationError), @@ -170,18 +176,44 @@ impl BackupClient { Ok(chunk_id) } - /// List backup generations known by the server. - pub async fn list_generations(&self) -> Result { - let (_, body) = self.get("", &[("generation", "true")]).await?; + /// Get current client trust chunk from repository, if there is one. + pub async fn get_client_trust(&self) -> Result, ClientError> { + let ids = self.find_client_trusts().await?; + let mut latest: Option = None; + for id in ids { + let chunk = self.fetch_chunk(&id).await?; + let new = ClientTrust::from_data_chunk(&chunk)?; + if let Some(t) = &latest { + if new.timestamp() > t.timestamp() { + latest = Some(new); + } + } else { + latest = Some(new); + } + } + Ok(latest) + } + + async fn find_client_trusts(&self) -> Result, ClientError> { + let body = match self.get("", &[("label", "client-trust")]).await { + Ok((_, body)) => body, + Err(err) => return Err(err), + }; - let map: HashMap = - serde_yaml::from_slice(&body).map_err(ClientError::YamlParse)?; - debug!("list_generations: map={:?}", map); - let finished = map + let hits: HashMap = + serde_json::from_slice(&body).map_err(ClientError::JsonParse)?; + let ids = hits.iter().map(|(id, _)| id.into()).collect(); + Ok(ids) + } + + /// List backup generations known by the server. + pub fn list_generations(&self, trust: &ClientTrust) -> GenerationList { + let finished = trust + .backups() .iter() - .map(|(id, meta)| FinishedGeneration::new(id, meta.ended().map_or("", |s| s))) + .map(|id| FinishedGeneration::new(&format!("{}", id), "")) .collect(); - Ok(GenerationList::new(finished)) + GenerationList::new(finished) } /// Fetch a data chunk from the server, given the chunk identifier. diff --git a/src/cmd/backup.rs b/src/cmd/backup.rs index 6983de4..db65da0 100644 --- a/src/cmd/backup.rs +++ b/src/cmd/backup.rs @@ -1,6 +1,7 @@ //! The `backup` subcommand. -use crate::backup_run::BackupRun; +use crate::backup_run::{current_timestamp, BackupRun}; +use crate::chunk::ClientTrust; use crate::client::BackupClient; use crate::config::ClientConfig; use crate::dbgen::{schema_version, FileId, DEFAULT_SCHEMA_MAJOR}; @@ -37,7 +38,12 @@ impl Backup { eprintln!("backup: schema: {}", schema); let client = BackupClient::new(config)?; - let genlist = client.list_generations().await?; + let trust = client + .get_client_trust() + .await? + .or_else(|| Some(ClientTrust::new("FIXME", None, current_timestamp(), vec![]))) + .unwrap(); + let genlist = client.list_generations(&trust); let temp = tempdir()?; let oldtemp = temp.path().join("old.db"); @@ -64,6 +70,13 @@ impl Backup { } }; + let mut trust = trust; + trust.append_backup(outcome.gen_id.as_chunk_id()); + trust.finalize(current_timestamp()); + let trust = trust.to_data_chunk()?; + let trust_id = client.upload_chunk(trust).await?; + info!("uploaded new client-trust {}", trust_id); + for w in outcome.warnings.iter() { println!("warning: {}", w); } diff --git a/src/cmd/gen_info.rs b/src/cmd/gen_info.rs index 2ce1f64..0aec103 100644 --- a/src/cmd/gen_info.rs +++ b/src/cmd/gen_info.rs @@ -1,5 +1,6 @@ //! The `gen-info` subcommand. +use crate::chunk::ClientTrust; use crate::client::BackupClient; use crate::config::ClientConfig; use crate::error::ObnamError; @@ -28,7 +29,13 @@ impl GenInfo { let client = BackupClient::new(config)?; - let genlist = client.list_generations().await?; + let trust = client + .get_client_trust() + .await? + .or_else(|| Some(ClientTrust::new("FIXME", None, "".to_string(), vec![]))) + .unwrap(); + + let genlist = client.list_generations(&trust); let gen_id = genlist.resolve(&self.gen_ref)?; info!("generation id is {}", gen_id.as_chunk_id()); diff --git a/src/cmd/inspect.rs b/src/cmd/inspect.rs index d5a75c6..02801ae 100644 --- a/src/cmd/inspect.rs +++ b/src/cmd/inspect.rs @@ -1,5 +1,7 @@ //! The `inspect` subcommand. +use crate::backup_run::current_timestamp; +use crate::chunk::ClientTrust; use crate::client::BackupClient; use crate::config::ClientConfig; use crate::error::ObnamError; @@ -27,7 +29,12 @@ impl Inspect { async fn run_async(&self, config: &ClientConfig) -> Result<(), ObnamError> { let temp = NamedTempFile::new()?; let client = BackupClient::new(config)?; - let genlist = client.list_generations().await?; + let trust = client + .get_client_trust() + .await? + .or_else(|| Some(ClientTrust::new("FIXME", None, current_timestamp(), vec![]))) + .unwrap(); + let genlist = client.list_generations(&trust); let gen_id = genlist.resolve(&self.gen_id)?; info!("generation id is {}", gen_id.as_chunk_id()); diff --git a/src/cmd/list.rs b/src/cmd/list.rs index f176594..bbb9c91 100644 --- a/src/cmd/list.rs +++ b/src/cmd/list.rs @@ -1,5 +1,6 @@ //! The `list` subcommand. +use crate::chunk::ClientTrust; use crate::client::BackupClient; use crate::config::ClientConfig; use crate::error::ObnamError; @@ -19,8 +20,13 @@ impl List { async fn run_async(&self, config: &ClientConfig) -> Result<(), ObnamError> { let client = BackupClient::new(config)?; + let trust = client + .get_client_trust() + .await? + .or_else(|| Some(ClientTrust::new("FIXME", None, "".to_string(), vec![]))) + .unwrap(); - let generations = client.list_generations().await?; + let generations = client.list_generations(&trust); for finished in generations.iter() { println!("{} {}", finished.id(), finished.ended()); } diff --git a/src/cmd/list_files.rs b/src/cmd/list_files.rs index 9126564..fb4764d 100644 --- a/src/cmd/list_files.rs +++ b/src/cmd/list_files.rs @@ -1,6 +1,7 @@ //! The `list-files` subcommand. use crate::backup_reason::Reason; +use crate::chunk::ClientTrust; use crate::client::BackupClient; use crate::config::ClientConfig; use crate::error::ObnamError; @@ -28,8 +29,13 @@ impl ListFiles { let temp = NamedTempFile::new()?; let client = BackupClient::new(config)?; + let trust = client + .get_client_trust() + .await? + .or_else(|| Some(ClientTrust::new("FIXME", None, "".to_string(), vec![]))) + .unwrap(); - let genlist = client.list_generations().await?; + let genlist = client.list_generations(&trust); let gen_id = genlist.resolve(&self.gen_id)?; let gen = client.fetch_generation(&gen_id, temp.path()).await?; diff --git a/src/cmd/resolve.rs b/src/cmd/resolve.rs index 3b9570a..12432cc 100644 --- a/src/cmd/resolve.rs +++ b/src/cmd/resolve.rs @@ -1,5 +1,6 @@ //! The `resolve` subcommand. +use crate::chunk::ClientTrust; use crate::client::BackupClient; use crate::config::ClientConfig; use crate::error::ObnamError; @@ -22,7 +23,12 @@ impl Resolve { async fn run_async(&self, config: &ClientConfig) -> Result<(), ObnamError> { let client = BackupClient::new(config)?; - let generations = client.list_generations().await?; + let trust = client + .get_client_trust() + .await? + .or_else(|| Some(ClientTrust::new("FIXME", None, "".to_string(), vec![]))) + .unwrap(); + let generations = client.list_generations(&trust); match generations.resolve(&self.generation) { Err(err) => { diff --git a/src/cmd/restore.rs b/src/cmd/restore.rs index 43d9679..4a637da 100644 --- a/src/cmd/restore.rs +++ b/src/cmd/restore.rs @@ -1,6 +1,7 @@ //! The `restore` subcommand. use crate::backup_reason::Reason; +use crate::chunk::ClientTrust; use crate::client::{BackupClient, ClientError}; use crate::config::ClientConfig; use crate::db::DatabaseError; @@ -46,8 +47,13 @@ impl Restore { let temp = NamedTempFile::new()?; let client = BackupClient::new(config)?; + let trust = client + .get_client_trust() + .await? + .or_else(|| Some(ClientTrust::new("FIXME", None, "".to_string(), vec![]))) + .unwrap(); - let genlist = client.list_generations().await?; + let genlist = client.list_generations(&trust); let gen_id = genlist.resolve(&self.gen_id)?; info!("generation id is {}", gen_id.as_chunk_id()); diff --git a/src/cmd/show_gen.rs b/src/cmd/show_gen.rs index 006e0e0..970a165 100644 --- a/src/cmd/show_gen.rs +++ b/src/cmd/show_gen.rs @@ -1,5 +1,6 @@ //! The `show-generation` subcommand. +use crate::chunk::ClientTrust; use crate::client::BackupClient; use crate::config::ClientConfig; use crate::error::ObnamError; @@ -27,8 +28,13 @@ impl ShowGeneration { async fn run_async(&self, config: &ClientConfig) -> Result<(), ObnamError> { let temp = NamedTempFile::new()?; let client = BackupClient::new(config)?; + let trust = client + .get_client_trust() + .await? + .or_else(|| Some(ClientTrust::new("FIXME", None, "".to_string(), vec![]))) + .unwrap(); - let genlist = client.list_generations().await?; + let genlist = client.list_generations(&trust); let gen_id = genlist.resolve(&self.gen_id)?; let gen = client.fetch_generation(&gen_id, temp.path()).await?; let mut files = gen.files()?; diff --git a/src/error.rs b/src/error.rs index 7812081..9c9b432 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,6 +1,7 @@ //! Errors from Obnam client. use crate::backup_run::BackupError; +use crate::chunk::ClientTrustError; use crate::cipher::CipherError; use crate::client::ClientError; use crate::cmd::restore::RestoreError; @@ -25,6 +26,10 @@ pub enum ObnamError { #[error(transparent)] GenerationListError(#[from] GenerationListError), + /// Error about client trust chunks. + #[error(transparent)] + ClientTrust(#[from] ClientTrustError), + /// Error saving passwords. #[error("couldn't save passwords to {0}: {1}")] PasswordSave(PathBuf, PasswordError), -- cgit v1.2.1