From 4c8afd2616ed35d7bdebd2069ed0909136fc598f Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 16 Jan 2022 16:41:30 +0200 Subject: test: deny dependencies with security issues or that are yanked Sponsored-by: author --- deny.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deny.toml b/deny.toml index 9db4685..08ad9ef 100644 --- a/deny.toml +++ b/deny.toml @@ -9,8 +9,8 @@ db-path = "~/.cargo/advisory-db" db-urls = ["https://github.com/rustsec/advisory-db"] vulnerability = "deny" unmaintained = "warn" -yanked = "allow" -notice = "warn" +yanked = "deny" +notice = "deny" ignore = [ "RUSTSEC-2020-0027", "RUSTSEC-2020-0071", -- cgit v1.2.1 From 39950794bf8d783c3db2fb245389baf06cb24709 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 16 Jan 2022 16:55:16 +0200 Subject: chore: document minimal rust version in Cargo.toml Sponsored-by: author --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index 676f306..17535db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ description = "a backup program" license = "AGPL-3.0-or-later" homepage = "https://obnam.org/" repository = "https://gitlab.com/larswirzenius/obnam" +rust-version = "1.56.0" [dependencies] -- cgit v1.2.1 From 579bacedf245a258c25c87d5da096816b2855b63 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 16 Jan 2022 17:04:07 +0200 Subject: chore: drop unused src/benhcmark.rs module Sponsored-by: author --- src/benchmark.rs | 38 -------------------------------------- src/lib.rs | 1 - 2 files changed, 39 deletions(-) delete mode 100644 src/benchmark.rs diff --git a/src/benchmark.rs b/src/benchmark.rs deleted file mode 100644 index d2d9003..0000000 --- a/src/benchmark.rs +++ /dev/null @@ -1,38 +0,0 @@ -//! Benchmark chunk generation. -//! -//! This is only for development. - -use crate::checksummer::Checksum; -use crate::chunk::DataChunk; -use crate::chunkid::ChunkId; -use crate::chunkmeta::ChunkMeta; - -/// Generate a desired number of empty data chunks with id and metadata. -pub struct ChunkGenerator { - goal: u32, - next: u32, -} - -impl ChunkGenerator { - /// Create a new ChunkGenerator. - pub fn new(goal: u32) -> Self { - Self { goal, next: 0 } - } -} - -impl Iterator for ChunkGenerator { - type Item = (ChunkId, Checksum, DataChunk); - - fn next(&mut self) -> Option { - if self.next >= self.goal { - None - } else { - let id = ChunkId::recreate(&format!("{}", self.next)); - let checksum = id.sha256(); - let meta = ChunkMeta::new(&checksum); - let chunk = DataChunk::new(vec![], meta); - self.next += 1; - Some((id, checksum, chunk)) - } - } -} diff --git a/src/lib.rs b/src/lib.rs index 957ec13..8961df4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,7 +8,6 @@ pub mod backup_progress; pub mod backup_reason; pub mod backup_run; -pub mod benchmark; pub mod checksummer; pub mod chunk; pub mod chunker; -- cgit v1.2.1 From d2f0fba9465c998d1416f985771c7c19292cc020 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 16 Jan 2022 17:15:52 +0200 Subject: refactor: rename AsyncBackupClient to just BackupClient There is only async. Sponsored-by: author --- src/backup_run.rs | 11 ++++------- src/client.rs | 14 ++++++-------- src/cmd/backup.rs | 4 ++-- src/cmd/gen_info.rs | 4 ++-- src/cmd/get_chunk.rs | 4 ++-- src/cmd/list.rs | 4 ++-- src/cmd/list_files.rs | 4 ++-- src/cmd/resolve.rs | 4 ++-- src/cmd/restore.rs | 8 ++++---- src/cmd/show_gen.rs | 4 ++-- 10 files changed, 28 insertions(+), 33 deletions(-) diff --git a/src/backup_run.rs b/src/backup_run.rs index ade5ee0..2286fca 100644 --- a/src/backup_run.rs +++ b/src/backup_run.rs @@ -5,7 +5,7 @@ use crate::backup_reason::Reason; use crate::chunk::{GenerationChunk, GenerationChunkError}; use crate::chunker::{Chunker, ChunkerError}; use crate::chunkid::ChunkId; -use crate::client::{AsyncBackupClient, ClientError}; +use crate::client::{BackupClient, ClientError}; use crate::config::ClientConfig; use crate::error::ObnamError; use crate::fsentry::{FilesystemEntry, FilesystemKind}; @@ -24,7 +24,7 @@ const SQLITE_CHUNK_SIZE: usize = MIB as usize; /// A running backup. pub struct BackupRun<'a> { - client: &'a AsyncBackupClient, + client: &'a BackupClient, policy: BackupPolicy, buffer_size: usize, progress: Option, @@ -95,10 +95,7 @@ pub struct RootsBackupOutcome { impl<'a> BackupRun<'a> { /// Create a new run for an initial backup. - pub fn initial( - config: &ClientConfig, - client: &'a AsyncBackupClient, - ) -> Result { + pub fn initial(config: &ClientConfig, client: &'a BackupClient) -> Result { Ok(Self { client, policy: BackupPolicy::default(), @@ -110,7 +107,7 @@ impl<'a> BackupRun<'a> { /// Create a new run for an incremental backup. pub fn incremental( config: &ClientConfig, - client: &'a AsyncBackupClient, + client: &'a BackupClient, ) -> Result { Ok(Self { client, diff --git a/src/client.rs b/src/client.rs index ed6b86b..e9b00cd 100644 --- a/src/client.rs +++ b/src/client.rs @@ -96,18 +96,16 @@ pub enum ClientError { } /// Client for the Obnam server HTTP API. -/// -/// This is the async version. -pub struct AsyncBackupClient { - chunk_client: AsyncChunkClient, +pub struct BackupClient { + chunk_client: ChunkClient, } -impl AsyncBackupClient { +impl BackupClient { /// Create a new backup client. pub fn new(config: &ClientConfig) -> Result { info!("creating backup client with config: {:#?}", config); Ok(Self { - chunk_client: AsyncChunkClient::new(config)?, + chunk_client: ChunkClient::new(config)?, }) } @@ -162,13 +160,13 @@ impl AsyncBackupClient { } /// Client for using chunk part of Obnam server HTTP API. -pub struct AsyncChunkClient { +pub struct ChunkClient { client: reqwest::Client, base_url: String, cipher: CipherEngine, } -impl AsyncChunkClient { +impl ChunkClient { /// Create a new chunk client. pub fn new(config: &ClientConfig) -> Result { let pass = config.passwords()?; diff --git a/src/cmd/backup.rs b/src/cmd/backup.rs index 6e09d37..92b0f40 100644 --- a/src/cmd/backup.rs +++ b/src/cmd/backup.rs @@ -1,7 +1,7 @@ //! The `backup` subcommand. use crate::backup_run::BackupRun; -use crate::client::AsyncBackupClient; +use crate::client::BackupClient; use crate::config::ClientConfig; use crate::error::ObnamError; use crate::generation::GenId; @@ -26,7 +26,7 @@ impl Backup { async fn run_async(&self, config: &ClientConfig) -> Result<(), ObnamError> { let runtime = SystemTime::now(); - let client = AsyncBackupClient::new(config)?; + let client = BackupClient::new(config)?; let genlist = client.list_generations().await?; let oldtemp = NamedTempFile::new()?; diff --git a/src/cmd/gen_info.rs b/src/cmd/gen_info.rs index 2663d9b..2ce1f64 100644 --- a/src/cmd/gen_info.rs +++ b/src/cmd/gen_info.rs @@ -1,6 +1,6 @@ //! The `gen-info` subcommand. -use crate::client::AsyncBackupClient; +use crate::client::BackupClient; use crate::config::ClientConfig; use crate::error::ObnamError; use log::info; @@ -26,7 +26,7 @@ impl GenInfo { async fn run_async(&self, config: &ClientConfig) -> Result<(), ObnamError> { let temp = NamedTempFile::new()?; - let client = AsyncBackupClient::new(config)?; + let client = BackupClient::new(config)?; let genlist = client.list_generations().await?; let gen_id = genlist.resolve(&self.gen_ref)?; diff --git a/src/cmd/get_chunk.rs b/src/cmd/get_chunk.rs index 905e997..0b27084 100644 --- a/src/cmd/get_chunk.rs +++ b/src/cmd/get_chunk.rs @@ -1,7 +1,7 @@ //! The `get-chunk` subcommand. use crate::chunkid::ChunkId; -use crate::client::AsyncBackupClient; +use crate::client::BackupClient; use crate::config::ClientConfig; use crate::error::ObnamError; use std::io::{stdout, Write}; @@ -24,7 +24,7 @@ impl GetChunk { } async fn run_async(&self, config: &ClientConfig) -> Result<(), ObnamError> { - let client = AsyncBackupClient::new(config)?; + let client = BackupClient::new(config)?; let chunk_id: ChunkId = self.chunk_id.parse().unwrap(); let chunk = client.fetch_chunk(&chunk_id).await?; let stdout = stdout(); diff --git a/src/cmd/list.rs b/src/cmd/list.rs index 6c58e30..f176594 100644 --- a/src/cmd/list.rs +++ b/src/cmd/list.rs @@ -1,6 +1,6 @@ //! The `list` subcommand. -use crate::client::AsyncBackupClient; +use crate::client::BackupClient; use crate::config::ClientConfig; use crate::error::ObnamError; use structopt::StructOpt; @@ -18,7 +18,7 @@ impl List { } async fn run_async(&self, config: &ClientConfig) -> Result<(), ObnamError> { - let client = AsyncBackupClient::new(config)?; + let client = BackupClient::new(config)?; let generations = client.list_generations().await?; for finished in generations.iter() { diff --git a/src/cmd/list_files.rs b/src/cmd/list_files.rs index 888943e..12d34b1 100644 --- a/src/cmd/list_files.rs +++ b/src/cmd/list_files.rs @@ -1,7 +1,7 @@ //! The `list-files` subcommand. use crate::backup_reason::Reason; -use crate::client::AsyncBackupClient; +use crate::client::BackupClient; use crate::config::ClientConfig; use crate::error::ObnamError; use crate::fsentry::{FilesystemEntry, FilesystemKind}; @@ -27,7 +27,7 @@ impl ListFiles { async fn run_async(&self, config: &ClientConfig) -> Result<(), ObnamError> { let temp = NamedTempFile::new()?; - let client = AsyncBackupClient::new(config)?; + let client = BackupClient::new(config)?; let genlist = client.list_generations().await?; let gen_id = genlist.resolve(&self.gen_id)?; diff --git a/src/cmd/resolve.rs b/src/cmd/resolve.rs index cd08908..3b9570a 100644 --- a/src/cmd/resolve.rs +++ b/src/cmd/resolve.rs @@ -1,6 +1,6 @@ //! The `resolve` subcommand. -use crate::client::AsyncBackupClient; +use crate::client::BackupClient; use crate::config::ClientConfig; use crate::error::ObnamError; use structopt::StructOpt; @@ -21,7 +21,7 @@ impl Resolve { } async fn run_async(&self, config: &ClientConfig) -> Result<(), ObnamError> { - let client = AsyncBackupClient::new(config)?; + let client = BackupClient::new(config)?; let generations = client.list_generations().await?; match generations.resolve(&self.generation) { diff --git a/src/cmd/restore.rs b/src/cmd/restore.rs index 2a36986..7b3d95e 100644 --- a/src/cmd/restore.rs +++ b/src/cmd/restore.rs @@ -1,7 +1,7 @@ //! The `restore` subcommand. use crate::backup_reason::Reason; -use crate::client::{AsyncBackupClient, ClientError}; +use crate::client::{BackupClient, ClientError}; use crate::config::ClientConfig; use crate::error::ObnamError; use crate::fsentry::{FilesystemEntry, FilesystemKind}; @@ -43,7 +43,7 @@ impl Restore { async fn run_async(&self, config: &ClientConfig) -> Result<(), ObnamError> { let temp = NamedTempFile::new()?; - let client = AsyncBackupClient::new(config)?; + let client = BackupClient::new(config)?; let genlist = client.list_generations().await?; let gen_id = genlist.resolve(&self.gen_id)?; @@ -130,7 +130,7 @@ pub enum RestoreError { } async fn restore_generation( - client: &AsyncBackupClient, + client: &BackupClient, gen: &LocalGeneration, fileid: i64, entry: &FilesystemEntry, @@ -182,7 +182,7 @@ fn restored_path(entry: &FilesystemEntry, to: &Path) -> Result Result<(), ObnamError> { let temp = NamedTempFile::new()?; - let client = AsyncBackupClient::new(config)?; + let client = BackupClient::new(config)?; let genlist = client.list_generations().await?; let gen_id = genlist.resolve(&self.gen_id)?; -- cgit v1.2.1 From 3ad93d94a347cef794798afbfbc588c295175147 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 16 Jan 2022 17:32:28 +0200 Subject: refactor: combine the chunk client into BackupClient The split doesn't seem to make sense any more. Sponsored-by: author --- src/client.rs | 105 ++++++++++++++++++++-------------------------------------- 1 file changed, 36 insertions(+), 69 deletions(-) diff --git a/src/client.rs b/src/client.rs index e9b00cd..bcc31b4 100644 --- a/src/client.rs +++ b/src/client.rs @@ -97,78 +97,16 @@ pub enum ClientError { /// Client for the Obnam server HTTP API. pub struct BackupClient { - chunk_client: ChunkClient, + client: reqwest::Client, + base_url: String, + cipher: CipherEngine, } impl BackupClient { /// Create a new backup client. pub fn new(config: &ClientConfig) -> Result { info!("creating backup client with config: {:#?}", config); - Ok(Self { - chunk_client: ChunkClient::new(config)?, - }) - } - - /// Does the server have a chunk? - pub async fn has_chunk(&self, meta: &ChunkMeta) -> Result, ClientError> { - self.chunk_client.has_chunk(meta).await - } - - /// Upload a data chunk to the srver. - pub async fn upload_chunk(&self, chunk: DataChunk) -> Result { - self.chunk_client.upload_chunk(chunk).await - } - - /// List backup generations known by the server. - pub async fn list_generations(&self) -> Result { - self.chunk_client.list_generations().await - } - - /// Fetch a data chunk from the server, given the chunk identifier. - pub async fn fetch_chunk(&self, chunk_id: &ChunkId) -> Result { - self.chunk_client.fetch_chunk(chunk_id).await - } - - async fn fetch_generation_chunk(&self, gen_id: &GenId) -> Result { - let chunk = self.fetch_chunk(gen_id.as_chunk_id()).await?; - let gen = GenerationChunk::from_data_chunk(&chunk)?; - Ok(gen) - } - /// Fetch a backup generation's metadata, given it's identifier. - pub async fn fetch_generation( - &self, - gen_id: &GenId, - dbname: &Path, - ) -> Result { - let gen = self.fetch_generation_chunk(gen_id).await?; - - // Fetch the SQLite file, storing it in the named file. - let mut dbfile = File::create(&dbname) - .map_err(|err| ClientError::FileCreate(dbname.to_path_buf(), err))?; - for id in gen.chunk_ids() { - let chunk = self.fetch_chunk(id).await?; - dbfile - .write_all(chunk.data()) - .map_err(|err| ClientError::FileWrite(dbname.to_path_buf(), err))?; - } - info!("downloaded generation to {}", dbname.display()); - - let gen = LocalGeneration::open(dbname)?; - Ok(gen) - } -} - -/// Client for using chunk part of Obnam server HTTP API. -pub struct ChunkClient { - client: reqwest::Client, - base_url: String, - cipher: CipherEngine, -} - -impl ChunkClient { - /// Create a new chunk client. - pub fn new(config: &ClientConfig) -> Result { let pass = config.passwords()?; let client = reqwest::Client::builder() @@ -190,7 +128,7 @@ impl ChunkClient { format!("{}/chunks", self.base_url()) } - /// Does server have a chunk? + /// Does the server have a chunk? pub async fn has_chunk(&self, meta: &ChunkMeta) -> Result, ClientError> { let body = match self.get("", &[("sha256", meta.sha256())]).await { Ok((_, body)) => body, @@ -209,7 +147,7 @@ impl ChunkClient { Ok(has) } - /// Upload a new chunk to the server. + /// Upload a data chunk to the srver. pub async fn upload_chunk(&self, chunk: DataChunk) -> Result { let enc = self.cipher.encrypt_chunk(&chunk)?; let res = self @@ -232,7 +170,7 @@ impl ChunkClient { Ok(chunk_id) } - /// List all generation chunks on the server. + /// List backup generations known by the server. pub async fn list_generations(&self) -> Result { let (_, body) = self.get("", &[("generation", "true")]).await?; @@ -246,7 +184,7 @@ impl ChunkClient { Ok(GenerationList::new(finished)) } - /// Fetch a chunk from the server, given its id. + /// Fetch a data chunk from the server, given the chunk identifier. pub async fn fetch_chunk(&self, chunk_id: &ChunkId) -> Result { let (headers, body) = self.get(&format!("/{}", chunk_id), &[]).await?; let meta = self.get_chunk_meta_header(chunk_id, &headers)?; @@ -257,6 +195,35 @@ impl ChunkClient { Ok(chunk) } + async fn fetch_generation_chunk(&self, gen_id: &GenId) -> Result { + let chunk = self.fetch_chunk(gen_id.as_chunk_id()).await?; + let gen = GenerationChunk::from_data_chunk(&chunk)?; + Ok(gen) + } + + /// Fetch a backup generation's metadata, given it's identifier. + pub async fn fetch_generation( + &self, + gen_id: &GenId, + dbname: &Path, + ) -> Result { + let gen = self.fetch_generation_chunk(gen_id).await?; + + // Fetch the SQLite file, storing it in the named file. + let mut dbfile = File::create(&dbname) + .map_err(|err| ClientError::FileCreate(dbname.to_path_buf(), err))?; + for id in gen.chunk_ids() { + let chunk = self.fetch_chunk(id).await?; + dbfile + .write_all(chunk.data()) + .map_err(|err| ClientError::FileWrite(dbname.to_path_buf(), err))?; + } + info!("downloaded generation to {}", dbname.display()); + + let gen = LocalGeneration::open(dbname)?; + Ok(gen) + } + async fn get( &self, path: &str, -- cgit v1.2.1 From 15338d80791e86647d420a29be538ec1c95fe52b Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 16 Jan 2022 17:37:42 +0200 Subject: docs: drop unnecessary YAML metadata from Markdown docs Sponsored-by: author --- DONE.md | 4 ---- NEWS.md | 4 +--- README.md | 4 ---- RELEASE.md | 4 ---- code-of-conduct.md | 4 ---- tutorial.md | 4 +--- 6 files changed, 2 insertions(+), 22 deletions(-) diff --git a/DONE.md b/DONE.md index 5836c59..641c703 100644 --- a/DONE.md +++ b/DONE.md @@ -1,7 +1,3 @@ ---- -title: Definition of done -... - # Definition of done This definition is not meant to be petty bureaucracy. It's meant to be diff --git a/NEWS.md b/NEWS.md index 61ca4b7..f595634 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,4 @@ ---- -title: Release notes for Obnam2 -... +# Release notes for Obnam2 This file summarizes changes between releases of the second generation of Obnam, the backup software. The software is technically called diff --git a/README.md b/README.md index 70691d5..a9583cd 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,3 @@ ---- -title: Obnam — a backup system -... - # Obnam — a backup system Obnam2 is a project to develop a backup system. diff --git a/RELEASE.md b/RELEASE.md index bf30633..f264d4f 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,7 +1,3 @@ ---- -title: Release checklist for Obnam -... - # Release checklist for Obnam Follow these steps to make a release of Obnam. diff --git a/code-of-conduct.md b/code-of-conduct.md index 8851723..ebc4a55 100644 --- a/code-of-conduct.md +++ b/code-of-conduct.md @@ -1,7 +1,3 @@ ---- -title: Contributor Covenant Code of Conduct -... - # Contributor Covenant Code of Conduct ## Our Pledge diff --git a/tutorial.md b/tutorial.md index 3435a64..b20c84e 100644 --- a/tutorial.md +++ b/tutorial.md @@ -1,6 +1,4 @@ ---- -title: Obnam tutorial -... +# Obnam tutorial With the help of this tutorial, you're going to set up Obnam, make your first backup, and check that you can restore files from it. -- cgit v1.2.1 From 6c462e17afcd9a13c9a2f4517db5dcaf88c80e77 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 16 Jan 2022 18:06:57 +0200 Subject: fix: if a file is skipped, don't include it in the new backup Sponsored-by: author --- src/backup_run.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/backup_run.rs b/src/backup_run.rs index 2286fca..bb7568f 100644 --- a/src/backup_run.rs +++ b/src/backup_run.rs @@ -234,7 +234,8 @@ impl<'a> BackupRun<'a> { Err(err) => { warnings.push(err); } - Ok(o) => { + Ok(None) => (), + Ok(Some(o)) => { if let Err(err) = new.insert(o.entry, &o.ids, o.reason, o.is_cachedir_tag) { @@ -257,16 +258,17 @@ impl<'a> BackupRun<'a> { &self, entry: AnnotatedFsEntry, old: &LocalGeneration, - ) -> Result { + ) -> Result, BackupError> { let path = &entry.inner.pathbuf(); info!("backup: {}", path.display()); self.found_live_file(path); let reason = self.policy.needs_backup(old, &entry.inner); match reason { Reason::IsNew | Reason::Changed | Reason::GenerationLookupError | Reason::Unknown => { - Ok(self.backup_one_entry(&entry, path, reason).await) + Ok(Some(self.backup_one_entry(&entry, path, reason).await)) } - Reason::Unchanged | Reason::Skipped | Reason::FileError => { + Reason::Skipped => Ok(None), + Reason::Unchanged | Reason::FileError => { let fileno = old.get_fileno(&entry.inner.pathbuf())?; let ids = if let Some(fileno) = fileno { let mut ids = vec![]; @@ -277,12 +279,12 @@ impl<'a> BackupRun<'a> { } else { vec![] }; - Ok(FsEntryBackupOutcome { + Ok(Some(FsEntryBackupOutcome { entry: entry.inner, ids, reason, is_cachedir_tag: entry.is_cachedir_tag, - }) + })) } } } -- cgit v1.2.1