From b672b2cdcb194f6fda77bdb76aa7b01a1eed0b2e Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 31 Dec 2020 09:35:08 +0200 Subject: refactor: add LocalGeneration type for read-only operations This splits the use of NascentGeneration to more cohesive "new generation being built" versus "existing generation being restored". --- src/client.rs | 31 ++++++++++++--- src/cmd/restore.rs | 25 +++--------- src/generation.rs | 114 +++++++++++++++++++++++++++-------------------------- 3 files changed, 90 insertions(+), 80 deletions(-) diff --git a/src/client.rs b/src/client.rs index 4d7ad7c..f76d817 100644 --- a/src/client.rs +++ b/src/client.rs @@ -5,13 +5,16 @@ use crate::chunker::Chunker; use crate::chunkid::ChunkId; use crate::chunkmeta::ChunkMeta; use crate::fsentry::{FilesystemEntry, FilesystemKind}; -use crate::generation::FinishedGeneration; +use crate::generation::{FinishedGeneration, LocalGeneration}; use crate::genlist::GenerationList; + use chrono::{DateTime, Local}; use log::{debug, error, info, trace}; use reqwest::blocking::Client; use serde::Deserialize; use std::collections::HashMap; +use std::fs::File; +use std::io::prelude::*; use std::path::{Path, PathBuf}; #[derive(Debug, Deserialize, Clone)] @@ -214,19 +217,35 @@ impl BackupClient { Ok(DataChunk::new(body.to_vec())) } - pub fn fetch_generation(&self, gen_id: &str) -> anyhow::Result { + fn fetch_generation_chunk(&self, gen_id: &str) -> anyhow::Result { let url = format!("{}/{}", &self.chunks_url(), gen_id); - trace!("fetch_generation: url={:?}", url); + trace!("fetch_generation_chunk: url={:?}", url); let req = self.client.get(&url).build()?; let res = self.client.execute(req)?; - debug!("fetch_generation: status={}", res.status()); + debug!("fetch_generation_chunk: status={}", res.status()); if res.status() != 200 { return Err(ClientError::GenerationNotFound(gen_id.to_string()).into()); } let text = res.text()?; - debug!("fetch_generation: text={:?}", text); - let gen = serde_json::from_str(&text)?; + debug!("fetch_generation_chunk: text={:?}", text); + let gen: GenerationChunk = serde_json::from_str(&text)?; + debug!("fetch_generation_chunk: {:?}", gen); + Ok(gen) + } + + pub fn fetch_generation(&self, gen_id: &str, dbname: &Path) -> anyhow::Result { + let gen = self.fetch_generation_chunk(gen_id)?; + + // Fetch the SQLite file, storing it in the named file. + let mut dbfile = File::create(&dbname)?; + for id in gen.chunk_ids() { + let chunk = self.fetch_chunk(id)?; + dbfile.write_all(chunk.data())?; + } + info!("downloaded generation to {}", dbname.display()); + + let gen = LocalGeneration::open(dbname)?; Ok(gen) } } diff --git a/src/cmd/restore.rs b/src/cmd/restore.rs index 9e137f2..a3594ea 100644 --- a/src/cmd/restore.rs +++ b/src/cmd/restore.rs @@ -2,7 +2,7 @@ use crate::client::BackupClient; use crate::client::ClientConfig; use crate::error::ObnamError; use crate::fsentry::{FilesystemEntry, FilesystemKind}; -use crate::generation::NascentGeneration; +use crate::generation::LocalGeneration; use indicatif::{ProgressBar, ProgressStyle}; use libc::{fchmod, futimens, timespec}; use log::{debug, error, info}; @@ -32,22 +32,9 @@ pub fn restore(config: &ClientConfig, gen_ref: &str, to: &Path) -> anyhow::Resul Some(id) => id, }; - let gen_chunk = client.fetch_generation(&gen_id)?; - debug!("gen: {:?}", gen_chunk); - - { - // Fetch the SQLite file, storing it in the temporary file. - let mut dbfile = File::create(&dbname)?; - for id in gen_chunk.chunk_ids() { - let chunk = client.fetch_chunk(id)?; - dbfile.write_all(chunk.data())?; - } - } - info!("downloaded generation to {}", dbname.display()); - - let gen = NascentGeneration::open(&dbname)?; - info!("restore file count: {}", gen.file_count()); - let progress = create_progress_bar(gen.file_count(), true); + let gen = client.fetch_generation(&gen_id, &dbname)?; + info!("restore file count: {}", gen.file_count()?); + let progress = create_progress_bar(gen.file_count()?.into(), true); for (fileid, entry) in gen.files()? { restore_generation(&client, &gen, fileid, &entry, &to, &progress)?; } @@ -82,7 +69,7 @@ struct Opt { fn restore_generation( client: &BackupClient, - gen: &NascentGeneration, + gen: &LocalGeneration, fileid: u64, entry: &FilesystemEntry, to: &Path, @@ -131,7 +118,7 @@ fn restored_path(entry: &FilesystemEntry, to: &Path) -> anyhow::Result fn restore_regular( client: &BackupClient, - gen: &NascentGeneration, + gen: &LocalGeneration, path: &Path, fileid: u64, entry: &FilesystemEntry, diff --git a/src/generation.rs b/src/generation.rs index 5d04a9f..4e7589c 100644 --- a/src/generation.rs +++ b/src/generation.rs @@ -6,8 +6,8 @@ use std::path::Path; /// A nascent backup generation. /// /// A nascent generation is one that is being prepared. It isn't -/// finished yet, and it's not actually on the server, yet, or not -/// completely. It can't be restored. +/// finished yet, and it's not actually on the server until the upload +/// of its generation chunk. pub struct NascentGeneration { conn: Connection, fileno: u64, @@ -32,16 +32,8 @@ impl NascentGeneration { Ok(Self { conn, fileno: 0 }) } - pub fn open

(filename: P) -> anyhow::Result - where - P: AsRef, - { - let flags = OpenFlags::SQLITE_OPEN_READ_WRITE; - let conn = Connection::open_with_flags(filename, flags)?; - conn.pragma_update(None, "journal_mode", &"WAL")?; - let fileno = find_max_fileno(&conn)?; - - Ok(Self { conn, fileno }) + pub fn file_count(&self) -> u64 { + self.fileno } pub fn insert(&mut self, e: FilesystemEntry, ids: &[ChunkId]) -> anyhow::Result<()> { @@ -65,36 +57,6 @@ impl NascentGeneration { t.commit()?; Ok(()) } - - pub fn file_count(&self) -> u64 { - self.fileno - } - - pub fn files(&self) -> anyhow::Result> { - let mut stmt = self.conn.prepare("SELECT * FROM files")?; - let iter = stmt.query_map(params![], |row| row_to_entry(row))?; - let mut files: Vec<(u64, FilesystemEntry)> = vec![]; - for x in iter { - let (fileno, json) = x?; - let entry = serde_json::from_str(&json)?; - files.push((fileno, entry)); - } - Ok(files) - } - - pub fn chunkids(&self, fileno: u64) -> anyhow::Result> { - let fileno = fileno as i64; - let mut stmt = self - .conn - .prepare("SELECT chunkid FROM chunks WHERE fileno = ?1")?; - let iter = stmt.query_map(params![fileno], |row| Ok(row.get(0)?))?; - let mut ids: Vec = vec![]; - for x in iter { - let fileno: String = x?; - ids.push(ChunkId::from(&fileno)); - } - Ok(ids) - } } fn row_to_entry(row: &Row) -> rusqlite::Result<(u64, String)> { @@ -125,19 +87,6 @@ fn insert_one( Ok(()) } -fn find_max_fileno(conn: &Connection) -> anyhow::Result { - let mut stmt = conn.prepare("SELECT fileno FROM files ORDER BY fileno")?; - let mut rows = stmt.query(params![])?; - let mut fileno: i64 = 0; - while let Some(row) = rows.next()? { - let x = row.get(0)?; - if x > fileno { - fileno = x; - } - } - Ok(fileno as u64) -} - #[cfg(test)] mod test { use super::NascentGeneration; @@ -181,3 +130,58 @@ impl FinishedGeneration { &self.ended } } + +/// A local representation of a finished generation. +/// +/// This is for querying an existing generation, and other read-only +/// operations. +pub struct LocalGeneration { + conn: Connection, +} + +impl LocalGeneration { + pub fn open

(filename: P) -> anyhow::Result + where + P: AsRef, + { + let flags = OpenFlags::SQLITE_OPEN_READ_WRITE; + let conn = Connection::open_with_flags(filename, flags)?; + conn.pragma_update(None, "journal_mode", &"WAL")?; + + Ok(Self { conn }) + } + + pub fn file_count(&self) -> anyhow::Result { + let mut stmt = self.conn.prepare("SELECT count(*) FROM files")?; + let mut iter = stmt.query_map(params![], |row| row.get(0))?; + let count = iter.next().expect("SQL count result"); + let count = count?; + Ok(count) + } + + pub fn files(&self) -> anyhow::Result> { + let mut stmt = self.conn.prepare("SELECT * FROM files")?; + let iter = stmt.query_map(params![], |row| row_to_entry(row))?; + let mut files: Vec<(u64, FilesystemEntry)> = vec![]; + for x in iter { + let (fileno, json) = x?; + let entry = serde_json::from_str(&json)?; + files.push((fileno, entry)); + } + Ok(files) + } + + pub fn chunkids(&self, fileno: u64) -> anyhow::Result> { + let fileno = fileno as i64; + let mut stmt = self + .conn + .prepare("SELECT chunkid FROM chunks WHERE fileno = ?1")?; + let iter = stmt.query_map(params![fileno], |row| Ok(row.get(0)?))?; + let mut ids: Vec = vec![]; + for x in iter { + let fileno: String = x?; + ids.push(ChunkId::from(&fileno)); + } + Ok(ids) + } +} -- cgit v1.2.1