From eeabe321112af8fcef1b4c1be7728684f9a7a9aa Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 23 Nov 2020 07:57:57 +0200 Subject: feat: progress reporting to backups and restores --- src/cmd/backup.rs | 33 +++++++++++++++++++++++++++++--- src/cmd/restore.rs | 23 +++++++++++++++++++++-- src/generation.rs | 55 ++++++++++++++++++++++++++++++++++++------------------ 3 files changed, 88 insertions(+), 23 deletions(-) diff --git a/src/cmd/backup.rs b/src/cmd/backup.rs index 926b2b1..8f195eb 100644 --- a/src/cmd/backup.rs +++ b/src/cmd/backup.rs @@ -1,9 +1,12 @@ use crate::client::{BackupClient, ClientConfig}; use crate::fsiter::FsIterator; use crate::generation::Generation; +use indicatif::{ProgressBar, ProgressStyle}; use std::path::Path; use tempfile::NamedTempFile; +const GUESS_FILE_COUNT: u64 = 0; + pub fn backup(config: &Path, buffer_size: usize) -> anyhow::Result<()> { let config = ClientConfig::read_config(config)?; let client = BackupClient::new(&config.server_url)?; @@ -21,10 +24,21 @@ pub fn backup(config: &Path, buffer_size: usize) -> anyhow::Result<()> { // The fetching is in its own block so that the file handles // get closed and data flushed to disk. let mut gen = Generation::create(&dbname)?; - gen.insert_iter(FsIterator::new(&config.root).map(|entry| match entry { - Err(err) => Err(err), - Ok(entry) => client.upload_filesystem_entry(entry, buffer_size), + let progress = create_progress_bar(GUESS_FILE_COUNT); + progress.enable_steady_tick(100); + gen.insert_iter(FsIterator::new(&config.root).map(|entry| { + progress.inc(1); + match entry { + Err(err) => Err(err), + Ok(entry) => { + progress.set_message(&format!("{}", entry.path().display())); + client.upload_filesystem_entry(entry, buffer_size) + } + } }))?; + progress.set_length(gen.file_count()); + progress.finish(); + println!("file count: {}", gen.file_count()); } // Upload the SQLite file, i.e., the named temporary file, which @@ -37,3 +51,16 @@ pub fn backup(config: &Path, buffer_size: usize) -> anyhow::Result<()> { Ok(()) } + +fn create_progress_bar(file_count: u64) -> ProgressBar { + let progress = ProgressBar::new(file_count); + let parts = vec![ + "{wide_bar}", + "elapsed: {elapsed}", + "files: {pos}/{len}", + "current: {wide_msg}", + "{spinner}", + ]; + progress.set_style(ProgressStyle::default_bar().template(&parts.join("\n"))); + progress +} diff --git a/src/cmd/restore.rs b/src/cmd/restore.rs index 0051d20..f2724c9 100644 --- a/src/cmd/restore.rs +++ b/src/cmd/restore.rs @@ -2,6 +2,7 @@ use crate::client::BackupClient; use crate::client::ClientConfig; use crate::fsentry::{FilesystemEntry, FilesystemKind}; use crate::generation::Generation; +use indicatif::{ProgressBar, ProgressStyle}; use log::{debug, info}; use std::fs::File; use std::io::prelude::*; @@ -35,9 +36,12 @@ pub fn restore(config: &Path, gen_id: &str, to: &Path) -> anyhow::Result<()> { info!("downloaded generation to {}", dbname.display()); let gen = Generation::open(&dbname)?; + println!("file count: {}", gen.file_count()); + let progress = create_progress_bar(gen.file_count()); for (fileid, entry) in gen.files()? { - restore_generation(&client, &gen, fileid, entry, &to)?; + restore_generation(&client, &gen, fileid, entry, &to, &progress)?; } + progress.finish(); // Delete the temporary file. std::fs::remove_file(&dbname)?; @@ -67,8 +71,10 @@ fn restore_generation( fileid: u64, entry: FilesystemEntry, to: &Path, + progress: &ProgressBar, ) -> anyhow::Result<()> { - println!("restoring {}:{}", fileid, entry.path().display()); + progress.set_message(&format!("{}", entry.path().display())); + progress.inc(1); let path = if entry.path().is_absolute() { entry.path().strip_prefix("/")? @@ -112,3 +118,16 @@ fn restore_regular( debug!("restored regular {}", path.display()); Ok(()) } + +fn create_progress_bar(file_count: u64) -> ProgressBar { + let progress = ProgressBar::new(file_count); + let parts = vec![ + "{wide_bar}", + "elapsed: {elapsed}", + "files: {pos}/{len}", + "current: {wide_msg}", + "{spinner}", + ]; + progress.set_style(ProgressStyle::default_bar().template(&parts.join("\n"))); + progress +} diff --git a/src/generation.rs b/src/generation.rs index b9edb74..a2a2cdb 100644 --- a/src/generation.rs +++ b/src/generation.rs @@ -21,11 +21,11 @@ impl Generation { let flags = OpenFlags::SQLITE_OPEN_CREATE | OpenFlags::SQLITE_OPEN_READ_WRITE; let conn = Connection::open_with_flags(filename, flags)?; conn.execute( - "CREATE TABLE files (fileid INTEGER PRIMARY KEY, path BLOB, kind INTEGER, len INTEGER)", + "CREATE TABLE files (fileno INTEGER PRIMARY KEY, path BLOB, kind INTEGER, len INTEGER)", params![], )?; conn.execute( - "CREATE TABLE chunks (fileid INTEGER, chunkid TEXT)", + "CREATE TABLE chunks (fileno INTEGER, chunkid TEXT)", params![], )?; conn.pragma_update(None, "journal_mode", &"WAL")?; @@ -39,13 +39,15 @@ impl Generation { let flags = OpenFlags::SQLITE_OPEN_READ_WRITE; let conn = Connection::open_with_flags(filename, flags)?; conn.pragma_update(None, "journal_mode", &"WAL")?; - Ok(Self { conn, fileno: 0 }) + let fileno = find_max_fileno(&conn)?; + + Ok(Self { conn, fileno }) } pub fn insert(&mut self, e: FilesystemEntry, ids: &[ChunkId]) -> anyhow::Result<()> { let t = self.conn.transaction()?; - insert_one(&t, e, self.fileno, ids)?; self.fileno += 1; + insert_one(&t, e, self.fileno, ids)?; t.commit()?; Ok(()) } @@ -57,42 +59,46 @@ impl Generation { let t = self.conn.transaction()?; for r in entries { let (e, ids) = r?; - insert_one(&t, e, self.fileno, &ids[..])?; self.fileno += 1; + insert_one(&t, e, self.fileno, &ids[..])?; } t.commit()?; Ok(()) } + pub fn file_count(&self) -> u64 { + self.fileno + } + pub fn files(&self) -> anyhow::Result> { let mut stmt = self.conn.prepare("SELECT * FROM files")?; let iter = stmt.query_map(params![], |row| row_to_entry(row))?; let mut files: Vec<(u64, FilesystemEntry)> = vec![]; for x in iter { - let (fileid, entry) = x?; - files.push((fileid, entry)); + let (fileno, entry) = x?; + files.push((fileno, entry)); } Ok(files) } - pub fn chunkids(&self, fileid: u64) -> anyhow::Result> { - let fileid = fileid as i64; + pub fn chunkids(&self, fileno: u64) -> anyhow::Result> { + let fileno = fileno as i64; let mut stmt = self .conn - .prepare("SELECT chunkid FROM chunks WHERE fileid = ?1")?; - let iter = stmt.query_map(params![fileid], |row| Ok(row.get(0)?))?; + .prepare("SELECT chunkid FROM chunks WHERE fileno = ?1")?; + let iter = stmt.query_map(params![fileno], |row| Ok(row.get(0)?))?; let mut ids: Vec = vec![]; for x in iter { - let fileid: String = x?; - ids.push(ChunkId::from(&fileid)); + let fileno: String = x?; + ids.push(ChunkId::from(&fileno)); } Ok(ids) } } fn row_to_entry(row: &Row) -> rusqlite::Result<(u64, FilesystemEntry)> { - let fileid: i64 = row.get(row.column_index("fileid")?)?; - let fileid = fileid as u64; + let fileno: i64 = row.get(row.column_index("fileno")?)?; + let fileno = fileno as u64; let path: Vec = row.get(row.column_index("path")?)?; let path: &OsStr = OsStrExt::from_bytes(&path); let path: PathBuf = PathBuf::from(path); @@ -102,7 +108,7 @@ fn row_to_entry(row: &Row) -> rusqlite::Result<(u64, FilesystemEntry)> { FilesystemKind::Regular => FilesystemEntry::regular(path, 0), FilesystemKind::Directory => FilesystemEntry::directory(path), }; - Ok((fileid, entry)) + Ok((fileno, entry)) } fn insert_one( @@ -116,18 +122,31 @@ fn insert_one( let len = e.len() as i64; let fileno = fileno as i64; t.execute( - "INSERT INTO files (fileid, path, kind, len) VALUES (?1, ?2, ?3, ?4)", + "INSERT INTO files (fileno, path, kind, len) VALUES (?1, ?2, ?3, ?4)", params![fileno, path, kind, len], )?; for id in ids { t.execute( - "INSERT INTO chunks (fileid, chunkid) VALUES (?1, ?2)", + "INSERT INTO chunks (fileno, chunkid) VALUES (?1, ?2)", params![fileno, id], )?; } Ok(()) } +fn find_max_fileno(conn: &Connection) -> anyhow::Result { + let mut stmt = conn.prepare("SELECT fileno FROM files ORDER BY fileno")?; + let mut rows = stmt.query(params![])?; + let mut fileno: i64 = 0; + while let Some(row) = rows.next()? { + let x = row.get(0)?; + if x > fileno { + fileno = x; + } + } + Ok(fileno as u64) +} + #[cfg(test)] mod test { use super::Generation; -- cgit v1.2.1