Merge branch 'progress' into 'main'

feat: progress reporting to backups and restores

See merge request larswirzenius/obnam!21

3 files changed, 88 insertions, 23 deletions

diff --git a/src/cmd/backup.rs b/src/cmd/backup.rs
index 926b2b1..8f195eb 100644
--- a/src/cmd/backup.rs
+++ b/src/cmd/backup.rs
@@ -1,9 +1,12 @@
 use crate::client::{BackupClient, ClientConfig};
 use crate::fsiter::FsIterator;
 use crate::generation::Generation;
+use indicatif::{ProgressBar, ProgressStyle};
 use std::path::Path;
 use tempfile::NamedTempFile;
 
+const GUESS_FILE_COUNT: u64 = 0;
+
 pub fn backup(config: &Path, buffer_size: usize) -> anyhow::Result<()> {
     let config = ClientConfig::read_config(config)?;
     let client = BackupClient::new(&config.server_url)?;
@@ -21,10 +24,21 @@ pub fn backup(config: &Path, buffer_size: usize) -> anyhow::Result<()> {
         // The fetching is in its own block so that the file handles
         // get closed and data flushed to disk.
         let mut gen = Generation::create(&dbname)?;
-        gen.insert_iter(FsIterator::new(&config.root).map(|entry| match entry {
-            Err(err) => Err(err),
-            Ok(entry) => client.upload_filesystem_entry(entry, buffer_size),
+        let progress = create_progress_bar(GUESS_FILE_COUNT);
+        progress.enable_steady_tick(100);
+        gen.insert_iter(FsIterator::new(&config.root).map(|entry| {
+            progress.inc(1);
+            match entry {
+                Err(err) => Err(err),
+                Ok(entry) => {
+                    progress.set_message(&format!("{}", entry.path().display()));
+                    client.upload_filesystem_entry(entry, buffer_size)
+                }
+            }
         }))?;
+        progress.set_length(gen.file_count());
+        progress.finish();
+        println!("file count: {}", gen.file_count());
     }
 
     // Upload the SQLite file, i.e., the named temporary file, which
@@ -37,3 +51,16 @@ pub fn backup(config: &Path, buffer_size: usize) -> anyhow::Result<()> {
 
     Ok(())
 }
+
+fn create_progress_bar(file_count: u64) -> ProgressBar {
+    let progress = ProgressBar::new(file_count);
+    let parts = vec![
+        "{wide_bar}",
+        "elapsed: {elapsed}",
+        "files: {pos}/{len}",
+        "current: {wide_msg}",
+        "{spinner}",
+    ];
+    progress.set_style(ProgressStyle::default_bar().template(&parts.join("\n")));
+    progress
+}
diff --git a/src/cmd/restore.rs b/src/cmd/restore.rs
index 0051d20..f2724c9 100644
--- a/src/cmd/restore.rs
+++ b/src/cmd/restore.rs
@@ -2,6 +2,7 @@ use crate::client::BackupClient;
 use crate::client::ClientConfig;
 use crate::fsentry::{FilesystemEntry, FilesystemKind};
 use crate::generation::Generation;
+use indicatif::{ProgressBar, ProgressStyle};
 use log::{debug, info};
 use std::fs::File;
 use std::io::prelude::*;
@@ -35,9 +36,12 @@ pub fn restore(config: &Path, gen_id: &str, to: &Path) -> anyhow::Result<()> {
     info!("downloaded generation to {}", dbname.display());
 
     let gen = Generation::open(&dbname)?;
+    println!("file count: {}", gen.file_count());
+    let progress = create_progress_bar(gen.file_count());
     for (fileid, entry) in gen.files()? {
-        restore_generation(&client, &gen, fileid, entry, &to)?;
+        restore_generation(&client, &gen, fileid, entry, &to, &progress)?;
     }
+    progress.finish();
 
     // Delete the temporary file.
     std::fs::remove_file(&dbname)?;
@@ -67,8 +71,10 @@ fn restore_generation(
     fileid: u64,
     entry: FilesystemEntry,
     to: &Path,
+    progress: &ProgressBar,
 ) -> anyhow::Result<()> {
-    println!("restoring {}:{}", fileid, entry.path().display());
+    progress.set_message(&format!("{}", entry.path().display()));
+    progress.inc(1);
 
     let path = if entry.path().is_absolute() {
         entry.path().strip_prefix("/")?
@@ -112,3 +118,16 @@ fn restore_regular(
     debug!("restored regular {}", path.display());
     Ok(())
 }
+
+fn create_progress_bar(file_count: u64) -> ProgressBar {
+    let progress = ProgressBar::new(file_count);
+    let parts = vec![
+        "{wide_bar}",
+        "elapsed: {elapsed}",
+        "files: {pos}/{len}",
+        "current: {wide_msg}",
+        "{spinner}",
+    ];
+    progress.set_style(ProgressStyle::default_bar().template(&parts.join("\n")));
+    progress
+}
diff --git a/src/generation.rs b/src/generation.rs
index b9edb74..a2a2cdb 100644
--- a/src/generation.rs
+++ b/src/generation.rs
@@ -21,11 +21,11 @@ impl Generation {
         let flags = OpenFlags::SQLITE_OPEN_CREATE | OpenFlags::SQLITE_OPEN_READ_WRITE;
         let conn = Connection::open_with_flags(filename, flags)?;
         conn.execute(
-            "CREATE TABLE files (fileid INTEGER PRIMARY KEY, path BLOB, kind INTEGER, len INTEGER)",
+            "CREATE TABLE files (fileno INTEGER PRIMARY KEY, path BLOB, kind INTEGER, len INTEGER)",
             params![],
         )?;
         conn.execute(
-            "CREATE TABLE chunks (fileid INTEGER, chunkid TEXT)",
+            "CREATE TABLE chunks (fileno INTEGER, chunkid TEXT)",
             params![],
         )?;
         conn.pragma_update(None, "journal_mode", &"WAL")?;
@@ -39,13 +39,15 @@ impl Generation {
         let flags = OpenFlags::SQLITE_OPEN_READ_WRITE;
         let conn = Connection::open_with_flags(filename, flags)?;
         conn.pragma_update(None, "journal_mode", &"WAL")?;
-        Ok(Self { conn, fileno: 0 })
+        let fileno = find_max_fileno(&conn)?;
+
+        Ok(Self { conn, fileno })
     }
 
     pub fn insert(&mut self, e: FilesystemEntry, ids: &[ChunkId]) -> anyhow::Result<()> {
         let t = self.conn.transaction()?;
-        insert_one(&t, e, self.fileno, ids)?;
         self.fileno += 1;
+        insert_one(&t, e, self.fileno, ids)?;
         t.commit()?;
         Ok(())
     }
@@ -57,42 +59,46 @@ impl Generation {
         let t = self.conn.transaction()?;
         for r in entries {
             let (e, ids) = r?;
-            insert_one(&t, e, self.fileno, &ids[..])?;
             self.fileno += 1;
+            insert_one(&t, e, self.fileno, &ids[..])?;
         }
         t.commit()?;
         Ok(())
     }
 
+    pub fn file_count(&self) -> u64 {
+        self.fileno
+    }
+
     pub fn files(&self) -> anyhow::Result<Vec<(u64, FilesystemEntry)>> {
         let mut stmt = self.conn.prepare("SELECT * FROM files")?;
         let iter = stmt.query_map(params![], |row| row_to_entry(row))?;
         let mut files: Vec<(u64, FilesystemEntry)> = vec![];
         for x in iter {
-            let (fileid, entry) = x?;
-            files.push((fileid, entry));
+            let (fileno, entry) = x?;
+            files.push((fileno, entry));
         }
         Ok(files)
     }
 
-    pub fn chunkids(&self, fileid: u64) -> anyhow::Result<Vec<ChunkId>> {
-        let fileid = fileid as i64;
+    pub fn chunkids(&self, fileno: u64) -> anyhow::Result<Vec<ChunkId>> {
+        let fileno = fileno as i64;
         let mut stmt = self
             .conn
-            .prepare("SELECT chunkid FROM chunks WHERE fileid = ?1")?;
-        let iter = stmt.query_map(params![fileid], |row| Ok(row.get(0)?))?;
+            .prepare("SELECT chunkid FROM chunks WHERE fileno = ?1")?;
+        let iter = stmt.query_map(params![fileno], |row| Ok(row.get(0)?))?;
         let mut ids: Vec<ChunkId> = vec![];
         for x in iter {
-            let fileid: String = x?;
-            ids.push(ChunkId::from(&fileid));
+            let fileno: String = x?;
+            ids.push(ChunkId::from(&fileno));
         }
         Ok(ids)
     }
 }
 
 fn row_to_entry(row: &Row) -> rusqlite::Result<(u64, FilesystemEntry)> {
-    let fileid: i64 = row.get(row.column_index("fileid")?)?;
-    let fileid = fileid as u64;
+    let fileno: i64 = row.get(row.column_index("fileno")?)?;
+    let fileno = fileno as u64;
     let path: Vec<u8> = row.get(row.column_index("path")?)?;
     let path: &OsStr = OsStrExt::from_bytes(&path);
     let path: PathBuf = PathBuf::from(path);
@@ -102,7 +108,7 @@ fn row_to_entry(row: &Row) -> rusqlite::Result<(u64, FilesystemEntry)> {
         FilesystemKind::Regular => FilesystemEntry::regular(path, 0),
         FilesystemKind::Directory => FilesystemEntry::directory(path),
     };
-    Ok((fileid, entry))
+    Ok((fileno, entry))
 }
 
 fn insert_one(
@@ -116,18 +122,31 @@ fn insert_one(
     let len = e.len() as i64;
     let fileno = fileno as i64;
     t.execute(
-        "INSERT INTO files (fileid, path, kind, len) VALUES (?1, ?2, ?3, ?4)",
+        "INSERT INTO files (fileno, path, kind, len) VALUES (?1, ?2, ?3, ?4)",
         params![fileno, path, kind, len],
     )?;
     for id in ids {
         t.execute(
-            "INSERT INTO chunks (fileid, chunkid) VALUES (?1, ?2)",
+            "INSERT INTO chunks (fileno, chunkid) VALUES (?1, ?2)",
             params![fileno, id],
         )?;
     }
     Ok(())
 }
 
+fn find_max_fileno(conn: &Connection) -> anyhow::Result<u64> {
+    let mut stmt = conn.prepare("SELECT fileno FROM files ORDER BY fileno")?;
+    let mut rows = stmt.query(params![])?;
+    let mut fileno: i64 = 0;
+    while let Some(row) = rows.next()? {
+        let x = row.get(0)?;
+        if x > fileno {
+            fileno = x;
+        }
+    }
+    Ok(fileno as u64)
+}
+
 #[cfg(test)]
 mod test {
     use super::Generation;