diff options
author | Lars Wirzenius <liw@liw.fi> | 2021-01-01 18:10:21 +0000 |
---|---|---|
committer | Lars Wirzenius <liw@liw.fi> | 2021-01-01 18:10:21 +0000 |
commit | e6a4eb6edf83a68a6f73094126804beb0c569937 (patch) | |
tree | 4d32fa1f41f3db39dd6ab169df2c6cfffe3e002e | |
parent | a7336cfa6cb3f5de69c994a5e61868d49756b1ad (diff) | |
parent | 5b204985983a9652d03d648fa24eb8e6fd37dfe7 (diff) | |
download | obnam2-e6a4eb6edf83a68a6f73094126804beb0c569937.tar.gz |
Merge branch 'sqlookup2' into 'main'
SQL lookup
See merge request larswirzenius/obnam!55
-rw-r--r-- | src/cmd/backup.rs | 4 | ||||
-rw-r--r-- | src/cmd/restore.rs | 10 | ||||
-rw-r--r-- | src/generation.rs | 186 |
3 files changed, 121 insertions, 79 deletions
diff --git a/src/cmd/backup.rs b/src/cmd/backup.rs index 2a8e086..4d13fe7 100644 --- a/src/cmd/backup.rs +++ b/src/cmd/backup.rs @@ -51,7 +51,7 @@ pub fn backup(config: &ClientConfig, buffer_size: usize) -> anyhow::Result<()> { Some(old) => { info!("incremental backup based on {}", old); let old = client.fetch_generation(&old, &oldname)?; - progress.set_length(old.file_count()?.into()); + progress.set_length(old.file_count()? as u64); new.insert_iter(iter.map(|entry| { progress.inc(1); match entry { @@ -76,7 +76,7 @@ pub fn backup(config: &ClientConfig, buffer_size: usize) -> anyhow::Result<()> { }))?; } } - progress.set_length(new.file_count()); + progress.set_length(new.file_count() as u64); progress.finish(); } diff --git a/src/cmd/restore.rs b/src/cmd/restore.rs index a3594ea..53e168a 100644 --- a/src/cmd/restore.rs +++ b/src/cmd/restore.rs @@ -34,7 +34,7 @@ pub fn restore(config: &ClientConfig, gen_ref: &str, to: &Path) -> anyhow::Resul let gen = client.fetch_generation(&gen_id, &dbname)?; info!("restore file count: {}", gen.file_count()?); - let progress = create_progress_bar(gen.file_count()?.into(), true); + let progress = create_progress_bar(gen.file_count()?, true); for (fileid, entry) in gen.files()? { restore_generation(&client, &gen, fileid, &entry, &to, &progress)?; } @@ -70,7 +70,7 @@ struct Opt { fn restore_generation( client: &BackupClient, gen: &LocalGeneration, - fileid: u64, + fileid: i64, entry: &FilesystemEntry, to: &Path, progress: &ProgressBar, @@ -120,7 +120,7 @@ fn restore_regular( client: &BackupClient, gen: &LocalGeneration, path: &Path, - fileid: u64, + fileid: i64, entry: &FilesystemEntry, ) -> anyhow::Result<()> { debug!("restoring regular {}", path.display()); @@ -191,9 +191,9 @@ fn restore_metadata(path: &Path, entry: &FilesystemEntry) -> anyhow::Result<()> Ok(()) } -fn create_progress_bar(file_count: u64, verbose: bool) -> ProgressBar { +fn create_progress_bar(file_count: i64, verbose: bool) -> ProgressBar { let progress = if verbose { - ProgressBar::new(file_count) + ProgressBar::new(file_count as u64) } else { ProgressBar::hidden() }; diff --git a/src/generation.rs b/src/generation.rs index 7a4b71b..e225fda 100644 --- a/src/generation.rs +++ b/src/generation.rs @@ -1,7 +1,6 @@ use crate::chunkid::ChunkId; -use crate::error::ObnamError; use crate::fsentry::FilesystemEntry; -use rusqlite::{params, Connection, OpenFlags, Row, Transaction}; +use rusqlite::Connection; use std::path::Path; /// A nascent backup generation. @@ -11,7 +10,7 @@ use std::path::Path; /// of its generation chunk. pub struct NascentGeneration { conn: Connection, - fileno: u64, + fileno: i64, } impl NascentGeneration { @@ -19,28 +18,18 @@ impl NascentGeneration { where P: AsRef<Path>, { - let flags = OpenFlags::SQLITE_OPEN_CREATE | OpenFlags::SQLITE_OPEN_READ_WRITE; - let conn = Connection::open_with_flags(filename, flags)?; - conn.execute( - "CREATE TABLE files (fileno INTEGER PRIMARY KEY, json TEXT)", - params![], - )?; - conn.execute( - "CREATE TABLE chunks (fileno INTEGER, chunkid TEXT)", - params![], - )?; - conn.pragma_update(None, "journal_mode", &"WAL")?; + let conn = sql::create_db(filename.as_ref())?; Ok(Self { conn, fileno: 0 }) } - pub fn file_count(&self) -> u64 { + pub fn file_count(&self) -> i64 { self.fileno } pub fn insert(&mut self, e: FilesystemEntry, ids: &[ChunkId]) -> anyhow::Result<()> { let t = self.conn.transaction()?; self.fileno += 1; - insert_one(&t, e, self.fileno, ids)?; + sql::insert_one(&t, e, self.fileno, ids)?; t.commit()?; Ok(()) } @@ -53,41 +42,13 @@ impl NascentGeneration { for r in entries { let (e, ids) = r?; self.fileno += 1; - insert_one(&t, e, self.fileno, &ids[..])?; + sql::insert_one(&t, e, self.fileno, &ids[..])?; } t.commit()?; Ok(()) } } -fn row_to_entry(row: &Row) -> rusqlite::Result<(u64, String)> { - let fileno: i64 = row.get(row.column_index("fileno")?)?; - let fileno = fileno as u64; - let json: String = row.get(row.column_index("json")?)?; - Ok((fileno, json)) -} - -fn insert_one( - t: &Transaction, - e: FilesystemEntry, - fileno: u64, - ids: &[ChunkId], -) -> anyhow::Result<()> { - let fileno = fileno as i64; - let json = serde_json::to_string(&e)?; - t.execute( - "INSERT INTO files (fileno, json) VALUES (?1, ?2)", - params![fileno, &json], - )?; - for id in ids { - t.execute( - "INSERT INTO chunks (fileno, chunkid) VALUES (?1, ?2)", - params![fileno, id], - )?; - } - Ok(()) -} - #[cfg(test)] mod test { use super::NascentGeneration; @@ -145,25 +106,104 @@ impl LocalGeneration { where P: AsRef<Path>, { + let conn = sql::open_db(filename.as_ref())?; + Ok(Self { conn }) + } + + pub fn file_count(&self) -> anyhow::Result<i64> { + Ok(sql::file_count(&self.conn)?) + } + + pub fn files(&self) -> anyhow::Result<Vec<(i64, FilesystemEntry)>> { + Ok(sql::files(&self.conn)?) + } + + pub fn chunkids(&self, fileno: i64) -> anyhow::Result<Vec<ChunkId>> { + Ok(sql::chunkids(&self.conn, fileno)?) + } + + pub fn get_file(&self, filename: &Path) -> anyhow::Result<Option<FilesystemEntry>> { + Ok(sql::get_file(&self.conn, filename)?) + } + + pub fn get_fileno(&self, filename: &Path) -> anyhow::Result<Option<i64>> { + Ok(sql::get_fileno(&self.conn, filename)?) + } +} + +mod sql { + use crate::chunkid::ChunkId; + use crate::error::ObnamError; + use crate::fsentry::FilesystemEntry; + use rusqlite::{params, Connection, OpenFlags, Row, Transaction}; + use std::os::unix::ffi::OsStrExt; + use std::path::Path; + + pub fn create_db(filename: &Path) -> anyhow::Result<Connection> { + let flags = OpenFlags::SQLITE_OPEN_CREATE | OpenFlags::SQLITE_OPEN_READ_WRITE; + let conn = Connection::open_with_flags(filename, flags)?; + conn.execute( + "CREATE TABLE files (fileno INTEGER PRIMARY KEY, filename BLOB, json TEXT)", + params![], + )?; + conn.execute( + "CREATE TABLE chunks (fileno INTEGER, chunkid TEXT)", + params![], + )?; + conn.execute("CREATE INDEX filenames ON files (filename)", params![])?; + conn.pragma_update(None, "journal_mode", &"WAL")?; + Ok(conn) + } + + pub fn open_db(filename: &Path) -> anyhow::Result<Connection> { let flags = OpenFlags::SQLITE_OPEN_READ_WRITE; let conn = Connection::open_with_flags(filename, flags)?; conn.pragma_update(None, "journal_mode", &"WAL")?; + Ok(conn) + } - Ok(Self { conn }) + pub fn insert_one( + t: &Transaction, + e: FilesystemEntry, + fileno: i64, + ids: &[ChunkId], + ) -> anyhow::Result<()> { + let json = serde_json::to_string(&e)?; + t.execute( + "INSERT INTO files (fileno, filename, json) VALUES (?1, ?2, ?3)", + params![fileno, path_into_blob(&e.pathbuf()), &json], + )?; + for id in ids { + t.execute( + "INSERT INTO chunks (fileno, chunkid) VALUES (?1, ?2)", + params![fileno, id], + )?; + } + Ok(()) + } + + fn path_into_blob(path: &Path) -> Vec<u8> { + path.as_os_str().as_bytes().to_vec() } - pub fn file_count(&self) -> anyhow::Result<u32> { - let mut stmt = self.conn.prepare("SELECT count(*) FROM files")?; + pub fn row_to_entry(row: &Row) -> rusqlite::Result<(i64, String)> { + let fileno: i64 = row.get(row.column_index("fileno")?)?; + let json: String = row.get(row.column_index("json")?)?; + Ok((fileno, json)) + } + + pub fn file_count(conn: &Connection) -> anyhow::Result<i64> { + let mut stmt = conn.prepare("SELECT count(*) FROM files")?; let mut iter = stmt.query_map(params![], |row| row.get(0))?; - let count = iter.next().expect("SQL count result"); + let count = iter.next().expect("SQL count result (1)"); let count = count?; Ok(count) } - pub fn files(&self) -> anyhow::Result<Vec<(u64, FilesystemEntry)>> { - let mut stmt = self.conn.prepare("SELECT * FROM files")?; + pub fn files(conn: &Connection) -> anyhow::Result<Vec<(i64, FilesystemEntry)>> { + let mut stmt = conn.prepare("SELECT * FROM files")?; let iter = stmt.query_map(params![], |row| row_to_entry(row))?; - let mut files: Vec<(u64, FilesystemEntry)> = vec![]; + let mut files: Vec<(i64, FilesystemEntry)> = vec![]; for x in iter { let (fileno, json) = x?; let entry = serde_json::from_str(&json)?; @@ -172,11 +212,9 @@ impl LocalGeneration { Ok(files) } - pub fn chunkids(&self, fileno: u64) -> anyhow::Result<Vec<ChunkId>> { + pub fn chunkids(conn: &Connection, fileno: i64) -> anyhow::Result<Vec<ChunkId>> { let fileno = fileno as i64; - let mut stmt = self - .conn - .prepare("SELECT chunkid FROM chunks WHERE fileno = ?1")?; + let mut stmt = conn.prepare("SELECT chunkid FROM chunks WHERE fileno = ?1")?; let iter = stmt.query_map(params![fileno], |row| Ok(row.get(0)?))?; let mut ids: Vec<ChunkId> = vec![]; for x in iter { @@ -186,34 +224,38 @@ impl LocalGeneration { Ok(ids) } - pub fn get_file(&self, filename: &Path) -> anyhow::Result<Option<FilesystemEntry>> { - match self.get_file_and_fileno(filename)? { + pub fn get_file(conn: &Connection, filename: &Path) -> anyhow::Result<Option<FilesystemEntry>> { + match get_file_and_fileno(conn, filename)? { None => Ok(None), Some((_, e)) => Ok(Some(e)), } } - pub fn get_fileno(&self, filename: &Path) -> anyhow::Result<Option<u64>> { - match self.get_file_and_fileno(filename)? { + pub fn get_fileno(conn: &Connection, filename: &Path) -> anyhow::Result<Option<i64>> { + match get_file_and_fileno(conn, filename)? { None => Ok(None), Some((id, _)) => Ok(Some(id)), } } fn get_file_and_fileno( - &self, + conn: &Connection, filename: &Path, - ) -> anyhow::Result<Option<(u64, FilesystemEntry)>> { - let files = self.files()?; - let files: Vec<(u64, FilesystemEntry)> = files - .iter() - .filter(|(_, e)| e.pathbuf() == filename) - .map(|(id, e)| (*id, e.clone())) - .collect(); - match files.len() { - 0 => Ok(None), - 1 => Ok(Some((files[0].0, files[0].1.clone()))), - _ => return Err(ObnamError::TooManyFiles(filename.to_path_buf()).into()), + ) -> anyhow::Result<Option<(i64, FilesystemEntry)>> { + let mut stmt = conn.prepare("SELECT fileno, json FROM files WHERE filename = ?1")?; + let mut iter = + stmt.query_map(params![path_into_blob(filename)], |row| row_to_entry(row))?; + match iter.next() { + None => Ok(None), + Some(Err(e)) => Err(e.into()), + Some(Ok((fileno, json))) => { + let entry = serde_json::from_str(&json)?; + if iter.next() == None { + Ok(Some((fileno, entry))) + } else { + Err(ObnamError::TooManyFiles(filename.to_path_buf()).into()) + } + } } } } |