From b5c4df5d31a4624d24171ba70db567327f3aae67 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 1 Jan 2021 16:21:36 +0200 Subject: refactor: move SQL use into sub-module This keeps all the SQL related functions closer together, making it easier to make changes to them. --- src/generation.rs | 150 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 92 insertions(+), 58 deletions(-) diff --git a/src/generation.rs b/src/generation.rs index 7a4b71b..5dc5f97 100644 --- a/src/generation.rs +++ b/src/generation.rs @@ -1,7 +1,6 @@ use crate::chunkid::ChunkId; -use crate::error::ObnamError; use crate::fsentry::FilesystemEntry; -use rusqlite::{params, Connection, OpenFlags, Row, Transaction}; +use rusqlite::Connection; use std::path::Path; /// A nascent backup generation. @@ -19,17 +18,7 @@ impl NascentGeneration { where P: AsRef, { - let flags = OpenFlags::SQLITE_OPEN_CREATE | OpenFlags::SQLITE_OPEN_READ_WRITE; - let conn = Connection::open_with_flags(filename, flags)?; - conn.execute( - "CREATE TABLE files (fileno INTEGER PRIMARY KEY, json TEXT)", - params![], - )?; - conn.execute( - "CREATE TABLE chunks (fileno INTEGER, chunkid TEXT)", - params![], - )?; - conn.pragma_update(None, "journal_mode", &"WAL")?; + let conn = sql::create_db(filename.as_ref())?; Ok(Self { conn, fileno: 0 }) } @@ -40,7 +29,7 @@ impl NascentGeneration { pub fn insert(&mut self, e: FilesystemEntry, ids: &[ChunkId]) -> anyhow::Result<()> { let t = self.conn.transaction()?; self.fileno += 1; - insert_one(&t, e, self.fileno, ids)?; + sql::insert_one(&t, e, self.fileno, ids)?; t.commit()?; Ok(()) } @@ -53,41 +42,13 @@ impl NascentGeneration { for r in entries { let (e, ids) = r?; self.fileno += 1; - insert_one(&t, e, self.fileno, &ids[..])?; + sql::insert_one(&t, e, self.fileno, &ids[..])?; } t.commit()?; Ok(()) } } -fn row_to_entry(row: &Row) -> rusqlite::Result<(u64, String)> { - let fileno: i64 = row.get(row.column_index("fileno")?)?; - let fileno = fileno as u64; - let json: String = row.get(row.column_index("json")?)?; - Ok((fileno, json)) -} - -fn insert_one( - t: &Transaction, - e: FilesystemEntry, - fileno: u64, - ids: &[ChunkId], -) -> anyhow::Result<()> { - let fileno = fileno as i64; - let json = serde_json::to_string(&e)?; - t.execute( - "INSERT INTO files (fileno, json) VALUES (?1, ?2)", - params![fileno, &json], - )?; - for id in ids { - t.execute( - "INSERT INTO chunks (fileno, chunkid) VALUES (?1, ?2)", - params![fileno, id], - )?; - } - Ok(()) -} - #[cfg(test)] mod test { use super::NascentGeneration; @@ -145,23 +106,98 @@ impl LocalGeneration { where P: AsRef, { + let conn = sql::open_db(filename.as_ref())?; + Ok(Self { conn }) + } + + pub fn file_count(&self) -> anyhow::Result { + Ok(sql::file_count(&self.conn)?) + } + + pub fn files(&self) -> anyhow::Result> { + Ok(sql::files(&self.conn)?) + } + + pub fn chunkids(&self, fileno: u64) -> anyhow::Result> { + Ok(sql::chunkids(&self.conn, fileno)?) + } + + pub fn get_file(&self, filename: &Path) -> anyhow::Result> { + Ok(sql::get_file(&self.conn, filename)?) + } + + pub fn get_fileno(&self, filename: &Path) -> anyhow::Result> { + Ok(sql::get_fileno(&self.conn, filename)?) + } +} + +mod sql { + use crate::chunkid::ChunkId; + use crate::error::ObnamError; + use crate::fsentry::FilesystemEntry; + use rusqlite::{params, Connection, OpenFlags, Row, Transaction}; + use std::path::Path; + + pub fn create_db(filename: &Path) -> anyhow::Result { + let flags = OpenFlags::SQLITE_OPEN_CREATE | OpenFlags::SQLITE_OPEN_READ_WRITE; + let conn = Connection::open_with_flags(filename, flags)?; + conn.execute( + "CREATE TABLE files (fileno INTEGER PRIMARY KEY, json TEXT)", + params![], + )?; + conn.execute( + "CREATE TABLE chunks (fileno INTEGER, chunkid TEXT)", + params![], + )?; + conn.pragma_update(None, "journal_mode", &"WAL")?; + Ok(conn) + } + + pub fn open_db(filename: &Path) -> anyhow::Result { let flags = OpenFlags::SQLITE_OPEN_READ_WRITE; let conn = Connection::open_with_flags(filename, flags)?; conn.pragma_update(None, "journal_mode", &"WAL")?; + Ok(conn) + } - Ok(Self { conn }) + pub fn insert_one( + t: &Transaction, + e: FilesystemEntry, + fileno: u64, + ids: &[ChunkId], + ) -> anyhow::Result<()> { + let fileno = fileno as i64; + let json = serde_json::to_string(&e)?; + t.execute( + "INSERT INTO files (fileno, json) VALUES (?1, ?2)", + params![fileno, &json], + )?; + for id in ids { + t.execute( + "INSERT INTO chunks (fileno, chunkid) VALUES (?1, ?2)", + params![fileno, id], + )?; + } + Ok(()) } - pub fn file_count(&self) -> anyhow::Result { - let mut stmt = self.conn.prepare("SELECT count(*) FROM files")?; + pub fn row_to_entry(row: &Row) -> rusqlite::Result<(u64, String)> { + let fileno: i64 = row.get(row.column_index("fileno")?)?; + let fileno = fileno as u64; + let json: String = row.get(row.column_index("json")?)?; + Ok((fileno, json)) + } + + pub fn file_count(conn: &Connection) -> anyhow::Result { + let mut stmt = conn.prepare("SELECT count(*) FROM files")?; let mut iter = stmt.query_map(params![], |row| row.get(0))?; let count = iter.next().expect("SQL count result"); let count = count?; Ok(count) } - pub fn files(&self) -> anyhow::Result> { - let mut stmt = self.conn.prepare("SELECT * FROM files")?; + pub fn files(conn: &Connection) -> anyhow::Result> { + let mut stmt = conn.prepare("SELECT * FROM files")?; let iter = stmt.query_map(params![], |row| row_to_entry(row))?; let mut files: Vec<(u64, FilesystemEntry)> = vec![]; for x in iter { @@ -172,11 +208,9 @@ impl LocalGeneration { Ok(files) } - pub fn chunkids(&self, fileno: u64) -> anyhow::Result> { + pub fn chunkids(conn: &Connection, fileno: u64) -> anyhow::Result> { let fileno = fileno as i64; - let mut stmt = self - .conn - .prepare("SELECT chunkid FROM chunks WHERE fileno = ?1")?; + let mut stmt = conn.prepare("SELECT chunkid FROM chunks WHERE fileno = ?1")?; let iter = stmt.query_map(params![fileno], |row| Ok(row.get(0)?))?; let mut ids: Vec = vec![]; for x in iter { @@ -186,25 +220,25 @@ impl LocalGeneration { Ok(ids) } - pub fn get_file(&self, filename: &Path) -> anyhow::Result> { - match self.get_file_and_fileno(filename)? { + pub fn get_file(conn: &Connection, filename: &Path) -> anyhow::Result> { + match get_file_and_fileno(conn, filename)? { None => Ok(None), Some((_, e)) => Ok(Some(e)), } } - pub fn get_fileno(&self, filename: &Path) -> anyhow::Result> { - match self.get_file_and_fileno(filename)? { + pub fn get_fileno(conn: &Connection, filename: &Path) -> anyhow::Result> { + match get_file_and_fileno(conn, filename)? { None => Ok(None), Some((id, _)) => Ok(Some(id)), } } fn get_file_and_fileno( - &self, + conn: &Connection, filename: &Path, ) -> anyhow::Result> { - let files = self.files()?; + let files = files(conn)?; let files: Vec<(u64, FilesystemEntry)> = files .iter() .filter(|(_, e)| e.pathbuf() == filename) -- cgit v1.2.1 From 3eefe4180855f1dd144545fd50eb71d96d798487 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 1 Jan 2021 16:36:22 +0200 Subject: feat! use signed 64 bit integers as file numbers, file counts --- src/cmd/backup.rs | 4 ++-- src/cmd/restore.rs | 10 +++++----- src/generation.rs | 32 +++++++++++++++----------------- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/src/cmd/backup.rs b/src/cmd/backup.rs index 2a8e086..4d13fe7 100644 --- a/src/cmd/backup.rs +++ b/src/cmd/backup.rs @@ -51,7 +51,7 @@ pub fn backup(config: &ClientConfig, buffer_size: usize) -> anyhow::Result<()> { Some(old) => { info!("incremental backup based on {}", old); let old = client.fetch_generation(&old, &oldname)?; - progress.set_length(old.file_count()?.into()); + progress.set_length(old.file_count()? as u64); new.insert_iter(iter.map(|entry| { progress.inc(1); match entry { @@ -76,7 +76,7 @@ pub fn backup(config: &ClientConfig, buffer_size: usize) -> anyhow::Result<()> { }))?; } } - progress.set_length(new.file_count()); + progress.set_length(new.file_count() as u64); progress.finish(); } diff --git a/src/cmd/restore.rs b/src/cmd/restore.rs index a3594ea..53e168a 100644 --- a/src/cmd/restore.rs +++ b/src/cmd/restore.rs @@ -34,7 +34,7 @@ pub fn restore(config: &ClientConfig, gen_ref: &str, to: &Path) -> anyhow::Resul let gen = client.fetch_generation(&gen_id, &dbname)?; info!("restore file count: {}", gen.file_count()?); - let progress = create_progress_bar(gen.file_count()?.into(), true); + let progress = create_progress_bar(gen.file_count()?, true); for (fileid, entry) in gen.files()? { restore_generation(&client, &gen, fileid, &entry, &to, &progress)?; } @@ -70,7 +70,7 @@ struct Opt { fn restore_generation( client: &BackupClient, gen: &LocalGeneration, - fileid: u64, + fileid: i64, entry: &FilesystemEntry, to: &Path, progress: &ProgressBar, @@ -120,7 +120,7 @@ fn restore_regular( client: &BackupClient, gen: &LocalGeneration, path: &Path, - fileid: u64, + fileid: i64, entry: &FilesystemEntry, ) -> anyhow::Result<()> { debug!("restoring regular {}", path.display()); @@ -191,9 +191,9 @@ fn restore_metadata(path: &Path, entry: &FilesystemEntry) -> anyhow::Result<()> Ok(()) } -fn create_progress_bar(file_count: u64, verbose: bool) -> ProgressBar { +fn create_progress_bar(file_count: i64, verbose: bool) -> ProgressBar { let progress = if verbose { - ProgressBar::new(file_count) + ProgressBar::new(file_count as u64) } else { ProgressBar::hidden() }; diff --git a/src/generation.rs b/src/generation.rs index 5dc5f97..e67d659 100644 --- a/src/generation.rs +++ b/src/generation.rs @@ -10,7 +10,7 @@ use std::path::Path; /// of its generation chunk. pub struct NascentGeneration { conn: Connection, - fileno: u64, + fileno: i64, } impl NascentGeneration { @@ -22,7 +22,7 @@ impl NascentGeneration { Ok(Self { conn, fileno: 0 }) } - pub fn file_count(&self) -> u64 { + pub fn file_count(&self) -> i64 { self.fileno } @@ -110,15 +110,15 @@ impl LocalGeneration { Ok(Self { conn }) } - pub fn file_count(&self) -> anyhow::Result { + pub fn file_count(&self) -> anyhow::Result { Ok(sql::file_count(&self.conn)?) } - pub fn files(&self) -> anyhow::Result> { + pub fn files(&self) -> anyhow::Result> { Ok(sql::files(&self.conn)?) } - pub fn chunkids(&self, fileno: u64) -> anyhow::Result> { + pub fn chunkids(&self, fileno: i64) -> anyhow::Result> { Ok(sql::chunkids(&self.conn, fileno)?) } @@ -126,7 +126,7 @@ impl LocalGeneration { Ok(sql::get_file(&self.conn, filename)?) } - pub fn get_fileno(&self, filename: &Path) -> anyhow::Result> { + pub fn get_fileno(&self, filename: &Path) -> anyhow::Result> { Ok(sql::get_fileno(&self.conn, filename)?) } } @@ -163,10 +163,9 @@ mod sql { pub fn insert_one( t: &Transaction, e: FilesystemEntry, - fileno: u64, + fileno: i64, ids: &[ChunkId], ) -> anyhow::Result<()> { - let fileno = fileno as i64; let json = serde_json::to_string(&e)?; t.execute( "INSERT INTO files (fileno, json) VALUES (?1, ?2)", @@ -181,14 +180,13 @@ mod sql { Ok(()) } - pub fn row_to_entry(row: &Row) -> rusqlite::Result<(u64, String)> { + pub fn row_to_entry(row: &Row) -> rusqlite::Result<(i64, String)> { let fileno: i64 = row.get(row.column_index("fileno")?)?; - let fileno = fileno as u64; let json: String = row.get(row.column_index("json")?)?; Ok((fileno, json)) } - pub fn file_count(conn: &Connection) -> anyhow::Result { + pub fn file_count(conn: &Connection) -> anyhow::Result { let mut stmt = conn.prepare("SELECT count(*) FROM files")?; let mut iter = stmt.query_map(params![], |row| row.get(0))?; let count = iter.next().expect("SQL count result"); @@ -196,10 +194,10 @@ mod sql { Ok(count) } - pub fn files(conn: &Connection) -> anyhow::Result> { + pub fn files(conn: &Connection) -> anyhow::Result> { let mut stmt = conn.prepare("SELECT * FROM files")?; let iter = stmt.query_map(params![], |row| row_to_entry(row))?; - let mut files: Vec<(u64, FilesystemEntry)> = vec![]; + let mut files: Vec<(i64, FilesystemEntry)> = vec![]; for x in iter { let (fileno, json) = x?; let entry = serde_json::from_str(&json)?; @@ -208,7 +206,7 @@ mod sql { Ok(files) } - pub fn chunkids(conn: &Connection, fileno: u64) -> anyhow::Result> { + pub fn chunkids(conn: &Connection, fileno: i64) -> anyhow::Result> { let fileno = fileno as i64; let mut stmt = conn.prepare("SELECT chunkid FROM chunks WHERE fileno = ?1")?; let iter = stmt.query_map(params![fileno], |row| Ok(row.get(0)?))?; @@ -227,7 +225,7 @@ mod sql { } } - pub fn get_fileno(conn: &Connection, filename: &Path) -> anyhow::Result> { + pub fn get_fileno(conn: &Connection, filename: &Path) -> anyhow::Result> { match get_file_and_fileno(conn, filename)? { None => Ok(None), Some((id, _)) => Ok(Some(id)), @@ -237,9 +235,9 @@ mod sql { fn get_file_and_fileno( conn: &Connection, filename: &Path, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let files = files(conn)?; - let files: Vec<(u64, FilesystemEntry)> = files + let files: Vec<(i64, FilesystemEntry)> = files .iter() .filter(|(_, e)| e.pathbuf() == filename) .map(|(id, e)| (*id, e.clone())) -- cgit v1.2.1 From 5b204985983a9652d03d648fa24eb8e6fd37dfe7 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 1 Jan 2021 16:45:49 +0200 Subject: feat: use SQL to look up information for a file, by filename This speeds things up a lot compared to iterating over all files. --- src/generation.rs | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/src/generation.rs b/src/generation.rs index e67d659..e225fda 100644 --- a/src/generation.rs +++ b/src/generation.rs @@ -136,19 +136,21 @@ mod sql { use crate::error::ObnamError; use crate::fsentry::FilesystemEntry; use rusqlite::{params, Connection, OpenFlags, Row, Transaction}; + use std::os::unix::ffi::OsStrExt; use std::path::Path; pub fn create_db(filename: &Path) -> anyhow::Result { let flags = OpenFlags::SQLITE_OPEN_CREATE | OpenFlags::SQLITE_OPEN_READ_WRITE; let conn = Connection::open_with_flags(filename, flags)?; conn.execute( - "CREATE TABLE files (fileno INTEGER PRIMARY KEY, json TEXT)", + "CREATE TABLE files (fileno INTEGER PRIMARY KEY, filename BLOB, json TEXT)", params![], )?; conn.execute( "CREATE TABLE chunks (fileno INTEGER, chunkid TEXT)", params![], )?; + conn.execute("CREATE INDEX filenames ON files (filename)", params![])?; conn.pragma_update(None, "journal_mode", &"WAL")?; Ok(conn) } @@ -168,8 +170,8 @@ mod sql { ) -> anyhow::Result<()> { let json = serde_json::to_string(&e)?; t.execute( - "INSERT INTO files (fileno, json) VALUES (?1, ?2)", - params![fileno, &json], + "INSERT INTO files (fileno, filename, json) VALUES (?1, ?2, ?3)", + params![fileno, path_into_blob(&e.pathbuf()), &json], )?; for id in ids { t.execute( @@ -180,6 +182,10 @@ mod sql { Ok(()) } + fn path_into_blob(path: &Path) -> Vec { + path.as_os_str().as_bytes().to_vec() + } + pub fn row_to_entry(row: &Row) -> rusqlite::Result<(i64, String)> { let fileno: i64 = row.get(row.column_index("fileno")?)?; let json: String = row.get(row.column_index("json")?)?; @@ -189,7 +195,7 @@ mod sql { pub fn file_count(conn: &Connection) -> anyhow::Result { let mut stmt = conn.prepare("SELECT count(*) FROM files")?; let mut iter = stmt.query_map(params![], |row| row.get(0))?; - let count = iter.next().expect("SQL count result"); + let count = iter.next().expect("SQL count result (1)"); let count = count?; Ok(count) } @@ -236,16 +242,20 @@ mod sql { conn: &Connection, filename: &Path, ) -> anyhow::Result> { - let files = files(conn)?; - let files: Vec<(i64, FilesystemEntry)> = files - .iter() - .filter(|(_, e)| e.pathbuf() == filename) - .map(|(id, e)| (*id, e.clone())) - .collect(); - match files.len() { - 0 => Ok(None), - 1 => Ok(Some((files[0].0, files[0].1.clone()))), - _ => return Err(ObnamError::TooManyFiles(filename.to_path_buf()).into()), + let mut stmt = conn.prepare("SELECT fileno, json FROM files WHERE filename = ?1")?; + let mut iter = + stmt.query_map(params![path_into_blob(filename)], |row| row_to_entry(row))?; + match iter.next() { + None => Ok(None), + Some(Err(e)) => Err(e.into()), + Some(Ok((fileno, json))) => { + let entry = serde_json::from_str(&json)?; + if iter.next() == None { + Ok(Some((fileno, entry))) + } else { + Err(ObnamError::TooManyFiles(filename.to_path_buf()).into()) + } + } } } } -- cgit v1.2.1