summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLars Wirzenius <liw@liw.fi>2021-01-01 18:10:21 +0000
committerLars Wirzenius <liw@liw.fi>2021-01-01 18:10:21 +0000
commite6a4eb6edf83a68a6f73094126804beb0c569937 (patch)
tree4d32fa1f41f3db39dd6ab169df2c6cfffe3e002e
parenta7336cfa6cb3f5de69c994a5e61868d49756b1ad (diff)
parent5b204985983a9652d03d648fa24eb8e6fd37dfe7 (diff)
downloadobnam2-e6a4eb6edf83a68a6f73094126804beb0c569937.tar.gz
Merge branch 'sqlookup2' into 'main'
SQL lookup See merge request larswirzenius/obnam!55
-rw-r--r--src/cmd/backup.rs4
-rw-r--r--src/cmd/restore.rs10
-rw-r--r--src/generation.rs186
3 files changed, 121 insertions, 79 deletions
diff --git a/src/cmd/backup.rs b/src/cmd/backup.rs
index 2a8e086..4d13fe7 100644
--- a/src/cmd/backup.rs
+++ b/src/cmd/backup.rs
@@ -51,7 +51,7 @@ pub fn backup(config: &ClientConfig, buffer_size: usize) -> anyhow::Result<()> {
Some(old) => {
info!("incremental backup based on {}", old);
let old = client.fetch_generation(&old, &oldname)?;
- progress.set_length(old.file_count()?.into());
+ progress.set_length(old.file_count()? as u64);
new.insert_iter(iter.map(|entry| {
progress.inc(1);
match entry {
@@ -76,7 +76,7 @@ pub fn backup(config: &ClientConfig, buffer_size: usize) -> anyhow::Result<()> {
}))?;
}
}
- progress.set_length(new.file_count());
+ progress.set_length(new.file_count() as u64);
progress.finish();
}
diff --git a/src/cmd/restore.rs b/src/cmd/restore.rs
index a3594ea..53e168a 100644
--- a/src/cmd/restore.rs
+++ b/src/cmd/restore.rs
@@ -34,7 +34,7 @@ pub fn restore(config: &ClientConfig, gen_ref: &str, to: &Path) -> anyhow::Resul
let gen = client.fetch_generation(&gen_id, &dbname)?;
info!("restore file count: {}", gen.file_count()?);
- let progress = create_progress_bar(gen.file_count()?.into(), true);
+ let progress = create_progress_bar(gen.file_count()?, true);
for (fileid, entry) in gen.files()? {
restore_generation(&client, &gen, fileid, &entry, &to, &progress)?;
}
@@ -70,7 +70,7 @@ struct Opt {
fn restore_generation(
client: &BackupClient,
gen: &LocalGeneration,
- fileid: u64,
+ fileid: i64,
entry: &FilesystemEntry,
to: &Path,
progress: &ProgressBar,
@@ -120,7 +120,7 @@ fn restore_regular(
client: &BackupClient,
gen: &LocalGeneration,
path: &Path,
- fileid: u64,
+ fileid: i64,
entry: &FilesystemEntry,
) -> anyhow::Result<()> {
debug!("restoring regular {}", path.display());
@@ -191,9 +191,9 @@ fn restore_metadata(path: &Path, entry: &FilesystemEntry) -> anyhow::Result<()>
Ok(())
}
-fn create_progress_bar(file_count: u64, verbose: bool) -> ProgressBar {
+fn create_progress_bar(file_count: i64, verbose: bool) -> ProgressBar {
let progress = if verbose {
- ProgressBar::new(file_count)
+ ProgressBar::new(file_count as u64)
} else {
ProgressBar::hidden()
};
diff --git a/src/generation.rs b/src/generation.rs
index 7a4b71b..e225fda 100644
--- a/src/generation.rs
+++ b/src/generation.rs
@@ -1,7 +1,6 @@
use crate::chunkid::ChunkId;
-use crate::error::ObnamError;
use crate::fsentry::FilesystemEntry;
-use rusqlite::{params, Connection, OpenFlags, Row, Transaction};
+use rusqlite::Connection;
use std::path::Path;
/// A nascent backup generation.
@@ -11,7 +10,7 @@ use std::path::Path;
/// of its generation chunk.
pub struct NascentGeneration {
conn: Connection,
- fileno: u64,
+ fileno: i64,
}
impl NascentGeneration {
@@ -19,28 +18,18 @@ impl NascentGeneration {
where
P: AsRef<Path>,
{
- let flags = OpenFlags::SQLITE_OPEN_CREATE | OpenFlags::SQLITE_OPEN_READ_WRITE;
- let conn = Connection::open_with_flags(filename, flags)?;
- conn.execute(
- "CREATE TABLE files (fileno INTEGER PRIMARY KEY, json TEXT)",
- params![],
- )?;
- conn.execute(
- "CREATE TABLE chunks (fileno INTEGER, chunkid TEXT)",
- params![],
- )?;
- conn.pragma_update(None, "journal_mode", &"WAL")?;
+ let conn = sql::create_db(filename.as_ref())?;
Ok(Self { conn, fileno: 0 })
}
- pub fn file_count(&self) -> u64 {
+ pub fn file_count(&self) -> i64 {
self.fileno
}
pub fn insert(&mut self, e: FilesystemEntry, ids: &[ChunkId]) -> anyhow::Result<()> {
let t = self.conn.transaction()?;
self.fileno += 1;
- insert_one(&t, e, self.fileno, ids)?;
+ sql::insert_one(&t, e, self.fileno, ids)?;
t.commit()?;
Ok(())
}
@@ -53,41 +42,13 @@ impl NascentGeneration {
for r in entries {
let (e, ids) = r?;
self.fileno += 1;
- insert_one(&t, e, self.fileno, &ids[..])?;
+ sql::insert_one(&t, e, self.fileno, &ids[..])?;
}
t.commit()?;
Ok(())
}
}
-fn row_to_entry(row: &Row) -> rusqlite::Result<(u64, String)> {
- let fileno: i64 = row.get(row.column_index("fileno")?)?;
- let fileno = fileno as u64;
- let json: String = row.get(row.column_index("json")?)?;
- Ok((fileno, json))
-}
-
-fn insert_one(
- t: &Transaction,
- e: FilesystemEntry,
- fileno: u64,
- ids: &[ChunkId],
-) -> anyhow::Result<()> {
- let fileno = fileno as i64;
- let json = serde_json::to_string(&e)?;
- t.execute(
- "INSERT INTO files (fileno, json) VALUES (?1, ?2)",
- params![fileno, &json],
- )?;
- for id in ids {
- t.execute(
- "INSERT INTO chunks (fileno, chunkid) VALUES (?1, ?2)",
- params![fileno, id],
- )?;
- }
- Ok(())
-}
-
#[cfg(test)]
mod test {
use super::NascentGeneration;
@@ -145,25 +106,104 @@ impl LocalGeneration {
where
P: AsRef<Path>,
{
+ let conn = sql::open_db(filename.as_ref())?;
+ Ok(Self { conn })
+ }
+
+ pub fn file_count(&self) -> anyhow::Result<i64> {
+ Ok(sql::file_count(&self.conn)?)
+ }
+
+ pub fn files(&self) -> anyhow::Result<Vec<(i64, FilesystemEntry)>> {
+ Ok(sql::files(&self.conn)?)
+ }
+
+ pub fn chunkids(&self, fileno: i64) -> anyhow::Result<Vec<ChunkId>> {
+ Ok(sql::chunkids(&self.conn, fileno)?)
+ }
+
+ pub fn get_file(&self, filename: &Path) -> anyhow::Result<Option<FilesystemEntry>> {
+ Ok(sql::get_file(&self.conn, filename)?)
+ }
+
+ pub fn get_fileno(&self, filename: &Path) -> anyhow::Result<Option<i64>> {
+ Ok(sql::get_fileno(&self.conn, filename)?)
+ }
+}
+
+mod sql {
+ use crate::chunkid::ChunkId;
+ use crate::error::ObnamError;
+ use crate::fsentry::FilesystemEntry;
+ use rusqlite::{params, Connection, OpenFlags, Row, Transaction};
+ use std::os::unix::ffi::OsStrExt;
+ use std::path::Path;
+
+ pub fn create_db(filename: &Path) -> anyhow::Result<Connection> {
+ let flags = OpenFlags::SQLITE_OPEN_CREATE | OpenFlags::SQLITE_OPEN_READ_WRITE;
+ let conn = Connection::open_with_flags(filename, flags)?;
+ conn.execute(
+ "CREATE TABLE files (fileno INTEGER PRIMARY KEY, filename BLOB, json TEXT)",
+ params![],
+ )?;
+ conn.execute(
+ "CREATE TABLE chunks (fileno INTEGER, chunkid TEXT)",
+ params![],
+ )?;
+ conn.execute("CREATE INDEX filenames ON files (filename)", params![])?;
+ conn.pragma_update(None, "journal_mode", &"WAL")?;
+ Ok(conn)
+ }
+
+ pub fn open_db(filename: &Path) -> anyhow::Result<Connection> {
let flags = OpenFlags::SQLITE_OPEN_READ_WRITE;
let conn = Connection::open_with_flags(filename, flags)?;
conn.pragma_update(None, "journal_mode", &"WAL")?;
+ Ok(conn)
+ }
- Ok(Self { conn })
+ pub fn insert_one(
+ t: &Transaction,
+ e: FilesystemEntry,
+ fileno: i64,
+ ids: &[ChunkId],
+ ) -> anyhow::Result<()> {
+ let json = serde_json::to_string(&e)?;
+ t.execute(
+ "INSERT INTO files (fileno, filename, json) VALUES (?1, ?2, ?3)",
+ params![fileno, path_into_blob(&e.pathbuf()), &json],
+ )?;
+ for id in ids {
+ t.execute(
+ "INSERT INTO chunks (fileno, chunkid) VALUES (?1, ?2)",
+ params![fileno, id],
+ )?;
+ }
+ Ok(())
+ }
+
+ fn path_into_blob(path: &Path) -> Vec<u8> {
+ path.as_os_str().as_bytes().to_vec()
}
- pub fn file_count(&self) -> anyhow::Result<u32> {
- let mut stmt = self.conn.prepare("SELECT count(*) FROM files")?;
+ pub fn row_to_entry(row: &Row) -> rusqlite::Result<(i64, String)> {
+ let fileno: i64 = row.get(row.column_index("fileno")?)?;
+ let json: String = row.get(row.column_index("json")?)?;
+ Ok((fileno, json))
+ }
+
+ pub fn file_count(conn: &Connection) -> anyhow::Result<i64> {
+ let mut stmt = conn.prepare("SELECT count(*) FROM files")?;
let mut iter = stmt.query_map(params![], |row| row.get(0))?;
- let count = iter.next().expect("SQL count result");
+ let count = iter.next().expect("SQL count result (1)");
let count = count?;
Ok(count)
}
- pub fn files(&self) -> anyhow::Result<Vec<(u64, FilesystemEntry)>> {
- let mut stmt = self.conn.prepare("SELECT * FROM files")?;
+ pub fn files(conn: &Connection) -> anyhow::Result<Vec<(i64, FilesystemEntry)>> {
+ let mut stmt = conn.prepare("SELECT * FROM files")?;
let iter = stmt.query_map(params![], |row| row_to_entry(row))?;
- let mut files: Vec<(u64, FilesystemEntry)> = vec![];
+ let mut files: Vec<(i64, FilesystemEntry)> = vec![];
for x in iter {
let (fileno, json) = x?;
let entry = serde_json::from_str(&json)?;
@@ -172,11 +212,9 @@ impl LocalGeneration {
Ok(files)
}
- pub fn chunkids(&self, fileno: u64) -> anyhow::Result<Vec<ChunkId>> {
+ pub fn chunkids(conn: &Connection, fileno: i64) -> anyhow::Result<Vec<ChunkId>> {
let fileno = fileno as i64;
- let mut stmt = self
- .conn
- .prepare("SELECT chunkid FROM chunks WHERE fileno = ?1")?;
+ let mut stmt = conn.prepare("SELECT chunkid FROM chunks WHERE fileno = ?1")?;
let iter = stmt.query_map(params![fileno], |row| Ok(row.get(0)?))?;
let mut ids: Vec<ChunkId> = vec![];
for x in iter {
@@ -186,34 +224,38 @@ impl LocalGeneration {
Ok(ids)
}
- pub fn get_file(&self, filename: &Path) -> anyhow::Result<Option<FilesystemEntry>> {
- match self.get_file_and_fileno(filename)? {
+ pub fn get_file(conn: &Connection, filename: &Path) -> anyhow::Result<Option<FilesystemEntry>> {
+ match get_file_and_fileno(conn, filename)? {
None => Ok(None),
Some((_, e)) => Ok(Some(e)),
}
}
- pub fn get_fileno(&self, filename: &Path) -> anyhow::Result<Option<u64>> {
- match self.get_file_and_fileno(filename)? {
+ pub fn get_fileno(conn: &Connection, filename: &Path) -> anyhow::Result<Option<i64>> {
+ match get_file_and_fileno(conn, filename)? {
None => Ok(None),
Some((id, _)) => Ok(Some(id)),
}
}
fn get_file_and_fileno(
- &self,
+ conn: &Connection,
filename: &Path,
- ) -> anyhow::Result<Option<(u64, FilesystemEntry)>> {
- let files = self.files()?;
- let files: Vec<(u64, FilesystemEntry)> = files
- .iter()
- .filter(|(_, e)| e.pathbuf() == filename)
- .map(|(id, e)| (*id, e.clone()))
- .collect();
- match files.len() {
- 0 => Ok(None),
- 1 => Ok(Some((files[0].0, files[0].1.clone()))),
- _ => return Err(ObnamError::TooManyFiles(filename.to_path_buf()).into()),
+ ) -> anyhow::Result<Option<(i64, FilesystemEntry)>> {
+ let mut stmt = conn.prepare("SELECT fileno, json FROM files WHERE filename = ?1")?;
+ let mut iter =
+ stmt.query_map(params![path_into_blob(filename)], |row| row_to_entry(row))?;
+ match iter.next() {
+ None => Ok(None),
+ Some(Err(e)) => Err(e.into()),
+ Some(Ok((fileno, json))) => {
+ let entry = serde_json::from_str(&json)?;
+ if iter.next() == None {
+ Ok(Some((fileno, entry)))
+ } else {
+ Err(ObnamError::TooManyFiles(filename.to_path_buf()).into())
+ }
+ }
}
}
}