diff options
author | Lars Wirzenius <liw@liw.fi> | 2021-04-27 16:09:27 +0000 |
---|---|---|
committer | Lars Wirzenius <liw@liw.fi> | 2021-04-27 16:09:27 +0000 |
commit | bf645f3645fd2ee57495eafd1ccfb4afbe917bec (patch) | |
tree | 314e48f95e781e0e3b65dab7467e2fed4afafa47 | |
parent | 80aaff3f70f790141fbc8caa8a2f4830cd5e3fee (diff) | |
parent | 4ae0960619537234d5591b40d05f91b131330618 (diff) | |
download | obnam2-bf645f3645fd2ee57495eafd1ccfb4afbe917bec.tar.gz |
Merge branch 'feature/28-vec-to-iter' into 'main'
Return iterators instead of huge vectors
Closes #28
See merge request larswirzenius/obnam!142
-rw-r--r-- | src/backup_run.rs | 6 | ||||
-rw-r--r-- | src/cmd/list_files.rs | 3 | ||||
-rw-r--r-- | src/cmd/restore.rs | 9 | ||||
-rw-r--r-- | src/cmd/show_gen.rs | 20 | ||||
-rw-r--r-- | src/generation.rs | 107 |
5 files changed, 111 insertions, 34 deletions
diff --git a/src/backup_run.rs b/src/backup_run.rs index e966855..23c97f6 100644 --- a/src/backup_run.rs +++ b/src/backup_run.rs @@ -149,7 +149,11 @@ impl<'a> IncrementalBackup<'a> { Reason::Unchanged | Reason::Skipped | Reason::FileError => { let fileno = old.get_fileno(&entry.pathbuf())?; let ids = if let Some(fileno) = fileno { - old.chunkids(fileno)? + let mut ids = vec![]; + for id in old.chunkids(fileno)?.iter()? { + ids.push(id?); + } + ids } else { vec![] }; diff --git a/src/cmd/list_files.rs b/src/cmd/list_files.rs index 22b102e..c5191f7 100644 --- a/src/cmd/list_files.rs +++ b/src/cmd/list_files.rs @@ -22,7 +22,8 @@ impl ListFiles { let gen_id: String = genlist.resolve(&self.gen_id)?; let gen = client.fetch_generation(&gen_id, temp.path())?; - for file in gen.files()? { + for file in gen.files()?.iter()? { + let file = file?; println!("{}", format_entry(&file.entry(), file.reason())); } diff --git a/src/cmd/restore.rs b/src/cmd/restore.rs index a321e80..4de0830 100644 --- a/src/cmd/restore.rs +++ b/src/cmd/restore.rs @@ -40,7 +40,8 @@ impl Restore { let gen = client.fetch_generation(&gen_id, temp.path())?; info!("restoring {} files", gen.file_count()?); let progress = create_progress_bar(gen.file_count()?, true); - for file in gen.files()? { + for file in gen.files()?.iter()? { + let file = file?; match file.reason() { Reason::FileError => (), _ => restore_generation( @@ -53,7 +54,8 @@ impl Restore { )?, } } - for file in gen.files()? { + for file in gen.files()?.iter()? { + let file = file?; if file.entry().is_dir() { restore_directory_metadata(file.entry(), &self.to)?; } @@ -170,7 +172,8 @@ fn restore_regular( std::fs::create_dir_all(parent)?; { let mut file = std::fs::File::create(path)?; - for chunkid in gen.chunkids(fileid)? { + for chunkid in gen.chunkids(fileid)?.iter()? { + let chunkid = chunkid?; let chunk = client.fetch_chunk(&chunkid)?; file.write_all(chunk.data())?; } diff --git a/src/cmd/show_gen.rs b/src/cmd/show_gen.rs index ba39809..df8a030 100644 --- a/src/cmd/show_gen.rs +++ b/src/cmd/show_gen.rs @@ -21,16 +21,20 @@ impl ShowGeneration { let genlist = client.list_generations()?; let gen_id: String = genlist.resolve(&self.gen_id)?; let gen = client.fetch_generation(&gen_id, temp.path())?; - let files = gen.files()?; + let mut files = gen.files()?; + let mut files = files.iter()?; - let total_bytes = files.iter().fold(0, |acc, file| { - let e = file.entry(); - if e.kind() == FilesystemKind::Regular { - acc + file.entry().len() - } else { - acc - } + let total_bytes = files.try_fold(0, |acc, file| { + file.map(|file| { + let e = file.entry(); + if e.kind() == FilesystemKind::Regular { + acc + e.len() + } else { + acc + } + }) }); + let total_bytes = total_bytes?; println!("generation-id: {}", gen_id); println!("file-count: {}", gen.file_count()?); diff --git a/src/generation.rs b/src/generation.rs index 1d0ee52..0055bfe 100644 --- a/src/generation.rs +++ b/src/generation.rs @@ -196,11 +196,11 @@ impl LocalGeneration { sql::file_count(&self.conn) } - pub fn files(&self) -> LocalGenerationResult<Vec<BackedUpFile>> { + pub fn files(&self) -> LocalGenerationResult<sql::SqlResults<BackedUpFile>> { sql::files(&self.conn) } - pub fn chunkids(&self, fileno: FileId) -> LocalGenerationResult<Vec<ChunkId>> { + pub fn chunkids(&self, fileno: FileId) -> LocalGenerationResult<sql::SqlResults<ChunkId>> { sql::chunkids(&self.conn, fileno) } @@ -221,7 +221,7 @@ mod sql { use crate::backup_reason::Reason; use crate::chunkid::ChunkId; use crate::fsentry::FilesystemEntry; - use rusqlite::{params, Connection, OpenFlags, Row, Transaction}; + use rusqlite::{params, Connection, OpenFlags, Row, Statement, Transaction}; use std::os::unix::ffi::OsStrExt; use std::path::Path; @@ -289,27 +289,92 @@ mod sql { Ok(count) } - pub fn files(conn: &Connection) -> LocalGenerationResult<Vec<BackedUpFile>> { - let mut stmt = conn.prepare("SELECT * FROM files")?; - let iter = stmt.query_map(params![], |row| row_to_entry(row))?; - let mut files = vec![]; - for x in iter { - let (fileno, json, reason) = x?; - let entry = serde_json::from_str(&json)?; - files.push(BackedUpFile::new(fileno, entry, &reason)); - } - Ok(files) + // A pointer to a "fallible iterator" over values of type `T`, which is to say it's an iterator + // over values of type `LocalGenerationResult<T>`. The iterator is only valid for the lifetime + // 'stmt. + // + // The fact that it's a pointer (`Box<dyn ...>`) means we don't care what the actual type of + // the iterator is, and who produces it. + type SqlResultsIterator<'stmt, T> = Box<dyn Iterator<Item = LocalGenerationResult<T>> + 'stmt>; + + // A pointer to a function which, when called on a prepared SQLite statement, would create + // a "fallible iterator" over values of type `ItemT`. (See above for an explanation of what + // a "fallible iterator" is.) + // + // The iterator is only valid for the lifetime of the associated SQLite statement; we + // call this lifetime 'stmt, and use it both both on the reference and the returned iterator. + // + // Now we're in a pickle: all named lifetimes have to be declared _somewhere_, but we can't add + // 'stmt to the signature of `CreateIterFn` because then we'll have to specify it when we + // define the function. Obviously, at that point we won't yet have a `Statement`, and thus we + // would have no idea what its lifetime is going to be. So we can't put the 'stmt lifetime into + // the signature of `CreateIterFn`. + // + // That's what `for<'stmt>` is for. This is a so-called ["higher-rank trait bound"][hrtb], and + // it enables us to say that a function is valid for *some* lifetime 'stmt that we pass into it + // at the call site. It lets Rust continue to track lifetimes even though `CreateIterFn` + // interferes by "hiding" the 'stmt lifetime from its signature. + // + // [hrtb]: https://doc.rust-lang.org/nomicon/hrtb.html + type CreateIterFn<'conn, ItemT> = Box< + dyn for<'stmt> Fn( + &'stmt mut Statement<'conn>, + ) -> LocalGenerationResult<SqlResultsIterator<'stmt, ItemT>>, + >; + + pub struct SqlResults<'conn, ItemT> { + stmt: Statement<'conn>, + create_iter: CreateIterFn<'conn, ItemT>, } - pub fn chunkids(conn: &Connection, fileno: FileId) -> LocalGenerationResult<Vec<ChunkId>> { - let mut stmt = conn.prepare("SELECT chunkid FROM chunks WHERE fileno = ?1")?; - let iter = stmt.query_map(params![fileno], |row| row.get(0))?; - let mut ids: Vec<ChunkId> = vec![]; - for x in iter { - let fileno: String = x?; - ids.push(ChunkId::from(&fileno)); + impl<'conn, ItemT> SqlResults<'conn, ItemT> { + fn new( + conn: &'conn Connection, + statement: &str, + create_iter: CreateIterFn<'conn, ItemT>, + ) -> LocalGenerationResult<Self> { + let stmt = conn.prepare(statement)?; + Ok(Self { stmt, create_iter }) + } + + pub fn iter(&'_ mut self) -> LocalGenerationResult<SqlResultsIterator<'_, ItemT>> { + (self.create_iter)(&mut self.stmt) } - Ok(ids) + } + + pub fn files(conn: &Connection) -> LocalGenerationResult<SqlResults<BackedUpFile>> { + SqlResults::new( + conn, + "SELECT * FROM files", + Box::new(|stmt| { + let iter = stmt.query_map(params![], |row| row_to_entry(row))?; + let iter = iter.map(|x| match x { + Ok((fileno, json, reason)) => serde_json::from_str(&json) + .map(|entry| BackedUpFile::new(fileno, entry, &reason)) + .map_err(|e| e.into()), + Err(e) => Err(e.into()), + }); + Ok(Box::new(iter)) + }), + ) + } + + pub fn chunkids( + conn: &Connection, + fileno: FileId, + ) -> LocalGenerationResult<SqlResults<ChunkId>> { + SqlResults::new( + conn, + "SELECT chunkid FROM chunks WHERE fileno = ?1", + Box::new(move |stmt| { + let iter = stmt.query_map(params![fileno], |row| row.get(0))?; + let iter = iter.map(|x| { + let fileno: String = x?; + Ok(ChunkId::from(&fileno)) + }); + Ok(Box::new(iter)) + }), + ) } pub fn get_file( |