summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLars Wirzenius <liw@liw.fi>2021-04-27 16:09:27 +0000
committerLars Wirzenius <liw@liw.fi>2021-04-27 16:09:27 +0000
commitbf645f3645fd2ee57495eafd1ccfb4afbe917bec (patch)
tree314e48f95e781e0e3b65dab7467e2fed4afafa47
parent80aaff3f70f790141fbc8caa8a2f4830cd5e3fee (diff)
parent4ae0960619537234d5591b40d05f91b131330618 (diff)
downloadobnam2-bf645f3645fd2ee57495eafd1ccfb4afbe917bec.tar.gz
Merge branch 'feature/28-vec-to-iter' into 'main'
Return iterators instead of huge vectors Closes #28 See merge request larswirzenius/obnam!142
-rw-r--r--src/backup_run.rs6
-rw-r--r--src/cmd/list_files.rs3
-rw-r--r--src/cmd/restore.rs9
-rw-r--r--src/cmd/show_gen.rs20
-rw-r--r--src/generation.rs107
5 files changed, 111 insertions, 34 deletions
diff --git a/src/backup_run.rs b/src/backup_run.rs
index e966855..23c97f6 100644
--- a/src/backup_run.rs
+++ b/src/backup_run.rs
@@ -149,7 +149,11 @@ impl<'a> IncrementalBackup<'a> {
Reason::Unchanged | Reason::Skipped | Reason::FileError => {
let fileno = old.get_fileno(&entry.pathbuf())?;
let ids = if let Some(fileno) = fileno {
- old.chunkids(fileno)?
+ let mut ids = vec![];
+ for id in old.chunkids(fileno)?.iter()? {
+ ids.push(id?);
+ }
+ ids
} else {
vec![]
};
diff --git a/src/cmd/list_files.rs b/src/cmd/list_files.rs
index 22b102e..c5191f7 100644
--- a/src/cmd/list_files.rs
+++ b/src/cmd/list_files.rs
@@ -22,7 +22,8 @@ impl ListFiles {
let gen_id: String = genlist.resolve(&self.gen_id)?;
let gen = client.fetch_generation(&gen_id, temp.path())?;
- for file in gen.files()? {
+ for file in gen.files()?.iter()? {
+ let file = file?;
println!("{}", format_entry(&file.entry(), file.reason()));
}
diff --git a/src/cmd/restore.rs b/src/cmd/restore.rs
index a321e80..4de0830 100644
--- a/src/cmd/restore.rs
+++ b/src/cmd/restore.rs
@@ -40,7 +40,8 @@ impl Restore {
let gen = client.fetch_generation(&gen_id, temp.path())?;
info!("restoring {} files", gen.file_count()?);
let progress = create_progress_bar(gen.file_count()?, true);
- for file in gen.files()? {
+ for file in gen.files()?.iter()? {
+ let file = file?;
match file.reason() {
Reason::FileError => (),
_ => restore_generation(
@@ -53,7 +54,8 @@ impl Restore {
)?,
}
}
- for file in gen.files()? {
+ for file in gen.files()?.iter()? {
+ let file = file?;
if file.entry().is_dir() {
restore_directory_metadata(file.entry(), &self.to)?;
}
@@ -170,7 +172,8 @@ fn restore_regular(
std::fs::create_dir_all(parent)?;
{
let mut file = std::fs::File::create(path)?;
- for chunkid in gen.chunkids(fileid)? {
+ for chunkid in gen.chunkids(fileid)?.iter()? {
+ let chunkid = chunkid?;
let chunk = client.fetch_chunk(&chunkid)?;
file.write_all(chunk.data())?;
}
diff --git a/src/cmd/show_gen.rs b/src/cmd/show_gen.rs
index ba39809..df8a030 100644
--- a/src/cmd/show_gen.rs
+++ b/src/cmd/show_gen.rs
@@ -21,16 +21,20 @@ impl ShowGeneration {
let genlist = client.list_generations()?;
let gen_id: String = genlist.resolve(&self.gen_id)?;
let gen = client.fetch_generation(&gen_id, temp.path())?;
- let files = gen.files()?;
+ let mut files = gen.files()?;
+ let mut files = files.iter()?;
- let total_bytes = files.iter().fold(0, |acc, file| {
- let e = file.entry();
- if e.kind() == FilesystemKind::Regular {
- acc + file.entry().len()
- } else {
- acc
- }
+ let total_bytes = files.try_fold(0, |acc, file| {
+ file.map(|file| {
+ let e = file.entry();
+ if e.kind() == FilesystemKind::Regular {
+ acc + e.len()
+ } else {
+ acc
+ }
+ })
});
+ let total_bytes = total_bytes?;
println!("generation-id: {}", gen_id);
println!("file-count: {}", gen.file_count()?);
diff --git a/src/generation.rs b/src/generation.rs
index 1d0ee52..0055bfe 100644
--- a/src/generation.rs
+++ b/src/generation.rs
@@ -196,11 +196,11 @@ impl LocalGeneration {
sql::file_count(&self.conn)
}
- pub fn files(&self) -> LocalGenerationResult<Vec<BackedUpFile>> {
+ pub fn files(&self) -> LocalGenerationResult<sql::SqlResults<BackedUpFile>> {
sql::files(&self.conn)
}
- pub fn chunkids(&self, fileno: FileId) -> LocalGenerationResult<Vec<ChunkId>> {
+ pub fn chunkids(&self, fileno: FileId) -> LocalGenerationResult<sql::SqlResults<ChunkId>> {
sql::chunkids(&self.conn, fileno)
}
@@ -221,7 +221,7 @@ mod sql {
use crate::backup_reason::Reason;
use crate::chunkid::ChunkId;
use crate::fsentry::FilesystemEntry;
- use rusqlite::{params, Connection, OpenFlags, Row, Transaction};
+ use rusqlite::{params, Connection, OpenFlags, Row, Statement, Transaction};
use std::os::unix::ffi::OsStrExt;
use std::path::Path;
@@ -289,27 +289,92 @@ mod sql {
Ok(count)
}
- pub fn files(conn: &Connection) -> LocalGenerationResult<Vec<BackedUpFile>> {
- let mut stmt = conn.prepare("SELECT * FROM files")?;
- let iter = stmt.query_map(params![], |row| row_to_entry(row))?;
- let mut files = vec![];
- for x in iter {
- let (fileno, json, reason) = x?;
- let entry = serde_json::from_str(&json)?;
- files.push(BackedUpFile::new(fileno, entry, &reason));
- }
- Ok(files)
+ // A pointer to a "fallible iterator" over values of type `T`, which is to say it's an iterator
+ // over values of type `LocalGenerationResult<T>`. The iterator is only valid for the lifetime
+ // 'stmt.
+ //
+ // The fact that it's a pointer (`Box<dyn ...>`) means we don't care what the actual type of
+ // the iterator is, and who produces it.
+ type SqlResultsIterator<'stmt, T> = Box<dyn Iterator<Item = LocalGenerationResult<T>> + 'stmt>;
+
+ // A pointer to a function which, when called on a prepared SQLite statement, would create
+ // a "fallible iterator" over values of type `ItemT`. (See above for an explanation of what
+ // a "fallible iterator" is.)
+ //
+ // The iterator is only valid for the lifetime of the associated SQLite statement; we
+ // call this lifetime 'stmt, and use it both both on the reference and the returned iterator.
+ //
+ // Now we're in a pickle: all named lifetimes have to be declared _somewhere_, but we can't add
+ // 'stmt to the signature of `CreateIterFn` because then we'll have to specify it when we
+ // define the function. Obviously, at that point we won't yet have a `Statement`, and thus we
+ // would have no idea what its lifetime is going to be. So we can't put the 'stmt lifetime into
+ // the signature of `CreateIterFn`.
+ //
+ // That's what `for<'stmt>` is for. This is a so-called ["higher-rank trait bound"][hrtb], and
+ // it enables us to say that a function is valid for *some* lifetime 'stmt that we pass into it
+ // at the call site. It lets Rust continue to track lifetimes even though `CreateIterFn`
+ // interferes by "hiding" the 'stmt lifetime from its signature.
+ //
+ // [hrtb]: https://doc.rust-lang.org/nomicon/hrtb.html
+ type CreateIterFn<'conn, ItemT> = Box<
+ dyn for<'stmt> Fn(
+ &'stmt mut Statement<'conn>,
+ ) -> LocalGenerationResult<SqlResultsIterator<'stmt, ItemT>>,
+ >;
+
+ pub struct SqlResults<'conn, ItemT> {
+ stmt: Statement<'conn>,
+ create_iter: CreateIterFn<'conn, ItemT>,
}
- pub fn chunkids(conn: &Connection, fileno: FileId) -> LocalGenerationResult<Vec<ChunkId>> {
- let mut stmt = conn.prepare("SELECT chunkid FROM chunks WHERE fileno = ?1")?;
- let iter = stmt.query_map(params![fileno], |row| row.get(0))?;
- let mut ids: Vec<ChunkId> = vec![];
- for x in iter {
- let fileno: String = x?;
- ids.push(ChunkId::from(&fileno));
+ impl<'conn, ItemT> SqlResults<'conn, ItemT> {
+ fn new(
+ conn: &'conn Connection,
+ statement: &str,
+ create_iter: CreateIterFn<'conn, ItemT>,
+ ) -> LocalGenerationResult<Self> {
+ let stmt = conn.prepare(statement)?;
+ Ok(Self { stmt, create_iter })
+ }
+
+ pub fn iter(&'_ mut self) -> LocalGenerationResult<SqlResultsIterator<'_, ItemT>> {
+ (self.create_iter)(&mut self.stmt)
}
- Ok(ids)
+ }
+
+ pub fn files(conn: &Connection) -> LocalGenerationResult<SqlResults<BackedUpFile>> {
+ SqlResults::new(
+ conn,
+ "SELECT * FROM files",
+ Box::new(|stmt| {
+ let iter = stmt.query_map(params![], |row| row_to_entry(row))?;
+ let iter = iter.map(|x| match x {
+ Ok((fileno, json, reason)) => serde_json::from_str(&json)
+ .map(|entry| BackedUpFile::new(fileno, entry, &reason))
+ .map_err(|e| e.into()),
+ Err(e) => Err(e.into()),
+ });
+ Ok(Box::new(iter))
+ }),
+ )
+ }
+
+ pub fn chunkids(
+ conn: &Connection,
+ fileno: FileId,
+ ) -> LocalGenerationResult<SqlResults<ChunkId>> {
+ SqlResults::new(
+ conn,
+ "SELECT chunkid FROM chunks WHERE fileno = ?1",
+ Box::new(move |stmt| {
+ let iter = stmt.query_map(params![fileno], |row| row.get(0))?;
+ let iter = iter.map(|x| {
+ let fileno: String = x?;
+ Ok(ChunkId::from(&fileno))
+ });
+ Ok(Box::new(iter))
+ }),
+ )
}
pub fn get_file(