Merge branch 'list' into 'main'

List

See merge request larswirzenius/obnam!7

9 files changed, 235 insertions, 93 deletions

diff --git a/src/bin/obnam-backup.rs b/src/bin/obnam-backup.rs
index f5d6b43..7d7e07e 100644
--- a/src/bin/obnam-backup.rs
+++ b/src/bin/obnam-backup.rs
@@ -17,7 +17,7 @@ fn main() -> anyhow::Result<()> {
     let client = BackupClient::new(&config.server_name, config.server_port)?;
 
     {
-        let mut gen = Generation::new(&config.dbname)?;
+        let mut gen = Generation::create(&config.dbname)?;
         gen.insert_iter(FsIterator::new(&config.root).map(|entry| match entry {
             Err(err) => Err(err),
             Ok(entry) => client.upload_filesystem_entry(entry, BUFFER_SIZE),
diff --git a/src/bin/obnam-list.rs b/src/bin/obnam-list.rs
new file mode 100644
index 0000000..18c49f1
--- /dev/null
+++ b/src/bin/obnam-list.rs
@@ -0,0 +1,27 @@
+use log::{debug, info};
+use obnam::client::{BackupClient, ClientConfig};
+use std::path::PathBuf;
+use structopt::StructOpt;
+
+fn main() -> anyhow::Result<()> {
+    pretty_env_logger::init();
+
+    let opt = Opt::from_args();
+    info!("obnam-list starts");
+    debug!("opt: {:?}", opt);
+    let config = ClientConfig::read_config(&opt.config)?;
+    let client = BackupClient::new(&config.server_name, config.server_port)?;
+
+    for gen_id in client.list_generations()? {
+        println!("{}", gen_id);
+    }
+
+    Ok(())
+}
+
+#[derive(Debug, StructOpt)]
+#[structopt(name = "obnam-backup", about = "Simplistic backup client")]
+struct Opt {
+    #[structopt(parse(from_os_str))]
+    config: PathBuf,
+}
diff --git a/src/bin/obnam-restore.rs b/src/bin/obnam-restore.rs
index 2014ef8..442104a 100644
--- a/src/bin/obnam-restore.rs
+++ b/src/bin/obnam-restore.rs
@@ -1,33 +1,14 @@
-// Fetch a backup generation's chunks, write to stdout.
-
-use log::{debug, info, trace};
-use obnam::chunk::{DataChunk, GenerationChunk};
-use obnam::chunkid::ChunkId;
+use log::{debug, info};
+use obnam::client::BackupClient;
+use obnam::fsentry::{FilesystemEntry, FilesystemKind};
+use obnam::generation::Generation;
 //use obnam::chunkmeta::ChunkMeta;
 use serde::Deserialize;
-use std::io::{stdout, Write};
+use std::fs::File;
+use std::io::prelude::*;
 use std::path::{Path, PathBuf};
 use structopt::StructOpt;
 
-#[derive(Debug, thiserror::Error)]
-enum ClientError {
-    #[error("Server does not have generation {0}")]
-    GenerationNotFound(String),
-
-    #[error("Server does not have chunk {0}")]
-    ChunkNotFound(String),
-}
-
-#[derive(Debug, StructOpt)]
-#[structopt(name = "obnam-backup", about = "Simplistic backup client")]
-struct Opt {
-    #[structopt(parse(from_os_str))]
-    config: PathBuf,
-
-    #[structopt()]
-    gen_id: String,
-}
-
 fn main() -> anyhow::Result<()> {
     pretty_env_logger::init();
 
@@ -38,22 +19,42 @@ fn main() -> anyhow::Result<()> {
     info!("opt: {:?}", opt);
     info!("config: {:?}", config);
 
-    let client = reqwest::blocking::Client::builder()
-        .danger_accept_invalid_certs(true)
-        .build()?;
-    let mut stdout = stdout();
-
-    let gen = fetch_generation(&client, &config, &opt.gen_id)?;
-    debug!("gen: {:?}", gen);
-    for id in gen.chunk_ids() {
-        let chunk = fetch_chunk(&client, &config, id)?;
-        debug!("got chunk: {}", id);
-        stdout.write_all(chunk.data())?;
+    let client = BackupClient::new(&config.server_name, config.server_port)?;
+    let gen_chunk = client.fetch_generation(&opt.gen_id)?;
+    debug!("gen: {:?}", gen_chunk);
+    {
+        let mut dbfile = File::create(&opt.dbname)?;
+        for id in gen_chunk.chunk_ids() {
+            let chunk = client.fetch_chunk(id)?;
+            dbfile.write_all(chunk.data())?;
+        }
+    }
+    info!("downloaded generation to {}", opt.dbname.display());
+
+    let gen = Generation::open(&opt.dbname)?;
+    for (fileid, entry) in gen.files()? {
+        restore(&client, &gen, fileid, entry, &opt.to)?;
     }
 
     Ok(())
 }
 
+#[derive(Debug, StructOpt)]
+#[structopt(name = "obnam-backup", about = "Simplistic backup client")]
+struct Opt {
+    #[structopt(parse(from_os_str))]
+    config: PathBuf,
+
+    #[structopt()]
+    gen_id: String,
+
+    #[structopt(parse(from_os_str))]
+    dbname: PathBuf,
+
+    #[structopt(parse(from_os_str))]
+    to: PathBuf,
+}
+
 #[derive(Debug, Deserialize, Clone)]
 pub struct Config {
     pub server_name: String,
@@ -68,52 +69,54 @@ impl Config {
     }
 }
 
-fn fetch_generation(
-    client: &reqwest::blocking::Client,
-    config: &Config,
-    gen_id: &str,
-) -> anyhow::Result<GenerationChunk> {
-    let url = format!(
-        "http://{}:{}/chunks/{}",
-        config.server_name, config.server_port, gen_id,
-    );
-
-    trace!("fetch_generation: url={:?}", url);
-    let req = client.get(&url).build()?;
-
-    let res = client.execute(req)?;
-    debug!("fetch_generation: status={}", res.status());
-    if res.status() != 200 {
-        debug!("fetch_generation: error from server");
-        return Err(ClientError::GenerationNotFound(gen_id.to_string()).into());
+fn restore(
+    client: &BackupClient,
+    gen: &Generation,
+    fileid: u64,
+    entry: FilesystemEntry,
+    to: &Path,
+) -> anyhow::Result<()> {
+    println!("restoring {}:{}", fileid, entry.path().display());
+
+    let path = if entry.path().is_absolute() {
+        entry.path().strip_prefix("/")?
+    } else {
+        entry.path()
+    };
+    let to = to.join(path);
+    debug!("  to: {}", to.display());
+
+    match entry.kind() {
+        FilesystemKind::Regular => restore_regular(client, &gen, &to, fileid, &entry)?,
+        FilesystemKind::Directory => restore_directory(&to)?,
     }
+    Ok(())
+}
 
-    let text = res.text()?;
-    debug!("fetch_generation: text={:?}", text);
-    let gen = serde_json::from_str(&text)?;
-    Ok(gen)
+fn restore_directory(path: &Path) -> anyhow::Result<()> {
+    debug!("restoring directory {}", path.display());
+    std::fs::create_dir_all(path)?;
+    Ok(())
 }
 
-fn fetch_chunk(
-    client: &reqwest::blocking::Client,
-    config: &Config,
-    chunk_id: &ChunkId,
-) -> anyhow::Result<DataChunk> {
-    let url = format!(
-        "http://{}:{}/chunks/{}",
-        config.server_name, config.server_port, chunk_id,
-    );
-
-    trace!("fetch_chunk: url={:?}", url);
-    let req = client.get(&url).build()?;
-
-    let res = client.execute(req)?;
-    debug!("fetch_chunk: status={}", res.status());
-    if res.status() != 200 {
-        debug!("fetch_chunk: error from server");
-        return Err(ClientError::ChunkNotFound(chunk_id.to_string()).into());
+fn restore_regular(
+    client: &BackupClient,
+    gen: &Generation,
+    path: &Path,
+    fileid: u64,
+    _entry: &FilesystemEntry,
+) -> anyhow::Result<()> {
+    debug!("restoring regular {}", path.display());
+    let parent = path.parent().unwrap();
+    debug!("  mkdir {}", parent.display());
+    std::fs::create_dir_all(parent)?;
+    {
+        let mut file = std::fs::File::create(path)?;
+        for chunkid in gen.chunkids(fileid)? {
+            let chunk = client.fetch_chunk(&chunkid)?;
+            file.write_all(chunk.data())?;
+        }
     }
-
-    let body = res.bytes()?;
-    Ok(DataChunk::new(body.to_vec()))
+    debug!("restored regular {}", path.display());
+    Ok(())
 }
diff --git a/src/bin/obnam-server.rs b/src/bin/obnam-server.rs
index cc21855..4e53953 100644
--- a/src/bin/obnam-server.rs
+++ b/src/bin/obnam-server.rs
@@ -195,7 +195,11 @@ pub async fn search_chunks(
             error!("search has more than one key to search for");
             return Ok(ChunkResult::BadRequest);
         }
-        index.find(&key, &value)
+        if key == "generation" && value == "true" {
+            index.find_generations()
+        } else {
+            index.find(&key, &value)
+        }
     } else {
         error!("search has no key to search for");
         return Ok(ChunkResult::BadRequest);
diff --git a/src/checksummer.rs b/src/checksummer.rs
new file mode 100644
index 0000000..162c26b
--- /dev/null
+++ b/src/checksummer.rs
@@ -0,0 +1,8 @@
+use sha2::{Digest, Sha256};
+
+pub fn sha256(data: &[u8]) -> String {
+    let mut hasher = Sha256::new();
+    hasher.update(data);
+    let hash = hasher.finalize();
+    format!("{:x}", hash)
+}
diff --git a/src/chunker.rs b/src/chunker.rs
index f4ca74c..145b1db 100644
--- a/src/chunker.rs
+++ b/src/chunker.rs
@@ -1,6 +1,6 @@
+use crate::checksummer::sha256;
 use crate::chunk::DataChunk;
 use crate::chunkmeta::ChunkMeta;
-use sha2::{Digest, Sha256};
 use std::io::prelude::*;
 
 pub struct Chunker {
@@ -36,12 +36,8 @@ impl Chunker {
         }
 
         let buffer = &self.buf.as_slice()[..used];
-        let mut hasher = Sha256::new();
-        hasher.update(buffer);
-        let hash = hasher.finalize();
-        let hash = format!("{:x}", hash);
+        let hash = sha256(buffer);
         let meta = ChunkMeta::new(&hash);
-
         let chunk = DataChunk::new(buffer.to_vec());
         Ok(Some((meta, chunk)))
     }
diff --git a/src/client.rs b/src/client.rs
index e7b5a3e..ecfc42c 100644
--- a/src/client.rs
+++ b/src/client.rs
@@ -1,3 +1,4 @@
+use crate::checksummer::sha256;
 use crate::chunk::DataChunk;
 use crate::chunk::GenerationChunk;
 use crate::chunker::Chunker;
@@ -31,6 +32,12 @@ impl ClientConfig {
 pub enum ClientError {
     #[error("Server successful response to creating chunk lacked chunk id")]
     NoCreatedChunkId,
+
+    #[error("Server does not have chunk {0}")]
+    ChunkNotFound(String),
+
+    #[error("Server does not have generation {0}")]
+    GenerationNotFound(String),
 }
 
 pub struct BackupClient {
@@ -62,7 +69,8 @@ impl BackupClient {
     pub fn upload_generation(&self, filename: &Path, size: usize) -> anyhow::Result<ChunkId> {
         let ids = self.read_file(filename, size)?;
         let gen = GenerationChunk::new(ids);
-        let meta = ChunkMeta::new_generation("checksum", "endtime");
+        let data = gen.to_data_chunk()?;
+        let meta = ChunkMeta::new_generation(&sha256(data.data()), "timestamp");
         let gen_id = self.upload_gen_chunk(meta, gen)?;
         Ok(gen_id)
     }
@@ -163,4 +171,47 @@ impl BackupClient {
 
         Ok(chunk_ids)
     }
+
+    pub fn list_generations(&self) -> anyhow::Result<Vec<ChunkId>> {
+        let url = format!("{}/?generation=true", self.base_url());
+        trace!("list_generations: url={:?}", url);
+        let req = self.client.get(&url).build()?;
+        let res = self.client.execute(req)?;
+        debug!("list_generations: status={}", res.status());
+        let body = res.bytes()?;
+        debug!("list_generationgs: body={:?}", body);
+        let map: HashMap<String, ChunkMeta> = serde_yaml::from_slice(&body)?;
+        debug!("list_generations: map={:?}", map);
+        Ok(map.keys().into_iter().map(|key| key.into()).collect())
+    }
+
+    pub fn fetch_chunk(&self, chunk_id: &ChunkId) -> anyhow::Result<DataChunk> {
+        let url = format!("{}/{}", self.base_url(), chunk_id);
+        trace!("fetch_chunk: url={:?}", url);
+        let req = self.client.get(&url).build()?;
+        let res = self.client.execute(req)?;
+        debug!("fetch_chunk: status={}", res.status());
+        if res.status() != 200 {
+            return Err(ClientError::ChunkNotFound(chunk_id.to_string()).into());
+        }
+
+        let body = res.bytes()?;
+        Ok(DataChunk::new(body.to_vec()))
+    }
+
+    pub fn fetch_generation(&self, gen_id: &str) -> anyhow::Result<GenerationChunk> {
+        let url = format!("{}/{}", self.base_url(), gen_id);
+        trace!("fetch_generation: url={:?}", url);
+        let req = self.client.get(&url).build()?;
+        let res = self.client.execute(req)?;
+        debug!("fetch_generation: status={}", res.status());
+        if res.status() != 200 {
+            return Err(ClientError::GenerationNotFound(gen_id.to_string()).into());
+        }
+
+        let text = res.text()?;
+        debug!("fetch_generation: text={:?}", text);
+        let gen = serde_json::from_str(&text)?;
+        Ok(gen)
+    }
 }
diff --git a/src/generation.rs b/src/generation.rs
index ca1f8d5..b9edb74 100644
--- a/src/generation.rs
+++ b/src/generation.rs
@@ -1,9 +1,11 @@
-use crate::fsentry::FilesystemEntry;
+use crate::fsentry::{FilesystemEntry, FilesystemKind};
+use std::ffi::OsStr;
+use std::os::unix::ffi::OsStrExt;
 //use crate::fsiter::FsIterator;
 use crate::chunkid::ChunkId;
-use rusqlite::{params, Connection, OpenFlags, Transaction};
+use rusqlite::{params, Connection, OpenFlags, Row, Transaction};
 use std::os::unix::ffi::OsStringExt;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
 /// A backup generation.
 pub struct Generation {
@@ -12,7 +14,7 @@ pub struct Generation {
 }
 
 impl Generation {
-    pub fn new<P>(filename: P) -> anyhow::Result<Self>
+    pub fn create<P>(filename: P) -> anyhow::Result<Self>
     where
         P: AsRef<Path>,
     {
@@ -30,6 +32,16 @@ impl Generation {
         Ok(Self { conn, fileno: 0 })
     }
 
+    pub fn open<P>(filename: P) -> anyhow::Result<Self>
+    where
+        P: AsRef<Path>,
+    {
+        let flags = OpenFlags::SQLITE_OPEN_READ_WRITE;
+        let conn = Connection::open_with_flags(filename, flags)?;
+        conn.pragma_update(None, "journal_mode", &"WAL")?;
+        Ok(Self { conn, fileno: 0 })
+    }
+
     pub fn insert(&mut self, e: FilesystemEntry, ids: &[ChunkId]) -> anyhow::Result<()> {
         let t = self.conn.transaction()?;
         insert_one(&t, e, self.fileno, ids)?;
@@ -51,6 +63,46 @@ impl Generation {
         t.commit()?;
         Ok(())
     }
+
+    pub fn files(&self) -> anyhow::Result<Vec<(u64, FilesystemEntry)>> {
+        let mut stmt = self.conn.prepare("SELECT * FROM files")?;
+        let iter = stmt.query_map(params![], |row| row_to_entry(row))?;
+        let mut files: Vec<(u64, FilesystemEntry)> = vec![];
+        for x in iter {
+            let (fileid, entry) = x?;
+            files.push((fileid, entry));
+        }
+        Ok(files)
+    }
+
+    pub fn chunkids(&self, fileid: u64) -> anyhow::Result<Vec<ChunkId>> {
+        let fileid = fileid as i64;
+        let mut stmt = self
+            .conn
+            .prepare("SELECT chunkid FROM chunks WHERE fileid = ?1")?;
+        let iter = stmt.query_map(params![fileid], |row| Ok(row.get(0)?))?;
+        let mut ids: Vec<ChunkId> = vec![];
+        for x in iter {
+            let fileid: String = x?;
+            ids.push(ChunkId::from(&fileid));
+        }
+        Ok(ids)
+    }
+}
+
+fn row_to_entry(row: &Row) -> rusqlite::Result<(u64, FilesystemEntry)> {
+    let fileid: i64 = row.get(row.column_index("fileid")?)?;
+    let fileid = fileid as u64;
+    let path: Vec<u8> = row.get(row.column_index("path")?)?;
+    let path: &OsStr = OsStrExt::from_bytes(&path);
+    let path: PathBuf = PathBuf::from(path);
+    let kind = row.get(row.column_index("kind")?)?;
+    let kind = FilesystemKind::from_code(kind).unwrap();
+    let entry = match kind {
+        FilesystemKind::Regular => FilesystemEntry::regular(path, 0),
+        FilesystemKind::Directory => FilesystemEntry::directory(path),
+    };
+    Ok((fileid, entry))
 }
 
 fn insert_one(
@@ -85,7 +137,7 @@ mod test {
     fn empty() {
         let filename = NamedTempFile::new().unwrap().path().to_path_buf();
         {
-            let mut _gen = Generation::new(&filename).unwrap();
+            let mut _gen = Generation::create(&filename).unwrap();
             // _gen is dropped here; the connection is close; the file
             // should not be removed.
         }
diff --git a/src/lib.rs b/src/lib.rs
index be9a508..f72ea56 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,3 +1,4 @@
+pub mod checksummer;
 pub mod chunk;
 pub mod chunker;
 pub mod chunkid;