From 566dd94d2e46c489b50d84a1fd24683460e5cfdc Mon Sep 17 00:00:00 2001
From: Lars Wirzenius <liw@liw.fi>
Date: Sun, 23 May 2021 11:40:24 +0300
Subject: refactor: split bare chunk server client into its own struct

This makes the code doing HTTP requests simpler for easier
comprehension and debugging.
---
 src/client.rs | 155 +++++++++++++++++++++++++++++++---------------------------
 1 file changed, 84 insertions(+), 71 deletions(-)

diff --git a/src/client.rs b/src/client.rs
index 0f8a72f..114574c 100644
--- a/src/client.rs
+++ b/src/client.rs
@@ -74,21 +74,14 @@ pub enum ClientError {
 pub type ClientResult<T> = Result<T, ClientError>;
 
 pub struct BackupClient {
-    client: Client,
-    base_url: String,
+    chunk_client: ChunkClient,
 }
 
 impl BackupClient {
     pub fn new(config: &ClientConfig) -> ClientResult<Self> {
         info!("creating backup client with config: {:#?}", config);
-        let config = config.config();
-        let client = Client::builder()
-            .danger_accept_invalid_certs(!config.verify_tls_cert)
-            .build()
-            .map_err(ClientError::ReqwestError)?;
         Ok(Self {
-            client,
-            base_url: config.server_url.to_string(),
+            chunk_client: ChunkClient::new(config)?,
         })
     }
 
@@ -130,6 +123,88 @@ impl BackupClient {
         Ok(chunk_ids)
     }
 
+    pub fn has_chunk(&self, meta: &ChunkMeta) -> ClientResult<Option<ChunkId>> {
+        self.chunk_client.has_chunk(meta)
+    }
+
+    pub fn upload_chunk(&self, meta: ChunkMeta, chunk: DataChunk) -> ClientResult<ChunkId> {
+        self.chunk_client.upload_chunk(meta, chunk)
+    }
+
+    pub fn upload_gen_chunk(&self, meta: ChunkMeta, gen: GenerationChunk) -> ClientResult<ChunkId> {
+        let data = gen.to_data_chunk()?;
+        self.upload_chunk(meta, data)
+    }
+
+    pub fn upload_new_file_chunks(&self, chunker: Chunker) -> ClientResult<Vec<ChunkId>> {
+        let mut chunk_ids = vec![];
+        for item in chunker {
+            let (meta, chunk) = item?;
+            if let Some(chunk_id) = self.has_chunk(&meta)? {
+                chunk_ids.push(chunk_id.clone());
+                info!("reusing existing chunk {}", chunk_id);
+            } else {
+                let chunk_id = self.upload_chunk(meta, chunk)?;
+                chunk_ids.push(chunk_id.clone());
+                info!("created new chunk {}", chunk_id);
+            }
+        }
+
+        Ok(chunk_ids)
+    }
+
+    pub fn list_generations(&self) -> ClientResult<GenerationList> {
+        self.chunk_client.list_generations()
+    }
+
+    pub fn fetch_chunk(&self, chunk_id: &ChunkId) -> ClientResult<DataChunk> {
+        self.chunk_client.fetch_chunk(chunk_id)
+    }
+
+    fn fetch_generation_chunk(&self, gen_id: &str) -> ClientResult<GenerationChunk> {
+        let chunk_id = ChunkId::recreate(gen_id);
+        let chunk = self.fetch_chunk(&chunk_id)?;
+        let gen = GenerationChunk::from_data_chunk(&chunk)?;
+        Ok(gen)
+    }
+
+    pub fn fetch_generation(&self, gen_id: &str, dbname: &Path) -> ClientResult<LocalGeneration> {
+        let gen = self.fetch_generation_chunk(gen_id)?;
+
+        // Fetch the SQLite file, storing it in the named file.
+        let mut dbfile = File::create(&dbname)
+            .map_err(|err| ClientError::FileCreate(dbname.to_path_buf(), err))?;
+        for id in gen.chunk_ids() {
+            let chunk = self.fetch_chunk(id)?;
+            dbfile
+                .write_all(chunk.data())
+                .map_err(|err| ClientError::FileWrite(dbname.to_path_buf(), err))?;
+        }
+        info!("downloaded generation to {}", dbname.display());
+
+        let gen = LocalGeneration::open(dbname)?;
+        Ok(gen)
+    }
+}
+
+pub struct ChunkClient {
+    client: Client,
+    base_url: String,
+}
+
+impl ChunkClient {
+    pub fn new(config: &ClientConfig) -> ClientResult<Self> {
+        let config = config.config();
+        let client = Client::builder()
+            .danger_accept_invalid_certs(!config.verify_tls_cert)
+            .build()
+            .map_err(ClientError::ReqwestError)?;
+        Ok(Self {
+            client,
+            base_url: config.server_url.to_string(),
+        })
+    }
+
     fn base_url(&self) -> &str {
         &self.base_url
     }
@@ -191,43 +266,6 @@ impl BackupClient {
         Ok(chunk_id)
     }
 
-    pub fn upload_gen_chunk(&self, meta: ChunkMeta, gen: GenerationChunk) -> ClientResult<ChunkId> {
-        let res = self
-            .client
-            .post(&self.chunks_url())
-            .header("chunk-meta", meta.to_json())
-            .body(serde_json::to_string(&gen).map_err(ClientError::JsonGenerate)?)
-            .send()
-            .map_err(ClientError::ReqwestError)?;
-        debug!("upload_chunk: res={:?}", res);
-        let res: HashMap<String, String> = res.json().map_err(ClientError::ReqwestError)?;
-        let chunk_id = if let Some(chunk_id) = res.get("chunk_id") {
-            debug!("upload_chunk: id={}", chunk_id);
-            chunk_id.parse().unwrap()
-        } else {
-            return Err(ClientError::NoCreatedChunkId);
-        };
-        info!("uploaded_generation chunk {}", chunk_id);
-        Ok(chunk_id)
-    }
-
-    pub fn upload_new_file_chunks(&self, chunker: Chunker) -> ClientResult<Vec<ChunkId>> {
-        let mut chunk_ids = vec![];
-        for item in chunker {
-            let (meta, chunk) = item?;
-            if let Some(chunk_id) = self.has_chunk(&meta)? {
-                chunk_ids.push(chunk_id.clone());
-                info!("reusing existing chunk {}", chunk_id);
-            } else {
-                let chunk_id = self.upload_chunk(meta, chunk)?;
-                chunk_ids.push(chunk_id.clone());
-                info!("created new chunk {}", chunk_id);
-            }
-        }
-
-        Ok(chunk_ids)
-    }
-
     pub fn list_generations(&self) -> ClientResult<GenerationList> {
         let url = format!("{}?generation=true", &self.chunks_url());
         trace!("list_generations: url={:?}", url);
@@ -301,31 +339,6 @@ impl BackupClient {
 
         Ok(chunk)
     }
-
-    fn fetch_generation_chunk(&self, gen_id: &str) -> ClientResult<GenerationChunk> {
-        let chunk_id = ChunkId::recreate(gen_id);
-        let chunk = self.fetch_chunk(&chunk_id)?;
-        let gen = GenerationChunk::from_data_chunk(&chunk)?;
-        Ok(gen)
-    }
-
-    pub fn fetch_generation(&self, gen_id: &str, dbname: &Path) -> ClientResult<LocalGeneration> {
-        let gen = self.fetch_generation_chunk(gen_id)?;
-
-        // Fetch the SQLite file, storing it in the named file.
-        let mut dbfile = File::create(&dbname)
-            .map_err(|err| ClientError::FileCreate(dbname.to_path_buf(), err))?;
-        for id in gen.chunk_ids() {
-            let chunk = self.fetch_chunk(id)?;
-            dbfile
-                .write_all(chunk.data())
-                .map_err(|err| ClientError::FileWrite(dbname.to_path_buf(), err))?;
-        }
-        info!("downloaded generation to {}", dbname.display());
-
-        let gen = LocalGeneration::open(dbname)?;
-        Ok(gen)
-    }
 }
 
 fn current_timestamp() -> String {
-- 
cgit v1.2.1


From 6de230c382a4329df00bc11cc1ffb90390b13159 Mon Sep 17 00:00:00 2001
From: Lars Wirzenius <liw@liw.fi>
Date: Mon, 24 May 2021 08:55:14 +0300
Subject: refactor: make metadata be part of datachunk

This makes it harder to lose the metadata for a chunk, or to use
unrelated metadata and chunk. Also, soon I will refactor things for
encrypting chunks, which will need metadata embedded in the encrypted
chunk.

Sponsored-by: author
---
 src/benchmark.rs                  |  6 +++---
 src/bin/benchmark-index.rs        |  7 ++++---
 src/bin/benchmark-indexedstore.rs |  8 ++++----
 src/bin/benchmark-null.rs         |  2 +-
 src/bin/benchmark-store.rs        |  4 ++--
 src/bin/obnam-server.rs           |  6 +++---
 src/chunk.rs                      | 22 +++++++++++++++++-----
 src/chunker.rs                    | 12 ++++++------
 src/client.rs                     | 30 ++++++++++++------------------
 src/indexedstore.rs               |  6 +++---
 src/store.rs                      | 12 +++++++-----
 11 files changed, 62 insertions(+), 53 deletions(-)

diff --git a/src/benchmark.rs b/src/benchmark.rs
index d214939..3c94f92 100644
--- a/src/benchmark.rs
+++ b/src/benchmark.rs
@@ -15,7 +15,7 @@ impl ChunkGenerator {
 }
 
 impl Iterator for ChunkGenerator {
-    type Item = (ChunkId, String, ChunkMeta, DataChunk);
+    type Item = (ChunkId, String, DataChunk);
 
     fn next(&mut self) -> Option<Self::Item> {
         if self.next >= self.goal {
@@ -24,9 +24,9 @@ impl Iterator for ChunkGenerator {
             let id = ChunkId::recreate(&format!("{}", self.next));
             let checksum = id.sha256();
             let meta = ChunkMeta::new(&checksum);
-            let chunk = DataChunk::new(vec![]);
+            let chunk = DataChunk::new(vec![], meta);
             self.next += 1;
-            Some((id, checksum, meta, chunk))
+            Some((id, checksum, chunk))
         }
     }
 }
diff --git a/src/bin/benchmark-index.rs b/src/bin/benchmark-index.rs
index 9baa327..b5a059c 100644
--- a/src/bin/benchmark-index.rs
+++ b/src/bin/benchmark-index.rs
@@ -60,7 +60,8 @@ fn create(chunks: &Path, num: u32) -> anyhow::Result<()> {
     let mut index = Index::new(chunks)?;
     let gen = ChunkGenerator::new(num);
 
-    for (id, _, meta, _) in gen {
+    for (id, _, chunk) in gen {
+        let meta = (*chunk.meta()).clone();
         index.insert_meta(id, meta)?;
     }
 
@@ -82,8 +83,8 @@ fn lookup(index: &mut Index, num: u32) -> anyhow::Result<()> {
 
     loop {
         let gen = ChunkGenerator::new(num);
-        for (_, _, meta, _) in gen {
-            index.find_by_sha256(&meta.sha256())?;
+        for (_, _, chunk) in gen {
+            index.find_by_sha256(&chunk.meta().sha256())?;
             done += 1;
             if done >= num {
                 return Ok(());
diff --git a/src/bin/benchmark-indexedstore.rs b/src/bin/benchmark-indexedstore.rs
index acc3bd3..5cd3ff1 100644
--- a/src/bin/benchmark-indexedstore.rs
+++ b/src/bin/benchmark-indexedstore.rs
@@ -60,8 +60,8 @@ fn create(chunks: &Path, num: u32) -> anyhow::Result<()> {
     let mut store = IndexedStore::new(chunks)?;
     let gen = ChunkGenerator::new(num);
 
-    for (_, _, meta, chunk) in gen {
-        store.save(&meta, &chunk)?;
+    for (_, _, chunk) in gen {
+        store.save(&chunk)?;
     }
 
     Ok(())
@@ -82,8 +82,8 @@ fn lookup(index: &mut IndexedStore, num: u32) -> anyhow::Result<()> {
 
     loop {
         let gen = ChunkGenerator::new(num);
-        for (_, _, meta, _) in gen {
-            index.find_by_sha256(&meta.sha256())?;
+        for (_, _, chunk) in gen {
+            index.find_by_sha256(&chunk.meta().sha256())?;
             done += 1;
             if done >= num {
                 return Ok(());
diff --git a/src/bin/benchmark-null.rs b/src/bin/benchmark-null.rs
index 259a837..fc60a77 100644
--- a/src/bin/benchmark-null.rs
+++ b/src/bin/benchmark-null.rs
@@ -23,5 +23,5 @@ fn main() {
     let opt = Opt::from_args();
     let gen = ChunkGenerator::new(opt.num);
 
-    for (_, _, _, _) in gen {}
+    for (_, _, _) in gen {}
 }
diff --git a/src/bin/benchmark-store.rs b/src/bin/benchmark-store.rs
index f7c82b1..7896f9d 100644
--- a/src/bin/benchmark-store.rs
+++ b/src/bin/benchmark-store.rs
@@ -20,8 +20,8 @@ fn main() -> anyhow::Result<()> {
     let gen = ChunkGenerator::new(opt.num);
 
     let store = Store::new(&opt.chunks);
-    for (id, _, meta, chunk) in gen {
-        store.save(&id, &meta, &chunk)?;
+    for (id, _, chunk) in gen {
+        store.save(&id, &&chunk)?;
     }
 
     Ok(())
diff --git a/src/bin/obnam-server.rs b/src/bin/obnam-server.rs
index 9a6540f..efee77e 100644
--- a/src/bin/obnam-server.rs
+++ b/src/bin/obnam-server.rs
@@ -109,9 +109,9 @@ pub async fn create_chunk(
         }
     };
 
-    let chunk = DataChunk::new(data.to_vec());
+    let chunk = DataChunk::new(data.to_vec(), meta);
 
-    let id = match store.save(&meta, &chunk) {
+    let id = match store.save(&chunk) {
         Ok(id) => id,
         Err(e) => {
             error!("couldn't save: {}", e);
@@ -119,7 +119,7 @@ pub async fn create_chunk(
         }
     };
 
-    info!("created chunk {}: {:?}", id, meta);
+    info!("created chunk {}", id);
     Ok(ChunkResult::Created(id))
 }
 
diff --git a/src/chunk.rs b/src/chunk.rs
index 0eed38a..50a2fc7 100644
--- a/src/chunk.rs
+++ b/src/chunk.rs
@@ -1,4 +1,6 @@
+use crate::checksummer::sha256;
 use crate::chunkid::ChunkId;
+use crate::chunkmeta::ChunkMeta;
 use serde::{Deserialize, Serialize};
 use std::default::Default;
 
@@ -11,18 +13,24 @@ use std::default::Default;
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct DataChunk {
     data: Vec<u8>,
+    meta: ChunkMeta,
 }
 
 impl DataChunk {
     /// Construct a new chunk.
-    pub fn new(data: Vec<u8>) -> Self {
-        Self { data }
+    pub fn new(data: Vec<u8>, meta: ChunkMeta) -> Self {
+        Self { data, meta }
     }
 
     /// Return a chunk's data.
     pub fn data(&self) -> &[u8] {
         &self.data
     }
+
+    /// Return a chunk's metadata.
+    pub fn meta(&self) -> &ChunkMeta {
+        &self.meta
+    }
 }
 
 #[derive(Default, Debug, Serialize, Deserialize)]
@@ -69,8 +77,12 @@ impl GenerationChunk {
         self.chunk_ids.iter()
     }
 
-    pub fn to_data_chunk(&self) -> GenerationChunkResult<DataChunk> {
-        let json = serde_json::to_string(self).map_err(GenerationChunkError::JsonGenerate)?;
-        Ok(DataChunk::new(json.as_bytes().to_vec()))
+    pub fn to_data_chunk(&self, ended: &str) -> GenerationChunkResult<DataChunk> {
+        let json: String =
+            serde_json::to_string(self).map_err(GenerationChunkError::JsonGenerate)?;
+        let bytes = json.as_bytes().to_vec();
+        let sha = sha256(&bytes);
+        let meta = ChunkMeta::new_generation(&sha, ended);
+        Ok(DataChunk::new(bytes, meta))
     }
 }
diff --git a/src/chunker.rs b/src/chunker.rs
index eeeed8d..a7a39f1 100644
--- a/src/chunker.rs
+++ b/src/chunker.rs
@@ -31,7 +31,7 @@ impl Chunker {
         }
     }
 
-    pub fn read_chunk(&mut self) -> ChunkerResult<Option<(ChunkMeta, DataChunk)>> {
+    pub fn read_chunk(&mut self) -> ChunkerResult<Option<DataChunk>> {
         let mut used = 0;
 
         loop {
@@ -52,18 +52,18 @@ impl Chunker {
         let buffer = &self.buf.as_slice()[..used];
         let hash = sha256(buffer);
         let meta = ChunkMeta::new(&hash);
-        let chunk = DataChunk::new(buffer.to_vec());
-        Ok(Some((meta, chunk)))
+        let chunk = DataChunk::new(buffer.to_vec(), meta);
+        Ok(Some(chunk))
     }
 }
 
 impl Iterator for Chunker {
-    type Item = ChunkerResult<(ChunkMeta, DataChunk)>;
+    type Item = ChunkerResult<DataChunk>;
 
-    fn next(&mut self) -> Option<ChunkerResult<(ChunkMeta, DataChunk)>> {
+    fn next(&mut self) -> Option<ChunkerResult<DataChunk>> {
         match self.read_chunk() {
             Ok(None) => None,
-            Ok(Some((meta, chunk))) => Some(Ok((meta, chunk))),
+            Ok(Some(chunk)) => Some(Ok(chunk)),
             Err(e) => Some(Err(e)),
         }
     }
diff --git a/src/client.rs b/src/client.rs
index 114574c..00f5bd7 100644
--- a/src/client.rs
+++ b/src/client.rs
@@ -107,10 +107,9 @@ impl BackupClient {
         info!("upload SQLite {}", filename.display());
         let ids = self.read_file(filename, size)?;
         let gen = GenerationChunk::new(ids);
-        let data = gen.to_data_chunk()?;
-        let meta = ChunkMeta::new_generation(&sha256(data.data()), &current_timestamp());
-        let gen_id = self.upload_gen_chunk(meta.clone(), gen)?;
-        info!("uploaded generation {}, meta {:?}", gen_id, meta);
+        let data = gen.to_data_chunk(&current_timestamp())?;
+        let gen_id = self.upload_chunk(data)?;
+        info!("uploaded generation {}", gen_id);
         Ok(gen_id)
     }
 
@@ -127,24 +126,19 @@ impl BackupClient {
         self.chunk_client.has_chunk(meta)
     }
 
-    pub fn upload_chunk(&self, meta: ChunkMeta, chunk: DataChunk) -> ClientResult<ChunkId> {
-        self.chunk_client.upload_chunk(meta, chunk)
-    }
-
-    pub fn upload_gen_chunk(&self, meta: ChunkMeta, gen: GenerationChunk) -> ClientResult<ChunkId> {
-        let data = gen.to_data_chunk()?;
-        self.upload_chunk(meta, data)
+    pub fn upload_chunk(&self, chunk: DataChunk) -> ClientResult<ChunkId> {
+        self.chunk_client.upload_chunk(chunk)
     }
 
     pub fn upload_new_file_chunks(&self, chunker: Chunker) -> ClientResult<Vec<ChunkId>> {
         let mut chunk_ids = vec![];
         for item in chunker {
-            let (meta, chunk) = item?;
-            if let Some(chunk_id) = self.has_chunk(&meta)? {
+            let chunk = item?;
+            if let Some(chunk_id) = self.has_chunk(chunk.meta())? {
                 chunk_ids.push(chunk_id.clone());
                 info!("reusing existing chunk {}", chunk_id);
             } else {
-                let chunk_id = self.upload_chunk(meta, chunk)?;
+                let chunk_id = self.upload_chunk(chunk)?;
                 chunk_ids.push(chunk_id.clone());
                 info!("created new chunk {}", chunk_id);
             }
@@ -246,11 +240,11 @@ impl ChunkClient {
         Ok(has)
     }
 
-    pub fn upload_chunk(&self, meta: ChunkMeta, chunk: DataChunk) -> ClientResult<ChunkId> {
+    pub fn upload_chunk(&self, chunk: DataChunk) -> ClientResult<ChunkId> {
         let res = self
             .client
             .post(&self.chunks_url())
-            .header("chunk-meta", meta.to_json())
+            .header("chunk-meta", chunk.meta().to_json())
             .body(chunk.data().to_vec())
             .send()
             .map_err(ClientError::ReqwestError)?;
@@ -262,7 +256,7 @@ impl ChunkClient {
         } else {
             return Err(ClientError::NoCreatedChunkId);
         };
-        info!("uploaded_chunk {} meta {:?}", chunk_id, meta);
+        info!("uploaded_chunk {}", chunk_id);
         Ok(chunk_id)
     }
 
@@ -335,7 +329,7 @@ impl ChunkClient {
             return Err(err);
         }
 
-        let chunk: DataChunk = DataChunk::new(body);
+        let chunk: DataChunk = DataChunk::new(body, meta);
 
         Ok(chunk)
     }
diff --git a/src/indexedstore.rs b/src/indexedstore.rs
index 7f67a1f..982e2d9 100644
--- a/src/indexedstore.rs
+++ b/src/indexedstore.rs
@@ -40,10 +40,10 @@ impl IndexedStore {
         Ok(Self { store, index })
     }
 
-    pub fn save(&mut self, meta: &ChunkMeta, chunk: &DataChunk) -> IndexedResult<ChunkId> {
+    pub fn save(&mut self, chunk: &DataChunk) -> IndexedResult<ChunkId> {
         let id = ChunkId::new();
-        self.store.save(&id, meta, chunk)?;
-        self.insert_meta(&id, meta)?;
+        self.store.save(&id, chunk)?;
+        self.insert_meta(&id, chunk.meta())?;
         Ok(id)
     }
 
diff --git a/src/store.rs b/src/store.rs
index fca2c13..bccecc7 100644
--- a/src/store.rs
+++ b/src/store.rs
@@ -1,6 +1,5 @@
 use crate::chunk::DataChunk;
 use crate::chunkid::ChunkId;
-use crate::chunkmeta::ChunkMeta;
 use std::path::{Path, PathBuf};
 
 /// Store chunks, with metadata, persistently.
@@ -43,23 +42,26 @@ impl Store {
     }
 
     /// Save a chunk into a store.
-    pub fn save(&self, id: &ChunkId, meta: &ChunkMeta, chunk: &DataChunk) -> StoreResult<()> {
+    pub fn save(&self, id: &ChunkId, chunk: &DataChunk) -> StoreResult<()> {
         let (dir, metaname, dataname) = &self.filenames(id);
 
         if !dir.exists() {
             std::fs::create_dir_all(dir)?;
         }
 
-        std::fs::write(&metaname, meta.to_json())?;
+        std::fs::write(&metaname, chunk.meta().to_json())?;
         std::fs::write(&dataname, chunk.data())?;
         Ok(())
     }
 
     /// Load a chunk from a store.
     pub fn load(&self, id: &ChunkId) -> StoreResult<DataChunk> {
-        let (_, _, dataname) = &self.filenames(id);
+        let (_, metaname, dataname) = &self.filenames(id);
+        let meta = std::fs::read(&metaname)?;
+        let meta = serde_json::from_slice(&meta)?;
+
         let data = std::fs::read(&dataname)?;
-        let data = DataChunk::new(data);
+        let data = DataChunk::new(data, meta);
         Ok(data)
     }
 
-- 
cgit v1.2.1


From 03a8b0f9cba08ad09cb8494579f6e318aee762f4 Mon Sep 17 00:00:00 2001
From: Lars Wirzenius <liw@liw.fi>
Date: Sat, 29 May 2021 11:37:18 +0300
Subject: update Cargo.toml, Cargo.lock with new dependencies

These aren't used yet, but they soon will be.
---
 Cargo.lock | 450 ++++++++++++++++++++++++++++++++++++++-----------------------
 Cargo.toml |   1 +
 2 files changed, 279 insertions(+), 172 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index dc8a614..3c3bc85 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1,5 +1,40 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+[[package]]
+name = "aead"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "922b33332f54fc0ad13fa3e514601e8d30fb54e1f3eadc36643f6526db645621"
+dependencies = [
+ "generic-array",
+]
+
+[[package]]
+name = "aes"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2333ac5777aaa1beb8589f5374976ae7dc8aa4f09fd21ae3d8662ca97f5247d"
+dependencies = [
+ "cfg-if 1.0.0",
+ "cipher",
+ "cpufeatures",
+ "opaque-debug",
+]
+
+[[package]]
+name = "aes-gcm"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ee2263805ba4537ccbb19db28525a7b1ebc7284c228eb5634c3124ca63eb03f"
+dependencies = [
+ "aead",
+ "aes",
+ "cipher",
+ "ctr",
+ "ghash",
+ "subtle",
+]
+
 [[package]]
 name = "ahash"
 version = "0.4.7"
@@ -8,9 +43,9 @@ checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e"
 
 [[package]]
 name = "aho-corasick"
-version = "0.7.15"
+version = "0.7.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5"
+checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
 dependencies = [
  "memchr",
 ]
@@ -26,9 +61,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.39"
+version = "1.0.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81cddc5f91628367664cc7c69714ff08deee8a3efc54623011c772544d7b2767"
+checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b"
 
 [[package]]
 name = "arc-swap"
@@ -98,9 +133,9 @@ dependencies = [
 
 [[package]]
 name = "bumpalo"
-version = "3.6.1"
+version = "3.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe"
+checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631"
 
 [[package]]
 name = "byteorder"
@@ -128,9 +163,9 @@ checksum = "81a18687293a1546b67c246452202bbbf143d239cb43494cc163da14979082da"
 
 [[package]]
 name = "cc"
-version = "1.0.67"
+version = "1.0.68"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd"
+checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787"
 
 [[package]]
 name = "cfg-if"
@@ -157,6 +192,15 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "cipher"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7"
+dependencies = [
+ "generic-array",
+]
+
 [[package]]
 name = "clap"
 version = "2.33.3"
@@ -204,10 +248,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b"
 
 [[package]]
-name = "cpuid-bool"
-version = "0.1.2"
+name = "cpufeatures"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634"
+checksum = "ed00c67cb5d0a7d64a44f6ad2668db7e7530311dd53ea79bcd4fb022c64911c8"
+dependencies = [
+ "libc",
+]
 
 [[package]]
 name = "crypto-mac"
@@ -219,6 +266,15 @@ dependencies = [
  "subtle",
 ]
 
+[[package]]
+name = "ctr"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a232f92a03f37dd7d7dd2adc67166c77e9cd88de5b019b9a9eecfaeaf7bfd481"
+dependencies = [
+ "cipher",
+]
+
 [[package]]
 name = "derivative"
 version = "2.2.0"
@@ -262,9 +318,9 @@ dependencies = [
 
 [[package]]
 name = "dtoa"
-version = "0.4.7"
+version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88d7ed2934d741c6b37e33e3832298e8850b53fd2d2bea03873375596c7cea4e"
+checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0"
 
 [[package]]
 name = "encode_unicode"
@@ -355,9 +411,9 @@ checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
 
 [[package]]
 name = "futures"
-version = "0.3.13"
+version = "0.3.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f55667319111d593ba876406af7c409c0ebb44dc4be6132a783ccf163ea14c1"
+checksum = "0e7e43a803dae2fa37c1f6a8fe121e1f7bf9548b4dfc0522a42f34145dadfc27"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -369,9 +425,9 @@ dependencies = [
 
 [[package]]
 name = "futures-channel"
-version = "0.3.13"
+version = "0.3.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c2dd2df839b57db9ab69c2c9d8f3e8c81984781937fe2807dc6dcf3b2ad2939"
+checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -379,34 +435,35 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.13"
+version = "0.3.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15496a72fabf0e62bdc3df11a59a3787429221dd0710ba8ef163d6f7a9112c94"
+checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1"
 
 [[package]]
 name = "futures-io"
-version = "0.3.13"
+version = "0.3.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d71c2c65c57704c32f5241c1223167c2c3294fd34ac020c807ddbe6db287ba59"
+checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1"
 
 [[package]]
 name = "futures-sink"
-version = "0.3.13"
+version = "0.3.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85754d98985841b7d4f5e8e6fbfa4a4ac847916893ec511a2917ccd8525b8bb3"
+checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282"
 
 [[package]]
 name = "futures-task"
-version = "0.3.13"
+version = "0.3.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa189ef211c15ee602667a6fcfe1c1fd9e07d42250d2156382820fba33c9df80"
+checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae"
 
 [[package]]
 name = "futures-util"
-version = "0.3.13"
+version = "0.3.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1812c7ab8aedf8d6f2701a43e1243acdbcc2b36ab26e2ad421eb99ac963d96d1"
+checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967"
 dependencies = [
+ "autocfg",
  "futures-core",
  "futures-io",
  "futures-sink",
@@ -440,15 +497,25 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.2"
+version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8"
+checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753"
 dependencies = [
  "cfg-if 1.0.0",
  "libc",
  "wasi 0.10.2+wasi-snapshot-preview1",
 ]
 
+[[package]]
+name = "ghash"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f6fb2a26dd2ebd268a68bc8e9acc9e67e487952f33384055a1cbe697514c64e"
+dependencies = [
+ "opaque-debug",
+ "polyval",
+]
+
 [[package]]
 name = "h2"
 version = "0.2.7"
@@ -471,9 +538,9 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.3.1"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d832b01df74254fe364568d6ddc294443f61cbec82816b60904303af87efae78"
+checksum = "825343c4eef0b63f541f8903f395dc5beb362a979b5799a84062527ef1e37726"
 dependencies = [
  "bytes 1.0.1",
  "fnv",
@@ -483,8 +550,8 @@ dependencies = [
  "http",
  "indexmap",
  "slab",
- "tokio 1.4.0",
- "tokio-util 0.6.5",
+ "tokio 1.6.1",
+ "tokio-util 0.6.7",
  "tracing",
 ]
 
@@ -561,9 +628,9 @@ dependencies = [
 
 [[package]]
 name = "http"
-version = "0.2.3"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747"
+checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11"
 dependencies = [
  "bytes 1.0.1",
  "fnv",
@@ -582,9 +649,9 @@ dependencies = [
 
 [[package]]
 name = "http-body"
-version = "0.4.1"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5dfb77c123b4e2f72a2069aeae0b4b4949cc7e966df277813fc16347e7549737"
+checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9"
 dependencies = [
  "bytes 1.0.1",
  "http",
@@ -593,9 +660,9 @@ dependencies = [
 
 [[package]]
 name = "httparse"
-version = "1.3.5"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "615caabe2c3160b313d52ccc905335f4ed5f10881dd63dc5699d47e90be85691"
+checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68"
 
 [[package]]
 name = "httpdate"
@@ -603,6 +670,12 @@ version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47"
 
+[[package]]
+name = "httpdate"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440"
+
 [[package]]
 name = "humantime"
 version = "1.3.0"
@@ -632,10 +705,10 @@ dependencies = [
  "http",
  "http-body 0.3.1",
  "httparse",
- "httpdate",
+ "httpdate 0.3.2",
  "itoa",
- "pin-project 1.0.5",
- "socket2",
+ "pin-project 1.0.7",
+ "socket2 0.3.19",
  "tokio 0.2.25",
  "tower-service",
  "tracing",
@@ -644,23 +717,23 @@ dependencies = [
 
 [[package]]
 name = "hyper"
-version = "0.14.4"
+version = "0.14.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8e946c2b1349055e0b72ae281b238baf1a3ea7307c7e9f9d64673bdd9c26ac7"
+checksum = "d3f71a7eea53a3f8257a7b4795373ff886397178cd634430ea94e12d7fe4fe34"
 dependencies = [
  "bytes 1.0.1",
  "futures-channel",
  "futures-core",
  "futures-util",
- "h2 0.3.1",
+ "h2 0.3.3",
  "http",
- "http-body 0.4.1",
+ "http-body 0.4.2",
  "httparse",
- "httpdate",
+ "httpdate 1.0.1",
  "itoa",
- "pin-project 1.0.5",
- "socket2",
- "tokio 1.4.0",
+ "pin-project 1.0.7",
+ "socket2 0.4.0",
+ "tokio 1.6.1",
  "tower-service",
  "tracing",
  "want",
@@ -673,17 +746,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
 dependencies = [
  "bytes 1.0.1",
- "hyper 0.14.4",
+ "hyper 0.14.8",
  "native-tls",
- "tokio 1.4.0",
+ "tokio 1.6.1",
  "tokio-native-tls",
 ]
 
 [[package]]
 name = "idna"
-version = "0.2.2"
+version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89829a5d69c23d348314a7ac337fe39173b61149a9864deabd260983aed48c21"
+checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8"
 dependencies = [
  "matches",
  "unicode-bidi",
@@ -753,9 +826,9 @@ checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"
 
 [[package]]
 name = "js-sys"
-version = "0.3.49"
+version = "0.3.51"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc15e39392125075f60c95ba416f5381ff6c3a948ff02ab12464715adf56c821"
+checksum = "83bdfbace3a0e81a4253f73b49e960b053e396a11012cbd49b9b74d6a2b67062"
 dependencies = [
  "wasm-bindgen",
 ]
@@ -778,9 +851,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.90"
+version = "0.2.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba4aede83fc3617411dc6993bc8c70919750c1c257c6ca6a502aed6e0e2394ae"
+checksum = "789da6d93f1b866ffe175afc5322a4d76c038605a1c3319bb57b06967ca98a36"
 
 [[package]]
 name = "libsqlite3-sys"
@@ -800,9 +873,9 @@ checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3"
 
 [[package]]
 name = "lock_api"
-version = "0.4.2"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312"
+checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb"
 dependencies = [
  "scopeguard",
 ]
@@ -858,9 +931,9 @@ checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
 
 [[package]]
 name = "memchr"
-version = "2.3.4"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
+checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
 
 [[package]]
 name = "mime"
@@ -1026,6 +1099,7 @@ checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
 name = "obnam"
 version = "0.3.1"
 dependencies = [
+ "aes-gcm",
  "anyhow",
  "bytesize",
  "chrono",
@@ -1068,9 +1142,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
 [[package]]
 name = "openssl"
-version = "0.10.33"
+version = "0.10.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a61075b62a23fef5a29815de7536d940aa35ce96d18ce0cc5076272db678a577"
+checksum = "6d7830286ad6a3973c0f1d9b73738f69c76b739301d0229c4b96501695cbe4c8"
 dependencies = [
  "bitflags",
  "cfg-if 1.0.0",
@@ -1082,15 +1156,15 @@ dependencies = [
 
 [[package]]
 name = "openssl-probe"
-version = "0.1.2"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de"
+checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a"
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.61"
+version = "0.9.63"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "313752393519e876837e09e1fa183ddef0be7735868dced3196f4472d536277f"
+checksum = "b6b0d6fb7d80f877617dfcb014e605e2b5ab2fb0afdf27935219bb6bd984cb98"
 dependencies = [
  "autocfg",
  "cc",
@@ -1101,11 +1175,12 @@ dependencies = [
 
 [[package]]
 name = "ordered-float"
-version = "2.1.1"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "766f840da25490628d8e63e529cd21c014f6600c6b8517add12a6fa6167a6218"
+checksum = "809348965973b261c3e504c8d0434e465274f78c880e10039914f2c5dcf49461"
 dependencies = [
  "num-traits",
+ "rand 0.8.3",
 ]
 
 [[package]]
@@ -1128,16 +1203,16 @@ dependencies = [
  "cfg-if 1.0.0",
  "instant",
  "libc",
- "redox_syscall 0.2.5",
+ "redox_syscall 0.2.8",
  "smallvec",
  "winapi 0.3.9",
 ]
 
 [[package]]
 name = "password-hash"
-version = "0.1.2"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85d8faea6c018131952a192ee55bd9394c51fc6f63294b668d97636e6f842d40"
+checksum = "54986aa4bfc9b98c6a5f40184223658d187159d7b3c6af33f2b2aa25ae1db0fa"
 dependencies = [
  "base64ct",
  "rand_core 0.6.2",
@@ -1164,27 +1239,27 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
 
 [[package]]
 name = "pin-project"
-version = "0.4.27"
+version = "0.4.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15"
+checksum = "918192b5c59119d51e0cd221f4d49dde9112824ba717369e903c97d076083d0f"
 dependencies = [
- "pin-project-internal 0.4.27",
+ "pin-project-internal 0.4.28",
 ]
 
 [[package]]
 name = "pin-project"
-version = "1.0.5"
+version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96fa8ebb90271c4477f144354485b8068bd8f6b78b428b01ba892ca26caf0b63"
+checksum = "c7509cc106041c40a4518d2af7a61530e1eed0e6285296a3d8c5472806ccc4a4"
 dependencies = [
- "pin-project-internal 1.0.5",
+ "pin-project-internal 1.0.7",
 ]
 
 [[package]]
 name = "pin-project-internal"
-version = "0.4.27"
+version = "0.4.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895"
+checksum = "3be26700300be6d9d23264c73211d8190e755b6b5ca7a1b28230025511b52a5e"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1193,9 +1268,9 @@ dependencies = [
 
 [[package]]
 name = "pin-project-internal"
-version = "1.0.5"
+version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "758669ae3558c6f74bd2a18b41f7ac0b5a195aea6639d6a9b5e5d1ad5ba24c0b"
+checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1226,6 +1301,17 @@ version = "0.3.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"
 
+[[package]]
+name = "polyval"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "864231b0b86ce05168a8e6da0fea2e67275dacf25f75b00a62cfd341aab904a9"
+dependencies = [
+ "cpufeatures",
+ "opaque-debug",
+ "universal-hash",
+]
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.10"
@@ -1268,9 +1354,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.24"
+version = "1.0.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71"
+checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
 dependencies = [
  "unicode-xid",
 ]
@@ -1350,7 +1436,7 @@ version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7"
 dependencies = [
- "getrandom 0.2.2",
+ "getrandom 0.2.3",
 ]
 
 [[package]]
@@ -1379,9 +1465,9 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
 
 [[package]]
 name = "redox_syscall"
-version = "0.2.5"
+version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9"
+checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc"
 dependencies = [
  "bitflags",
 ]
@@ -1392,15 +1478,15 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64"
 dependencies = [
- "getrandom 0.2.2",
- "redox_syscall 0.2.5",
+ "getrandom 0.2.3",
+ "redox_syscall 0.2.8",
 ]
 
 [[package]]
 name = "regex"
-version = "1.4.5"
+version = "1.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19"
+checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -1409,9 +1495,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.6.23"
+version = "0.6.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548"
+checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
 
 [[package]]
 name = "remove_dir_all"
@@ -1424,9 +1510,9 @@ dependencies = [
 
 [[package]]
 name = "reqwest"
-version = "0.11.2"
+version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf12057f289428dbf5c591c74bf10392e4a8003f993405a902f20117019022d4"
+checksum = "2296f2fac53979e8ccbc4a1136b25dcefd37be9ed7e4a1f6b05a6029c84ff124"
 dependencies = [
  "base64 0.13.0",
  "bytes 1.0.1",
@@ -1434,8 +1520,8 @@ dependencies = [
  "futures-core",
  "futures-util",
  "http",
- "http-body 0.4.1",
- "hyper 0.14.4",
+ "http-body 0.4.2",
+ "hyper 0.14.8",
  "hyper-tls",
  "ipnet",
  "js-sys",
@@ -1448,7 +1534,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_urlencoded 0.7.0",
- "tokio 1.4.0",
+ "tokio 1.6.1",
  "tokio-native-tls",
  "url",
  "wasm-bindgen",
@@ -1555,9 +1641,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
 [[package]]
 name = "sct"
-version = "0.6.0"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c"
+checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce"
 dependencies = [
  "ring",
  "untrusted",
@@ -1565,9 +1651,9 @@ dependencies = [
 
 [[package]]
 name = "security-framework"
-version = "2.1.2"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d493c5f39e02dfb062cd8f33301f90f9b13b650e8c1b1d0fd75c19dd64bff69d"
+checksum = "3670b1d2fdf6084d192bc71ead7aabe6c06aa2ea3fbd9cc3ac111fa5c2b1bd84"
 dependencies = [
  "bitflags",
  "core-foundation",
@@ -1578,9 +1664,9 @@ dependencies = [
 
 [[package]]
 name = "security-framework-sys"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dee48cdde5ed250b0d3252818f646e174ab414036edb884dde62d80a3ac6082d"
+checksum = "3676258fd3cfe2c9a0ec99ce3038798d847ce3e4bb17746373eb9f0f1ac16339"
 dependencies = [
  "core-foundation-sys",
  "libc",
@@ -1588,9 +1674,9 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.124"
+version = "1.0.126"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd761ff957cb2a45fbb9ab3da6512de9de55872866160b23c25f1a841e99d29f"
+checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03"
 dependencies = [
  "serde_derive",
 ]
@@ -1607,9 +1693,9 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.124"
+version = "1.0.126"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1800f7693e94e186f5e25a28291ae1570da908aff7d97a095dec1e56ff99069b"
+checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1665,35 +1751,35 @@ dependencies = [
 
 [[package]]
 name = "sha-1"
-version = "0.9.4"
+version = "0.9.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfebf75d25bd900fd1e7d11501efab59bc846dbc76196839663e6637bba9f25f"
+checksum = "8c4cfa741c5832d0ef7fab46cabed29c2aae926db0b11bb2069edd8db5e64e16"
 dependencies = [
  "block-buffer",
  "cfg-if 1.0.0",
- "cpuid-bool",
+ "cpufeatures",
  "digest",
  "opaque-debug",
 ]
 
 [[package]]
 name = "sha2"
-version = "0.9.3"
+version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de"
+checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12"
 dependencies = [
  "block-buffer",
  "cfg-if 1.0.0",
- "cpuid-bool",
+ "cpufeatures",
  "digest",
  "opaque-debug",
 ]
 
 [[package]]
 name = "slab"
-version = "0.4.2"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
+checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527"
 
 [[package]]
 name = "smallvec"
@@ -1712,6 +1798,16 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "socket2"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2"
+dependencies = [
+ "libc",
+ "winapi 0.3.9",
+]
+
 [[package]]
 name = "spin"
 version = "0.5.2"
@@ -1756,9 +1852,9 @@ checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2"
 
 [[package]]
 name = "syn"
-version = "1.0.64"
+version = "1.0.72"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fd9d1e9976102a03c542daa2eff1b43f9d72306342f3f8b3ed5fb8908195d6f"
+checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1774,7 +1870,7 @@ dependencies = [
  "cfg-if 1.0.0",
  "libc",
  "rand 0.8.3",
- "redox_syscall 0.2.5",
+ "redox_syscall 0.2.8",
  "remove_dir_all",
  "winapi 0.3.9",
 ]
@@ -1790,9 +1886,9 @@ dependencies = [
 
 [[package]]
 name = "terminal_size"
-version = "0.1.16"
+version = "0.1.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86ca8ced750734db02076f44132d802af0b33b09942331f4459dde8636fd2406"
+checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df"
 dependencies = [
  "libc",
  "winapi 0.3.9",
@@ -1809,18 +1905,18 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.24"
+version = "1.0.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e"
+checksum = "fa6f76457f59514c7eeb4e59d891395fab0b2fd1d40723ae737d64153392e9c6"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.24"
+version = "1.0.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0"
+checksum = "8a36768c0fbf1bb15eca10defa29526bda730a2376c2ab4393ccfa16fb1a318d"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1850,9 +1946,9 @@ dependencies = [
 
 [[package]]
 name = "tinyvec"
-version = "1.1.1"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "317cca572a0e89c3ce0ca1f1bdc9369547fe318a683418e42ac8f59d14701023"
+checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342"
 dependencies = [
  "tinyvec_macros",
 ]
@@ -1883,9 +1979,9 @@ dependencies = [
 
 [[package]]
 name = "tokio"
-version = "1.4.0"
+version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "134af885d758d645f0f0505c9a8b3f9bf8a348fd822e112ab5248138348f1722"
+checksum = "0a38d31d7831c6ed7aad00aa4c12d9375fd225a6dd77da1d25b707346319a975"
 dependencies = [
  "autocfg",
  "bytes 1.0.1",
@@ -1914,7 +2010,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
 dependencies = [
  "native-tls",
- "tokio 1.4.0",
+ "tokio 1.6.1",
 ]
 
 [[package]]
@@ -1937,7 +2033,7 @@ checksum = "6d9e878ad426ca286e4dcae09cbd4e1973a7f8987d97570e2469703dd7f5720c"
 dependencies = [
  "futures-util",
  "log",
- "pin-project 0.4.27",
+ "pin-project 0.4.28",
  "tokio 0.2.25",
  "tungstenite",
 ]
@@ -1958,16 +2054,16 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.6.5"
+version = "0.6.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5143d049e85af7fbc36f5454d990e62c2df705b3589f123b71f441b6b59f443f"
+checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592"
 dependencies = [
  "bytes 1.0.1",
  "futures-core",
  "futures-sink",
  "log",
  "pin-project-lite 0.2.6",
- "tokio 1.4.0",
+ "tokio 1.6.1",
 ]
 
 [[package]]
@@ -1978,9 +2074,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6"
 
 [[package]]
 name = "tracing"
-version = "0.1.25"
+version = "0.1.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f"
+checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d"
 dependencies = [
  "cfg-if 1.0.0",
  "log",
@@ -1990,9 +2086,9 @@ dependencies = [
 
 [[package]]
 name = "tracing-core"
-version = "0.1.17"
+version = "0.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f"
+checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052"
 dependencies = [
  "lazy_static",
 ]
@@ -2003,7 +2099,7 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"
 dependencies = [
- "pin-project 1.0.5",
+ "pin-project 1.0.7",
  "tracing",
 ]
 
@@ -2073,18 +2169,18 @@ dependencies = [
 
 [[package]]
 name = "unicode-bidi"
-version = "0.3.4"
+version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5"
+checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0"
 dependencies = [
  "matches",
 ]
 
 [[package]]
 name = "unicode-normalization"
-version = "0.1.17"
+version = "0.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef"
+checksum = "33717dca7ac877f497014e10d73f3acf948c342bee31b5ca7892faf94ccc6b49"
 dependencies = [
  "tinyvec",
 ]
@@ -2103,9 +2199,19 @@ checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
 
 [[package]]
 name = "unicode-xid"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
+checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
+
+[[package]]
+name = "universal-hash"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402"
+dependencies = [
+ "generic-array",
+ "subtle",
+]
 
 [[package]]
 name = "unsafe-any"
@@ -2124,9 +2230,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
 
 [[package]]
 name = "url"
-version = "2.2.1"
+version = "2.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b"
+checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c"
 dependencies = [
  "form_urlencoded",
  "idna",
@@ -2136,9 +2242,9 @@ dependencies = [
 
 [[package]]
 name = "urlencoding"
-version = "1.1.1"
+version = "1.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9232eb53352b4442e40d7900465dfc534e8cb2dc8f18656fcb2ac16112b5593"
+checksum = "5a1f0175e03a0973cf4afd476bef05c26e228520400eb1fd473ad417b1c00ffb"
 
 [[package]]
 name = "users"
@@ -2152,9 +2258,9 @@ dependencies = [
 
 [[package]]
 name = "utf-8"
-version = "0.7.5"
+version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05e42f7c18b8f902290b009cde6d651262f956c98bc51bca4cd1d511c9cd85c7"
+checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
 
 [[package]]
 name = "uuid"
@@ -2162,14 +2268,14 @@ version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
 dependencies = [
- "getrandom 0.2.2",
+ "getrandom 0.2.3",
 ]
 
 [[package]]
 name = "vcpkg"
-version = "0.2.11"
+version = "0.2.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb"
+checksum = "025ce40a007e1907e58d5bc1a594def78e5573bb0b1160bc389634e8f12e4faa"
 
 [[package]]
 name = "vec_map"
@@ -2185,9 +2291,9 @@ checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
 
 [[package]]
 name = "walkdir"
-version = "2.3.1"
+version = "2.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d"
+checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56"
 dependencies = [
  "same-file",
  "winapi 0.3.9",
@@ -2219,7 +2325,7 @@ dependencies = [
  "mime",
  "mime_guess",
  "multipart",
- "pin-project 0.4.27",
+ "pin-project 0.4.28",
  "scoped-tls",
  "serde",
  "serde_json",
@@ -2247,9 +2353,9 @@ checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.72"
+version = "0.2.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fe8f61dba8e5d645a4d8132dc7a0a66861ed5e1045d2c0ed940fab33bac0fbe"
+checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd"
 dependencies = [
  "cfg-if 1.0.0",
  "serde",
@@ -2259,9 +2365,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.72"
+version = "0.2.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "046ceba58ff062da072c7cb4ba5b22a37f00a302483f7e2a6cdc18fedbdc1fd3"
+checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900"
 dependencies = [
  "bumpalo",
  "lazy_static",
@@ -2274,9 +2380,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.22"
+version = "0.4.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73157efb9af26fb564bb59a009afd1c7c334a44db171d280690d0c3faaec3468"
+checksum = "5fba7978c679d53ce2d0ac80c8c175840feb849a161664365d1287b41f2e67f1"
 dependencies = [
  "cfg-if 1.0.0",
  "js-sys",
@@ -2286,9 +2392,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.72"
+version = "0.2.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ef9aa01d36cda046f797c57959ff5f3c615c9cc63997a8d545831ec7976819b"
+checksum = "088169ca61430fe1e58b8096c24975251700e7b1f6fd91cc9d59b04fb9b18bd4"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -2296,9 +2402,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.72"
+version = "0.2.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96eb45c1b2ee33545a813a92dbb53856418bf7eb54ab34f7f7ff1448a5b3735d"
+checksum = "be2241542ff3d9f241f5e2cb6dd09b37efe786df8851c54957683a49f0987a97"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2309,15 +2415,15 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.72"
+version = "0.2.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7148f4696fb4960a346eaa60bbfb42a1ac4ebba21f750f75fc1375b098d5ffa"
+checksum = "d7cff876b8f18eed75a66cf49b65e7f967cb354a7aa16003fb55dbfd25b44b4f"
 
 [[package]]
 name = "web-sys"
-version = "0.3.49"
+version = "0.3.51"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59fe19d70f5dacc03f6e46777213facae5ac3801575d56ca6cbd4c93dcd12310"
+checksum = "e828417b379f3df7111d3a2a9e5753706cae29c41f7c4029ee9fd77f3e09e582"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
diff --git a/Cargo.toml b/Cargo.toml
index 180d342..5349b92 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -10,6 +10,7 @@ repository = "https://gitlab.com/larswirzenius/obnam"
 
 
 [dependencies]
+aes-gcm = "0.9.1"
 anyhow = "1"
 bytesize = "1"
 chrono = "0.4"
-- 
cgit v1.2.1


From e839c5f1c93e1fe024a2656d319721a7b23c6461 Mon Sep 17 00:00:00 2001
From: Lars Wirzenius <liw@liw.fi>
Date: Sat, 29 May 2021 11:39:31 +0300
Subject: refactor: count chunks via file system, not via chunk server API

---
 src/bin/obnam-server.rs |  2 --
 src/indexedstore.rs     | 25 +------------------------
 subplot/server.py       | 18 +++++++++++++-----
 subplot/server.yaml     |  4 ++--
 4 files changed, 16 insertions(+), 33 deletions(-)

diff --git a/src/bin/obnam-server.rs b/src/bin/obnam-server.rs
index efee77e..29ea9ff 100644
--- a/src/bin/obnam-server.rs
+++ b/src/bin/obnam-server.rs
@@ -155,8 +155,6 @@ pub async fn search_chunks(
         }
         if key == "generation" && value == "true" {
             store.find_generations().expect("SQL lookup failed")
-        } else if key == "data" && value == "true" {
-            store.find_file_chunks().expect("SQL lookup failed")
         } else if key == "sha256" {
             store.find_by_sha256(value).expect("SQL lookup failed")
         } else {
diff --git a/src/indexedstore.rs b/src/indexedstore.rs
index 982e2d9..b05cfba 100644
--- a/src/indexedstore.rs
+++ b/src/indexedstore.rs
@@ -1,9 +1,8 @@
-use crate::chunk::{DataChunk, GenerationChunk, GenerationChunkError};
+use crate::chunk::{DataChunk, GenerationChunkError};
 use crate::chunkid::ChunkId;
 use crate::chunkmeta::ChunkMeta;
 use crate::index::{Index, IndexError};
 use crate::store::{Store, StoreError};
-use std::collections::HashSet;
 use std::path::Path;
 
 /// A store for chunks and their metadata.
@@ -68,28 +67,6 @@ impl IndexedStore {
         Ok(self.index.find_generations()?)
     }
 
-    pub fn find_file_chunks(&self) -> IndexedResult<Vec<ChunkId>> {
-        let gen_ids = self.find_generations()?;
-
-        let mut sql_chunks: HashSet<ChunkId> = HashSet::new();
-        for id in gen_ids {
-            let gen_chunk = self.store.load(&id)?;
-            let gen = GenerationChunk::from_data_chunk(&gen_chunk)?;
-            for sqlite_chunk_id in gen.chunk_ids() {
-                sql_chunks.insert(sqlite_chunk_id.clone());
-            }
-        }
-
-        let all_chunk_ids = self.index.all_chunks()?;
-        let file_chunks = all_chunk_ids
-            .iter()
-            .filter(|id| !sql_chunks.contains(id))
-            .cloned()
-            .collect();
-
-        Ok(file_chunks)
-    }
-
     pub fn remove(&mut self, id: &ChunkId) -> IndexedResult<()> {
         self.index.remove_meta(id)?;
         self.store.delete(id)?;
diff --git a/subplot/server.py b/subplot/server.py
index df594f7..cfe91ab 100644
--- a/subplot/server.py
+++ b/subplot/server.py
@@ -134,13 +134,21 @@ def json_body_matches(ctx, wanted=None):
         assert_eq(body.get(key, "not.there"), wanted[key])
 
 
-def server_has_n_file_chunks(ctx, n=None):
+def server_has_n_chunks(ctx, n=None):
     assert_eq = globals()["assert_eq"]
     n = int(n)
-    url = f"{ctx['server_url']}/chunks?data=true"
-    _request(ctx, requests.get, url)
-    num_chunks = len(ctx["http.json"])
-    assert_eq(n, num_chunks)
+    files = find_files(ctx["config"]["chunks"])
+    files = [json.load(open(x)) for x in files if x.endswith(".meta")]
+    logging.debug(f"server_has_n_file_chunks: n={n}")
+    logging.debug(f"server_has_n_file_chunks: len(files)={len(files)}")
+    logging.debug(f"server_has_n_file_chunks: files={files}")
+    assert_eq(n, len(files))
+
+
+def find_files(root):
+    for dirname, _, names in os.walk(root):
+        for name in names:
+            yield os.path.join(dirname, name)
 
 
 # Make an HTTP request.
diff --git a/subplot/server.yaml b/subplot/server.yaml
index 60f8a44..5b8a242 100644
--- a/subplot/server.yaml
+++ b/subplot/server.yaml
@@ -44,5 +44,5 @@
 - then: "the body matches file {filename}"
   function: body_matches_file
 
-- then: "server has {n:int} file chunks"
-  function: server_has_n_file_chunks
+- then: "server has {n:int} chunks"
+  function: server_has_n_chunks
-- 
cgit v1.2.1


From 08b71890de3407acc323f09330ebe8a8ee2782ec Mon Sep 17 00:00:00 2001
From: Lars Wirzenius <liw@liw.fi>
Date: Sat, 29 May 2021 11:39:31 +0300
Subject: refactor: only have client config without passwords

This means the config is always the config, and not sometimes the
config or the config and passwords.

Also, there's no config option for encrypting, anymore. It will not be
optional now.
---
 src/backup_run.rs      |  2 --
 src/bin/obnam.rs       | 39 ++++++++-------------
 src/cmd/backup.rs      |  2 --
 src/cmd/init.rs        |  8 ++---
 src/cmd/show_config.rs |  2 +-
 src/config.rs          | 92 +++++++++++++++++---------------------------------
 subplot/client.py      | 11 +++++-
 subplot/client.yaml    |  5 ++-
 8 files changed, 63 insertions(+), 98 deletions(-)

diff --git a/src/backup_run.rs b/src/backup_run.rs
index 23c97f6..16d6700 100644
--- a/src/backup_run.rs
+++ b/src/backup_run.rs
@@ -41,7 +41,6 @@ pub type BackupResult<T> = Result<T, BackupError>;
 impl<'a> InitialBackup<'a> {
     pub fn new(config: &ClientConfig, client: &'a BackupClient) -> BackupResult<Self> {
         let progress = BackupProgress::initial();
-        let config = config.config();
         Ok(Self {
             client,
             buffer_size: config.chunk_size,
@@ -81,7 +80,6 @@ impl<'a> InitialBackup<'a> {
 
 impl<'a> IncrementalBackup<'a> {
     pub fn new(config: &ClientConfig, client: &'a BackupClient) -> BackupResult<Self> {
-        let config = config.config();
         let policy = BackupPolicy::default();
         Ok(Self {
             client,
diff --git a/src/bin/obnam.rs b/src/bin/obnam.rs
index cdb5179..c8da6c2 100644
--- a/src/bin/obnam.rs
+++ b/src/bin/obnam.rs
@@ -3,6 +3,7 @@ use log::{debug, error, info, LevelFilter};
 use log4rs::append::file::FileAppender;
 use log4rs::config::{Appender, Logger, Root};
 use obnam::cmd::backup::Backup;
+use obnam::cmd::chunk::{DecryptChunk, EncryptChunk};
 use obnam::cmd::get_chunk::GetChunk;
 use obnam::cmd::init::Init;
 use obnam::cmd::list::List;
@@ -20,28 +21,24 @@ const APPLICATION: &str = "obnam";
 
 fn main() -> anyhow::Result<()> {
     let opt = Opt::from_args();
-    let config = load_config_without_passwords(&opt)?;
-    setup_logging(&config.config().log)?;
+    let config = ClientConfig::read(&config_filename(&opt))?;
+    setup_logging(&config.log)?;
 
     info!("client starts");
     debug!("{:?}", opt);
     debug!("configuration: {:#?}", config);
 
     let result = match opt.cmd {
-        Command::Init(x) => x.run(config.config()),
-        _ => {
-            let config = load_config_with_passwords(&opt)?;
-            match opt.cmd {
-                Command::Init(_) => panic!("this can't happen"),
-                Command::Backup(x) => x.run(&config),
-                Command::List(x) => x.run(&config),
-                Command::ShowGeneration(x) => x.run(&config),
-                Command::ListFiles(x) => x.run(&config),
-                Command::Restore(x) => x.run(&config),
-                Command::GetChunk(x) => x.run(&config),
-                Command::Config(x) => x.run(&config),
-            }
-        }
+        Command::Init(x) => x.run(&config),
+        Command::Backup(x) => x.run(&config),
+        Command::List(x) => x.run(&config),
+        Command::ShowGeneration(x) => x.run(&config),
+        Command::ListFiles(x) => x.run(&config),
+        Command::Restore(x) => x.run(&config),
+        Command::GetChunk(x) => x.run(&config),
+        Command::Config(x) => x.run(&config),
+        Command::EncryptChunk(x) => x.run(&config),
+        Command::DecryptChunk(x) => x.run(&config),
     };
 
     if let Err(ref e) = result {
@@ -66,14 +63,6 @@ fn setup_logging(filename: &Path) -> anyhow::Result<()> {
     Ok(())
 }
 
-fn load_config_with_passwords(opt: &Opt) -> Result<ClientConfig, anyhow::Error> {
-    Ok(ClientConfig::read_with_passwords(&config_filename(opt))?)
-}
-
-fn load_config_without_passwords(opt: &Opt) -> Result<ClientConfig, anyhow::Error> {
-    Ok(ClientConfig::read_without_passwords(&config_filename(opt))?)
-}
-
 fn config_filename(opt: &Opt) -> PathBuf {
     match opt.config {
         None => default_config(),
@@ -109,4 +98,6 @@ enum Command {
     ShowGeneration(ShowGeneration),
     GetChunk(GetChunk),
     Config(ShowConfig),
+    EncryptChunk(EncryptChunk),
+    DecryptChunk(DecryptChunk),
 }
diff --git a/src/cmd/backup.rs b/src/cmd/backup.rs
index 0479844..22afd6e 100644
--- a/src/cmd/backup.rs
+++ b/src/cmd/backup.rs
@@ -60,7 +60,6 @@ fn initial_backup(
     info!("fresh backup without a previous generation");
     let newtemp = NamedTempFile::new()?;
     let run = InitialBackup::new(config, &client)?;
-    let config = config.config();
     let mut all_warnings = vec![];
     let count = {
         let mut new = NascentGeneration::create(newtemp.path())?;
@@ -87,7 +86,6 @@ fn incremental_backup(
     info!("incremental backup based on {}", old_ref);
     let newtemp = NamedTempFile::new()?;
     let mut run = IncrementalBackup::new(config, &client)?;
-    let config = config.config();
     let mut all_warnings = vec![];
     let count = {
         let oldtemp = NamedTempFile::new()?;
diff --git a/src/cmd/init.rs b/src/cmd/init.rs
index cb61fba..08060f7 100644
--- a/src/cmd/init.rs
+++ b/src/cmd/init.rs
@@ -1,4 +1,4 @@
-use crate::config::ClientConfigWithoutPasswords;
+use crate::config::ClientConfig;
 use crate::error::ObnamError;
 use crate::passwords::{passwords_filename, Passwords};
 use structopt::StructOpt;
@@ -12,11 +12,7 @@ pub struct Init {
 }
 
 impl Init {
-    pub fn run(&self, config: &ClientConfigWithoutPasswords) -> Result<(), ObnamError> {
-        if !config.encrypt {
-            panic!("no encryption specified");
-        }
-
+    pub fn run(&self, config: &ClientConfig) -> Result<(), ObnamError> {
         let passphrase = match &self.insecure_passphrase {
             Some(x) => x.to_string(),
             None => rpassword::read_password_from_tty(Some(PROMPT)).unwrap(),
diff --git a/src/cmd/show_config.rs b/src/cmd/show_config.rs
index 424e2ed..05e83c1 100644
--- a/src/cmd/show_config.rs
+++ b/src/cmd/show_config.rs
@@ -7,7 +7,7 @@ pub struct ShowConfig {}
 
 impl ShowConfig {
     pub fn run(&self, config: &ClientConfig) -> Result<(), ObnamError> {
-        println!("{}", serde_json::to_string_pretty(&config.config())?);
+        println!("{}", serde_json::to_string_pretty(config)?);
         Ok(())
     }
 }
diff --git a/src/config.rs b/src/config.rs
index 33e08a2..0d4e9de 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -16,78 +16,22 @@ struct TentativeClientConfig {
     chunk_size: Option<usize>,
     roots: Vec<PathBuf>,
     log: Option<PathBuf>,
-    encrypt: Option<bool>,
     exclude_cache_tag_directories: Option<bool>,
 }
 
 #[derive(Debug, Serialize, Clone)]
-pub enum ClientConfig {
-    Plain(ClientConfigWithoutPasswords),
-    WithPasswords(ClientConfigWithoutPasswords, Passwords),
-}
-
-impl ClientConfig {
-    pub fn read_without_passwords(filename: &Path) -> Result<Self, ClientConfigError> {
-        let config = ClientConfigWithoutPasswords::read_config(filename)?;
-        Ok(ClientConfig::Plain(config))
-    }
-
-    pub fn read_with_passwords(filename: &Path) -> Result<Self, ClientConfigError> {
-        let config = ClientConfigWithoutPasswords::read_config(filename)?;
-        if config.encrypt {
-            let passwords = Passwords::load(&passwords_filename(filename))
-                .map_err(ClientConfigError::PasswordsMissing)?;
-            Ok(ClientConfig::WithPasswords(config, passwords))
-        } else {
-            Ok(ClientConfig::Plain(config))
-        }
-    }
-
-    pub fn config(&self) -> &ClientConfigWithoutPasswords {
-        match self {
-            Self::Plain(config) => &config,
-            Self::WithPasswords(config, _) => &config,
-        }
-    }
-}
-
-#[derive(Debug, Serialize, Clone)]
-pub struct ClientConfigWithoutPasswords {
+pub struct ClientConfig {
     pub filename: PathBuf,
     pub server_url: String,
     pub verify_tls_cert: bool,
     pub chunk_size: usize,
     pub roots: Vec<PathBuf>,
     pub log: PathBuf,
-    pub encrypt: bool,
     pub exclude_cache_tag_directories: bool,
 }
 
-#[derive(Debug, thiserror::Error)]
-pub enum ClientConfigError {
-    #[error("server_url is empty")]
-    ServerUrlIsEmpty,
-
-    #[error("No backup roots in config; at least one is needed")]
-    NoBackupRoot,
-
-    #[error("server URL doesn't use https: {0}")]
-    NotHttps(String),
-
-    #[error("No passwords are set: you may need to run 'obnam init': {0}")]
-    PasswordsMissing(PasswordError),
-
-    #[error("failed to read configuration file {0}: {1}")]
-    Read(PathBuf, std::io::Error),
-
-    #[error("failed to parse configuration file {0} as YAML: {1}")]
-    YamlParse(PathBuf, serde_yaml::Error),
-}
-
-pub type ClientConfigResult<T> = Result<T, ClientConfigError>;
-
-impl ClientConfigWithoutPasswords {
-    pub fn read_config(filename: &Path) -> ClientConfigResult<Self> {
+impl ClientConfig {
+    pub fn read(filename: &Path) -> ClientConfigResult<Self> {
         trace!("read_config: filename={:?}", filename);
         let config = std::fs::read_to_string(filename)
             .map_err(|err| ClientConfigError::Read(filename.to_path_buf(), err))?;
@@ -102,12 +46,10 @@ impl ClientConfigWithoutPasswords {
             .log
             .map(|path| expand_tilde(&path))
             .unwrap_or_else(|| PathBuf::from(DEVNULL));
-        let encrypt = tentative.encrypt.or(Some(false)).unwrap();
         let exclude_cache_tag_directories = tentative.exclude_cache_tag_directories.unwrap_or(true);
 
         let config = Self {
             chunk_size: tentative.chunk_size.or(Some(DEFAULT_CHUNK_SIZE)).unwrap(),
-            encrypt,
             filename: filename.to_path_buf(),
             roots,
             server_url: tentative.server_url,
@@ -132,8 +74,36 @@ impl ClientConfigWithoutPasswords {
         }
         Ok(())
     }
+
+    pub fn passwords(&self) -> Result<Passwords, ClientConfigError> {
+        Passwords::load(&passwords_filename(&self.filename))
+            .map_err(ClientConfigError::PasswordsMissing)
+    }
 }
 
+#[derive(Debug, thiserror::Error)]
+pub enum ClientConfigError {
+    #[error("server_url is empty")]
+    ServerUrlIsEmpty,
+
+    #[error("No backup roots in config; at least one is needed")]
+    NoBackupRoot,
+
+    #[error("server URL doesn't use https: {0}")]
+    NotHttps(String),
+
+    #[error("No passwords are set: you may need to run 'obnam init': {0}")]
+    PasswordsMissing(PasswordError),
+
+    #[error("failed to read configuration file {0}: {1}")]
+    Read(PathBuf, std::io::Error),
+
+    #[error("failed to parse configuration file {0} as YAML: {1}")]
+    YamlParse(PathBuf, serde_yaml::Error),
+}
+
+pub type ClientConfigResult<T> = Result<T, ClientConfigError>;
+
 fn expand_tilde(path: &Path) -> PathBuf {
     if path.starts_with("~/") {
         if let Some(home) = std::env::var_os("HOME") {
diff --git a/subplot/client.py b/subplot/client.py
index be0a6d6..d0beba5 100644
--- a/subplot/client.py
+++ b/subplot/client.py
@@ -16,7 +16,7 @@ def uninstall_obnam(ctx):
     runcmd_run(ctx, ["chmod", "-R", "u+rwX", "."])
 
 
-def configure_client(ctx, filename=None):
+def configure_client_without_init(ctx, filename=None):
     get_file = globals()["get_file"]
 
     assert ctx.get("server_url") is not None
@@ -35,6 +35,15 @@ def configure_client(ctx, filename=None):
         yaml.safe_dump(config, stream=f)
 
 
+def configure_client_with_init(ctx, filename=None):
+    runcmd_run = globals()["runcmd_run"]
+    runcmd_exit_code_is_zero = globals()["runcmd_exit_code_is_zero"]
+
+    configure_client_without_init(ctx, filename=filename)
+    runcmd_run(ctx, ["obnam", "init", "--insecure-passphrase=hunter2"])
+    runcmd_exit_code_is_zero(ctx)
+
+
 def run_obnam_restore(ctx, genid=None, todir=None):
     runcmd_run = globals()["runcmd_run"]
 
diff --git a/subplot/client.yaml b/subplot/client.yaml
index 6de04c9..d660089 100644
--- a/subplot/client.yaml
+++ b/subplot/client.yaml
@@ -3,7 +3,10 @@
   cleanup: uninstall_obnam
 
 - given: "a client config based on {filename}"
-  function: configure_client
+  function: configure_client_with_init
+
+- given: "a client config, without passphrase, based on {filename}"
+  function: configure_client_without_init
 
 - when: "I invoke obnam restore <{genid}> {todir}"
   function: run_obnam_restore
-- 
cgit v1.2.1


From cc62aac1a3ebdb1bf48a6520d430ad80948bcd51 Mon Sep 17 00:00:00 2001
From: Lars Wirzenius <liw@liw.fi>
Date: Sat, 29 May 2021 11:38:06 +0300
Subject: refactor: make metadata part of a chunk

This makes is harder to accidentally use the wrong metadata for a
chunk.
---
 src/chunk.rs     |  2 +-
 src/chunkmeta.rs | 21 ++++++++++++++++++++-
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/src/chunk.rs b/src/chunk.rs
index 50a2fc7..8631fd9 100644
--- a/src/chunk.rs
+++ b/src/chunk.rs
@@ -10,7 +10,7 @@ use std::default::Default;
 ///
 /// A chunk also contains its associated metadata, except its
 /// identifier.
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 pub struct DataChunk {
     data: Vec<u8>,
     meta: ChunkMeta,
diff --git a/src/chunkmeta.rs b/src/chunkmeta.rs
index 37e2ed5..73d9007 100644
--- a/src/chunkmeta.rs
+++ b/src/chunkmeta.rs
@@ -80,10 +80,20 @@ impl ChunkMeta {
         &self.sha256
     }
 
+    /// Serialize from a textual JSON representation.
+    pub fn from_json(json: &str) -> Result<Self, serde_json::Error> {
+        serde_json::from_str(json)
+    }
+
     /// Serialize as JSON.
     pub fn to_json(&self) -> String {
         serde_json::to_string(self).unwrap()
     }
+
+    /// Serialize as JSON, as a byte vector.
+    pub fn to_json_vec(&self) -> Vec<u8> {
+        self.to_json().as_bytes().to_vec()
+    }
 }
 
 impl FromStr for ChunkMeta {
@@ -135,10 +145,19 @@ mod test {
     }
 
     #[test]
-    fn json_roundtrip() {
+    fn generation_json_roundtrip() {
         let meta = ChunkMeta::new_generation("abcdef", "2020-09-17T08:17:13+03:00");
         let json = serde_json::to_string(&meta).unwrap();
         let meta2 = serde_json::from_str(&json).unwrap();
         assert_eq!(meta, meta2);
     }
+
+    #[test]
+    fn data_json_roundtrip() {
+        let meta = ChunkMeta::new("abcdef");
+        let json = meta.to_json_vec();
+        let meta2 = serde_json::from_slice(&json).unwrap();
+        assert_eq!(meta, meta2);
+        assert_eq!(meta.to_json_vec(), meta2.to_json_vec());
+    }
 }
-- 
cgit v1.2.1


From 426e6acb7129756aea295c01b56a902d655efa6d Mon Sep 17 00:00:00 2001
From: Lars Wirzenius <liw@liw.fi>
Date: Sat, 29 May 2021 11:39:31 +0300
Subject: feat! drop MAC passphrase, fix key derivation

Previously we were deriving a key that was of the wrong length for the
aead crate. Now we make it the right length.
---
 src/passwords.rs | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/passwords.rs b/src/passwords.rs
index b8ca3f5..a1cf42e 100644
--- a/src/passwords.rs
+++ b/src/passwords.rs
@@ -8,18 +8,23 @@ use std::io::prelude::Write;
 use std::os::unix::fs::PermissionsExt;
 use std::path::{Path, PathBuf};
 
+const KEY_LEN: usize = 32; // Only size accepted by aead crate?
+
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct Passwords {
     encryption: String,
-    mac: String,
 }
 
 impl Passwords {
     pub fn new(passphrase: &str) -> Self {
-        Self {
-            encryption: derive_password(passphrase),
-            mac: derive_password(passphrase),
-        }
+        let mut key = derive_password(passphrase);
+        let _ = key.split_off(KEY_LEN);
+        assert_eq!(key.len(), KEY_LEN);
+        Self { encryption: key }
+    }
+
+    pub fn encryption_key(&self) -> &[u8] {
+        self.encryption.as_bytes()
     }
 
     pub fn load(filename: &Path) -> Result<Self, PasswordError> {
-- 
cgit v1.2.1


From 4e95e1003c4f2c89a807977b34f287d2f200f5bc Mon Sep 17 00:00:00 2001
From: Lars Wirzenius <liw@liw.fi>
Date: Sat, 29 May 2021 11:39:52 +0300
Subject: feat: add chunk encryption

---
 src/cipher.rs | 200 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 src/error.rs  |   4 ++
 src/lib.rs    |   1 +
 3 files changed, 205 insertions(+)
 create mode 100644 src/cipher.rs

diff --git a/src/cipher.rs b/src/cipher.rs
new file mode 100644
index 0000000..550fafd
--- /dev/null
+++ b/src/cipher.rs
@@ -0,0 +1,200 @@
+use crate::chunk::DataChunk;
+use crate::chunkmeta::ChunkMeta;
+use crate::passwords::Passwords;
+
+use aes_gcm::aead::{generic_array::GenericArray, Aead, NewAead, Payload};
+use aes_gcm::Aes256Gcm; // Or `Aes128Gcm`
+use rand::Rng;
+
+use std::str::FromStr;
+
+const CHUNK_V1: &[u8] = b"0001";
+
+pub struct EncryptedChunk {
+    ciphertext: Vec<u8>,
+    aad: Vec<u8>,
+}
+
+impl EncryptedChunk {
+    fn new(ciphertext: Vec<u8>, aad: Vec<u8>) -> Self {
+        Self { ciphertext, aad }
+    }
+
+    pub fn ciphertext(&self) -> &[u8] {
+        &self.ciphertext
+    }
+
+    pub fn aad(&self) -> &[u8] {
+        &self.aad
+    }
+}
+
+pub struct CipherEngine {
+    cipher: Aes256Gcm,
+}
+
+impl CipherEngine {
+    pub fn new(pass: &Passwords) -> Self {
+        let key = GenericArray::from_slice(pass.encryption_key());
+        Self {
+            cipher: Aes256Gcm::new(key),
+        }
+    }
+
+    pub fn encrypt_chunk(&self, chunk: &DataChunk) -> Result<EncryptedChunk, CipherError> {
+        // Payload with metadata as associated data, to be encrypted.
+        //
+        // The metadata will be stored in cleartext after encryption.
+        let aad = chunk.meta().to_json_vec();
+        let payload = Payload {
+            msg: chunk.data(),
+            aad: &aad,
+        };
+
+        // Unique random key for each encryption.
+        let nonce = Nonce::new();
+        let nonce_arr = GenericArray::from_slice(nonce.as_bytes());
+
+        // Encrypt the sensitive part.
+        let ciphertext = self
+            .cipher
+            .encrypt(nonce_arr, payload)
+            .map_err(CipherError::EncryptError)?;
+
+        // Construct the blob to be stored on the server.
+        let mut vec: Vec<u8> = vec![];
+        push_bytes(&mut vec, CHUNK_V1);
+        push_bytes(&mut vec, nonce.as_bytes());
+        push_bytes(&mut vec, &ciphertext);
+
+        Ok(EncryptedChunk::new(vec, aad))
+    }
+
+    pub fn decrypt_chunk(&self, bytes: &[u8], meta: &[u8]) -> Result<DataChunk, CipherError> {
+        // Does encrypted chunk start with the right version?
+        if !bytes.starts_with(CHUNK_V1) {
+            return Err(CipherError::UnknownChunkVersion);
+        }
+        let version_len = CHUNK_V1.len();
+        let bytes = &bytes[version_len..];
+
+        // Get nonce.
+        let nonce = &bytes[..NONCE_SIZE];
+        if nonce.len() != NONCE_SIZE {
+            return Err(CipherError::NoNonce);
+        }
+        let nonce = GenericArray::from_slice(nonce);
+        let ciphertext = &bytes[NONCE_SIZE..];
+
+        let payload = Payload {
+            msg: ciphertext,
+            aad: meta,
+        };
+
+        let payload = self
+            .cipher
+            .decrypt(nonce, payload)
+            .map_err(CipherError::DecryptError)?;
+        let payload = Payload::from(payload.as_slice());
+
+        let meta = std::str::from_utf8(meta)?;
+        let meta = ChunkMeta::from_str(&meta)?;
+
+        let chunk = DataChunk::new(payload.msg.to_vec(), meta);
+
+        Ok(chunk)
+    }
+}
+
+fn push_bytes(vec: &mut Vec<u8>, bytes: &[u8]) {
+    for byte in bytes.iter() {
+        vec.push(*byte);
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum CipherError {
+    #[error("failed to encrypt with AES-GEM: {0}")]
+    EncryptError(aes_gcm::Error),
+
+    #[error("encrypted chunk does not start with correct version")]
+    UnknownChunkVersion,
+
+    #[error("encrypted chunk does not have a complete nonce")]
+    NoNonce,
+
+    #[error("failed to decrypt with AES-GEM: {0}")]
+    DecryptError(aes_gcm::Error),
+
+    #[error("failed to parse decrypted data as a DataChunk: {0}")]
+    Parse(serde_yaml::Error),
+
+    #[error(transparent)]
+    Utf8Error(#[from] std::str::Utf8Error),
+
+    #[error("failed to parse JSON: {0}")]
+    JsonParse(#[from] serde_json::Error),
+}
+
+const NONCE_SIZE: usize = 12;
+
+#[derive(Debug)]
+struct Nonce {
+    nonce: Vec<u8>,
+}
+
+impl Nonce {
+    fn from_bytes(bytes: &[u8]) -> Self {
+        assert_eq!(bytes.len(), NONCE_SIZE);
+        Self {
+            nonce: bytes.to_vec(),
+        }
+    }
+
+    fn new() -> Self {
+        let mut bytes: Vec<u8> = vec![0; NONCE_SIZE];
+        let mut rng = rand::thread_rng();
+        for x in bytes.iter_mut() {
+            *x = rng.gen();
+        }
+        Self::from_bytes(&bytes)
+    }
+
+    fn as_bytes(&self) -> &[u8] {
+        &self.nonce
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use crate::chunk::DataChunk;
+    use crate::chunkmeta::ChunkMeta;
+    use crate::cipher::CipherEngine;
+    use crate::passwords::Passwords;
+
+    #[test]
+    fn metadata_as_aad() {
+        let meta = ChunkMeta::new("dummy-checksum");
+        let meta_as_aad = meta.to_json_vec();
+        let chunk = DataChunk::new("hello".as_bytes().to_vec(), meta);
+        let pass = Passwords::new("secret");
+        let cipher = CipherEngine::new(&pass);
+        let enc = cipher.encrypt_chunk(&chunk).unwrap();
+
+        assert_eq!(meta_as_aad, enc.aad());
+    }
+
+    #[test]
+    fn round_trip() {
+        let meta = ChunkMeta::new("dummy-checksum");
+        let chunk = DataChunk::new("hello".as_bytes().to_vec(), meta);
+        let pass = Passwords::new("secret");
+
+        let cipher = CipherEngine::new(&pass);
+        let enc = cipher.encrypt_chunk(&chunk).unwrap();
+
+        let bytes: Vec<u8> = enc.ciphertext().to_vec();
+        let dec = cipher.decrypt_chunk(&bytes, enc.aad()).unwrap();
+        assert_eq!(chunk, dec);
+    }
+}
diff --git a/src/error.rs b/src/error.rs
index 8241d5d..e4d77d3 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -1,4 +1,5 @@
 use crate::backup_run::BackupError;
+use crate::cipher::CipherError;
 use crate::client::ClientError;
 use crate::cmd::restore::RestoreError;
 use crate::config::ClientConfigError;
@@ -31,6 +32,9 @@ pub enum ObnamError {
     #[error(transparent)]
     NascentError(#[from] NascentError),
 
+    #[error(transparent)]
+    CipherError(#[from] CipherError),
+
     #[error(transparent)]
     LocalGenerationError(#[from] LocalGenerationError),
 
diff --git a/src/lib.rs b/src/lib.rs
index 82dab15..7d7afdc 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -7,6 +7,7 @@ pub mod chunk;
 pub mod chunker;
 pub mod chunkid;
 pub mod chunkmeta;
+pub mod cipher;
 pub mod client;
 pub mod cmd;
 pub mod config;
-- 
cgit v1.2.1


From 7a88987b3c10c81845fce749b7b86b62770aecca Mon Sep 17 00:00:00 2001
From: Lars Wirzenius <liw@liw.fi>
Date: Sat, 29 May 2021 11:38:06 +0300
Subject: refactor: only send/receive encrypted chunks with server

Also, refactor the client code to split out the HTTP bits more
clearly. The refactor should've been separately, but got entangled
with other changes during frantic debugging.
---
 src/client.rs | 130 +++++++++++++++++++++++++++++-----------------------------
 1 file changed, 65 insertions(+), 65 deletions(-)

diff --git a/src/client.rs b/src/client.rs
index 00f5bd7..b1f9976 100644
--- a/src/client.rs
+++ b/src/client.rs
@@ -1,17 +1,18 @@
-use crate::checksummer::sha256;
 use crate::chunk::DataChunk;
 use crate::chunk::{GenerationChunk, GenerationChunkError};
 use crate::chunker::{Chunker, ChunkerError};
 use crate::chunkid::ChunkId;
 use crate::chunkmeta::ChunkMeta;
-use crate::config::ClientConfig;
+use crate::cipher::{CipherEngine, CipherError};
+use crate::config::{ClientConfig, ClientConfigError};
 use crate::fsentry::{FilesystemEntry, FilesystemKind};
 use crate::generation::{FinishedGeneration, LocalGeneration, LocalGenerationError};
 use crate::genlist::GenerationList;
 
 use chrono::{DateTime, Local};
-use log::{debug, error, info, trace};
+use log::{debug, error, info};
 use reqwest::blocking::Client;
+use reqwest::header::HeaderMap;
 use std::collections::HashMap;
 use std::fs::File;
 use std::io::prelude::*;
@@ -22,6 +23,9 @@ pub enum ClientError {
     #[error("Server response claimed it had created a chunk, but lacked chunk id")]
     NoCreatedChunkId,
 
+    #[error("Server does not have {0}")]
+    NotFound(String),
+
     #[error("Server does not have chunk {0}")]
     ChunkNotFound(String),
 
@@ -34,6 +38,12 @@ pub enum ClientError {
     #[error("Wrong checksum for chunk {0}, got {1}, expected {2}")]
     WrongChecksum(ChunkId, String, String),
 
+    #[error(transparent)]
+    ClientConfigError(#[from] ClientConfigError),
+
+    #[error(transparent)]
+    CipherError(#[from] CipherError),
+
     #[error(transparent)]
     GenerationChunkError(#[from] GenerationChunkError),
 
@@ -184,11 +194,13 @@ impl BackupClient {
 pub struct ChunkClient {
     client: Client,
     base_url: String,
+    cipher: CipherEngine,
 }
 
 impl ChunkClient {
     pub fn new(config: &ClientConfig) -> ClientResult<Self> {
-        let config = config.config();
+        let pass = config.passwords()?;
+
         let client = Client::builder()
             .danger_accept_invalid_certs(!config.verify_tls_cert)
             .build()
@@ -196,6 +208,7 @@ impl ChunkClient {
         Ok(Self {
             client,
             base_url: config.server_url.to_string(),
+            cipher: CipherEngine::new(&pass),
         })
     }
 
@@ -208,44 +221,30 @@ impl ChunkClient {
     }
 
     pub fn has_chunk(&self, meta: &ChunkMeta) -> ClientResult<Option<ChunkId>> {
-        trace!("has_chunk: url={:?}", self.base_url());
-        let req = self
-            .client
-            .get(&self.chunks_url())
-            .query(&[("sha256", meta.sha256())])
-            .build()
-            .map_err(ClientError::ReqwestError)?;
+        let body = match self.get("", &[("sha256", meta.sha256())]) {
+            Ok((_, body)) => body,
+            Err(err) => return Err(err),
+        };
 
-        let res = self.client.execute(req).map_err(ClientError::ChunkExists)?;
-        debug!("has_chunk: status={}", res.status());
-        let has = if res.status() != 200 {
-            debug!("has_chunk: error from server");
-            None
+        let hits: HashMap<String, ChunkMeta> =
+            serde_json::from_slice(&body).map_err(ClientError::JsonParse)?;
+        let mut iter = hits.iter();
+        let has = if let Some((chunk_id, _)) = iter.next() {
+            Some(chunk_id.into())
         } else {
-            let text = res.text().map_err(ClientError::ReqwestError)?;
-            debug!("has_chunk: text={:?}", text);
-            let hits: HashMap<String, ChunkMeta> =
-                serde_json::from_str(&text).map_err(ClientError::JsonParse)?;
-            debug!("has_chunk: hits={:?}", hits);
-            let mut iter = hits.iter();
-            if let Some((chunk_id, _)) = iter.next() {
-                debug!("has_chunk: chunk_id={:?}", chunk_id);
-                Some(chunk_id.into())
-            } else {
-                None
-            }
+            None
         };
 
-        info!("has_chunk result: {:?}", has);
         Ok(has)
     }
 
     pub fn upload_chunk(&self, chunk: DataChunk) -> ClientResult<ChunkId> {
+        let enc = self.cipher.encrypt_chunk(&chunk)?;
         let res = self
             .client
             .post(&self.chunks_url())
             .header("chunk-meta", chunk.meta().to_json())
-            .body(chunk.data().to_vec())
+            .body(enc.ciphertext().to_vec())
             .send()
             .map_err(ClientError::ReqwestError)?;
         debug!("upload_chunk: res={:?}", res);
@@ -261,20 +260,8 @@ impl ChunkClient {
     }
 
     pub fn list_generations(&self) -> ClientResult<GenerationList> {
-        let url = format!("{}?generation=true", &self.chunks_url());
-        trace!("list_generations: url={:?}", url);
-        let req = self
-            .client
-            .get(&url)
-            .build()
-            .map_err(ClientError::ReqwestError)?;
-        let res = self
-            .client
-            .execute(req)
-            .map_err(ClientError::ReqwestError)?;
-        debug!("list_generations: status={}", res.status());
-        let body = res.bytes().map_err(ClientError::ReqwestError)?;
-        debug!("list_generations: body={:?}", body);
+        let (_, body) = self.get("", &[("generation", "true")])?;
+
         let map: HashMap<String, ChunkMeta> =
             serde_yaml::from_slice(&body).map_err(ClientError::YamlParse)?;
         debug!("list_generations: map={:?}", map);
@@ -286,52 +273,65 @@ impl ChunkClient {
     }
 
     pub fn fetch_chunk(&self, chunk_id: &ChunkId) -> ClientResult<DataChunk> {
-        info!("fetch chunk {}", chunk_id);
+        let (headers, body) = self.get(&format!("/{}", chunk_id), &[])?;
+        let meta = self.get_chunk_meta_header(chunk_id, &headers)?;
+
+        let meta_bytes = meta.to_json_vec();
+        let chunk = self.cipher.decrypt_chunk(&body, &meta_bytes)?;
 
-        let url = format!("{}/{}", &self.chunks_url(), chunk_id);
+        Ok(chunk)
+    }
+
+    fn get(&self, path: &str, query: &[(&str, &str)]) -> ClientResult<(HeaderMap, Vec<u8>)> {
+        let url = format!("{}{}", &self.chunks_url(), path);
+        info!("GET {}", url);
+
+        // Build HTTP request structure.
         let req = self
             .client
             .get(&url)
+            .query(query)
             .build()
             .map_err(ClientError::ReqwestError)?;
+
+        // Make HTTP request.
         let res = self
             .client
             .execute(req)
             .map_err(ClientError::ReqwestError)?;
+
+        // Did it work?
         if res.status() != 200 {
-            let err = ClientError::ChunkNotFound(chunk_id.to_string());
-            error!("fetching chunk {} failed: {}", chunk_id, err);
-            return Err(err);
+            return Err(ClientError::NotFound(path.to_string()));
         }
 
-        let headers = res.headers();
+        // Return headers and body.
+        let headers = res.headers().clone();
+        let body = res.bytes().map_err(ClientError::ReqwestError)?;
+        let body = body.to_vec();
+        Ok((headers, body))
+    }
+
+    fn get_chunk_meta_header(
+        &self,
+        chunk_id: &ChunkId,
+        headers: &HeaderMap,
+    ) -> ClientResult<ChunkMeta> {
         let meta = headers.get("chunk-meta");
+
         if meta.is_none() {
             let err = ClientError::NoChunkMeta(chunk_id.clone());
             error!("fetching chunk {} failed: {}", chunk_id, err);
             return Err(err);
         }
+
         let meta = meta
             .unwrap()
             .to_str()
             .map_err(ClientError::MetaHeaderToString)?;
-        debug!("fetching chunk {}: meta={:?}", chunk_id, meta);
         let meta: ChunkMeta = serde_json::from_str(meta).map_err(ClientError::JsonParse)?;
-        debug!("fetching chunk {}: meta={:?}", chunk_id, meta);
-
-        let body = res.bytes().map_err(ClientError::ReqwestError)?;
-        let body = body.to_vec();
-        let actual = sha256(&body);
-        if actual != meta.sha256() {
-            let err =
-                ClientError::WrongChecksum(chunk_id.clone(), actual, meta.sha256().to_string());
-            error!("fetching chunk {} failed: {}", chunk_id, err);
-            return Err(err);
-        }
-
-        let chunk: DataChunk = DataChunk::new(body, meta);
 
-        Ok(chunk)
+        Ok(meta)
     }
 }
 
-- 
cgit v1.2.1


From cccb998288d05273d709cda75766928e052cca2a Mon Sep 17 00:00:00 2001
From: Lars Wirzenius <liw@liw.fi>
Date: Sat, 29 May 2021 11:40:03 +0300
Subject: feat: add subcommands for encrypting, decrypting individual chunks

---
 src/cmd/chunk.rs  | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 src/cmd/mod.rs    |  1 +
 subplot/data.py   | 16 ++++++++++++++
 subplot/data.yaml |  6 ++++++
 4 files changed, 87 insertions(+)
 create mode 100644 src/cmd/chunk.rs

diff --git a/src/cmd/chunk.rs b/src/cmd/chunk.rs
new file mode 100644
index 0000000..e0e91b1
--- /dev/null
+++ b/src/cmd/chunk.rs
@@ -0,0 +1,64 @@
+use crate::chunk::DataChunk;
+use crate::chunkmeta::ChunkMeta;
+use crate::cipher::CipherEngine;
+use crate::config::ClientConfig;
+use crate::error::ObnamError;
+use std::path::PathBuf;
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+pub struct EncryptChunk {
+    #[structopt(parse(from_os_str))]
+    filename: PathBuf,
+
+    #[structopt(parse(from_os_str))]
+    output: PathBuf,
+
+    #[structopt()]
+    json: String,
+}
+
+impl EncryptChunk {
+    pub fn run(&self, config: &ClientConfig) -> Result<(), ObnamError> {
+        let pass = config.passwords()?;
+        let cipher = CipherEngine::new(&pass);
+
+        let meta = ChunkMeta::from_json(&self.json)?;
+
+        let cleartext = std::fs::read(&self.filename)?;
+        let chunk = DataChunk::new(cleartext, meta);
+        let encrypted = cipher.encrypt_chunk(&chunk)?;
+
+        std::fs::write(&self.output, encrypted.ciphertext())?;
+
+        Ok(())
+    }
+}
+
+#[derive(Debug, StructOpt)]
+pub struct DecryptChunk {
+    #[structopt(parse(from_os_str))]
+    filename: PathBuf,
+
+    #[structopt(parse(from_os_str))]
+    output: PathBuf,
+
+    #[structopt()]
+    json: String,
+}
+
+impl DecryptChunk {
+    pub fn run(&self, config: &ClientConfig) -> Result<(), ObnamError> {
+        let pass = config.passwords()?;
+        let cipher = CipherEngine::new(&pass);
+
+        let meta = ChunkMeta::from_json(&self.json)?;
+
+        let encrypted = std::fs::read(&self.filename)?;
+        let chunk = cipher.decrypt_chunk(&encrypted, &meta.to_json_vec())?;
+
+        std::fs::write(&self.output, chunk.data())?;
+
+        Ok(())
+    }
+}
diff --git a/src/cmd/mod.rs b/src/cmd/mod.rs
index 890e176..bd101da 100644
--- a/src/cmd/mod.rs
+++ b/src/cmd/mod.rs
@@ -1,4 +1,5 @@
 pub mod backup;
+pub mod chunk;
 pub mod get_chunk;
 pub mod init;
 pub mod list;
diff --git a/subplot/data.py b/subplot/data.py
index d134e5f..13b6d2b 100644
--- a/subplot/data.py
+++ b/subplot/data.py
@@ -170,3 +170,19 @@ def file_is_readable_by_owner(ctx, filename=None):
 def file_does_not_contain(ctx, filename=None, pattern=None):
     data = open(filename).read()
     assert pattern not in data
+
+
+def files_are_different(ctx, filename1=None, filename2=None):
+    assert_ne = globals()["assert_ne"]
+
+    data1 = open(filename1, "rb").read()
+    data2 = open(filename2, "rb").read()
+    assert_ne(data1, data2)
+
+
+def files_are_identical(ctx, filename1=None, filename2=None):
+    assert_eq = globals()["assert_eq"]
+
+    data1 = open(filename1, "rb").read()
+    data2 = open(filename2, "rb").read()
+    assert_eq(data1, data2)
diff --git a/subplot/data.yaml b/subplot/data.yaml
index dcc6807..41a563f 100644
--- a/subplot/data.yaml
+++ b/subplot/data.yaml
@@ -48,3 +48,9 @@
 
 - then: "file {filename} does not contain \"{pattern:text}\""
   function: file_does_not_contain
+
+- then: "files {filename1} and {filename2} are different"
+  function: files_are_different
+
+- then: "files {filename1} and {filename2} are identical"
+  function: files_are_identical
-- 
cgit v1.2.1


From 9c2590d2428f0d3de882686ec2ec5832e7123c62 Mon Sep 17 00:00:00 2001
From: Lars Wirzenius <liw@liw.fi>
Date: Sat, 29 May 2021 11:38:06 +0300
Subject: test: update subplot to use new encryption stuff

---
 obnam.md | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/obnam.md b/obnam.md
index b9ae57a..7fa4a19 100644
--- a/obnam.md
+++ b/obnam.md
@@ -994,7 +994,7 @@ when I POST data.dat to /chunks, with chunk-meta: {"sha256":"abc"}
 then HTTP status code is 201
 and content-type is application/json
 and the JSON body has a field chunk_id, henceforth ID
-and server has 1 file chunks
+and server has 1 chunks
 ~~~
 
 We must be able to retrieve it.
@@ -1136,7 +1136,6 @@ then stdout, as JSON, matches file config.json
 roots: [live]
 server_url: https://backup.example.com
 verify_tls_cert: true
-encrypt: false
 ~~~
 
 
@@ -1159,7 +1158,6 @@ roots: [~/important]
 log: ~/obnam.log
 server_url: https://backup.example.com
 verify_tls_cert: true
-encrypt: false
 ~~~
 
 
@@ -1205,6 +1203,19 @@ roots: [live]
 ~~~
 
 
+## Encrypt and decrypt chunk locally
+
+~~~scenario
+given an installed obnam
+given a running chunk server
+given a client config based on smoke.yaml
+given a file cleartext.dat containing some random data
+when I run obnam encrypt-chunk cleartext.dat encrypted.dat '{"sha256":"fake"}'
+when I run obnam decrypt-chunk encrypted.dat decrypted.dat '{"sha256":"fake"}'
+then files cleartext.dat and encrypted.dat are different
+then files cleartext.dat and decrypted.dat are identical
+~~~
+
 # Acceptance criteria for Obnam as a whole
 
 The scenarios in this chapter apply to Obnam as a whole: the client
@@ -1313,13 +1324,18 @@ This scenario verifies that the user can set the chunk size in the
 configuration file. The chunk size only affects the chunks of live
 data.
 
+The backup uses a chunk size of one byte, and backs up a file with
+three bytes. This results in three chunks for the file data, plus one
+for the generation SQLite file (not split into chunks of one byte),
+plus a chunk for the generation itself. A total of five chunks.
+
 ~~~scenario
 given an installed obnam
 given a running chunk server
 given a client config based on tiny-chunk-size.yaml
 given a file live/data.dat containing "abc"
 when I run obnam backup
-then server has 3 file chunks
+then server has 5 chunks
 ~~~
 
 ~~~{#tiny-chunk-size.yaml .file .yaml .numberLines}
@@ -1636,7 +1652,7 @@ passphrase.
 ~~~scenario
 given an installed obnam
 and a running chunk server
-and a client config based on encryption.yaml
+and a client config, without passphrase, based on encryption.yaml
 and a file live/data.dat containing some random data
 and a manifest of the directory live in live.yaml
 when I try to run obnam backup
@@ -1647,7 +1663,6 @@ then stderr contains "obnam init"
 ~~~{#encryption.yaml .file .yaml .numberLines}
 verify_tls_cert: false
 roots: [live]
-encrypt: true
 ~~~
 
 ## A passphrase can be set
@@ -1658,7 +1673,7 @@ readable by it owner. Verify that a backup can be made.
 ~~~scenario
 given an installed obnam
 and a running chunk server
-and a client config based on encryption.yaml
+and a client config, without passphrase, based on encryption.yaml
 and a file live/data.dat containing some random data
 and a manifest of the directory live in live.yaml
 when I run obnam init --insecure-passphrase=hunter2
-- 
cgit v1.2.1