diff options
author | Lars Wirzenius <liw@liw.fi> | 2021-06-04 06:35:14 +0000 |
---|---|---|
committer | Lars Wirzenius <liw@liw.fi> | 2021-06-04 06:35:14 +0000 |
commit | f2a274ee1291531c1154176bca5b9a47e9c234bd (patch) | |
tree | 6721b515739c6e5f9236ffd6f4ecc2dfecc471d2 | |
parent | cb33088dbedf4b772013f83b8226047cc4355dd2 (diff) | |
parent | 9c2590d2428f0d3de882686ec2ec5832e7123c62 (diff) | |
download | obnam2-f2a274ee1291531c1154176bca5b9a47e9c234bd.tar.gz |
Merge branch 'aead' into 'main'
add encryption of individual chunks
Closes #110
See merge request larswirzenius/obnam!146
-rw-r--r-- | Cargo.lock | 450 | ||||
-rw-r--r-- | Cargo.toml | 1 | ||||
-rw-r--r-- | obnam.md | 29 | ||||
-rw-r--r-- | src/backup_run.rs | 2 | ||||
-rw-r--r-- | src/benchmark.rs | 6 | ||||
-rw-r--r-- | src/bin/benchmark-index.rs | 7 | ||||
-rw-r--r-- | src/bin/benchmark-indexedstore.rs | 8 | ||||
-rw-r--r-- | src/bin/benchmark-null.rs | 2 | ||||
-rw-r--r-- | src/bin/benchmark-store.rs | 4 | ||||
-rw-r--r-- | src/bin/obnam-server.rs | 8 | ||||
-rw-r--r-- | src/bin/obnam.rs | 39 | ||||
-rw-r--r-- | src/chunk.rs | 24 | ||||
-rw-r--r-- | src/chunker.rs | 12 | ||||
-rw-r--r-- | src/chunkmeta.rs | 21 | ||||
-rw-r--r-- | src/cipher.rs | 200 | ||||
-rw-r--r-- | src/client.rs | 291 | ||||
-rw-r--r-- | src/cmd/backup.rs | 2 | ||||
-rw-r--r-- | src/cmd/chunk.rs | 64 | ||||
-rw-r--r-- | src/cmd/init.rs | 8 | ||||
-rw-r--r-- | src/cmd/mod.rs | 1 | ||||
-rw-r--r-- | src/cmd/show_config.rs | 2 | ||||
-rw-r--r-- | src/config.rs | 92 | ||||
-rw-r--r-- | src/error.rs | 4 | ||||
-rw-r--r-- | src/indexedstore.rs | 31 | ||||
-rw-r--r-- | src/lib.rs | 1 | ||||
-rw-r--r-- | src/passwords.rs | 15 | ||||
-rw-r--r-- | src/store.rs | 12 | ||||
-rw-r--r-- | subplot/client.py | 11 | ||||
-rw-r--r-- | subplot/client.yaml | 5 | ||||
-rw-r--r-- | subplot/data.py | 16 | ||||
-rw-r--r-- | subplot/data.yaml | 6 | ||||
-rw-r--r-- | subplot/server.py | 18 | ||||
-rw-r--r-- | subplot/server.yaml | 4 |
33 files changed, 902 insertions, 494 deletions
@@ -1,6 +1,41 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. [[package]] +name = "aead" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "922b33332f54fc0ad13fa3e514601e8d30fb54e1f3eadc36643f6526db645621" +dependencies = [ + "generic-array", +] + +[[package]] +name = "aes" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2333ac5777aaa1beb8589f5374976ae7dc8aa4f09fd21ae3d8662ca97f5247d" +dependencies = [ + "cfg-if 1.0.0", + "cipher", + "cpufeatures", + "opaque-debug", +] + +[[package]] +name = "aes-gcm" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ee2263805ba4537ccbb19db28525a7b1ebc7284c228eb5634c3124ca63eb03f" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + +[[package]] name = "ahash" version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -8,9 +43,9 @@ checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" [[package]] name = "aho-corasick" -version = "0.7.15" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" dependencies = [ "memchr", ] @@ -26,9 +61,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.39" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cddc5f91628367664cc7c69714ff08deee8a3efc54623011c772544d7b2767" +checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b" [[package]] name = "arc-swap" @@ -98,9 +133,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.6.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" +checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" [[package]] name = "byteorder" @@ -128,9 +163,9 @@ checksum = "81a18687293a1546b67c246452202bbbf143d239cb43494cc163da14979082da" [[package]] name = "cc" -version = "1.0.67" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" +checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787" [[package]] name = "cfg-if" @@ -158,6 +193,15 @@ dependencies = [ ] [[package]] +name = "cipher" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" +dependencies = [ + "generic-array", +] + +[[package]] name = "clap" version = "2.33.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -204,10 +248,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" [[package]] -name = "cpuid-bool" -version = "0.1.2" +name = "cpufeatures" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" +checksum = "ed00c67cb5d0a7d64a44f6ad2668db7e7530311dd53ea79bcd4fb022c64911c8" +dependencies = [ + "libc", +] [[package]] name = "crypto-mac" @@ -220,6 +267,15 @@ dependencies = [ ] [[package]] +name = "ctr" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a232f92a03f37dd7d7dd2adc67166c77e9cd88de5b019b9a9eecfaeaf7bfd481" +dependencies = [ + "cipher", +] + +[[package]] name = "derivative" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -262,9 +318,9 @@ dependencies = [ [[package]] name = "dtoa" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d7ed2934d741c6b37e33e3832298e8850b53fd2d2bea03873375596c7cea4e" +checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" [[package]] name = "encode_unicode" @@ -355,9 +411,9 @@ checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" [[package]] name = "futures" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f55667319111d593ba876406af7c409c0ebb44dc4be6132a783ccf163ea14c1" +checksum = "0e7e43a803dae2fa37c1f6a8fe121e1f7bf9548b4dfc0522a42f34145dadfc27" dependencies = [ "futures-channel", "futures-core", @@ -369,9 +425,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c2dd2df839b57db9ab69c2c9d8f3e8c81984781937fe2807dc6dcf3b2ad2939" +checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2" dependencies = [ "futures-core", "futures-sink", @@ -379,34 +435,35 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15496a72fabf0e62bdc3df11a59a3787429221dd0710ba8ef163d6f7a9112c94" +checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1" [[package]] name = "futures-io" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71c2c65c57704c32f5241c1223167c2c3294fd34ac020c807ddbe6db287ba59" +checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1" [[package]] name = "futures-sink" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85754d98985841b7d4f5e8e6fbfa4a4ac847916893ec511a2917ccd8525b8bb3" +checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282" [[package]] name = "futures-task" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa189ef211c15ee602667a6fcfe1c1fd9e07d42250d2156382820fba33c9df80" +checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae" [[package]] name = "futures-util" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1812c7ab8aedf8d6f2701a43e1243acdbcc2b36ab26e2ad421eb99ac963d96d1" +checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967" dependencies = [ + "autocfg", "futures-core", "futures-io", "futures-sink", @@ -440,9 +497,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" dependencies = [ "cfg-if 1.0.0", "libc", @@ -450,6 +507,16 @@ dependencies = [ ] [[package]] +name = "ghash" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6fb2a26dd2ebd268a68bc8e9acc9e67e487952f33384055a1cbe697514c64e" +dependencies = [ + "opaque-debug", + "polyval", +] + +[[package]] name = "h2" version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -471,9 +538,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d832b01df74254fe364568d6ddc294443f61cbec82816b60904303af87efae78" +checksum = "825343c4eef0b63f541f8903f395dc5beb362a979b5799a84062527ef1e37726" dependencies = [ "bytes 1.0.1", "fnv", @@ -483,8 +550,8 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 1.4.0", - "tokio-util 0.6.5", + "tokio 1.6.1", + "tokio-util 0.6.7", "tracing", ] @@ -561,9 +628,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" +checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11" dependencies = [ "bytes 1.0.1", "fnv", @@ -582,9 +649,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfb77c123b4e2f72a2069aeae0b4b4949cc7e966df277813fc16347e7549737" +checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9" dependencies = [ "bytes 1.0.1", "http", @@ -593,9 +660,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.3.5" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "615caabe2c3160b313d52ccc905335f4ed5f10881dd63dc5699d47e90be85691" +checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68" [[package]] name = "httpdate" @@ -604,6 +671,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" [[package]] +name = "httpdate" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440" + +[[package]] name = "humantime" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -632,10 +705,10 @@ dependencies = [ "http", "http-body 0.3.1", "httparse", - "httpdate", + "httpdate 0.3.2", "itoa", - "pin-project 1.0.5", - "socket2", + "pin-project 1.0.7", + "socket2 0.3.19", "tokio 0.2.25", "tower-service", "tracing", @@ -644,23 +717,23 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.4" +version = "0.14.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8e946c2b1349055e0b72ae281b238baf1a3ea7307c7e9f9d64673bdd9c26ac7" +checksum = "d3f71a7eea53a3f8257a7b4795373ff886397178cd634430ea94e12d7fe4fe34" dependencies = [ "bytes 1.0.1", "futures-channel", "futures-core", "futures-util", - "h2 0.3.1", + "h2 0.3.3", "http", - "http-body 0.4.1", + "http-body 0.4.2", "httparse", - "httpdate", + "httpdate 1.0.1", "itoa", - "pin-project 1.0.5", - "socket2", - "tokio 1.4.0", + "pin-project 1.0.7", + "socket2 0.4.0", + "tokio 1.6.1", "tower-service", "tracing", "want", @@ -673,17 +746,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes 1.0.1", - "hyper 0.14.4", + "hyper 0.14.8", "native-tls", - "tokio 1.4.0", + "tokio 1.6.1", "tokio-native-tls", ] [[package]] name = "idna" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89829a5d69c23d348314a7ac337fe39173b61149a9864deabd260983aed48c21" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" dependencies = [ "matches", "unicode-bidi", @@ -753,9 +826,9 @@ checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" [[package]] name = "js-sys" -version = "0.3.49" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc15e39392125075f60c95ba416f5381ff6c3a948ff02ab12464715adf56c821" +checksum = "83bdfbace3a0e81a4253f73b49e960b053e396a11012cbd49b9b74d6a2b67062" dependencies = [ "wasm-bindgen", ] @@ -778,9 +851,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.90" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba4aede83fc3617411dc6993bc8c70919750c1c257c6ca6a502aed6e0e2394ae" +checksum = "789da6d93f1b866ffe175afc5322a4d76c038605a1c3319bb57b06967ca98a36" [[package]] name = "libsqlite3-sys" @@ -800,9 +873,9 @@ checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" [[package]] name = "lock_api" -version = "0.4.2" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" +checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb" dependencies = [ "scopeguard", ] @@ -858,9 +931,9 @@ checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] name = "memchr" -version = "2.3.4" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" [[package]] name = "mime" @@ -1026,6 +1099,7 @@ checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" name = "obnam" version = "0.3.1" dependencies = [ + "aes-gcm", "anyhow", "bytesize", "chrono", @@ -1068,9 +1142,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.33" +version = "0.10.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a61075b62a23fef5a29815de7536d940aa35ce96d18ce0cc5076272db678a577" +checksum = "6d7830286ad6a3973c0f1d9b73738f69c76b739301d0229c4b96501695cbe4c8" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -1082,15 +1156,15 @@ dependencies = [ [[package]] name = "openssl-probe" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" +checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" [[package]] name = "openssl-sys" -version = "0.9.61" +version = "0.9.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "313752393519e876837e09e1fa183ddef0be7735868dced3196f4472d536277f" +checksum = "b6b0d6fb7d80f877617dfcb014e605e2b5ab2fb0afdf27935219bb6bd984cb98" dependencies = [ "autocfg", "cc", @@ -1101,11 +1175,12 @@ dependencies = [ [[package]] name = "ordered-float" -version = "2.1.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "766f840da25490628d8e63e529cd21c014f6600c6b8517add12a6fa6167a6218" +checksum = "809348965973b261c3e504c8d0434e465274f78c880e10039914f2c5dcf49461" dependencies = [ "num-traits", + "rand 0.8.3", ] [[package]] @@ -1128,16 +1203,16 @@ dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall 0.2.5", + "redox_syscall 0.2.8", "smallvec", "winapi 0.3.9", ] [[package]] name = "password-hash" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85d8faea6c018131952a192ee55bd9394c51fc6f63294b668d97636e6f842d40" +checksum = "54986aa4bfc9b98c6a5f40184223658d187159d7b3c6af33f2b2aa25ae1db0fa" dependencies = [ "base64ct", "rand_core 0.6.2", @@ -1164,27 +1239,27 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pin-project" -version = "0.4.27" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" +checksum = "918192b5c59119d51e0cd221f4d49dde9112824ba717369e903c97d076083d0f" dependencies = [ - "pin-project-internal 0.4.27", + "pin-project-internal 0.4.28", ] [[package]] name = "pin-project" -version = "1.0.5" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96fa8ebb90271c4477f144354485b8068bd8f6b78b428b01ba892ca26caf0b63" +checksum = "c7509cc106041c40a4518d2af7a61530e1eed0e6285296a3d8c5472806ccc4a4" dependencies = [ - "pin-project-internal 1.0.5", + "pin-project-internal 1.0.7", ] [[package]] name = "pin-project-internal" -version = "0.4.27" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" +checksum = "3be26700300be6d9d23264c73211d8190e755b6b5ca7a1b28230025511b52a5e" dependencies = [ "proc-macro2", "quote", @@ -1193,9 +1268,9 @@ dependencies = [ [[package]] name = "pin-project-internal" -version = "1.0.5" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758669ae3558c6f74bd2a18b41f7ac0b5a195aea6639d6a9b5e5d1ad5ba24c0b" +checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f" dependencies = [ "proc-macro2", "quote", @@ -1227,6 +1302,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" [[package]] +name = "polyval" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "864231b0b86ce05168a8e6da0fea2e67275dacf25f75b00a62cfd341aab904a9" +dependencies = [ + "cpufeatures", + "opaque-debug", + "universal-hash", +] + +[[package]] name = "ppv-lite86" version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1268,9 +1354,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.24" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038" dependencies = [ "unicode-xid", ] @@ -1350,7 +1436,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" dependencies = [ - "getrandom 0.2.2", + "getrandom 0.2.3", ] [[package]] @@ -1379,9 +1465,9 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_syscall" -version = "0.2.5" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc" dependencies = [ "bitflags", ] @@ -1392,15 +1478,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" dependencies = [ - "getrandom 0.2.2", - "redox_syscall 0.2.5", + "getrandom 0.2.3", + "redox_syscall 0.2.8", ] [[package]] name = "regex" -version = "1.4.5" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" dependencies = [ "aho-corasick", "memchr", @@ -1409,9 +1495,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.23" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" [[package]] name = "remove_dir_all" @@ -1424,9 +1510,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf12057f289428dbf5c591c74bf10392e4a8003f993405a902f20117019022d4" +checksum = "2296f2fac53979e8ccbc4a1136b25dcefd37be9ed7e4a1f6b05a6029c84ff124" dependencies = [ "base64 0.13.0", "bytes 1.0.1", @@ -1434,8 +1520,8 @@ dependencies = [ "futures-core", "futures-util", "http", - "http-body 0.4.1", - "hyper 0.14.4", + "http-body 0.4.2", + "hyper 0.14.8", "hyper-tls", "ipnet", "js-sys", @@ -1448,7 +1534,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded 0.7.0", - "tokio 1.4.0", + "tokio 1.6.1", "tokio-native-tls", "url", "wasm-bindgen", @@ -1555,9 +1641,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "sct" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" dependencies = [ "ring", "untrusted", @@ -1565,9 +1651,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d493c5f39e02dfb062cd8f33301f90f9b13b650e8c1b1d0fd75c19dd64bff69d" +checksum = "3670b1d2fdf6084d192bc71ead7aabe6c06aa2ea3fbd9cc3ac111fa5c2b1bd84" dependencies = [ "bitflags", "core-foundation", @@ -1578,9 +1664,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee48cdde5ed250b0d3252818f646e174ab414036edb884dde62d80a3ac6082d" +checksum = "3676258fd3cfe2c9a0ec99ce3038798d847ce3e4bb17746373eb9f0f1ac16339" dependencies = [ "core-foundation-sys", "libc", @@ -1588,9 +1674,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.124" +version = "1.0.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd761ff957cb2a45fbb9ab3da6512de9de55872866160b23c25f1a841e99d29f" +checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03" dependencies = [ "serde_derive", ] @@ -1607,9 +1693,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.124" +version = "1.0.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1800f7693e94e186f5e25a28291ae1570da908aff7d97a095dec1e56ff99069b" +checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43" dependencies = [ "proc-macro2", "quote", @@ -1665,35 +1751,35 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.9.4" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfebf75d25bd900fd1e7d11501efab59bc846dbc76196839663e6637bba9f25f" +checksum = "8c4cfa741c5832d0ef7fab46cabed29c2aae926db0b11bb2069edd8db5e64e16" dependencies = [ "block-buffer", "cfg-if 1.0.0", - "cpuid-bool", + "cpufeatures", "digest", "opaque-debug", ] [[package]] name = "sha2" -version = "0.9.3" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de" +checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12" dependencies = [ "block-buffer", "cfg-if 1.0.0", - "cpuid-bool", + "cpufeatures", "digest", "opaque-debug", ] [[package]] name = "slab" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527" [[package]] name = "smallvec" @@ -1713,6 +1799,16 @@ dependencies = [ ] [[package]] +name = "socket2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] name = "spin" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1756,9 +1852,9 @@ checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" [[package]] name = "syn" -version = "1.0.64" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fd9d1e9976102a03c542daa2eff1b43f9d72306342f3f8b3ed5fb8908195d6f" +checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82" dependencies = [ "proc-macro2", "quote", @@ -1774,7 +1870,7 @@ dependencies = [ "cfg-if 1.0.0", "libc", "rand 0.8.3", - "redox_syscall 0.2.5", + "redox_syscall 0.2.8", "remove_dir_all", "winapi 0.3.9", ] @@ -1790,9 +1886,9 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ca8ced750734db02076f44132d802af0b33b09942331f4459dde8636fd2406" +checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" dependencies = [ "libc", "winapi 0.3.9", @@ -1809,18 +1905,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" +checksum = "fa6f76457f59514c7eeb4e59d891395fab0b2fd1d40723ae737d64153392e9c6" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" +checksum = "8a36768c0fbf1bb15eca10defa29526bda730a2376c2ab4393ccfa16fb1a318d" dependencies = [ "proc-macro2", "quote", @@ -1850,9 +1946,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317cca572a0e89c3ce0ca1f1bdc9369547fe318a683418e42ac8f59d14701023" +checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342" dependencies = [ "tinyvec_macros", ] @@ -1883,9 +1979,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.4.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134af885d758d645f0f0505c9a8b3f9bf8a348fd822e112ab5248138348f1722" +checksum = "0a38d31d7831c6ed7aad00aa4c12d9375fd225a6dd77da1d25b707346319a975" dependencies = [ "autocfg", "bytes 1.0.1", @@ -1914,7 +2010,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" dependencies = [ "native-tls", - "tokio 1.4.0", + "tokio 1.6.1", ] [[package]] @@ -1937,7 +2033,7 @@ checksum = "6d9e878ad426ca286e4dcae09cbd4e1973a7f8987d97570e2469703dd7f5720c" dependencies = [ "futures-util", "log", - "pin-project 0.4.27", + "pin-project 0.4.28", "tokio 0.2.25", "tungstenite", ] @@ -1958,16 +2054,16 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.6.5" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5143d049e85af7fbc36f5454d990e62c2df705b3589f123b71f441b6b59f443f" +checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" dependencies = [ "bytes 1.0.1", "futures-core", "futures-sink", "log", "pin-project-lite 0.2.6", - "tokio 1.4.0", + "tokio 1.6.1", ] [[package]] @@ -1978,9 +2074,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" +checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" dependencies = [ "cfg-if 1.0.0", "log", @@ -1990,9 +2086,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052" dependencies = [ "lazy_static", ] @@ -2003,7 +2099,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" dependencies = [ - "pin-project 1.0.5", + "pin-project 1.0.7", "tracing", ] @@ -2073,18 +2169,18 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0" dependencies = [ "matches", ] [[package]] name = "unicode-normalization" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" +checksum = "33717dca7ac877f497014e10d73f3acf948c342bee31b5ca7892faf94ccc6b49" dependencies = [ "tinyvec", ] @@ -2103,9 +2199,19 @@ checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" [[package]] name = "unicode-xid" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "universal-hash" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402" +dependencies = [ + "generic-array", + "subtle", +] [[package]] name = "unsafe-any" @@ -2124,9 +2230,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.2.1" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" dependencies = [ "form_urlencoded", "idna", @@ -2136,9 +2242,9 @@ dependencies = [ [[package]] name = "urlencoding" -version = "1.1.1" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9232eb53352b4442e40d7900465dfc534e8cb2dc8f18656fcb2ac16112b5593" +checksum = "5a1f0175e03a0973cf4afd476bef05c26e228520400eb1fd473ad417b1c00ffb" [[package]] name = "users" @@ -2152,9 +2258,9 @@ dependencies = [ [[package]] name = "utf-8" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05e42f7c18b8f902290b009cde6d651262f956c98bc51bca4cd1d511c9cd85c7" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] name = "uuid" @@ -2162,14 +2268,14 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ - "getrandom 0.2.2", + "getrandom 0.2.3", ] [[package]] name = "vcpkg" -version = "0.2.11" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb" +checksum = "025ce40a007e1907e58d5bc1a594def78e5573bb0b1160bc389634e8f12e4faa" [[package]] name = "vec_map" @@ -2185,9 +2291,9 @@ checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" [[package]] name = "walkdir" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" dependencies = [ "same-file", "winapi 0.3.9", @@ -2219,7 +2325,7 @@ dependencies = [ "mime", "mime_guess", "multipart", - "pin-project 0.4.27", + "pin-project 0.4.28", "scoped-tls", "serde", "serde_json", @@ -2247,9 +2353,9 @@ checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" [[package]] name = "wasm-bindgen" -version = "0.2.72" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fe8f61dba8e5d645a4d8132dc7a0a66861ed5e1045d2c0ed940fab33bac0fbe" +checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd" dependencies = [ "cfg-if 1.0.0", "serde", @@ -2259,9 +2365,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.72" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046ceba58ff062da072c7cb4ba5b22a37f00a302483f7e2a6cdc18fedbdc1fd3" +checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900" dependencies = [ "bumpalo", "lazy_static", @@ -2274,9 +2380,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.22" +version = "0.4.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73157efb9af26fb564bb59a009afd1c7c334a44db171d280690d0c3faaec3468" +checksum = "5fba7978c679d53ce2d0ac80c8c175840feb849a161664365d1287b41f2e67f1" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -2286,9 +2392,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.72" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef9aa01d36cda046f797c57959ff5f3c615c9cc63997a8d545831ec7976819b" +checksum = "088169ca61430fe1e58b8096c24975251700e7b1f6fd91cc9d59b04fb9b18bd4" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2296,9 +2402,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.72" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96eb45c1b2ee33545a813a92dbb53856418bf7eb54ab34f7f7ff1448a5b3735d" +checksum = "be2241542ff3d9f241f5e2cb6dd09b37efe786df8851c54957683a49f0987a97" dependencies = [ "proc-macro2", "quote", @@ -2309,15 +2415,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.72" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7148f4696fb4960a346eaa60bbfb42a1ac4ebba21f750f75fc1375b098d5ffa" +checksum = "d7cff876b8f18eed75a66cf49b65e7f967cb354a7aa16003fb55dbfd25b44b4f" [[package]] name = "web-sys" -version = "0.3.49" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59fe19d70f5dacc03f6e46777213facae5ac3801575d56ca6cbd4c93dcd12310" +checksum = "e828417b379f3df7111d3a2a9e5753706cae29c41f7c4029ee9fd77f3e09e582" dependencies = [ "js-sys", "wasm-bindgen", @@ -10,6 +10,7 @@ repository = "https://gitlab.com/larswirzenius/obnam" [dependencies] +aes-gcm = "0.9.1" anyhow = "1" bytesize = "1" chrono = "0.4" @@ -994,7 +994,7 @@ when I POST data.dat to /chunks, with chunk-meta: {"sha256":"abc"} then HTTP status code is 201 and content-type is application/json and the JSON body has a field chunk_id, henceforth ID -and server has 1 file chunks +and server has 1 chunks ~~~ We must be able to retrieve it. @@ -1136,7 +1136,6 @@ then stdout, as JSON, matches file config.json roots: [live] server_url: https://backup.example.com verify_tls_cert: true -encrypt: false ~~~ @@ -1159,7 +1158,6 @@ roots: [~/important] log: ~/obnam.log server_url: https://backup.example.com verify_tls_cert: true -encrypt: false ~~~ @@ -1205,6 +1203,19 @@ roots: [live] ~~~ +## Encrypt and decrypt chunk locally + +~~~scenario +given an installed obnam +given a running chunk server +given a client config based on smoke.yaml +given a file cleartext.dat containing some random data +when I run obnam encrypt-chunk cleartext.dat encrypted.dat '{"sha256":"fake"}' +when I run obnam decrypt-chunk encrypted.dat decrypted.dat '{"sha256":"fake"}' +then files cleartext.dat and encrypted.dat are different +then files cleartext.dat and decrypted.dat are identical +~~~ + # Acceptance criteria for Obnam as a whole The scenarios in this chapter apply to Obnam as a whole: the client @@ -1313,13 +1324,18 @@ This scenario verifies that the user can set the chunk size in the configuration file. The chunk size only affects the chunks of live data. +The backup uses a chunk size of one byte, and backs up a file with +three bytes. This results in three chunks for the file data, plus one +for the generation SQLite file (not split into chunks of one byte), +plus a chunk for the generation itself. A total of five chunks. + ~~~scenario given an installed obnam given a running chunk server given a client config based on tiny-chunk-size.yaml given a file live/data.dat containing "abc" when I run obnam backup -then server has 3 file chunks +then server has 5 chunks ~~~ ~~~{#tiny-chunk-size.yaml .file .yaml .numberLines} @@ -1636,7 +1652,7 @@ passphrase. ~~~scenario given an installed obnam and a running chunk server -and a client config based on encryption.yaml +and a client config, without passphrase, based on encryption.yaml and a file live/data.dat containing some random data and a manifest of the directory live in live.yaml when I try to run obnam backup @@ -1647,7 +1663,6 @@ then stderr contains "obnam init" ~~~{#encryption.yaml .file .yaml .numberLines} verify_tls_cert: false roots: [live] -encrypt: true ~~~ ## A passphrase can be set @@ -1658,7 +1673,7 @@ readable by it owner. Verify that a backup can be made. ~~~scenario given an installed obnam and a running chunk server -and a client config based on encryption.yaml +and a client config, without passphrase, based on encryption.yaml and a file live/data.dat containing some random data and a manifest of the directory live in live.yaml when I run obnam init --insecure-passphrase=hunter2 diff --git a/src/backup_run.rs b/src/backup_run.rs index 23c97f6..16d6700 100644 --- a/src/backup_run.rs +++ b/src/backup_run.rs @@ -41,7 +41,6 @@ pub type BackupResult<T> = Result<T, BackupError>; impl<'a> InitialBackup<'a> { pub fn new(config: &ClientConfig, client: &'a BackupClient) -> BackupResult<Self> { let progress = BackupProgress::initial(); - let config = config.config(); Ok(Self { client, buffer_size: config.chunk_size, @@ -81,7 +80,6 @@ impl<'a> InitialBackup<'a> { impl<'a> IncrementalBackup<'a> { pub fn new(config: &ClientConfig, client: &'a BackupClient) -> BackupResult<Self> { - let config = config.config(); let policy = BackupPolicy::default(); Ok(Self { client, diff --git a/src/benchmark.rs b/src/benchmark.rs index d214939..3c94f92 100644 --- a/src/benchmark.rs +++ b/src/benchmark.rs @@ -15,7 +15,7 @@ impl ChunkGenerator { } impl Iterator for ChunkGenerator { - type Item = (ChunkId, String, ChunkMeta, DataChunk); + type Item = (ChunkId, String, DataChunk); fn next(&mut self) -> Option<Self::Item> { if self.next >= self.goal { @@ -24,9 +24,9 @@ impl Iterator for ChunkGenerator { let id = ChunkId::recreate(&format!("{}", self.next)); let checksum = id.sha256(); let meta = ChunkMeta::new(&checksum); - let chunk = DataChunk::new(vec![]); + let chunk = DataChunk::new(vec![], meta); self.next += 1; - Some((id, checksum, meta, chunk)) + Some((id, checksum, chunk)) } } } diff --git a/src/bin/benchmark-index.rs b/src/bin/benchmark-index.rs index 9baa327..b5a059c 100644 --- a/src/bin/benchmark-index.rs +++ b/src/bin/benchmark-index.rs @@ -60,7 +60,8 @@ fn create(chunks: &Path, num: u32) -> anyhow::Result<()> { let mut index = Index::new(chunks)?; let gen = ChunkGenerator::new(num); - for (id, _, meta, _) in gen { + for (id, _, chunk) in gen { + let meta = (*chunk.meta()).clone(); index.insert_meta(id, meta)?; } @@ -82,8 +83,8 @@ fn lookup(index: &mut Index, num: u32) -> anyhow::Result<()> { loop { let gen = ChunkGenerator::new(num); - for (_, _, meta, _) in gen { - index.find_by_sha256(&meta.sha256())?; + for (_, _, chunk) in gen { + index.find_by_sha256(&chunk.meta().sha256())?; done += 1; if done >= num { return Ok(()); diff --git a/src/bin/benchmark-indexedstore.rs b/src/bin/benchmark-indexedstore.rs index acc3bd3..5cd3ff1 100644 --- a/src/bin/benchmark-indexedstore.rs +++ b/src/bin/benchmark-indexedstore.rs @@ -60,8 +60,8 @@ fn create(chunks: &Path, num: u32) -> anyhow::Result<()> { let mut store = IndexedStore::new(chunks)?; let gen = ChunkGenerator::new(num); - for (_, _, meta, chunk) in gen { - store.save(&meta, &chunk)?; + for (_, _, chunk) in gen { + store.save(&chunk)?; } Ok(()) @@ -82,8 +82,8 @@ fn lookup(index: &mut IndexedStore, num: u32) -> anyhow::Result<()> { loop { let gen = ChunkGenerator::new(num); - for (_, _, meta, _) in gen { - index.find_by_sha256(&meta.sha256())?; + for (_, _, chunk) in gen { + index.find_by_sha256(&chunk.meta().sha256())?; done += 1; if done >= num { return Ok(()); diff --git a/src/bin/benchmark-null.rs b/src/bin/benchmark-null.rs index 259a837..fc60a77 100644 --- a/src/bin/benchmark-null.rs +++ b/src/bin/benchmark-null.rs @@ -23,5 +23,5 @@ fn main() { let opt = Opt::from_args(); let gen = ChunkGenerator::new(opt.num); - for (_, _, _, _) in gen {} + for (_, _, _) in gen {} } diff --git a/src/bin/benchmark-store.rs b/src/bin/benchmark-store.rs index f7c82b1..7896f9d 100644 --- a/src/bin/benchmark-store.rs +++ b/src/bin/benchmark-store.rs @@ -20,8 +20,8 @@ fn main() -> anyhow::Result<()> { let gen = ChunkGenerator::new(opt.num); let store = Store::new(&opt.chunks); - for (id, _, meta, chunk) in gen { - store.save(&id, &meta, &chunk)?; + for (id, _, chunk) in gen { + store.save(&id, &&chunk)?; } Ok(()) diff --git a/src/bin/obnam-server.rs b/src/bin/obnam-server.rs index 9a6540f..29ea9ff 100644 --- a/src/bin/obnam-server.rs +++ b/src/bin/obnam-server.rs @@ -109,9 +109,9 @@ pub async fn create_chunk( } }; - let chunk = DataChunk::new(data.to_vec()); + let chunk = DataChunk::new(data.to_vec(), meta); - let id = match store.save(&meta, &chunk) { + let id = match store.save(&chunk) { Ok(id) => id, Err(e) => { error!("couldn't save: {}", e); @@ -119,7 +119,7 @@ pub async fn create_chunk( } }; - info!("created chunk {}: {:?}", id, meta); + info!("created chunk {}", id); Ok(ChunkResult::Created(id)) } @@ -155,8 +155,6 @@ pub async fn search_chunks( } if key == "generation" && value == "true" { store.find_generations().expect("SQL lookup failed") - } else if key == "data" && value == "true" { - store.find_file_chunks().expect("SQL lookup failed") } else if key == "sha256" { store.find_by_sha256(value).expect("SQL lookup failed") } else { diff --git a/src/bin/obnam.rs b/src/bin/obnam.rs index cdb5179..c8da6c2 100644 --- a/src/bin/obnam.rs +++ b/src/bin/obnam.rs @@ -3,6 +3,7 @@ use log::{debug, error, info, LevelFilter}; use log4rs::append::file::FileAppender; use log4rs::config::{Appender, Logger, Root}; use obnam::cmd::backup::Backup; +use obnam::cmd::chunk::{DecryptChunk, EncryptChunk}; use obnam::cmd::get_chunk::GetChunk; use obnam::cmd::init::Init; use obnam::cmd::list::List; @@ -20,28 +21,24 @@ const APPLICATION: &str = "obnam"; fn main() -> anyhow::Result<()> { let opt = Opt::from_args(); - let config = load_config_without_passwords(&opt)?; - setup_logging(&config.config().log)?; + let config = ClientConfig::read(&config_filename(&opt))?; + setup_logging(&config.log)?; info!("client starts"); debug!("{:?}", opt); debug!("configuration: {:#?}", config); let result = match opt.cmd { - Command::Init(x) => x.run(config.config()), - _ => { - let config = load_config_with_passwords(&opt)?; - match opt.cmd { - Command::Init(_) => panic!("this can't happen"), - Command::Backup(x) => x.run(&config), - Command::List(x) => x.run(&config), - Command::ShowGeneration(x) => x.run(&config), - Command::ListFiles(x) => x.run(&config), - Command::Restore(x) => x.run(&config), - Command::GetChunk(x) => x.run(&config), - Command::Config(x) => x.run(&config), - } - } + Command::Init(x) => x.run(&config), + Command::Backup(x) => x.run(&config), + Command::List(x) => x.run(&config), + Command::ShowGeneration(x) => x.run(&config), + Command::ListFiles(x) => x.run(&config), + Command::Restore(x) => x.run(&config), + Command::GetChunk(x) => x.run(&config), + Command::Config(x) => x.run(&config), + Command::EncryptChunk(x) => x.run(&config), + Command::DecryptChunk(x) => x.run(&config), }; if let Err(ref e) = result { @@ -66,14 +63,6 @@ fn setup_logging(filename: &Path) -> anyhow::Result<()> { Ok(()) } -fn load_config_with_passwords(opt: &Opt) -> Result<ClientConfig, anyhow::Error> { - Ok(ClientConfig::read_with_passwords(&config_filename(opt))?) -} - -fn load_config_without_passwords(opt: &Opt) -> Result<ClientConfig, anyhow::Error> { - Ok(ClientConfig::read_without_passwords(&config_filename(opt))?) -} - fn config_filename(opt: &Opt) -> PathBuf { match opt.config { None => default_config(), @@ -109,4 +98,6 @@ enum Command { ShowGeneration(ShowGeneration), GetChunk(GetChunk), Config(ShowConfig), + EncryptChunk(EncryptChunk), + DecryptChunk(DecryptChunk), } diff --git a/src/chunk.rs b/src/chunk.rs index 0eed38a..8631fd9 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -1,4 +1,6 @@ +use crate::checksummer::sha256; use crate::chunkid::ChunkId; +use crate::chunkmeta::ChunkMeta; use serde::{Deserialize, Serialize}; use std::default::Default; @@ -8,21 +10,27 @@ use std::default::Default; /// /// A chunk also contains its associated metadata, except its /// identifier. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct DataChunk { data: Vec<u8>, + meta: ChunkMeta, } impl DataChunk { /// Construct a new chunk. - pub fn new(data: Vec<u8>) -> Self { - Self { data } + pub fn new(data: Vec<u8>, meta: ChunkMeta) -> Self { + Self { data, meta } } /// Return a chunk's data. pub fn data(&self) -> &[u8] { &self.data } + + /// Return a chunk's metadata. + pub fn meta(&self) -> &ChunkMeta { + &self.meta + } } #[derive(Default, Debug, Serialize, Deserialize)] @@ -69,8 +77,12 @@ impl GenerationChunk { self.chunk_ids.iter() } - pub fn to_data_chunk(&self) -> GenerationChunkResult<DataChunk> { - let json = serde_json::to_string(self).map_err(GenerationChunkError::JsonGenerate)?; - Ok(DataChunk::new(json.as_bytes().to_vec())) + pub fn to_data_chunk(&self, ended: &str) -> GenerationChunkResult<DataChunk> { + let json: String = + serde_json::to_string(self).map_err(GenerationChunkError::JsonGenerate)?; + let bytes = json.as_bytes().to_vec(); + let sha = sha256(&bytes); + let meta = ChunkMeta::new_generation(&sha, ended); + Ok(DataChunk::new(bytes, meta)) } } diff --git a/src/chunker.rs b/src/chunker.rs index eeeed8d..a7a39f1 100644 --- a/src/chunker.rs +++ b/src/chunker.rs @@ -31,7 +31,7 @@ impl Chunker { } } - pub fn read_chunk(&mut self) -> ChunkerResult<Option<(ChunkMeta, DataChunk)>> { + pub fn read_chunk(&mut self) -> ChunkerResult<Option<DataChunk>> { let mut used = 0; loop { @@ -52,18 +52,18 @@ impl Chunker { let buffer = &self.buf.as_slice()[..used]; let hash = sha256(buffer); let meta = ChunkMeta::new(&hash); - let chunk = DataChunk::new(buffer.to_vec()); - Ok(Some((meta, chunk))) + let chunk = DataChunk::new(buffer.to_vec(), meta); + Ok(Some(chunk)) } } impl Iterator for Chunker { - type Item = ChunkerResult<(ChunkMeta, DataChunk)>; + type Item = ChunkerResult<DataChunk>; - fn next(&mut self) -> Option<ChunkerResult<(ChunkMeta, DataChunk)>> { + fn next(&mut self) -> Option<ChunkerResult<DataChunk>> { match self.read_chunk() { Ok(None) => None, - Ok(Some((meta, chunk))) => Some(Ok((meta, chunk))), + Ok(Some(chunk)) => Some(Ok(chunk)), Err(e) => Some(Err(e)), } } diff --git a/src/chunkmeta.rs b/src/chunkmeta.rs index 37e2ed5..73d9007 100644 --- a/src/chunkmeta.rs +++ b/src/chunkmeta.rs @@ -80,10 +80,20 @@ impl ChunkMeta { &self.sha256 } + /// Serialize from a textual JSON representation. + pub fn from_json(json: &str) -> Result<Self, serde_json::Error> { + serde_json::from_str(json) + } + /// Serialize as JSON. pub fn to_json(&self) -> String { serde_json::to_string(self).unwrap() } + + /// Serialize as JSON, as a byte vector. + pub fn to_json_vec(&self) -> Vec<u8> { + self.to_json().as_bytes().to_vec() + } } impl FromStr for ChunkMeta { @@ -135,10 +145,19 @@ mod test { } #[test] - fn json_roundtrip() { + fn generation_json_roundtrip() { let meta = ChunkMeta::new_generation("abcdef", "2020-09-17T08:17:13+03:00"); let json = serde_json::to_string(&meta).unwrap(); let meta2 = serde_json::from_str(&json).unwrap(); assert_eq!(meta, meta2); } + + #[test] + fn data_json_roundtrip() { + let meta = ChunkMeta::new("abcdef"); + let json = meta.to_json_vec(); + let meta2 = serde_json::from_slice(&json).unwrap(); + assert_eq!(meta, meta2); + assert_eq!(meta.to_json_vec(), meta2.to_json_vec()); + } } diff --git a/src/cipher.rs b/src/cipher.rs new file mode 100644 index 0000000..550fafd --- /dev/null +++ b/src/cipher.rs @@ -0,0 +1,200 @@ +use crate::chunk::DataChunk; +use crate::chunkmeta::ChunkMeta; +use crate::passwords::Passwords; + +use aes_gcm::aead::{generic_array::GenericArray, Aead, NewAead, Payload}; +use aes_gcm::Aes256Gcm; // Or `Aes128Gcm` +use rand::Rng; + +use std::str::FromStr; + +const CHUNK_V1: &[u8] = b"0001"; + +pub struct EncryptedChunk { + ciphertext: Vec<u8>, + aad: Vec<u8>, +} + +impl EncryptedChunk { + fn new(ciphertext: Vec<u8>, aad: Vec<u8>) -> Self { + Self { ciphertext, aad } + } + + pub fn ciphertext(&self) -> &[u8] { + &self.ciphertext + } + + pub fn aad(&self) -> &[u8] { + &self.aad + } +} + +pub struct CipherEngine { + cipher: Aes256Gcm, +} + +impl CipherEngine { + pub fn new(pass: &Passwords) -> Self { + let key = GenericArray::from_slice(pass.encryption_key()); + Self { + cipher: Aes256Gcm::new(key), + } + } + + pub fn encrypt_chunk(&self, chunk: &DataChunk) -> Result<EncryptedChunk, CipherError> { + // Payload with metadata as associated data, to be encrypted. + // + // The metadata will be stored in cleartext after encryption. + let aad = chunk.meta().to_json_vec(); + let payload = Payload { + msg: chunk.data(), + aad: &aad, + }; + + // Unique random key for each encryption. + let nonce = Nonce::new(); + let nonce_arr = GenericArray::from_slice(nonce.as_bytes()); + + // Encrypt the sensitive part. + let ciphertext = self + .cipher + .encrypt(nonce_arr, payload) + .map_err(CipherError::EncryptError)?; + + // Construct the blob to be stored on the server. + let mut vec: Vec<u8> = vec![]; + push_bytes(&mut vec, CHUNK_V1); + push_bytes(&mut vec, nonce.as_bytes()); + push_bytes(&mut vec, &ciphertext); + + Ok(EncryptedChunk::new(vec, aad)) + } + + pub fn decrypt_chunk(&self, bytes: &[u8], meta: &[u8]) -> Result<DataChunk, CipherError> { + // Does encrypted chunk start with the right version? + if !bytes.starts_with(CHUNK_V1) { + return Err(CipherError::UnknownChunkVersion); + } + let version_len = CHUNK_V1.len(); + let bytes = &bytes[version_len..]; + + // Get nonce. + let nonce = &bytes[..NONCE_SIZE]; + if nonce.len() != NONCE_SIZE { + return Err(CipherError::NoNonce); + } + let nonce = GenericArray::from_slice(nonce); + let ciphertext = &bytes[NONCE_SIZE..]; + + let payload = Payload { + msg: ciphertext, + aad: meta, + }; + + let payload = self + .cipher + .decrypt(nonce, payload) + .map_err(CipherError::DecryptError)?; + let payload = Payload::from(payload.as_slice()); + + let meta = std::str::from_utf8(meta)?; + let meta = ChunkMeta::from_str(&meta)?; + + let chunk = DataChunk::new(payload.msg.to_vec(), meta); + + Ok(chunk) + } +} + +fn push_bytes(vec: &mut Vec<u8>, bytes: &[u8]) { + for byte in bytes.iter() { + vec.push(*byte); + } +} + +#[derive(Debug, thiserror::Error)] +pub enum CipherError { + #[error("failed to encrypt with AES-GEM: {0}")] + EncryptError(aes_gcm::Error), + + #[error("encrypted chunk does not start with correct version")] + UnknownChunkVersion, + + #[error("encrypted chunk does not have a complete nonce")] + NoNonce, + + #[error("failed to decrypt with AES-GEM: {0}")] + DecryptError(aes_gcm::Error), + + #[error("failed to parse decrypted data as a DataChunk: {0}")] + Parse(serde_yaml::Error), + + #[error(transparent)] + Utf8Error(#[from] std::str::Utf8Error), + + #[error("failed to parse JSON: {0}")] + JsonParse(#[from] serde_json::Error), +} + +const NONCE_SIZE: usize = 12; + +#[derive(Debug)] +struct Nonce { + nonce: Vec<u8>, +} + +impl Nonce { + fn from_bytes(bytes: &[u8]) -> Self { + assert_eq!(bytes.len(), NONCE_SIZE); + Self { + nonce: bytes.to_vec(), + } + } + + fn new() -> Self { + let mut bytes: Vec<u8> = vec![0; NONCE_SIZE]; + let mut rng = rand::thread_rng(); + for x in bytes.iter_mut() { + *x = rng.gen(); + } + Self::from_bytes(&bytes) + } + + fn as_bytes(&self) -> &[u8] { + &self.nonce + } +} + +#[cfg(test)] +mod test { + use crate::chunk::DataChunk; + use crate::chunkmeta::ChunkMeta; + use crate::cipher::CipherEngine; + use crate::passwords::Passwords; + + #[test] + fn metadata_as_aad() { + let meta = ChunkMeta::new("dummy-checksum"); + let meta_as_aad = meta.to_json_vec(); + let chunk = DataChunk::new("hello".as_bytes().to_vec(), meta); + let pass = Passwords::new("secret"); + let cipher = CipherEngine::new(&pass); + let enc = cipher.encrypt_chunk(&chunk).unwrap(); + + assert_eq!(meta_as_aad, enc.aad()); + } + + #[test] + fn round_trip() { + let meta = ChunkMeta::new("dummy-checksum"); + let chunk = DataChunk::new("hello".as_bytes().to_vec(), meta); + let pass = Passwords::new("secret"); + + let cipher = CipherEngine::new(&pass); + let enc = cipher.encrypt_chunk(&chunk).unwrap(); + + let bytes: Vec<u8> = enc.ciphertext().to_vec(); + let dec = cipher.decrypt_chunk(&bytes, enc.aad()).unwrap(); + assert_eq!(chunk, dec); + } +} diff --git a/src/client.rs b/src/client.rs index 0f8a72f..b1f9976 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,17 +1,18 @@ -use crate::checksummer::sha256; use crate::chunk::DataChunk; use crate::chunk::{GenerationChunk, GenerationChunkError}; use crate::chunker::{Chunker, ChunkerError}; use crate::chunkid::ChunkId; use crate::chunkmeta::ChunkMeta; -use crate::config::ClientConfig; +use crate::cipher::{CipherEngine, CipherError}; +use crate::config::{ClientConfig, ClientConfigError}; use crate::fsentry::{FilesystemEntry, FilesystemKind}; use crate::generation::{FinishedGeneration, LocalGeneration, LocalGenerationError}; use crate::genlist::GenerationList; use chrono::{DateTime, Local}; -use log::{debug, error, info, trace}; +use log::{debug, error, info}; use reqwest::blocking::Client; +use reqwest::header::HeaderMap; use std::collections::HashMap; use std::fs::File; use std::io::prelude::*; @@ -22,6 +23,9 @@ pub enum ClientError { #[error("Server response claimed it had created a chunk, but lacked chunk id")] NoCreatedChunkId, + #[error("Server does not have {0}")] + NotFound(String), + #[error("Server does not have chunk {0}")] ChunkNotFound(String), @@ -35,6 +39,12 @@ pub enum ClientError { WrongChecksum(ChunkId, String, String), #[error(transparent)] + ClientConfigError(#[from] ClientConfigError), + + #[error(transparent)] + CipherError(#[from] CipherError), + + #[error(transparent)] GenerationChunkError(#[from] GenerationChunkError), #[error(transparent)] @@ -74,21 +84,14 @@ pub enum ClientError { pub type ClientResult<T> = Result<T, ClientError>; pub struct BackupClient { - client: Client, - base_url: String, + chunk_client: ChunkClient, } impl BackupClient { pub fn new(config: &ClientConfig) -> ClientResult<Self> { info!("creating backup client with config: {:#?}", config); - let config = config.config(); - let client = Client::builder() - .danger_accept_invalid_certs(!config.verify_tls_cert) - .build() - .map_err(ClientError::ReqwestError)?; Ok(Self { - client, - base_url: config.server_url.to_string(), + chunk_client: ChunkClient::new(config)?, }) } @@ -114,10 +117,9 @@ impl BackupClient { info!("upload SQLite {}", filename.display()); let ids = self.read_file(filename, size)?; let gen = GenerationChunk::new(ids); - let data = gen.to_data_chunk()?; - let meta = ChunkMeta::new_generation(&sha256(data.data()), ¤t_timestamp()); - let gen_id = self.upload_gen_chunk(meta.clone(), gen)?; - info!("uploaded generation {}, meta {:?}", gen_id, meta); + let data = gen.to_data_chunk(¤t_timestamp())?; + let gen_id = self.upload_chunk(data)?; + info!("uploaded generation {}", gen_id); Ok(gen_id) } @@ -130,6 +132,86 @@ impl BackupClient { Ok(chunk_ids) } + pub fn has_chunk(&self, meta: &ChunkMeta) -> ClientResult<Option<ChunkId>> { + self.chunk_client.has_chunk(meta) + } + + pub fn upload_chunk(&self, chunk: DataChunk) -> ClientResult<ChunkId> { + self.chunk_client.upload_chunk(chunk) + } + + pub fn upload_new_file_chunks(&self, chunker: Chunker) -> ClientResult<Vec<ChunkId>> { + let mut chunk_ids = vec![]; + for item in chunker { + let chunk = item?; + if let Some(chunk_id) = self.has_chunk(chunk.meta())? { + chunk_ids.push(chunk_id.clone()); + info!("reusing existing chunk {}", chunk_id); + } else { + let chunk_id = self.upload_chunk(chunk)?; + chunk_ids.push(chunk_id.clone()); + info!("created new chunk {}", chunk_id); + } + } + + Ok(chunk_ids) + } + + pub fn list_generations(&self) -> ClientResult<GenerationList> { + self.chunk_client.list_generations() + } + + pub fn fetch_chunk(&self, chunk_id: &ChunkId) -> ClientResult<DataChunk> { + self.chunk_client.fetch_chunk(chunk_id) + } + + fn fetch_generation_chunk(&self, gen_id: &str) -> ClientResult<GenerationChunk> { + let chunk_id = ChunkId::recreate(gen_id); + let chunk = self.fetch_chunk(&chunk_id)?; + let gen = GenerationChunk::from_data_chunk(&chunk)?; + Ok(gen) + } + + pub fn fetch_generation(&self, gen_id: &str, dbname: &Path) -> ClientResult<LocalGeneration> { + let gen = self.fetch_generation_chunk(gen_id)?; + + // Fetch the SQLite file, storing it in the named file. + let mut dbfile = File::create(&dbname) + .map_err(|err| ClientError::FileCreate(dbname.to_path_buf(), err))?; + for id in gen.chunk_ids() { + let chunk = self.fetch_chunk(id)?; + dbfile + .write_all(chunk.data()) + .map_err(|err| ClientError::FileWrite(dbname.to_path_buf(), err))?; + } + info!("downloaded generation to {}", dbname.display()); + + let gen = LocalGeneration::open(dbname)?; + Ok(gen) + } +} + +pub struct ChunkClient { + client: Client, + base_url: String, + cipher: CipherEngine, +} + +impl ChunkClient { + pub fn new(config: &ClientConfig) -> ClientResult<Self> { + let pass = config.passwords()?; + + let client = Client::builder() + .danger_accept_invalid_certs(!config.verify_tls_cert) + .build() + .map_err(ClientError::ReqwestError)?; + Ok(Self { + client, + base_url: config.server_url.to_string(), + cipher: CipherEngine::new(&pass), + }) + } + fn base_url(&self) -> &str { &self.base_url } @@ -139,44 +221,30 @@ impl BackupClient { } pub fn has_chunk(&self, meta: &ChunkMeta) -> ClientResult<Option<ChunkId>> { - trace!("has_chunk: url={:?}", self.base_url()); - let req = self - .client - .get(&self.chunks_url()) - .query(&[("sha256", meta.sha256())]) - .build() - .map_err(ClientError::ReqwestError)?; + let body = match self.get("", &[("sha256", meta.sha256())]) { + Ok((_, body)) => body, + Err(err) => return Err(err), + }; - let res = self.client.execute(req).map_err(ClientError::ChunkExists)?; - debug!("has_chunk: status={}", res.status()); - let has = if res.status() != 200 { - debug!("has_chunk: error from server"); - None + let hits: HashMap<String, ChunkMeta> = + serde_json::from_slice(&body).map_err(ClientError::JsonParse)?; + let mut iter = hits.iter(); + let has = if let Some((chunk_id, _)) = iter.next() { + Some(chunk_id.into()) } else { - let text = res.text().map_err(ClientError::ReqwestError)?; - debug!("has_chunk: text={:?}", text); - let hits: HashMap<String, ChunkMeta> = - serde_json::from_str(&text).map_err(ClientError::JsonParse)?; - debug!("has_chunk: hits={:?}", hits); - let mut iter = hits.iter(); - if let Some((chunk_id, _)) = iter.next() { - debug!("has_chunk: chunk_id={:?}", chunk_id); - Some(chunk_id.into()) - } else { - None - } + None }; - info!("has_chunk result: {:?}", has); Ok(has) } - pub fn upload_chunk(&self, meta: ChunkMeta, chunk: DataChunk) -> ClientResult<ChunkId> { + pub fn upload_chunk(&self, chunk: DataChunk) -> ClientResult<ChunkId> { + let enc = self.cipher.encrypt_chunk(&chunk)?; let res = self .client .post(&self.chunks_url()) - .header("chunk-meta", meta.to_json()) - .body(chunk.data().to_vec()) + .header("chunk-meta", chunk.meta().to_json()) + .body(enc.ciphertext().to_vec()) .send() .map_err(ClientError::ReqwestError)?; debug!("upload_chunk: res={:?}", res); @@ -187,62 +255,13 @@ impl BackupClient { } else { return Err(ClientError::NoCreatedChunkId); }; - info!("uploaded_chunk {} meta {:?}", chunk_id, meta); + info!("uploaded_chunk {}", chunk_id); Ok(chunk_id) } - pub fn upload_gen_chunk(&self, meta: ChunkMeta, gen: GenerationChunk) -> ClientResult<ChunkId> { - let res = self - .client - .post(&self.chunks_url()) - .header("chunk-meta", meta.to_json()) - .body(serde_json::to_string(&gen).map_err(ClientError::JsonGenerate)?) - .send() - .map_err(ClientError::ReqwestError)?; - debug!("upload_chunk: res={:?}", res); - let res: HashMap<String, String> = res.json().map_err(ClientError::ReqwestError)?; - let chunk_id = if let Some(chunk_id) = res.get("chunk_id") { - debug!("upload_chunk: id={}", chunk_id); - chunk_id.parse().unwrap() - } else { - return Err(ClientError::NoCreatedChunkId); - }; - info!("uploaded_generation chunk {}", chunk_id); - Ok(chunk_id) - } - - pub fn upload_new_file_chunks(&self, chunker: Chunker) -> ClientResult<Vec<ChunkId>> { - let mut chunk_ids = vec![]; - for item in chunker { - let (meta, chunk) = item?; - if let Some(chunk_id) = self.has_chunk(&meta)? { - chunk_ids.push(chunk_id.clone()); - info!("reusing existing chunk {}", chunk_id); - } else { - let chunk_id = self.upload_chunk(meta, chunk)?; - chunk_ids.push(chunk_id.clone()); - info!("created new chunk {}", chunk_id); - } - } - - Ok(chunk_ids) - } - pub fn list_generations(&self) -> ClientResult<GenerationList> { - let url = format!("{}?generation=true", &self.chunks_url()); - trace!("list_generations: url={:?}", url); - let req = self - .client - .get(&url) - .build() - .map_err(ClientError::ReqwestError)?; - let res = self - .client - .execute(req) - .map_err(ClientError::ReqwestError)?; - debug!("list_generations: status={}", res.status()); - let body = res.bytes().map_err(ClientError::ReqwestError)?; - debug!("list_generations: body={:?}", body); + let (_, body) = self.get("", &[("generation", "true")])?; + let map: HashMap<String, ChunkMeta> = serde_yaml::from_slice(&body).map_err(ClientError::YamlParse)?; debug!("list_generations: map={:?}", map); @@ -254,77 +273,65 @@ impl BackupClient { } pub fn fetch_chunk(&self, chunk_id: &ChunkId) -> ClientResult<DataChunk> { - info!("fetch chunk {}", chunk_id); + let (headers, body) = self.get(&format!("/{}", chunk_id), &[])?; + let meta = self.get_chunk_meta_header(chunk_id, &headers)?; + + let meta_bytes = meta.to_json_vec(); + let chunk = self.cipher.decrypt_chunk(&body, &meta_bytes)?; + + Ok(chunk) + } + + fn get(&self, path: &str, query: &[(&str, &str)]) -> ClientResult<(HeaderMap, Vec<u8>)> { + let url = format!("{}{}", &self.chunks_url(), path); + info!("GET {}", url); - let url = format!("{}/{}", &self.chunks_url(), chunk_id); + // Build HTTP request structure. let req = self .client .get(&url) + .query(query) .build() .map_err(ClientError::ReqwestError)?; + + // Make HTTP request. let res = self .client .execute(req) .map_err(ClientError::ReqwestError)?; + + // Did it work? if res.status() != 200 { - let err = ClientError::ChunkNotFound(chunk_id.to_string()); - error!("fetching chunk {} failed: {}", chunk_id, err); - return Err(err); + return Err(ClientError::NotFound(path.to_string())); } - let headers = res.headers(); + // Return headers and body. + let headers = res.headers().clone(); + let body = res.bytes().map_err(ClientError::ReqwestError)?; + let body = body.to_vec(); + Ok((headers, body)) + } + + fn get_chunk_meta_header( + &self, + chunk_id: &ChunkId, + headers: &HeaderMap, + ) -> ClientResult<ChunkMeta> { let meta = headers.get("chunk-meta"); + if meta.is_none() { let err = ClientError::NoChunkMeta(chunk_id.clone()); error!("fetching chunk {} failed: {}", chunk_id, err); return Err(err); } + let meta = meta .unwrap() .to_str() .map_err(ClientError::MetaHeaderToString)?; - debug!("fetching chunk {}: meta={:?}", chunk_id, meta); let meta: ChunkMeta = serde_json::from_str(meta).map_err(ClientError::JsonParse)?; - debug!("fetching chunk {}: meta={:?}", chunk_id, meta); - - let body = res.bytes().map_err(ClientError::ReqwestError)?; - let body = body.to_vec(); - let actual = sha256(&body); - if actual != meta.sha256() { - let err = - ClientError::WrongChecksum(chunk_id.clone(), actual, meta.sha256().to_string()); - error!("fetching chunk {} failed: {}", chunk_id, err); - return Err(err); - } - - let chunk: DataChunk = DataChunk::new(body); - - Ok(chunk) - } - - fn fetch_generation_chunk(&self, gen_id: &str) -> ClientResult<GenerationChunk> { - let chunk_id = ChunkId::recreate(gen_id); - let chunk = self.fetch_chunk(&chunk_id)?; - let gen = GenerationChunk::from_data_chunk(&chunk)?; - Ok(gen) - } - - pub fn fetch_generation(&self, gen_id: &str, dbname: &Path) -> ClientResult<LocalGeneration> { - let gen = self.fetch_generation_chunk(gen_id)?; - - // Fetch the SQLite file, storing it in the named file. - let mut dbfile = File::create(&dbname) - .map_err(|err| ClientError::FileCreate(dbname.to_path_buf(), err))?; - for id in gen.chunk_ids() { - let chunk = self.fetch_chunk(id)?; - dbfile - .write_all(chunk.data()) - .map_err(|err| ClientError::FileWrite(dbname.to_path_buf(), err))?; - } - info!("downloaded generation to {}", dbname.display()); - let gen = LocalGeneration::open(dbname)?; - Ok(gen) + Ok(meta) } } diff --git a/src/cmd/backup.rs b/src/cmd/backup.rs index 0479844..22afd6e 100644 --- a/src/cmd/backup.rs +++ b/src/cmd/backup.rs @@ -60,7 +60,6 @@ fn initial_backup( info!("fresh backup without a previous generation"); let newtemp = NamedTempFile::new()?; let run = InitialBackup::new(config, &client)?; - let config = config.config(); let mut all_warnings = vec![]; let count = { let mut new = NascentGeneration::create(newtemp.path())?; @@ -87,7 +86,6 @@ fn incremental_backup( info!("incremental backup based on {}", old_ref); let newtemp = NamedTempFile::new()?; let mut run = IncrementalBackup::new(config, &client)?; - let config = config.config(); let mut all_warnings = vec![]; let count = { let oldtemp = NamedTempFile::new()?; diff --git a/src/cmd/chunk.rs b/src/cmd/chunk.rs new file mode 100644 index 0000000..e0e91b1 --- /dev/null +++ b/src/cmd/chunk.rs @@ -0,0 +1,64 @@ +use crate::chunk::DataChunk; +use crate::chunkmeta::ChunkMeta; +use crate::cipher::CipherEngine; +use crate::config::ClientConfig; +use crate::error::ObnamError; +use std::path::PathBuf; +use structopt::StructOpt; + +#[derive(Debug, StructOpt)] +pub struct EncryptChunk { + #[structopt(parse(from_os_str))] + filename: PathBuf, + + #[structopt(parse(from_os_str))] + output: PathBuf, + + #[structopt()] + json: String, +} + +impl EncryptChunk { + pub fn run(&self, config: &ClientConfig) -> Result<(), ObnamError> { + let pass = config.passwords()?; + let cipher = CipherEngine::new(&pass); + + let meta = ChunkMeta::from_json(&self.json)?; + + let cleartext = std::fs::read(&self.filename)?; + let chunk = DataChunk::new(cleartext, meta); + let encrypted = cipher.encrypt_chunk(&chunk)?; + + std::fs::write(&self.output, encrypted.ciphertext())?; + + Ok(()) + } +} + +#[derive(Debug, StructOpt)] +pub struct DecryptChunk { + #[structopt(parse(from_os_str))] + filename: PathBuf, + + #[structopt(parse(from_os_str))] + output: PathBuf, + + #[structopt()] + json: String, +} + +impl DecryptChunk { + pub fn run(&self, config: &ClientConfig) -> Result<(), ObnamError> { + let pass = config.passwords()?; + let cipher = CipherEngine::new(&pass); + + let meta = ChunkMeta::from_json(&self.json)?; + + let encrypted = std::fs::read(&self.filename)?; + let chunk = cipher.decrypt_chunk(&encrypted, &meta.to_json_vec())?; + + std::fs::write(&self.output, chunk.data())?; + + Ok(()) + } +} diff --git a/src/cmd/init.rs b/src/cmd/init.rs index cb61fba..08060f7 100644 --- a/src/cmd/init.rs +++ b/src/cmd/init.rs @@ -1,4 +1,4 @@ -use crate::config::ClientConfigWithoutPasswords; +use crate::config::ClientConfig; use crate::error::ObnamError; use crate::passwords::{passwords_filename, Passwords}; use structopt::StructOpt; @@ -12,11 +12,7 @@ pub struct Init { } impl Init { - pub fn run(&self, config: &ClientConfigWithoutPasswords) -> Result<(), ObnamError> { - if !config.encrypt { - panic!("no encryption specified"); - } - + pub fn run(&self, config: &ClientConfig) -> Result<(), ObnamError> { let passphrase = match &self.insecure_passphrase { Some(x) => x.to_string(), None => rpassword::read_password_from_tty(Some(PROMPT)).unwrap(), diff --git a/src/cmd/mod.rs b/src/cmd/mod.rs index 890e176..bd101da 100644 --- a/src/cmd/mod.rs +++ b/src/cmd/mod.rs @@ -1,4 +1,5 @@ pub mod backup; +pub mod chunk; pub mod get_chunk; pub mod init; pub mod list; diff --git a/src/cmd/show_config.rs b/src/cmd/show_config.rs index 424e2ed..05e83c1 100644 --- a/src/cmd/show_config.rs +++ b/src/cmd/show_config.rs @@ -7,7 +7,7 @@ pub struct ShowConfig {} impl ShowConfig { pub fn run(&self, config: &ClientConfig) -> Result<(), ObnamError> { - println!("{}", serde_json::to_string_pretty(&config.config())?); + println!("{}", serde_json::to_string_pretty(config)?); Ok(()) } } diff --git a/src/config.rs b/src/config.rs index 33e08a2..0d4e9de 100644 --- a/src/config.rs +++ b/src/config.rs @@ -16,78 +16,22 @@ struct TentativeClientConfig { chunk_size: Option<usize>, roots: Vec<PathBuf>, log: Option<PathBuf>, - encrypt: Option<bool>, exclude_cache_tag_directories: Option<bool>, } #[derive(Debug, Serialize, Clone)] -pub enum ClientConfig { - Plain(ClientConfigWithoutPasswords), - WithPasswords(ClientConfigWithoutPasswords, Passwords), -} - -impl ClientConfig { - pub fn read_without_passwords(filename: &Path) -> Result<Self, ClientConfigError> { - let config = ClientConfigWithoutPasswords::read_config(filename)?; - Ok(ClientConfig::Plain(config)) - } - - pub fn read_with_passwords(filename: &Path) -> Result<Self, ClientConfigError> { - let config = ClientConfigWithoutPasswords::read_config(filename)?; - if config.encrypt { - let passwords = Passwords::load(&passwords_filename(filename)) - .map_err(ClientConfigError::PasswordsMissing)?; - Ok(ClientConfig::WithPasswords(config, passwords)) - } else { - Ok(ClientConfig::Plain(config)) - } - } - - pub fn config(&self) -> &ClientConfigWithoutPasswords { - match self { - Self::Plain(config) => &config, - Self::WithPasswords(config, _) => &config, - } - } -} - -#[derive(Debug, Serialize, Clone)] -pub struct ClientConfigWithoutPasswords { +pub struct ClientConfig { pub filename: PathBuf, pub server_url: String, pub verify_tls_cert: bool, pub chunk_size: usize, pub roots: Vec<PathBuf>, pub log: PathBuf, - pub encrypt: bool, pub exclude_cache_tag_directories: bool, } -#[derive(Debug, thiserror::Error)] -pub enum ClientConfigError { - #[error("server_url is empty")] - ServerUrlIsEmpty, - - #[error("No backup roots in config; at least one is needed")] - NoBackupRoot, - - #[error("server URL doesn't use https: {0}")] - NotHttps(String), - - #[error("No passwords are set: you may need to run 'obnam init': {0}")] - PasswordsMissing(PasswordError), - - #[error("failed to read configuration file {0}: {1}")] - Read(PathBuf, std::io::Error), - - #[error("failed to parse configuration file {0} as YAML: {1}")] - YamlParse(PathBuf, serde_yaml::Error), -} - -pub type ClientConfigResult<T> = Result<T, ClientConfigError>; - -impl ClientConfigWithoutPasswords { - pub fn read_config(filename: &Path) -> ClientConfigResult<Self> { +impl ClientConfig { + pub fn read(filename: &Path) -> ClientConfigResult<Self> { trace!("read_config: filename={:?}", filename); let config = std::fs::read_to_string(filename) .map_err(|err| ClientConfigError::Read(filename.to_path_buf(), err))?; @@ -102,12 +46,10 @@ impl ClientConfigWithoutPasswords { .log .map(|path| expand_tilde(&path)) .unwrap_or_else(|| PathBuf::from(DEVNULL)); - let encrypt = tentative.encrypt.or(Some(false)).unwrap(); let exclude_cache_tag_directories = tentative.exclude_cache_tag_directories.unwrap_or(true); let config = Self { chunk_size: tentative.chunk_size.or(Some(DEFAULT_CHUNK_SIZE)).unwrap(), - encrypt, filename: filename.to_path_buf(), roots, server_url: tentative.server_url, @@ -132,8 +74,36 @@ impl ClientConfigWithoutPasswords { } Ok(()) } + + pub fn passwords(&self) -> Result<Passwords, ClientConfigError> { + Passwords::load(&passwords_filename(&self.filename)) + .map_err(ClientConfigError::PasswordsMissing) + } } +#[derive(Debug, thiserror::Error)] +pub enum ClientConfigError { + #[error("server_url is empty")] + ServerUrlIsEmpty, + + #[error("No backup roots in config; at least one is needed")] + NoBackupRoot, + + #[error("server URL doesn't use https: {0}")] + NotHttps(String), + + #[error("No passwords are set: you may need to run 'obnam init': {0}")] + PasswordsMissing(PasswordError), + + #[error("failed to read configuration file {0}: {1}")] + Read(PathBuf, std::io::Error), + + #[error("failed to parse configuration file {0} as YAML: {1}")] + YamlParse(PathBuf, serde_yaml::Error), +} + +pub type ClientConfigResult<T> = Result<T, ClientConfigError>; + fn expand_tilde(path: &Path) -> PathBuf { if path.starts_with("~/") { if let Some(home) = std::env::var_os("HOME") { diff --git a/src/error.rs b/src/error.rs index 8241d5d..e4d77d3 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,4 +1,5 @@ use crate::backup_run::BackupError; +use crate::cipher::CipherError; use crate::client::ClientError; use crate::cmd::restore::RestoreError; use crate::config::ClientConfigError; @@ -32,6 +33,9 @@ pub enum ObnamError { NascentError(#[from] NascentError), #[error(transparent)] + CipherError(#[from] CipherError), + + #[error(transparent)] LocalGenerationError(#[from] LocalGenerationError), #[error(transparent)] diff --git a/src/indexedstore.rs b/src/indexedstore.rs index 7f67a1f..b05cfba 100644 --- a/src/indexedstore.rs +++ b/src/indexedstore.rs @@ -1,9 +1,8 @@ -use crate::chunk::{DataChunk, GenerationChunk, GenerationChunkError}; +use crate::chunk::{DataChunk, GenerationChunkError}; use crate::chunkid::ChunkId; use crate::chunkmeta::ChunkMeta; use crate::index::{Index, IndexError}; use crate::store::{Store, StoreError}; -use std::collections::HashSet; use std::path::Path; /// A store for chunks and their metadata. @@ -40,10 +39,10 @@ impl IndexedStore { Ok(Self { store, index }) } - pub fn save(&mut self, meta: &ChunkMeta, chunk: &DataChunk) -> IndexedResult<ChunkId> { + pub fn save(&mut self, chunk: &DataChunk) -> IndexedResult<ChunkId> { let id = ChunkId::new(); - self.store.save(&id, meta, chunk)?; - self.insert_meta(&id, meta)?; + self.store.save(&id, chunk)?; + self.insert_meta(&id, chunk.meta())?; Ok(id) } @@ -68,28 +67,6 @@ impl IndexedStore { Ok(self.index.find_generations()?) } - pub fn find_file_chunks(&self) -> IndexedResult<Vec<ChunkId>> { - let gen_ids = self.find_generations()?; - - let mut sql_chunks: HashSet<ChunkId> = HashSet::new(); - for id in gen_ids { - let gen_chunk = self.store.load(&id)?; - let gen = GenerationChunk::from_data_chunk(&gen_chunk)?; - for sqlite_chunk_id in gen.chunk_ids() { - sql_chunks.insert(sqlite_chunk_id.clone()); - } - } - - let all_chunk_ids = self.index.all_chunks()?; - let file_chunks = all_chunk_ids - .iter() - .filter(|id| !sql_chunks.contains(id)) - .cloned() - .collect(); - - Ok(file_chunks) - } - pub fn remove(&mut self, id: &ChunkId) -> IndexedResult<()> { self.index.remove_meta(id)?; self.store.delete(id)?; @@ -7,6 +7,7 @@ pub mod chunk; pub mod chunker; pub mod chunkid; pub mod chunkmeta; +pub mod cipher; pub mod client; pub mod cmd; pub mod config; diff --git a/src/passwords.rs b/src/passwords.rs index b8ca3f5..a1cf42e 100644 --- a/src/passwords.rs +++ b/src/passwords.rs @@ -8,18 +8,23 @@ use std::io::prelude::Write; use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; +const KEY_LEN: usize = 32; // Only size accepted by aead crate? + #[derive(Debug, Serialize, Deserialize, Clone)] pub struct Passwords { encryption: String, - mac: String, } impl Passwords { pub fn new(passphrase: &str) -> Self { - Self { - encryption: derive_password(passphrase), - mac: derive_password(passphrase), - } + let mut key = derive_password(passphrase); + let _ = key.split_off(KEY_LEN); + assert_eq!(key.len(), KEY_LEN); + Self { encryption: key } + } + + pub fn encryption_key(&self) -> &[u8] { + self.encryption.as_bytes() } pub fn load(filename: &Path) -> Result<Self, PasswordError> { diff --git a/src/store.rs b/src/store.rs index fca2c13..bccecc7 100644 --- a/src/store.rs +++ b/src/store.rs @@ -1,6 +1,5 @@ use crate::chunk::DataChunk; use crate::chunkid::ChunkId; -use crate::chunkmeta::ChunkMeta; use std::path::{Path, PathBuf}; /// Store chunks, with metadata, persistently. @@ -43,23 +42,26 @@ impl Store { } /// Save a chunk into a store. - pub fn save(&self, id: &ChunkId, meta: &ChunkMeta, chunk: &DataChunk) -> StoreResult<()> { + pub fn save(&self, id: &ChunkId, chunk: &DataChunk) -> StoreResult<()> { let (dir, metaname, dataname) = &self.filenames(id); if !dir.exists() { std::fs::create_dir_all(dir)?; } - std::fs::write(&metaname, meta.to_json())?; + std::fs::write(&metaname, chunk.meta().to_json())?; std::fs::write(&dataname, chunk.data())?; Ok(()) } /// Load a chunk from a store. pub fn load(&self, id: &ChunkId) -> StoreResult<DataChunk> { - let (_, _, dataname) = &self.filenames(id); + let (_, metaname, dataname) = &self.filenames(id); + let meta = std::fs::read(&metaname)?; + let meta = serde_json::from_slice(&meta)?; + let data = std::fs::read(&dataname)?; - let data = DataChunk::new(data); + let data = DataChunk::new(data, meta); Ok(data) } diff --git a/subplot/client.py b/subplot/client.py index be0a6d6..d0beba5 100644 --- a/subplot/client.py +++ b/subplot/client.py @@ -16,7 +16,7 @@ def uninstall_obnam(ctx): runcmd_run(ctx, ["chmod", "-R", "u+rwX", "."]) -def configure_client(ctx, filename=None): +def configure_client_without_init(ctx, filename=None): get_file = globals()["get_file"] assert ctx.get("server_url") is not None @@ -35,6 +35,15 @@ def configure_client(ctx, filename=None): yaml.safe_dump(config, stream=f) +def configure_client_with_init(ctx, filename=None): + runcmd_run = globals()["runcmd_run"] + runcmd_exit_code_is_zero = globals()["runcmd_exit_code_is_zero"] + + configure_client_without_init(ctx, filename=filename) + runcmd_run(ctx, ["obnam", "init", "--insecure-passphrase=hunter2"]) + runcmd_exit_code_is_zero(ctx) + + def run_obnam_restore(ctx, genid=None, todir=None): runcmd_run = globals()["runcmd_run"] diff --git a/subplot/client.yaml b/subplot/client.yaml index 6de04c9..d660089 100644 --- a/subplot/client.yaml +++ b/subplot/client.yaml @@ -3,7 +3,10 @@ cleanup: uninstall_obnam - given: "a client config based on {filename}" - function: configure_client + function: configure_client_with_init + +- given: "a client config, without passphrase, based on {filename}" + function: configure_client_without_init - when: "I invoke obnam restore <{genid}> {todir}" function: run_obnam_restore diff --git a/subplot/data.py b/subplot/data.py index d134e5f..13b6d2b 100644 --- a/subplot/data.py +++ b/subplot/data.py @@ -170,3 +170,19 @@ def file_is_readable_by_owner(ctx, filename=None): def file_does_not_contain(ctx, filename=None, pattern=None): data = open(filename).read() assert pattern not in data + + +def files_are_different(ctx, filename1=None, filename2=None): + assert_ne = globals()["assert_ne"] + + data1 = open(filename1, "rb").read() + data2 = open(filename2, "rb").read() + assert_ne(data1, data2) + + +def files_are_identical(ctx, filename1=None, filename2=None): + assert_eq = globals()["assert_eq"] + + data1 = open(filename1, "rb").read() + data2 = open(filename2, "rb").read() + assert_eq(data1, data2) diff --git a/subplot/data.yaml b/subplot/data.yaml index dcc6807..41a563f 100644 --- a/subplot/data.yaml +++ b/subplot/data.yaml @@ -48,3 +48,9 @@ - then: "file {filename} does not contain \"{pattern:text}\"" function: file_does_not_contain + +- then: "files {filename1} and {filename2} are different" + function: files_are_different + +- then: "files {filename1} and {filename2} are identical" + function: files_are_identical diff --git a/subplot/server.py b/subplot/server.py index df594f7..cfe91ab 100644 --- a/subplot/server.py +++ b/subplot/server.py @@ -134,13 +134,21 @@ def json_body_matches(ctx, wanted=None): assert_eq(body.get(key, "not.there"), wanted[key]) -def server_has_n_file_chunks(ctx, n=None): +def server_has_n_chunks(ctx, n=None): assert_eq = globals()["assert_eq"] n = int(n) - url = f"{ctx['server_url']}/chunks?data=true" - _request(ctx, requests.get, url) - num_chunks = len(ctx["http.json"]) - assert_eq(n, num_chunks) + files = find_files(ctx["config"]["chunks"]) + files = [json.load(open(x)) for x in files if x.endswith(".meta")] + logging.debug(f"server_has_n_file_chunks: n={n}") + logging.debug(f"server_has_n_file_chunks: len(files)={len(files)}") + logging.debug(f"server_has_n_file_chunks: files={files}") + assert_eq(n, len(files)) + + +def find_files(root): + for dirname, _, names in os.walk(root): + for name in names: + yield os.path.join(dirname, name) # Make an HTTP request. diff --git a/subplot/server.yaml b/subplot/server.yaml index 60f8a44..5b8a242 100644 --- a/subplot/server.yaml +++ b/subplot/server.yaml @@ -44,5 +44,5 @@ - then: "the body matches file {filename}" function: body_matches_file -- then: "server has {n:int} file chunks" - function: server_has_n_file_chunks +- then: "server has {n:int} chunks" + function: server_has_n_chunks |