From c23941a070ce2f0c39b6b720f6a693e6187d7c7e Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 4 Jan 2022 17:56:13 +0200 Subject: test: be quiet unless test.py fails Sponsored-by: author --- check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check b/check index c928b06..cc8ef6c 100755 --- a/check +++ b/check @@ -8,6 +8,6 @@ chronic cargo test -q subplot docgen obnam-benchmark.md -o obnam-benchmark.html subplot docgen obnam-benchmark.md -o obnam-benchmark.pdf subplot codegen obnam-benchmark.md -o test.py -python3 test.py --log test.log "$@" +chronic python3 test.py --log test.log "$@" cargo fmt -- --check echo A-OK -- cgit v1.2.1 From 64a11eca7f3cf75d735436366d8489ab93964dbf Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 4 Jan 2022 18:04:57 +0200 Subject: feat: add "steps" subcommand Sponsored-by: author --- src/bin/obnam-benchmark.rs | 27 ++++++++++++++-- src/lib.rs | 1 + src/result.rs | 3 ++ src/specification.rs | 10 ++++++ src/step.rs | 17 ++++++++++ src/suite.rs | 80 +++++++++++++++++++++++++++++++++++++++++++++- src/summain.rs | 24 ++++++++++++++ 7 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 src/summain.rs diff --git a/src/bin/obnam-benchmark.rs b/src/bin/obnam-benchmark.rs index e6a476c..f079d75 100644 --- a/src/bin/obnam-benchmark.rs +++ b/src/bin/obnam-benchmark.rs @@ -10,7 +10,6 @@ use structopt::StructOpt; fn main() { pretty_env_logger::init_custom_env("OBNAM_BENCHMARK_LOG"); - println!("START"); info!("obnam-benchmark starts"); if let Err(err) = real_main() { eprintln!("ERROR: {}", err); @@ -18,7 +17,6 @@ fn main() { exit(1); } info!("obnam-benchmark ends successfully"); - println!("END"); } fn real_main() -> anyhow::Result<()> { @@ -30,6 +28,7 @@ fn real_main() -> anyhow::Result<()> { Command::GenerateJunk(x) => x.run()?, Command::Run(x) => x.run()?, Command::Spec(x) => x.run()?, + Command::Steps(x) => x.run()?, } Ok(()) @@ -51,6 +50,9 @@ enum Command { /// Generate some junk data in a file. GenerateJunk(GenerateJunk), + + /// Show the steps for running a benchmark. + Steps(Steps), } #[derive(Debug, StructOpt)] @@ -129,3 +131,24 @@ impl GenerateJunk { Ok(()) } } + +#[derive(Debug, StructOpt)] +struct Steps { + /// Name of the specification file + #[structopt(parse(from_os_str))] + spec: PathBuf, +} + +impl Steps { + fn run(&self) -> anyhow::Result<()> { + info!( + "showing steps to run benchmarks from {}", + self.spec.display() + ); + let spec = Specification::from_file(&self.spec)?; + for step in spec.steps().iter() { + println!("{:?}", step); + } + Ok(()) + } +} diff --git a/src/lib.rs b/src/lib.rs index e8c6d82..f491ae7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -37,4 +37,5 @@ pub mod server; pub mod specification; pub mod step; pub mod suite; +pub mod summain; pub mod tlsgen; diff --git a/src/result.rs b/src/result.rs index cf6c0ac..9e4119b 100644 --- a/src/result.rs +++ b/src/result.rs @@ -28,6 +28,9 @@ pub enum Operation { Delete, Backup, Restore, + ManifestLive, + ManifestRestored, + CompareManiests, } impl Result { diff --git a/src/specification.rs b/src/specification.rs index 2932f41..9e315f7 100644 --- a/src/specification.rs +++ b/src/specification.rs @@ -113,16 +113,26 @@ impl Specification { /// Serialize the specification into a sequence of steps to execute it. pub fn steps(&self) -> Vec { let mut steps = vec![]; + let num_benchmarks = self.benchmarks.len(); + let after_base = num_benchmarks; + let restore_base = after_base + num_benchmarks; for b in self.benchmarks.iter() { steps.push(Step::Start(b.benchmark.to_string())); for (i, backup) in b.backups.iter().enumerate() { for change in backup.changes.iter() { steps.push(Step::from(change)); } + steps.push(Step::ManifestLive(i)); steps.push(Step::Backup(i)); + let after = after_base + i; + steps.push(Step::ManifestLive(after)); + steps.push(Step::CompareManifests(i, after)); } for (i, _) in b.backups.iter().enumerate() { steps.push(Step::Restore(i)); + let restored = restore_base + i; + // steps.push(Step::ManifestRestored(restored)); + // steps.push(Step::CompareManifests(i, restored)); } steps.push(Step::Stop(b.benchmark.to_string())); } diff --git a/src/step.rs b/src/step.rs index fa7abd6..d18b475 100644 --- a/src/step.rs +++ b/src/step.rs @@ -29,6 +29,23 @@ pub enum Step { /// n usize, ), + /// Create and remember a manifest of current live data. Call it id. + ManifestLive( + /// id + usize, + ), + /// Create and remember a manifest of latest restored data. Call it id. + ManifestRestored( + /// id + usize, + ), + /// Compare two manifests for equality. + CompareManifests( + /// First + usize, + /// Second. + usize, + ), } /// Possible errors from executing a benchmark step. diff --git a/src/suite.rs b/src/suite.rs index e900330..b4e8abb 100644 --- a/src/suite.rs +++ b/src/suite.rs @@ -5,7 +5,9 @@ use crate::result::{Measurement, OpMeasurements, Operation}; use crate::server::{ObnamServer, ObnamServerError}; use crate::specification::{Create, FileCount}; use crate::step::Step; -use log::{debug, info}; +use crate::summain::{summain, SummainError}; +use log::{debug, error, info}; +use std::collections::HashMap; use std::fs::File; use std::path::{Path, PathBuf}; use std::time::Instant; @@ -47,6 +49,22 @@ pub enum SuiteError { /// Error managing an Obnam server. #[error(transparent)] Server(#[from] ObnamServerError), + + /// Suite already has a manifest with a given id. + #[error("Suite already has manifest {0}: this is a bug")] + ManifestExists(usize), + + /// Suite doesn't have a manifest with a given id. + #[error("Suite doesn't have a manifest {0}: this is a bug")] + ManifestMissing(usize), + + /// Manifests are not identical. + #[error("Manifests {0} and {1} are not identical, as expected")] + ManifestsDiffer(usize, usize), + + /// Error running summain. + #[error(transparent)] + Summain(SummainError), } impl Suite { @@ -98,6 +116,21 @@ impl Suite { assert!(self.benchmark.is_some()); self.benchmark.as_mut().unwrap().restore(*x)? } + Step::ManifestLive(id) => { + assert!(self.benchmark.is_some()); + self.benchmark.as_mut().unwrap().manifest_live(*id)? + } + Step::ManifestRestored(id) => { + assert!(self.benchmark.is_some()); + self.benchmark.as_mut().unwrap().manifest_restored(*id)? + } + Step::CompareManifests(first, second) => { + assert!(self.benchmark.is_some()); + self.benchmark + .as_mut() + .unwrap() + .compare_manifests(*first, *second)? + } }; let t = std::time::Duration::from_millis(10); @@ -115,6 +148,7 @@ struct Benchmark { client: ObnamClient, server: ObnamServer, live: TempDir, + manifests: HashMap, } impl Benchmark { @@ -127,6 +161,7 @@ impl Benchmark { client, server, live, + manifests: HashMap::new(), }) } @@ -194,6 +229,49 @@ impl Benchmark { info!("restoring backup {} in benchmark {}", n, self.name()); Ok(OpMeasurements::new(self.name(), Operation::Restore)) } + + fn manifest_live(&mut self, id: usize) -> Result { + info!("make manifest {} of current test data", id); + if self.manifests.contains_key(&id) { + return Err(SuiteError::ManifestExists(id)); + } + let m = summain(self.live.path()).map_err(SuiteError::Summain)?; + self.manifests.insert(id, m); + Ok(OpMeasurements::new(self.name(), Operation::ManifestLive)) + } + + fn manifest_restored(&mut self, id: usize) -> Result { + info!("make manifest {} of latest restored data", id); + if self.manifests.contains_key(&id) { + return Err(SuiteError::ManifestExists(id)); + } + self.manifests.insert(id, "dummy".to_string()); + Ok(OpMeasurements::new(self.name(), Operation::ManifestLive)) + } + + fn compare_manifests( + &mut self, + first: usize, + second: usize, + ) -> Result { + info!("compare manifests {} and {}", first, second); + let m1 = self.manifest(first)?; + let m2 = self.manifest(second)?; + if m1 != m2 { + error!("first manifest:\n{}", m1); + error!("second manifest:\n{}", m2); + return Err(SuiteError::ManifestsDiffer(first, second)); + } + Ok(OpMeasurements::new(self.name(), Operation::ManifestLive)) + } + + fn manifest(&self, id: usize) -> Result { + if let Some(m) = self.manifests.get(&id) { + Ok(m.clone()) + } else { + Err(SuiteError::ManifestMissing(id)) + } + } } #[derive(Debug, Default)] diff --git a/src/summain.rs b/src/summain.rs new file mode 100644 index 0000000..ea6044a --- /dev/null +++ b/src/summain.rs @@ -0,0 +1,24 @@ +use std::path::Path; +use std::process::Command; + +#[derive(Debug, thiserror::Error)] +pub enum SummainError { + #[error("failed to run summain: {0}")] + Run(std::io::Error), +} + +pub fn summain(root: &Path) -> Result { + let output = Command::new("summain") + .arg(root) + .output() + .map_err(SummainError::Run)?; + if output.status.code() != Some(0) { + eprintln!("{}", String::from_utf8_lossy(&output.stdout)); + eprintln!("{}", String::from_utf8_lossy(&output.stderr)); + std::process::exit(1); + } + + Ok(String::from_utf8_lossy(&output.stdout) + .to_owned() + .to_string()) +} -- cgit v1.2.1 From f368595170ccfe9689b7ce2da9e26454d9bd4edb Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 4 Jan 2022 19:14:18 +0200 Subject: feat: implement step to restore data Sponsored-by: author --- obnam-benchmark.md | 33 +++++++++++++++++++++++++++++++++ src/specification.rs | 11 ++++++----- src/suite.rs | 42 +++++++++++++++++++++++++++++++++++++++++- src/summain.rs | 3 ++- 4 files changed, 82 insertions(+), 7 deletions(-) diff --git a/obnam-benchmark.md b/obnam-benchmark.md index 4440b57..5fc3164 100644 --- a/obnam-benchmark.md +++ b/obnam-benchmark.md @@ -157,3 +157,36 @@ benchmarks: backups: - changes: [] ``` + + +## Benchmark with several backups + +_Requirement: The benchmark tool can benchmarks with more than one +backup._ + +We verify this by running a benchmark with three backup generations. + +~~~scenario +given an installed Rust program obnam-benchmark +given file three.yaml +when I run obnam-benchmark run three.yaml --output three.json +then file three.json is valid JSON +~~~ + +```{#three.yaml .yaml .file .numberLines} +benchmarks: +- benchmark: three + backups: + - changes: + - create: + files: 1 + file_size: 1024 + - changes: + - create: + files: 2 + file_size: 2048 + - changes: + - create: + files: 4 + file_size: 4096 +``` diff --git a/src/specification.rs b/src/specification.rs index 9e315f7..4eeb430 100644 --- a/src/specification.rs +++ b/src/specification.rs @@ -113,10 +113,11 @@ impl Specification { /// Serialize the specification into a sequence of steps to execute it. pub fn steps(&self) -> Vec { let mut steps = vec![]; - let num_benchmarks = self.benchmarks.len(); - let after_base = num_benchmarks; - let restore_base = after_base + num_benchmarks; for b in self.benchmarks.iter() { + let n = b.backups.len(); + let after_base = n; + let restore_base = 2 * n; + steps.push(Step::Start(b.benchmark.to_string())); for (i, backup) in b.backups.iter().enumerate() { for change in backup.changes.iter() { @@ -131,8 +132,8 @@ impl Specification { for (i, _) in b.backups.iter().enumerate() { steps.push(Step::Restore(i)); let restored = restore_base + i; - // steps.push(Step::ManifestRestored(restored)); - // steps.push(Step::CompareManifests(i, restored)); + steps.push(Step::ManifestRestored(restored)); + steps.push(Step::CompareManifests(i, restored)); } steps.push(Step::Stop(b.benchmark.to_string())); } diff --git a/src/suite.rs b/src/suite.rs index b4e8abb..224238a 100644 --- a/src/suite.rs +++ b/src/suite.rs @@ -42,6 +42,10 @@ pub enum SuiteError { #[error("Error looking up file metadata: {0}: {1}")] FileMeta(PathBuf, walkdir::Error), + /// Error removing restored data. + #[error("Error removing temporary directory: {0}: {1}")] + RemoveRestored(PathBuf, std::io::Error), + /// Error using an Obnam client. #[error(transparent)] Client(#[from] ObnamClientError), @@ -148,6 +152,8 @@ struct Benchmark { client: ObnamClient, server: ObnamServer, live: TempDir, + restored: TempDir, + gen_ids: HashMap, manifests: HashMap, } @@ -155,12 +161,15 @@ impl Benchmark { fn new(name: &str, manager: &DaemonManager) -> Result { let server = ObnamServer::new(manager)?; let live = tempdir().map_err(SuiteError::TempDir)?; + let restored = tempdir().map_err(SuiteError::TempDir)?; let client = ObnamClient::new(server.url(), live.path().to_path_buf())?; Ok(Self { name: name.to_string(), client, server, live, + restored, + gen_ids: HashMap::new(), manifests: HashMap::new(), }) } @@ -173,6 +182,10 @@ impl Benchmark { self.live.path().to_path_buf() } + fn restored(&self) -> PathBuf { + self.restored.path().join("restored") + } + fn start(&mut self) -> Result { info!("starting benchmark {}", self.name()); self.client @@ -218,6 +231,15 @@ impl Benchmark { fn backup(&mut self, n: usize) -> Result { info!("making backup {} in benchmark {}", n, self.name()); self.client.run(&["backup"])?; + let gen_id = self + .client + .run(&["resolve", "latest"])? + .strip_suffix('\n') + .or(Some("")) + .unwrap() + .to_string(); + debug!("backed up generation {}", gen_id); + self.gen_ids.insert(n, gen_id); let mut om = OpMeasurements::new(self.name(), Operation::Backup); let stats = filestats(&self.live())?; om.push(Measurement::TotalFiles(stats.count)); @@ -227,6 +249,15 @@ impl Benchmark { fn restore(&mut self, n: usize) -> Result { info!("restoring backup {} in benchmark {}", n, self.name()); + debug!("first removing all data from restore directory"); + let restored = self.restored(); + if restored.exists() { + std::fs::remove_dir_all(&restored) + .map_err(|err| SuiteError::RemoveRestored(restored, err))?; + } + let gen_id = self.gen_ids.get(&n).unwrap(); + let path = self.restored().display().to_string(); + self.client.run(&["restore", gen_id, &path])?; Ok(OpMeasurements::new(self.name(), Operation::Restore)) } @@ -245,7 +276,16 @@ impl Benchmark { if self.manifests.contains_key(&id) { return Err(SuiteError::ManifestExists(id)); } - self.manifests.insert(id, "dummy".to_string()); + debug!("self.restored()={}", self.restored().display()); + let restored = format!( + "{}{}", + self.restored().display(), + self.live.path().display() + ); + let restored = Path::new(&restored); + debug!("restored directory is {}", restored.display()); + let m = summain(restored).map_err(SuiteError::Summain)?; + self.manifests.insert(id, m); Ok(OpMeasurements::new(self.name(), Operation::ManifestLive)) } diff --git a/src/summain.rs b/src/summain.rs index ea6044a..ce9cf0f 100644 --- a/src/summain.rs +++ b/src/summain.rs @@ -9,7 +9,8 @@ pub enum SummainError { pub fn summain(root: &Path) -> Result { let output = Command::new("summain") - .arg(root) + .arg(".") + .current_dir(root) .output() .map_err(SummainError::Run)?; if output.status.code() != Some(0) { -- cgit v1.2.1