From 64a11eca7f3cf75d735436366d8489ab93964dbf Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 4 Jan 2022 18:04:57 +0200 Subject: feat: add "steps" subcommand Sponsored-by: author --- src/bin/obnam-benchmark.rs | 27 ++++++++++++++-- src/lib.rs | 1 + src/result.rs | 3 ++ src/specification.rs | 10 ++++++ src/step.rs | 17 ++++++++++ src/suite.rs | 80 +++++++++++++++++++++++++++++++++++++++++++++- src/summain.rs | 24 ++++++++++++++ 7 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 src/summain.rs diff --git a/src/bin/obnam-benchmark.rs b/src/bin/obnam-benchmark.rs index e6a476c..f079d75 100644 --- a/src/bin/obnam-benchmark.rs +++ b/src/bin/obnam-benchmark.rs @@ -10,7 +10,6 @@ use structopt::StructOpt; fn main() { pretty_env_logger::init_custom_env("OBNAM_BENCHMARK_LOG"); - println!("START"); info!("obnam-benchmark starts"); if let Err(err) = real_main() { eprintln!("ERROR: {}", err); @@ -18,7 +17,6 @@ fn main() { exit(1); } info!("obnam-benchmark ends successfully"); - println!("END"); } fn real_main() -> anyhow::Result<()> { @@ -30,6 +28,7 @@ fn real_main() -> anyhow::Result<()> { Command::GenerateJunk(x) => x.run()?, Command::Run(x) => x.run()?, Command::Spec(x) => x.run()?, + Command::Steps(x) => x.run()?, } Ok(()) @@ -51,6 +50,9 @@ enum Command { /// Generate some junk data in a file. GenerateJunk(GenerateJunk), + + /// Show the steps for running a benchmark. + Steps(Steps), } #[derive(Debug, StructOpt)] @@ -129,3 +131,24 @@ impl GenerateJunk { Ok(()) } } + +#[derive(Debug, StructOpt)] +struct Steps { + /// Name of the specification file + #[structopt(parse(from_os_str))] + spec: PathBuf, +} + +impl Steps { + fn run(&self) -> anyhow::Result<()> { + info!( + "showing steps to run benchmarks from {}", + self.spec.display() + ); + let spec = Specification::from_file(&self.spec)?; + for step in spec.steps().iter() { + println!("{:?}", step); + } + Ok(()) + } +} diff --git a/src/lib.rs b/src/lib.rs index e8c6d82..f491ae7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -37,4 +37,5 @@ pub mod server; pub mod specification; pub mod step; pub mod suite; +pub mod summain; pub mod tlsgen; diff --git a/src/result.rs b/src/result.rs index cf6c0ac..9e4119b 100644 --- a/src/result.rs +++ b/src/result.rs @@ -28,6 +28,9 @@ pub enum Operation { Delete, Backup, Restore, + ManifestLive, + ManifestRestored, + CompareManiests, } impl Result { diff --git a/src/specification.rs b/src/specification.rs index 2932f41..9e315f7 100644 --- a/src/specification.rs +++ b/src/specification.rs @@ -113,16 +113,26 @@ impl Specification { /// Serialize the specification into a sequence of steps to execute it. pub fn steps(&self) -> Vec { let mut steps = vec![]; + let num_benchmarks = self.benchmarks.len(); + let after_base = num_benchmarks; + let restore_base = after_base + num_benchmarks; for b in self.benchmarks.iter() { steps.push(Step::Start(b.benchmark.to_string())); for (i, backup) in b.backups.iter().enumerate() { for change in backup.changes.iter() { steps.push(Step::from(change)); } + steps.push(Step::ManifestLive(i)); steps.push(Step::Backup(i)); + let after = after_base + i; + steps.push(Step::ManifestLive(after)); + steps.push(Step::CompareManifests(i, after)); } for (i, _) in b.backups.iter().enumerate() { steps.push(Step::Restore(i)); + let restored = restore_base + i; + // steps.push(Step::ManifestRestored(restored)); + // steps.push(Step::CompareManifests(i, restored)); } steps.push(Step::Stop(b.benchmark.to_string())); } diff --git a/src/step.rs b/src/step.rs index fa7abd6..d18b475 100644 --- a/src/step.rs +++ b/src/step.rs @@ -29,6 +29,23 @@ pub enum Step { /// n usize, ), + /// Create and remember a manifest of current live data. Call it id. + ManifestLive( + /// id + usize, + ), + /// Create and remember a manifest of latest restored data. Call it id. + ManifestRestored( + /// id + usize, + ), + /// Compare two manifests for equality. + CompareManifests( + /// First + usize, + /// Second. + usize, + ), } /// Possible errors from executing a benchmark step. diff --git a/src/suite.rs b/src/suite.rs index e900330..b4e8abb 100644 --- a/src/suite.rs +++ b/src/suite.rs @@ -5,7 +5,9 @@ use crate::result::{Measurement, OpMeasurements, Operation}; use crate::server::{ObnamServer, ObnamServerError}; use crate::specification::{Create, FileCount}; use crate::step::Step; -use log::{debug, info}; +use crate::summain::{summain, SummainError}; +use log::{debug, error, info}; +use std::collections::HashMap; use std::fs::File; use std::path::{Path, PathBuf}; use std::time::Instant; @@ -47,6 +49,22 @@ pub enum SuiteError { /// Error managing an Obnam server. #[error(transparent)] Server(#[from] ObnamServerError), + + /// Suite already has a manifest with a given id. + #[error("Suite already has manifest {0}: this is a bug")] + ManifestExists(usize), + + /// Suite doesn't have a manifest with a given id. + #[error("Suite doesn't have a manifest {0}: this is a bug")] + ManifestMissing(usize), + + /// Manifests are not identical. + #[error("Manifests {0} and {1} are not identical, as expected")] + ManifestsDiffer(usize, usize), + + /// Error running summain. + #[error(transparent)] + Summain(SummainError), } impl Suite { @@ -98,6 +116,21 @@ impl Suite { assert!(self.benchmark.is_some()); self.benchmark.as_mut().unwrap().restore(*x)? } + Step::ManifestLive(id) => { + assert!(self.benchmark.is_some()); + self.benchmark.as_mut().unwrap().manifest_live(*id)? + } + Step::ManifestRestored(id) => { + assert!(self.benchmark.is_some()); + self.benchmark.as_mut().unwrap().manifest_restored(*id)? + } + Step::CompareManifests(first, second) => { + assert!(self.benchmark.is_some()); + self.benchmark + .as_mut() + .unwrap() + .compare_manifests(*first, *second)? + } }; let t = std::time::Duration::from_millis(10); @@ -115,6 +148,7 @@ struct Benchmark { client: ObnamClient, server: ObnamServer, live: TempDir, + manifests: HashMap, } impl Benchmark { @@ -127,6 +161,7 @@ impl Benchmark { client, server, live, + manifests: HashMap::new(), }) } @@ -194,6 +229,49 @@ impl Benchmark { info!("restoring backup {} in benchmark {}", n, self.name()); Ok(OpMeasurements::new(self.name(), Operation::Restore)) } + + fn manifest_live(&mut self, id: usize) -> Result { + info!("make manifest {} of current test data", id); + if self.manifests.contains_key(&id) { + return Err(SuiteError::ManifestExists(id)); + } + let m = summain(self.live.path()).map_err(SuiteError::Summain)?; + self.manifests.insert(id, m); + Ok(OpMeasurements::new(self.name(), Operation::ManifestLive)) + } + + fn manifest_restored(&mut self, id: usize) -> Result { + info!("make manifest {} of latest restored data", id); + if self.manifests.contains_key(&id) { + return Err(SuiteError::ManifestExists(id)); + } + self.manifests.insert(id, "dummy".to_string()); + Ok(OpMeasurements::new(self.name(), Operation::ManifestLive)) + } + + fn compare_manifests( + &mut self, + first: usize, + second: usize, + ) -> Result { + info!("compare manifests {} and {}", first, second); + let m1 = self.manifest(first)?; + let m2 = self.manifest(second)?; + if m1 != m2 { + error!("first manifest:\n{}", m1); + error!("second manifest:\n{}", m2); + return Err(SuiteError::ManifestsDiffer(first, second)); + } + Ok(OpMeasurements::new(self.name(), Operation::ManifestLive)) + } + + fn manifest(&self, id: usize) -> Result { + if let Some(m) = self.manifests.get(&id) { + Ok(m.clone()) + } else { + Err(SuiteError::ManifestMissing(id)) + } + } } #[derive(Debug, Default)] diff --git a/src/summain.rs b/src/summain.rs new file mode 100644 index 0000000..ea6044a --- /dev/null +++ b/src/summain.rs @@ -0,0 +1,24 @@ +use std::path::Path; +use std::process::Command; + +#[derive(Debug, thiserror::Error)] +pub enum SummainError { + #[error("failed to run summain: {0}")] + Run(std::io::Error), +} + +pub fn summain(root: &Path) -> Result { + let output = Command::new("summain") + .arg(root) + .output() + .map_err(SummainError::Run)?; + if output.status.code() != Some(0) { + eprintln!("{}", String::from_utf8_lossy(&output.stdout)); + eprintln!("{}", String::from_utf8_lossy(&output.stderr)); + std::process::exit(1); + } + + Ok(String::from_utf8_lossy(&output.stdout) + .to_owned() + .to_string()) +} -- cgit v1.2.1