feat: add "steps" subcommand

Sponsored-by: author

7 files changed, 159 insertions, 3 deletions

diff --git a/src/bin/obnam-benchmark.rs b/src/bin/obnam-benchmark.rs
index e6a476c..f079d75 100644
--- a/src/bin/obnam-benchmark.rs
+++ b/src/bin/obnam-benchmark.rs
@@ -10,7 +10,6 @@ use structopt::StructOpt;
 
 fn main() {
     pretty_env_logger::init_custom_env("OBNAM_BENCHMARK_LOG");
-    println!("START");
     info!("obnam-benchmark starts");
     if let Err(err) = real_main() {
         eprintln!("ERROR: {}", err);
@@ -18,7 +17,6 @@ fn main() {
         exit(1);
     }
     info!("obnam-benchmark ends successfully");
-    println!("END");
 }
 
 fn real_main() -> anyhow::Result<()> {
@@ -30,6 +28,7 @@ fn real_main() -> anyhow::Result<()> {
         Command::GenerateJunk(x) => x.run()?,
         Command::Run(x) => x.run()?,
         Command::Spec(x) => x.run()?,
+        Command::Steps(x) => x.run()?,
     }
 
     Ok(())
@@ -51,6 +50,9 @@ enum Command {
 
     /// Generate some junk data in a file.
     GenerateJunk(GenerateJunk),
+
+    /// Show the steps for running a benchmark.
+    Steps(Steps),
 }
 
 #[derive(Debug, StructOpt)]
@@ -129,3 +131,24 @@ impl GenerateJunk {
         Ok(())
     }
 }
+
+#[derive(Debug, StructOpt)]
+struct Steps {
+    /// Name of the specification file
+    #[structopt(parse(from_os_str))]
+    spec: PathBuf,
+}
+
+impl Steps {
+    fn run(&self) -> anyhow::Result<()> {
+        info!(
+            "showing steps to run benchmarks from {}",
+            self.spec.display()
+        );
+        let spec = Specification::from_file(&self.spec)?;
+        for step in spec.steps().iter() {
+            println!("{:?}", step);
+        }
+        Ok(())
+    }
+}
diff --git a/src/lib.rs b/src/lib.rs
index e8c6d82..f491ae7 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -37,4 +37,5 @@ pub mod server;
 pub mod specification;
 pub mod step;
 pub mod suite;
+pub mod summain;
 pub mod tlsgen;
diff --git a/src/result.rs b/src/result.rs
index cf6c0ac..9e4119b 100644
--- a/src/result.rs
+++ b/src/result.rs
@@ -28,6 +28,9 @@ pub enum Operation {
     Delete,
     Backup,
     Restore,
+    ManifestLive,
+    ManifestRestored,
+    CompareManiests,
 }
 
 impl Result {
diff --git a/src/specification.rs b/src/specification.rs
index 2932f41..9e315f7 100644
--- a/src/specification.rs
+++ b/src/specification.rs
@@ -113,16 +113,26 @@ impl Specification {
     /// Serialize the specification into a sequence of steps to execute it.
     pub fn steps(&self) -> Vec<Step> {
         let mut steps = vec![];
+        let num_benchmarks = self.benchmarks.len();
+        let after_base = num_benchmarks;
+        let restore_base = after_base + num_benchmarks;
         for b in self.benchmarks.iter() {
             steps.push(Step::Start(b.benchmark.to_string()));
             for (i, backup) in b.backups.iter().enumerate() {
                 for change in backup.changes.iter() {
                     steps.push(Step::from(change));
                 }
+                steps.push(Step::ManifestLive(i));
                 steps.push(Step::Backup(i));
+                let after = after_base + i;
+                steps.push(Step::ManifestLive(after));
+                steps.push(Step::CompareManifests(i, after));
             }
             for (i, _) in b.backups.iter().enumerate() {
                 steps.push(Step::Restore(i));
+                let restored = restore_base + i;
+                //                steps.push(Step::ManifestRestored(restored));
+                //                steps.push(Step::CompareManifests(i, restored));
             }
             steps.push(Step::Stop(b.benchmark.to_string()));
         }
diff --git a/src/step.rs b/src/step.rs
index fa7abd6..d18b475 100644
--- a/src/step.rs
+++ b/src/step.rs
@@ -29,6 +29,23 @@ pub enum Step {
         /// n
         usize,
     ),
+    /// Create and remember a manifest of current live data. Call it id.
+    ManifestLive(
+        /// id
+        usize,
+    ),
+    /// Create and remember a manifest of latest restored data. Call it id.
+    ManifestRestored(
+        /// id
+        usize,
+    ),
+    /// Compare two manifests for equality.
+    CompareManifests(
+        /// First
+        usize,
+        /// Second.
+        usize,
+    ),
 }
 
 /// Possible errors from executing a benchmark step.
diff --git a/src/suite.rs b/src/suite.rs
index e900330..b4e8abb 100644
--- a/src/suite.rs
+++ b/src/suite.rs
@@ -5,7 +5,9 @@ use crate::result::{Measurement, OpMeasurements, Operation};
 use crate::server::{ObnamServer, ObnamServerError};
 use crate::specification::{Create, FileCount};
 use crate::step::Step;
-use log::{debug, info};
+use crate::summain::{summain, SummainError};
+use log::{debug, error, info};
+use std::collections::HashMap;
 use std::fs::File;
 use std::path::{Path, PathBuf};
 use std::time::Instant;
@@ -47,6 +49,22 @@ pub enum SuiteError {
     /// Error managing an Obnam server.
     #[error(transparent)]
     Server(#[from] ObnamServerError),
+
+    /// Suite already has a manifest with a given id.
+    #[error("Suite already has manifest {0}: this is a bug")]
+    ManifestExists(usize),
+
+    /// Suite doesn't have a manifest with a given id.
+    #[error("Suite doesn't have a manifest {0}: this is a bug")]
+    ManifestMissing(usize),
+
+    /// Manifests are not identical.
+    #[error("Manifests {0} and {1} are not identical, as expected")]
+    ManifestsDiffer(usize, usize),
+
+    /// Error running summain.
+    #[error(transparent)]
+    Summain(SummainError),
 }
 
 impl Suite {
@@ -98,6 +116,21 @@ impl Suite {
                 assert!(self.benchmark.is_some());
                 self.benchmark.as_mut().unwrap().restore(*x)?
             }
+            Step::ManifestLive(id) => {
+                assert!(self.benchmark.is_some());
+                self.benchmark.as_mut().unwrap().manifest_live(*id)?
+            }
+            Step::ManifestRestored(id) => {
+                assert!(self.benchmark.is_some());
+                self.benchmark.as_mut().unwrap().manifest_restored(*id)?
+            }
+            Step::CompareManifests(first, second) => {
+                assert!(self.benchmark.is_some());
+                self.benchmark
+                    .as_mut()
+                    .unwrap()
+                    .compare_manifests(*first, *second)?
+            }
         };
 
         let t = std::time::Duration::from_millis(10);
@@ -115,6 +148,7 @@ struct Benchmark {
     client: ObnamClient,
     server: ObnamServer,
     live: TempDir,
+    manifests: HashMap<usize, String>,
 }
 
 impl Benchmark {
@@ -127,6 +161,7 @@ impl Benchmark {
             client,
             server,
             live,
+            manifests: HashMap::new(),
         })
     }
 
@@ -194,6 +229,49 @@ impl Benchmark {
         info!("restoring backup {} in benchmark {}", n, self.name());
         Ok(OpMeasurements::new(self.name(), Operation::Restore))
     }
+
+    fn manifest_live(&mut self, id: usize) -> Result<OpMeasurements, SuiteError> {
+        info!("make manifest {} of current test data", id);
+        if self.manifests.contains_key(&id) {
+            return Err(SuiteError::ManifestExists(id));
+        }
+        let m = summain(self.live.path()).map_err(SuiteError::Summain)?;
+        self.manifests.insert(id, m);
+        Ok(OpMeasurements::new(self.name(), Operation::ManifestLive))
+    }
+
+    fn manifest_restored(&mut self, id: usize) -> Result<OpMeasurements, SuiteError> {
+        info!("make manifest {} of latest restored data", id);
+        if self.manifests.contains_key(&id) {
+            return Err(SuiteError::ManifestExists(id));
+        }
+        self.manifests.insert(id, "dummy".to_string());
+        Ok(OpMeasurements::new(self.name(), Operation::ManifestLive))
+    }
+
+    fn compare_manifests(
+        &mut self,
+        first: usize,
+        second: usize,
+    ) -> Result<OpMeasurements, SuiteError> {
+        info!("compare manifests {} and {}", first, second);
+        let m1 = self.manifest(first)?;
+        let m2 = self.manifest(second)?;
+        if m1 != m2 {
+            error!("first manifest:\n{}", m1);
+            error!("second manifest:\n{}", m2);
+            return Err(SuiteError::ManifestsDiffer(first, second));
+        }
+        Ok(OpMeasurements::new(self.name(), Operation::ManifestLive))
+    }
+
+    fn manifest(&self, id: usize) -> Result<String, SuiteError> {
+        if let Some(m) = self.manifests.get(&id) {
+            Ok(m.clone())
+        } else {
+            Err(SuiteError::ManifestMissing(id))
+        }
+    }
 }
 
 #[derive(Debug, Default)]
diff --git a/src/summain.rs b/src/summain.rs
new file mode 100644
index 0000000..ea6044a
--- /dev/null
+++ b/src/summain.rs
@@ -0,0 +1,24 @@
+use std::path::Path;
+use std::process::Command;
+
+#[derive(Debug, thiserror::Error)]
+pub enum SummainError {
+    #[error("failed to run summain: {0}")]
+    Run(std::io::Error),
+}
+
+pub fn summain(root: &Path) -> Result<String, SummainError> {
+    let output = Command::new("summain")
+        .arg(root)
+        .output()
+        .map_err(SummainError::Run)?;
+    if output.status.code() != Some(0) {
+        eprintln!("{}", String::from_utf8_lossy(&output.stdout));
+        eprintln!("{}", String::from_utf8_lossy(&output.stderr));
+        std::process::exit(1);
+    }
+
+    Ok(String::from_utf8_lossy(&output.stdout)
+        .to_owned()
+        .to_string())
+}