summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLars Wirzenius <liw@liw.fi>2022-01-05 08:39:37 +0000
committerLars Wirzenius <liw@liw.fi>2022-01-05 08:39:37 +0000
commit5fbd5a2ee98badc71867d2e7fce4d1429189267a (patch)
treee16886fb2e0e443d6bb02b99375c66d8f40cd96b
parent945b7cb605691889cef4d81e421816e55243c95b (diff)
parentf368595170ccfe9689b7ce2da9e26454d9bd4edb (diff)
downloadobnam-benchmark-5fbd5a2ee98badc71867d2e7fce4d1429189267a.tar.gz
Merge branch 'summain-try2' into 'main'
test: be quiet unless test.py fails See merge request obnam/obnam-benchmark!5
-rwxr-xr-xcheck2
-rw-r--r--obnam-benchmark.md33
-rw-r--r--src/bin/obnam-benchmark.rs27
-rw-r--r--src/lib.rs1
-rw-r--r--src/result.rs3
-rw-r--r--src/specification.rs11
-rw-r--r--src/step.rs17
-rw-r--r--src/suite.rs120
-rw-r--r--src/summain.rs25
9 files changed, 235 insertions, 4 deletions
diff --git a/check b/check
index c928b06..cc8ef6c 100755
--- a/check
+++ b/check
@@ -8,6 +8,6 @@ chronic cargo test -q
subplot docgen obnam-benchmark.md -o obnam-benchmark.html
subplot docgen obnam-benchmark.md -o obnam-benchmark.pdf
subplot codegen obnam-benchmark.md -o test.py
-python3 test.py --log test.log "$@"
+chronic python3 test.py --log test.log "$@"
cargo fmt -- --check
echo A-OK
diff --git a/obnam-benchmark.md b/obnam-benchmark.md
index 4440b57..5fc3164 100644
--- a/obnam-benchmark.md
+++ b/obnam-benchmark.md
@@ -157,3 +157,36 @@ benchmarks:
backups:
- changes: []
```
+
+
+## Benchmark with several backups
+
+_Requirement: The benchmark tool can benchmarks with more than one
+backup._
+
+We verify this by running a benchmark with three backup generations.
+
+~~~scenario
+given an installed Rust program obnam-benchmark
+given file three.yaml
+when I run obnam-benchmark run three.yaml --output three.json
+then file three.json is valid JSON
+~~~
+
+```{#three.yaml .yaml .file .numberLines}
+benchmarks:
+- benchmark: three
+ backups:
+ - changes:
+ - create:
+ files: 1
+ file_size: 1024
+ - changes:
+ - create:
+ files: 2
+ file_size: 2048
+ - changes:
+ - create:
+ files: 4
+ file_size: 4096
+```
diff --git a/src/bin/obnam-benchmark.rs b/src/bin/obnam-benchmark.rs
index e6a476c..f079d75 100644
--- a/src/bin/obnam-benchmark.rs
+++ b/src/bin/obnam-benchmark.rs
@@ -10,7 +10,6 @@ use structopt::StructOpt;
fn main() {
pretty_env_logger::init_custom_env("OBNAM_BENCHMARK_LOG");
- println!("START");
info!("obnam-benchmark starts");
if let Err(err) = real_main() {
eprintln!("ERROR: {}", err);
@@ -18,7 +17,6 @@ fn main() {
exit(1);
}
info!("obnam-benchmark ends successfully");
- println!("END");
}
fn real_main() -> anyhow::Result<()> {
@@ -30,6 +28,7 @@ fn real_main() -> anyhow::Result<()> {
Command::GenerateJunk(x) => x.run()?,
Command::Run(x) => x.run()?,
Command::Spec(x) => x.run()?,
+ Command::Steps(x) => x.run()?,
}
Ok(())
@@ -51,6 +50,9 @@ enum Command {
/// Generate some junk data in a file.
GenerateJunk(GenerateJunk),
+
+ /// Show the steps for running a benchmark.
+ Steps(Steps),
}
#[derive(Debug, StructOpt)]
@@ -129,3 +131,24 @@ impl GenerateJunk {
Ok(())
}
}
+
+#[derive(Debug, StructOpt)]
+struct Steps {
+ /// Name of the specification file
+ #[structopt(parse(from_os_str))]
+ spec: PathBuf,
+}
+
+impl Steps {
+ fn run(&self) -> anyhow::Result<()> {
+ info!(
+ "showing steps to run benchmarks from {}",
+ self.spec.display()
+ );
+ let spec = Specification::from_file(&self.spec)?;
+ for step in spec.steps().iter() {
+ println!("{:?}", step);
+ }
+ Ok(())
+ }
+}
diff --git a/src/lib.rs b/src/lib.rs
index e8c6d82..f491ae7 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -37,4 +37,5 @@ pub mod server;
pub mod specification;
pub mod step;
pub mod suite;
+pub mod summain;
pub mod tlsgen;
diff --git a/src/result.rs b/src/result.rs
index cf6c0ac..9e4119b 100644
--- a/src/result.rs
+++ b/src/result.rs
@@ -28,6 +28,9 @@ pub enum Operation {
Delete,
Backup,
Restore,
+ ManifestLive,
+ ManifestRestored,
+ CompareManiests,
}
impl Result {
diff --git a/src/specification.rs b/src/specification.rs
index 2932f41..4eeb430 100644
--- a/src/specification.rs
+++ b/src/specification.rs
@@ -114,15 +114,26 @@ impl Specification {
pub fn steps(&self) -> Vec<Step> {
let mut steps = vec![];
for b in self.benchmarks.iter() {
+ let n = b.backups.len();
+ let after_base = n;
+ let restore_base = 2 * n;
+
steps.push(Step::Start(b.benchmark.to_string()));
for (i, backup) in b.backups.iter().enumerate() {
for change in backup.changes.iter() {
steps.push(Step::from(change));
}
+ steps.push(Step::ManifestLive(i));
steps.push(Step::Backup(i));
+ let after = after_base + i;
+ steps.push(Step::ManifestLive(after));
+ steps.push(Step::CompareManifests(i, after));
}
for (i, _) in b.backups.iter().enumerate() {
steps.push(Step::Restore(i));
+ let restored = restore_base + i;
+ steps.push(Step::ManifestRestored(restored));
+ steps.push(Step::CompareManifests(i, restored));
}
steps.push(Step::Stop(b.benchmark.to_string()));
}
diff --git a/src/step.rs b/src/step.rs
index fa7abd6..d18b475 100644
--- a/src/step.rs
+++ b/src/step.rs
@@ -29,6 +29,23 @@ pub enum Step {
/// n
usize,
),
+ /// Create and remember a manifest of current live data. Call it id.
+ ManifestLive(
+ /// id
+ usize,
+ ),
+ /// Create and remember a manifest of latest restored data. Call it id.
+ ManifestRestored(
+ /// id
+ usize,
+ ),
+ /// Compare two manifests for equality.
+ CompareManifests(
+ /// First
+ usize,
+ /// Second.
+ usize,
+ ),
}
/// Possible errors from executing a benchmark step.
diff --git a/src/suite.rs b/src/suite.rs
index e900330..224238a 100644
--- a/src/suite.rs
+++ b/src/suite.rs
@@ -5,7 +5,9 @@ use crate::result::{Measurement, OpMeasurements, Operation};
use crate::server::{ObnamServer, ObnamServerError};
use crate::specification::{Create, FileCount};
use crate::step::Step;
-use log::{debug, info};
+use crate::summain::{summain, SummainError};
+use log::{debug, error, info};
+use std::collections::HashMap;
use std::fs::File;
use std::path::{Path, PathBuf};
use std::time::Instant;
@@ -40,6 +42,10 @@ pub enum SuiteError {
#[error("Error looking up file metadata: {0}: {1}")]
FileMeta(PathBuf, walkdir::Error),
+ /// Error removing restored data.
+ #[error("Error removing temporary directory: {0}: {1}")]
+ RemoveRestored(PathBuf, std::io::Error),
+
/// Error using an Obnam client.
#[error(transparent)]
Client(#[from] ObnamClientError),
@@ -47,6 +53,22 @@ pub enum SuiteError {
/// Error managing an Obnam server.
#[error(transparent)]
Server(#[from] ObnamServerError),
+
+ /// Suite already has a manifest with a given id.
+ #[error("Suite already has manifest {0}: this is a bug")]
+ ManifestExists(usize),
+
+ /// Suite doesn't have a manifest with a given id.
+ #[error("Suite doesn't have a manifest {0}: this is a bug")]
+ ManifestMissing(usize),
+
+ /// Manifests are not identical.
+ #[error("Manifests {0} and {1} are not identical, as expected")]
+ ManifestsDiffer(usize, usize),
+
+ /// Error running summain.
+ #[error(transparent)]
+ Summain(SummainError),
}
impl Suite {
@@ -98,6 +120,21 @@ impl Suite {
assert!(self.benchmark.is_some());
self.benchmark.as_mut().unwrap().restore(*x)?
}
+ Step::ManifestLive(id) => {
+ assert!(self.benchmark.is_some());
+ self.benchmark.as_mut().unwrap().manifest_live(*id)?
+ }
+ Step::ManifestRestored(id) => {
+ assert!(self.benchmark.is_some());
+ self.benchmark.as_mut().unwrap().manifest_restored(*id)?
+ }
+ Step::CompareManifests(first, second) => {
+ assert!(self.benchmark.is_some());
+ self.benchmark
+ .as_mut()
+ .unwrap()
+ .compare_manifests(*first, *second)?
+ }
};
let t = std::time::Duration::from_millis(10);
@@ -115,18 +152,25 @@ struct Benchmark {
client: ObnamClient,
server: ObnamServer,
live: TempDir,
+ restored: TempDir,
+ gen_ids: HashMap<usize, String>,
+ manifests: HashMap<usize, String>,
}
impl Benchmark {
fn new(name: &str, manager: &DaemonManager) -> Result<Self, SuiteError> {
let server = ObnamServer::new(manager)?;
let live = tempdir().map_err(SuiteError::TempDir)?;
+ let restored = tempdir().map_err(SuiteError::TempDir)?;
let client = ObnamClient::new(server.url(), live.path().to_path_buf())?;
Ok(Self {
name: name.to_string(),
client,
server,
live,
+ restored,
+ gen_ids: HashMap::new(),
+ manifests: HashMap::new(),
})
}
@@ -138,6 +182,10 @@ impl Benchmark {
self.live.path().to_path_buf()
}
+ fn restored(&self) -> PathBuf {
+ self.restored.path().join("restored")
+ }
+
fn start(&mut self) -> Result<OpMeasurements, SuiteError> {
info!("starting benchmark {}", self.name());
self.client
@@ -183,6 +231,15 @@ impl Benchmark {
fn backup(&mut self, n: usize) -> Result<OpMeasurements, SuiteError> {
info!("making backup {} in benchmark {}", n, self.name());
self.client.run(&["backup"])?;
+ let gen_id = self
+ .client
+ .run(&["resolve", "latest"])?
+ .strip_suffix('\n')
+ .or(Some(""))
+ .unwrap()
+ .to_string();
+ debug!("backed up generation {}", gen_id);
+ self.gen_ids.insert(n, gen_id);
let mut om = OpMeasurements::new(self.name(), Operation::Backup);
let stats = filestats(&self.live())?;
om.push(Measurement::TotalFiles(stats.count));
@@ -192,8 +249,69 @@ impl Benchmark {
fn restore(&mut self, n: usize) -> Result<OpMeasurements, SuiteError> {
info!("restoring backup {} in benchmark {}", n, self.name());
+ debug!("first removing all data from restore directory");
+ let restored = self.restored();
+ if restored.exists() {
+ std::fs::remove_dir_all(&restored)
+ .map_err(|err| SuiteError::RemoveRestored(restored, err))?;
+ }
+ let gen_id = self.gen_ids.get(&n).unwrap();
+ let path = self.restored().display().to_string();
+ self.client.run(&["restore", gen_id, &path])?;
Ok(OpMeasurements::new(self.name(), Operation::Restore))
}
+
+ fn manifest_live(&mut self, id: usize) -> Result<OpMeasurements, SuiteError> {
+ info!("make manifest {} of current test data", id);
+ if self.manifests.contains_key(&id) {
+ return Err(SuiteError::ManifestExists(id));
+ }
+ let m = summain(self.live.path()).map_err(SuiteError::Summain)?;
+ self.manifests.insert(id, m);
+ Ok(OpMeasurements::new(self.name(), Operation::ManifestLive))
+ }
+
+ fn manifest_restored(&mut self, id: usize) -> Result<OpMeasurements, SuiteError> {
+ info!("make manifest {} of latest restored data", id);
+ if self.manifests.contains_key(&id) {
+ return Err(SuiteError::ManifestExists(id));
+ }
+ debug!("self.restored()={}", self.restored().display());
+ let restored = format!(
+ "{}{}",
+ self.restored().display(),
+ self.live.path().display()
+ );
+ let restored = Path::new(&restored);
+ debug!("restored directory is {}", restored.display());
+ let m = summain(restored).map_err(SuiteError::Summain)?;
+ self.manifests.insert(id, m);
+ Ok(OpMeasurements::new(self.name(), Operation::ManifestLive))
+ }
+
+ fn compare_manifests(
+ &mut self,
+ first: usize,
+ second: usize,
+ ) -> Result<OpMeasurements, SuiteError> {
+ info!("compare manifests {} and {}", first, second);
+ let m1 = self.manifest(first)?;
+ let m2 = self.manifest(second)?;
+ if m1 != m2 {
+ error!("first manifest:\n{}", m1);
+ error!("second manifest:\n{}", m2);
+ return Err(SuiteError::ManifestsDiffer(first, second));
+ }
+ Ok(OpMeasurements::new(self.name(), Operation::ManifestLive))
+ }
+
+ fn manifest(&self, id: usize) -> Result<String, SuiteError> {
+ if let Some(m) = self.manifests.get(&id) {
+ Ok(m.clone())
+ } else {
+ Err(SuiteError::ManifestMissing(id))
+ }
+ }
}
#[derive(Debug, Default)]
diff --git a/src/summain.rs b/src/summain.rs
new file mode 100644
index 0000000..ce9cf0f
--- /dev/null
+++ b/src/summain.rs
@@ -0,0 +1,25 @@
+use std::path::Path;
+use std::process::Command;
+
+#[derive(Debug, thiserror::Error)]
+pub enum SummainError {
+ #[error("failed to run summain: {0}")]
+ Run(std::io::Error),
+}
+
+pub fn summain(root: &Path) -> Result<String, SummainError> {
+ let output = Command::new("summain")
+ .arg(".")
+ .current_dir(root)
+ .output()
+ .map_err(SummainError::Run)?;
+ if output.status.code() != Some(0) {
+ eprintln!("{}", String::from_utf8_lossy(&output.stdout));
+ eprintln!("{}", String::from_utf8_lossy(&output.stderr));
+ std::process::exit(1);
+ }
+
+ Ok(String::from_utf8_lossy(&output.stdout)
+ .to_owned()
+ .to_string())
+}