summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLars Wirzenius <liw@liw.fi>2022-01-05 15:28:33 +0000
committerLars Wirzenius <liw@liw.fi>2022-01-05 15:28:33 +0000
commit5701859650fc77f4a5832eddb7c045caf4571017 (patch)
treee2e6421be7d0fd3bed0e5251e7a72e988ff992c8
parentb6c4e5e0301158ac821e876aa0255168ed4043dc (diff)
parent3a635e26d57573828163cafcc5240fa8c670203d (diff)
downloadobnam-benchmark-5701859650fc77f4a5832eddb7c045caf4571017.tar.gz
Merge branch 'reporting' into 'main'
feat: add report generation Closes #3 See merge request obnam/obnam-benchmark!8
-rw-r--r--obnam-benchmark.css27
-rwxr-xr-xreport.sh7
-rw-r--r--src/bin/obnam-benchmark.rs24
-rw-r--r--src/lib.rs1
-rw-r--r--src/report.rs164
-rw-r--r--src/result.rs75
-rw-r--r--src/suite.rs7
7 files changed, 298 insertions, 7 deletions
diff --git a/obnam-benchmark.css b/obnam-benchmark.css
new file mode 100644
index 0000000..9f0c891
--- /dev/null
+++ b/obnam-benchmark.css
@@ -0,0 +1,27 @@
+<style>
+table {
+ border: 0px;
+ font-weight: normal;
+ border-collapse: collapse;
+ width: 98%;
+ margin-top: 2em;
+}
+
+table caption {
+ text-align: center;
+}
+
+th, td {
+ text-align: right;
+ font-weight: normal;
+ border-top: 1px solid black;
+ border-bottom: 1px solid black;
+ border-left: 1px solid black;
+ border-right: 1px solid black;
+ width: 12em;
+}
+
+tr:nth-child(even) {
+ background-color: #f2f2f2;
+}
+</style>
diff --git a/report.sh b/report.sh
new file mode 100755
index 0000000..4c426db
--- /dev/null
+++ b/report.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -euo pipefail
+
+obnam-benchmark report "$@" |
+ pandoc -f markdown+pipe_tables -t html -H obnam-benchmark.css \
+ --metadata title="Obnam benchmarks" - -o /dev/stdout
diff --git a/src/bin/obnam-benchmark.rs b/src/bin/obnam-benchmark.rs
index 41118cb..4ac3e4d 100644
--- a/src/bin/obnam-benchmark.rs
+++ b/src/bin/obnam-benchmark.rs
@@ -1,5 +1,6 @@
use log::{debug, error, info};
use obnam_benchmark::junk::junk;
+use obnam_benchmark::report::Reporter;
use obnam_benchmark::result::SuiteMeasurements;
use obnam_benchmark::specification::Specification;
use obnam_benchmark::suite::Suite;
@@ -30,6 +31,7 @@ fn real_main() -> anyhow::Result<()> {
Command::Run(x) => x.run()?,
Command::Spec(x) => x.run()?,
Command::Steps(x) => x.run()?,
+ Command::Report(x) => x.run()?,
}
Ok(())
@@ -54,6 +56,9 @@ enum Command {
/// Show the steps for running a benchmark.
Steps(Steps),
+
+ /// Report results for several benchmarks.
+ Report(Report),
}
#[derive(Debug, StructOpt)]
@@ -166,3 +171,22 @@ impl Steps {
Ok(())
}
}
+
+#[derive(Debug, StructOpt)]
+struct Report {
+ /// Names of the results file for which to produce a report.
+ #[structopt(parse(from_os_str))]
+ filenames: Vec<PathBuf>,
+}
+
+impl Report {
+ fn run(&self) -> anyhow::Result<()> {
+ info!("Reporting results");
+ let mut report = Reporter::default();
+ for filename in self.filenames.iter() {
+ report.push(SuiteMeasurements::from_file(filename)?);
+ }
+ report.write(&mut std::io::stdout())?;
+ Ok(())
+ }
+}
diff --git a/src/lib.rs b/src/lib.rs
index f491ae7..5f0a522 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -32,6 +32,7 @@ pub mod client;
pub mod daemon;
pub mod junk;
pub mod obnam;
+pub mod report;
pub mod result;
pub mod server;
pub mod specification;
diff --git a/src/report.rs b/src/report.rs
new file mode 100644
index 0000000..0538760
--- /dev/null
+++ b/src/report.rs
@@ -0,0 +1,164 @@
+use crate::result::{Measurement, Operation, SuiteMeasurements};
+use std::collections::HashSet;
+use std::io::Write;
+use std::iter::FromIterator;
+
+#[derive(Debug, Default)]
+pub struct Reporter {
+ results: Vec<SuiteMeasurements>,
+}
+
+impl Reporter {
+ pub fn push(&mut self, result: SuiteMeasurements) {
+ self.results.push(result);
+ }
+
+ pub fn write(&self, f: &mut dyn Write) -> Result<(), std::io::Error> {
+ // Summary tables of all benchmarks.
+ writeln!(f, "# Summaries of all benchmark runs")?;
+ let hosts = self.hosts();
+ for bench in self.benchmarks() {
+ writeln!(f)?;
+ writeln!(f, "## Benchmark {}", bench)?;
+ for op in [Operation::Backup, Operation::Restore] {
+ for host in hosts.iter() {
+ writeln!(f)?;
+ writeln!(f, "Table: {:?} on host {}, times in ms", op, host)?;
+ writeln!(f)?;
+ let mut want_headings = true;
+ for r in self.results_for(host) {
+ // Column data
+ let cols = self.durations(r, &bench, op);
+
+ if want_headings {
+ want_headings = false;
+ self.headings(f, &cols)?;
+ }
+ self.rows(f, r, &cols)?;
+ }
+ }
+ }
+ }
+
+ writeln!(f)?;
+ writeln!(f, "# Individual benchmark runs")?;
+ for host in self.hosts() {
+ writeln!(f)?;
+ writeln!(f, "## Host `{}`", host)?;
+ for r in self.results_for(&host) {
+ writeln!(f)?;
+ writeln!(f, "### Benchmark run {}", r.timestamp())?;
+ writeln!(f)?;
+ writeln!(f, "* CPUs: {}", r.cpus())?;
+ writeln!(f, "* RAM: {} MiB", r.ram() / 1024 / 1024)?;
+ for bench in r.benchmark_names() {
+ writeln!(f)?;
+ writeln!(f, "#### Benchmark `{}`", bench)?;
+ writeln!(f)?;
+ for opm in r.ops().filter(|o| o.name() == bench) {
+ let op = opm.op();
+ match op {
+ Operation::Backup => {
+ writeln!(f, "* Backup: {} ms", opm.millis())?;
+ }
+ Operation::Restore => {
+ writeln!(f, "* Restore: {} ms", opm.millis())?;
+ }
+ _ => continue,
+ }
+ for m in opm.iter() {
+ match m {
+ Measurement::DurationMs(_) => (),
+ Measurement::TotalFiles(n) => {
+ writeln!(f, " * files: {}", n)?;
+ }
+ &Measurement::TotalData(n) => {
+ writeln!(f, " * data: {} bytes", n)?;
+ }
+ }
+ }
+ }
+
+ for op in [Operation::Backup, Operation::Restore] {
+ writeln!(f)?;
+ writeln!(f, "Table: {:?}", op)?;
+ writeln!(f)?;
+ let cols = self.durations(r, &bench, op);
+ self.headings(f, &cols)?;
+ self.rows(f, r, &cols)?;
+ }
+ }
+ }
+ }
+
+ Ok(())
+ }
+
+ fn durations(&self, r: &SuiteMeasurements, bench: &str, op: Operation) -> Vec<u128> {
+ r.ops()
+ .filter(|r| r.name() == bench)
+ .filter(|r| r.op() == op)
+ .map(|r| r.millis())
+ .collect()
+ }
+
+ fn headings(&self, f: &mut dyn Write, durations: &[u128]) -> Result<(), std::io::Error> {
+ // Column headings.
+ let mut headings: Vec<String> = durations
+ .iter()
+ .enumerate()
+ .map(|(i, _)| format!("step {}", i))
+ .collect();
+ headings.insert(0, "Version".to_string());
+ for h in headings.iter() {
+ write!(f, "| {}", h)?;
+ }
+ writeln!(f, "|")?;
+
+ // Heading separator.
+ for _ in headings.iter() {
+ write!(f, "|----")?;
+ }
+ writeln!(f, "|")?;
+ Ok(())
+ }
+
+ fn rows(
+ &self,
+ f: &mut dyn Write,
+ r: &SuiteMeasurements,
+ durations: &[u128],
+ ) -> Result<(), std::io::Error> {
+ write!(f, "| {}", r.obnam_version())?;
+ for ms in durations.iter() {
+ write!(f, "| {}", ms)?;
+ }
+ writeln!(f, "|")?;
+ Ok(())
+ }
+
+ fn benchmarks(&self) -> Vec<String> {
+ let mut names = HashSet::new();
+ for r in self.results.iter() {
+ for opm in r.ops() {
+ names.insert(opm.name());
+ }
+ }
+ let mut names: Vec<String> = names.iter().map(|x| x.to_string()).collect();
+ names.sort();
+ names
+ }
+
+ fn hosts(&self) -> Vec<String> {
+ let names: HashSet<&str> = HashSet::from_iter(self.results.iter().map(|r| r.hostname()));
+ let mut names: Vec<String> = names.iter().map(|x| x.to_string()).collect();
+ names.sort();
+ names
+ }
+
+ fn results_for<'a>(&'a self, hostname: &'a str) -> impl Iterator<Item = &'a SuiteMeasurements> {
+ self.results
+ .iter()
+ .filter(move |r| r.hostname() == hostname)
+ }
+}
diff --git a/src/result.rs b/src/result.rs
index 22a49a0..cce5d2d 100644
--- a/src/result.rs
+++ b/src/result.rs
@@ -1,10 +1,14 @@
use chrono::prelude::*;
use git_testament::{git_testament, render_testament};
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
+use std::collections::HashSet;
+use std::fs::File;
+use std::iter::FromIterator;
+use std::path::{Path, PathBuf};
git_testament!(TESTAMENT);
-#[derive(Debug, Serialize)]
+#[derive(Debug, Deserialize, Serialize)]
pub struct SuiteMeasurements {
measurements: Vec<OpMeasurements>,
obnam_version: String,
@@ -15,21 +19,21 @@ pub struct SuiteMeasurements {
host_ram: u64,
}
-#[derive(Debug, Serialize)]
+#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct OpMeasurements {
benchmark: String,
op: Operation,
measurements: Vec<Measurement>,
}
-#[derive(Debug, Serialize)]
+#[derive(Debug, Clone, Deserialize, Serialize)]
pub enum Measurement {
TotalFiles(u64),
TotalData(u64),
DurationMs(u128),
}
-#[derive(Debug, Clone, Copy, Serialize)]
+#[derive(Debug, Clone, Copy, PartialEq, Deserialize, Serialize)]
pub enum Operation {
Start,
Stop,
@@ -53,6 +57,12 @@ pub enum SuiteMeasurementsError {
#[error("failed to get hostname: {0}")]
Hostname(nix::Error),
+
+ #[error("failed to open result file {0} for reading: {1}")]
+ Open(PathBuf, std::io::Error),
+
+ #[error("failed to read result file {0}: {1}")]
+ Read(PathBuf, serde_json::Error),
}
impl SuiteMeasurements {
@@ -74,6 +84,14 @@ impl SuiteMeasurements {
})
}
+ pub fn from_file(filename: &Path) -> Result<Self, SuiteMeasurementsError> {
+ let data = File::open(filename)
+ .map_err(|err| SuiteMeasurementsError::Open(filename.to_path_buf(), err))?;
+ let m: Self = serde_json::from_reader(&data)
+ .map_err(|err| SuiteMeasurementsError::Read(filename.to_path_buf(), err))?;
+ Ok(m)
+ }
+
pub fn hostname(&self) -> &str {
&self.hostname
}
@@ -82,9 +100,35 @@ impl SuiteMeasurements {
&self.benchmark_started
}
+ pub fn cpus(&self) -> usize {
+ self.host_cpus
+ }
+
+ pub fn ram(&self) -> u64 {
+ self.host_ram
+ }
+
+ pub fn obnam_version(&self) -> &str {
+ self.obnam_version
+ .strip_prefix("obnam-backup ")
+ .or(Some(""))
+ .unwrap()
+ }
+
pub fn push(&mut self, m: OpMeasurements) {
self.measurements.push(m);
}
+
+ pub fn benchmark_names(&self) -> Vec<String> {
+ let names: HashSet<&str> = HashSet::from_iter(self.measurements.iter().map(|m| m.name()));
+ let mut names: Vec<String> = names.iter().map(|x| x.to_string()).collect();
+ names.sort();
+ names
+ }
+
+ pub fn ops(&self) -> impl Iterator<Item = &OpMeasurements> {
+ self.measurements.iter()
+ }
}
impl OpMeasurements {
@@ -97,7 +141,28 @@ impl OpMeasurements {
}
}
+ pub fn name(&self) -> &str {
+ &self.benchmark
+ }
+
pub fn push(&mut self, m: Measurement) {
self.measurements.push(m);
}
+
+ pub fn op(&self) -> Operation {
+ self.op
+ }
+
+ pub fn iter(&self) -> impl Iterator<Item = &Measurement> {
+ self.measurements.iter()
+ }
+
+ pub fn millis(&self) -> u128 {
+ for m in self.iter() {
+ if let Measurement::DurationMs(ms) = m {
+ return *ms;
+ }
+ }
+ 0
+ }
}
diff --git a/src/suite.rs b/src/suite.rs
index 224238a..43fca8f 100644
--- a/src/suite.rs
+++ b/src/suite.rs
@@ -286,7 +286,10 @@ impl Benchmark {
debug!("restored directory is {}", restored.display());
let m = summain(restored).map_err(SuiteError::Summain)?;
self.manifests.insert(id, m);
- Ok(OpMeasurements::new(self.name(), Operation::ManifestLive))
+ Ok(OpMeasurements::new(
+ self.name(),
+ Operation::ManifestRestored,
+ ))
}
fn compare_manifests(
@@ -302,7 +305,7 @@ impl Benchmark {
error!("second manifest:\n{}", m2);
return Err(SuiteError::ManifestsDiffer(first, second));
}
- Ok(OpMeasurements::new(self.name(), Operation::ManifestLive))
+ Ok(OpMeasurements::new(self.name(), Operation::CompareManiests))
}
fn manifest(&self, id: usize) -> Result<String, SuiteError> {