From 3a635e26d57573828163cafcc5240fa8c670203d Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 5 Jan 2022 13:37:07 +0200 Subject: feat: add report generation Sponsored-by: author --- obnam-benchmark.css | 27 ++++++++ report.sh | 7 ++ src/bin/obnam-benchmark.rs | 24 +++++++ src/lib.rs | 1 + src/report.rs | 164 +++++++++++++++++++++++++++++++++++++++++++++ src/result.rs | 75 +++++++++++++++++++-- src/suite.rs | 7 +- 7 files changed, 298 insertions(+), 7 deletions(-) create mode 100644 obnam-benchmark.css create mode 100755 report.sh create mode 100644 src/report.rs diff --git a/obnam-benchmark.css b/obnam-benchmark.css new file mode 100644 index 0000000..9f0c891 --- /dev/null +++ b/obnam-benchmark.css @@ -0,0 +1,27 @@ + diff --git a/report.sh b/report.sh new file mode 100755 index 0000000..4c426db --- /dev/null +++ b/report.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -euo pipefail + +obnam-benchmark report "$@" | + pandoc -f markdown+pipe_tables -t html -H obnam-benchmark.css \ + --metadata title="Obnam benchmarks" - -o /dev/stdout diff --git a/src/bin/obnam-benchmark.rs b/src/bin/obnam-benchmark.rs index 41118cb..4ac3e4d 100644 --- a/src/bin/obnam-benchmark.rs +++ b/src/bin/obnam-benchmark.rs @@ -1,5 +1,6 @@ use log::{debug, error, info}; use obnam_benchmark::junk::junk; +use obnam_benchmark::report::Reporter; use obnam_benchmark::result::SuiteMeasurements; use obnam_benchmark::specification::Specification; use obnam_benchmark::suite::Suite; @@ -30,6 +31,7 @@ fn real_main() -> anyhow::Result<()> { Command::Run(x) => x.run()?, Command::Spec(x) => x.run()?, Command::Steps(x) => x.run()?, + Command::Report(x) => x.run()?, } Ok(()) @@ -54,6 +56,9 @@ enum Command { /// Show the steps for running a benchmark. Steps(Steps), + + /// Report results for several benchmarks. + Report(Report), } #[derive(Debug, StructOpt)] @@ -166,3 +171,22 @@ impl Steps { Ok(()) } } + +#[derive(Debug, StructOpt)] +struct Report { + /// Names of the results file for which to produce a report. + #[structopt(parse(from_os_str))] + filenames: Vec, +} + +impl Report { + fn run(&self) -> anyhow::Result<()> { + info!("Reporting results"); + let mut report = Reporter::default(); + for filename in self.filenames.iter() { + report.push(SuiteMeasurements::from_file(filename)?); + } + report.write(&mut std::io::stdout())?; + Ok(()) + } +} diff --git a/src/lib.rs b/src/lib.rs index f491ae7..5f0a522 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -32,6 +32,7 @@ pub mod client; pub mod daemon; pub mod junk; pub mod obnam; +pub mod report; pub mod result; pub mod server; pub mod specification; diff --git a/src/report.rs b/src/report.rs new file mode 100644 index 0000000..0538760 --- /dev/null +++ b/src/report.rs @@ -0,0 +1,164 @@ +use crate::result::{Measurement, Operation, SuiteMeasurements}; +use std::collections::HashSet; +use std::io::Write; +use std::iter::FromIterator; + +#[derive(Debug, Default)] +pub struct Reporter { + results: Vec, +} + +impl Reporter { + pub fn push(&mut self, result: SuiteMeasurements) { + self.results.push(result); + } + + pub fn write(&self, f: &mut dyn Write) -> Result<(), std::io::Error> { + // Summary tables of all benchmarks. + writeln!(f, "# Summaries of all benchmark runs")?; + let hosts = self.hosts(); + for bench in self.benchmarks() { + writeln!(f)?; + writeln!(f, "## Benchmark {}", bench)?; + for op in [Operation::Backup, Operation::Restore] { + for host in hosts.iter() { + writeln!(f)?; + writeln!(f, "Table: {:?} on host {}, times in ms", op, host)?; + writeln!(f)?; + let mut want_headings = true; + for r in self.results_for(host) { + // Column data + let cols = self.durations(r, &bench, op); + + if want_headings { + want_headings = false; + self.headings(f, &cols)?; + } + self.rows(f, r, &cols)?; + } + } + } + } + + writeln!(f)?; + writeln!(f, "# Individual benchmark runs")?; + for host in self.hosts() { + writeln!(f)?; + writeln!(f, "## Host `{}`", host)?; + for r in self.results_for(&host) { + writeln!(f)?; + writeln!(f, "### Benchmark run {}", r.timestamp())?; + writeln!(f)?; + writeln!(f, "* CPUs: {}", r.cpus())?; + writeln!(f, "* RAM: {} MiB", r.ram() / 1024 / 1024)?; + for bench in r.benchmark_names() { + writeln!(f)?; + writeln!(f, "#### Benchmark `{}`", bench)?; + writeln!(f)?; + for opm in r.ops().filter(|o| o.name() == bench) { + let op = opm.op(); + match op { + Operation::Backup => { + writeln!(f, "* Backup: {} ms", opm.millis())?; + } + Operation::Restore => { + writeln!(f, "* Restore: {} ms", opm.millis())?; + } + _ => continue, + } + for m in opm.iter() { + match m { + Measurement::DurationMs(_) => (), + Measurement::TotalFiles(n) => { + writeln!(f, " * files: {}", n)?; + } + &Measurement::TotalData(n) => { + writeln!(f, " * data: {} bytes", n)?; + } + } + } + } + + for op in [Operation::Backup, Operation::Restore] { + writeln!(f)?; + writeln!(f, "Table: {:?}", op)?; + writeln!(f)?; + let cols = self.durations(r, &bench, op); + self.headings(f, &cols)?; + self.rows(f, r, &cols)?; + } + } + } + } + + Ok(()) + } + + fn durations(&self, r: &SuiteMeasurements, bench: &str, op: Operation) -> Vec { + r.ops() + .filter(|r| r.name() == bench) + .filter(|r| r.op() == op) + .map(|r| r.millis()) + .collect() + } + + fn headings(&self, f: &mut dyn Write, durations: &[u128]) -> Result<(), std::io::Error> { + // Column headings. + let mut headings: Vec = durations + .iter() + .enumerate() + .map(|(i, _)| format!("step {}", i)) + .collect(); + headings.insert(0, "Version".to_string()); + for h in headings.iter() { + write!(f, "| {}", h)?; + } + writeln!(f, "|")?; + + // Heading separator. + for _ in headings.iter() { + write!(f, "|----")?; + } + writeln!(f, "|")?; + Ok(()) + } + + fn rows( + &self, + f: &mut dyn Write, + r: &SuiteMeasurements, + durations: &[u128], + ) -> Result<(), std::io::Error> { + write!(f, "| {}", r.obnam_version())?; + for ms in durations.iter() { + write!(f, "| {}", ms)?; + } + writeln!(f, "|")?; + Ok(()) + } + + fn benchmarks(&self) -> Vec { + let mut names = HashSet::new(); + for r in self.results.iter() { + for opm in r.ops() { + names.insert(opm.name()); + } + } + let mut names: Vec = names.iter().map(|x| x.to_string()).collect(); + names.sort(); + names + } + + fn hosts(&self) -> Vec { + let names: HashSet<&str> = HashSet::from_iter(self.results.iter().map(|r| r.hostname())); + let mut names: Vec = names.iter().map(|x| x.to_string()).collect(); + names.sort(); + names + } + + fn results_for<'a>(&'a self, hostname: &'a str) -> impl Iterator { + self.results + .iter() + .filter(move |r| r.hostname() == hostname) + } +} diff --git a/src/result.rs b/src/result.rs index 22a49a0..cce5d2d 100644 --- a/src/result.rs +++ b/src/result.rs @@ -1,10 +1,14 @@ use chrono::prelude::*; use git_testament::{git_testament, render_testament}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use std::fs::File; +use std::iter::FromIterator; +use std::path::{Path, PathBuf}; git_testament!(TESTAMENT); -#[derive(Debug, Serialize)] +#[derive(Debug, Deserialize, Serialize)] pub struct SuiteMeasurements { measurements: Vec, obnam_version: String, @@ -15,21 +19,21 @@ pub struct SuiteMeasurements { host_ram: u64, } -#[derive(Debug, Serialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct OpMeasurements { benchmark: String, op: Operation, measurements: Vec, } -#[derive(Debug, Serialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub enum Measurement { TotalFiles(u64), TotalData(u64), DurationMs(u128), } -#[derive(Debug, Clone, Copy, Serialize)] +#[derive(Debug, Clone, Copy, PartialEq, Deserialize, Serialize)] pub enum Operation { Start, Stop, @@ -53,6 +57,12 @@ pub enum SuiteMeasurementsError { #[error("failed to get hostname: {0}")] Hostname(nix::Error), + + #[error("failed to open result file {0} for reading: {1}")] + Open(PathBuf, std::io::Error), + + #[error("failed to read result file {0}: {1}")] + Read(PathBuf, serde_json::Error), } impl SuiteMeasurements { @@ -74,6 +84,14 @@ impl SuiteMeasurements { }) } + pub fn from_file(filename: &Path) -> Result { + let data = File::open(filename) + .map_err(|err| SuiteMeasurementsError::Open(filename.to_path_buf(), err))?; + let m: Self = serde_json::from_reader(&data) + .map_err(|err| SuiteMeasurementsError::Read(filename.to_path_buf(), err))?; + Ok(m) + } + pub fn hostname(&self) -> &str { &self.hostname } @@ -82,9 +100,35 @@ impl SuiteMeasurements { &self.benchmark_started } + pub fn cpus(&self) -> usize { + self.host_cpus + } + + pub fn ram(&self) -> u64 { + self.host_ram + } + + pub fn obnam_version(&self) -> &str { + self.obnam_version + .strip_prefix("obnam-backup ") + .or(Some("")) + .unwrap() + } + pub fn push(&mut self, m: OpMeasurements) { self.measurements.push(m); } + + pub fn benchmark_names(&self) -> Vec { + let names: HashSet<&str> = HashSet::from_iter(self.measurements.iter().map(|m| m.name())); + let mut names: Vec = names.iter().map(|x| x.to_string()).collect(); + names.sort(); + names + } + + pub fn ops(&self) -> impl Iterator { + self.measurements.iter() + } } impl OpMeasurements { @@ -97,7 +141,28 @@ impl OpMeasurements { } } + pub fn name(&self) -> &str { + &self.benchmark + } + pub fn push(&mut self, m: Measurement) { self.measurements.push(m); } + + pub fn op(&self) -> Operation { + self.op + } + + pub fn iter(&self) -> impl Iterator { + self.measurements.iter() + } + + pub fn millis(&self) -> u128 { + for m in self.iter() { + if let Measurement::DurationMs(ms) = m { + return *ms; + } + } + 0 + } } diff --git a/src/suite.rs b/src/suite.rs index 224238a..43fca8f 100644 --- a/src/suite.rs +++ b/src/suite.rs @@ -286,7 +286,10 @@ impl Benchmark { debug!("restored directory is {}", restored.display()); let m = summain(restored).map_err(SuiteError::Summain)?; self.manifests.insert(id, m); - Ok(OpMeasurements::new(self.name(), Operation::ManifestLive)) + Ok(OpMeasurements::new( + self.name(), + Operation::ManifestRestored, + )) } fn compare_manifests( @@ -302,7 +305,7 @@ impl Benchmark { error!("second manifest:\n{}", m2); return Err(SuiteError::ManifestsDiffer(first, second)); } - Ok(OpMeasurements::new(self.name(), Operation::ManifestLive)) + Ok(OpMeasurements::new(self.name(), Operation::CompareManiests)) } fn manifest(&self, id: usize) -> Result { -- cgit v1.2.1