use crate::builder::{ObnamBuilder, ObnamBuilderError}; use crate::client::{ObnamClient, ObnamClientError}; use crate::daemon::DaemonManager; use crate::junk::junk; use crate::result::{Measurement, OpMeasurements, Operation}; use crate::server::{ObnamServer, ObnamServerError}; use crate::specification::{Create, FileCount}; use crate::step::Step; use crate::summain::{summain, SummainError}; use log::{debug, error, info}; use std::collections::HashMap; use std::fs::File; use std::path::{Path, PathBuf}; use std::time::Instant; use tempfile::{tempdir, TempDir}; use walkdir::WalkDir; /// A running benchmark suite. /// /// This manages temporary data created for the benchmarks, and /// executes individual steps in the suite. pub struct Suite { client: PathBuf, server: PathBuf, manager: DaemonManager, benchmark: Option, } /// Possible errors from running a benchmark suite. #[derive(Debug, thiserror::Error)] pub enum SuiteError { /// Error building Obnam. #[error(transparent)] Build(ObnamBuilderError), /// Error creating a temporary directory. #[error(transparent)] TempDir(#[from] std::io::Error), /// File creation failed. #[error("Failed to create file {0}: {1}")] CreateFile(PathBuf, std::io::Error), /// Error from counting files. #[error("Error counting files in {0}: {1}")] FileCount(PathBuf, walkdir::Error), /// Error looking up file metadata. #[error("Error looking up file metadata: {0}: {1}")] FileMeta(PathBuf, walkdir::Error), /// Error removing restored data. #[error("Error removing temporary directory: {0}: {1}")] RemoveRestored(PathBuf, std::io::Error), /// Error using an Obnam client. #[error(transparent)] Client(#[from] ObnamClientError), /// Error managing an Obnam server. #[error(transparent)] Server(#[from] ObnamServerError), /// Suite already has a manifest with a given id. #[error("Suite already has manifest {0}: this is a bug")] ManifestExists(usize), /// Suite doesn't have a manifest with a given id. #[error("Suite doesn't have a manifest {0}: this is a bug")] ManifestMissing(usize), /// Manifests are not identical. #[error("Manifests {0} and {1} are not identical, as expected")] ManifestsDiffer(usize, usize), /// Error running summain. #[error(transparent)] Summain(SummainError), } impl Suite { pub fn new(builder: &ObnamBuilder) -> Result { Ok(Self { client: builder.client_binary().to_path_buf(), server: builder.server_binary().to_path_buf(), manager: DaemonManager::new(), benchmark: None, }) } /// Execute one step in the benchmark suite. /// /// Return a measurement of the step. pub fn execute(&mut self, step: &Step) -> Result { let time = Instant::now(); eprintln!("step: {:?}", step); let mut om = match step { Step::Start(name) => { assert!(self.benchmark.is_none()); let mut benchmark = Benchmark::new(&self.client, &self.server, name, &self.manager)?; let om = benchmark.start()?; self.benchmark = Some(benchmark); om } Step::Stop(name) => { assert!(self.benchmark.is_some()); assert_eq!(name, self.benchmark.as_ref().unwrap().name()); let om = self.benchmark.as_mut().unwrap().stop()?; self.benchmark = None; om } Step::Create(x) => { assert!(self.benchmark.is_some()); self.benchmark.as_mut().unwrap().create(x)? } Step::Rename(x) => { assert!(self.benchmark.is_some()); self.benchmark.as_mut().unwrap().rename(x)? } Step::Delete(x) => { assert!(self.benchmark.is_some()); self.benchmark.as_mut().unwrap().delete(x)? } Step::Backup(x) => { assert!(self.benchmark.is_some()); self.benchmark.as_mut().unwrap().backup(*x)? } Step::Restore(x) => { assert!(self.benchmark.is_some()); self.benchmark.as_mut().unwrap().restore(*x)? } Step::ManifestLive(id) => { assert!(self.benchmark.is_some()); self.benchmark.as_mut().unwrap().manifest_live(*id)? } Step::ManifestRestored(id) => { assert!(self.benchmark.is_some()); self.benchmark.as_mut().unwrap().manifest_restored(*id)? } Step::CompareManifests(first, second) => { assert!(self.benchmark.is_some()); self.benchmark .as_mut() .unwrap() .compare_manifests(*first, *second)? } }; let t = std::time::Duration::from_millis(10); std::thread::sleep(t); let ms = time.elapsed().as_millis(); debug!("step duration was {} ms", ms); om.push(Measurement::DurationMs(ms)); Ok(om) } } struct Benchmark { name: String, client: ObnamClient, server: ObnamServer, live: TempDir, restored: TempDir, gen_ids: HashMap, manifests: HashMap, } impl Benchmark { fn new( client: &Path, server: &Path, name: &str, manager: &DaemonManager, ) -> Result { let server = ObnamServer::new(server, manager)?; let live = tempdir().map_err(SuiteError::TempDir)?; let restored = tempdir().map_err(SuiteError::TempDir)?; let client = ObnamClient::new(client, server.url(), live.path().to_path_buf())?; Ok(Self { name: name.to_string(), client, server, live, restored, gen_ids: HashMap::new(), manifests: HashMap::new(), }) } fn name(&self) -> &str { &self.name } fn live(&self) -> PathBuf { self.live.path().to_path_buf() } fn restored(&self) -> PathBuf { self.restored.path().join("restored") } fn start(&mut self) -> Result { info!("starting benchmark {}", self.name()); self.client .run(&["init", "--insecure-passphrase=hunter2"])?; Ok(OpMeasurements::new(self.name(), Operation::Start)) } fn stop(&mut self) -> Result { info!("ending benchmark {}", self.name); self.server.stop(); Ok(OpMeasurements::new(self.name(), Operation::Stop)) } fn create(&mut self, create: &Create) -> Result { let root = self.live(); info!( "creating {} files of {} bytes each in {}", create.files, create.file_size, root.display() ); for i in 0..create.files { let filename = root.join(format!("{}", i)); let mut f = File::create(&filename).map_err(|err| SuiteError::CreateFile(filename, err))?; junk(&mut f, create.file_size)?; } Ok(OpMeasurements::new(self.name(), Operation::Create)) } fn rename(&mut self, count: &FileCount) -> Result { info!("renaming {} test data files", count.files); Ok(OpMeasurements::new(self.name(), Operation::Rename)) } fn delete(&mut self, count: &FileCount) -> Result { info!("deleting {} test data files", count.files); Ok(OpMeasurements::new(self.name(), Operation::Delete)) } fn backup(&mut self, n: usize) -> Result { info!("making backup {} in benchmark {}", n, self.name()); self.client.run(&["backup"])?; let gen_id = self .client .run(&["resolve", "latest"])? .strip_suffix('\n') .unwrap_or("") .to_string(); debug!("backed up generation {}", gen_id); self.gen_ids.insert(n, gen_id); let mut om = OpMeasurements::new(self.name(), Operation::Backup); let stats = filestats(&self.live())?; om.push(Measurement::TotalFiles(stats.count)); om.push(Measurement::TotalData(stats.size)); Ok(om) } fn restore(&mut self, n: usize) -> Result { info!("restoring backup {} in benchmark {}", n, self.name()); debug!("first removing all data from restore directory"); let restored = self.restored(); if restored.exists() { std::fs::remove_dir_all(&restored) .map_err(|err| SuiteError::RemoveRestored(restored, err))?; } let gen_id = self.gen_ids.get(&n).unwrap(); let path = self.restored().display().to_string(); self.client.run(&["restore", gen_id, &path])?; Ok(OpMeasurements::new(self.name(), Operation::Restore)) } fn manifest_live(&mut self, id: usize) -> Result { info!("make manifest {} of current test data", id); if self.manifests.contains_key(&id) { return Err(SuiteError::ManifestExists(id)); } let m = summain(self.live.path()).map_err(SuiteError::Summain)?; self.manifests.insert(id, m); Ok(OpMeasurements::new(self.name(), Operation::ManifestLive)) } fn manifest_restored(&mut self, id: usize) -> Result { info!("make manifest {} of latest restored data", id); if self.manifests.contains_key(&id) { return Err(SuiteError::ManifestExists(id)); } debug!("self.restored()={}", self.restored().display()); let restored = format!( "{}{}", self.restored().display(), self.live.path().display() ); let restored = Path::new(&restored); debug!("restored directory is {}", restored.display()); let m = summain(restored).map_err(SuiteError::Summain)?; self.manifests.insert(id, m); Ok(OpMeasurements::new( self.name(), Operation::ManifestRestored, )) } fn compare_manifests( &mut self, first: usize, second: usize, ) -> Result { info!("compare manifests {} and {}", first, second); let m1 = self.manifest(first)?; let m2 = self.manifest(second)?; if m1 != m2 { error!("first manifest:\n{}", m1); error!("second manifest:\n{}", m2); return Err(SuiteError::ManifestsDiffer(first, second)); } Ok(OpMeasurements::new(self.name(), Operation::CompareManiests)) } fn manifest(&self, id: usize) -> Result { if let Some(m) = self.manifests.get(&id) { Ok(m.clone()) } else { Err(SuiteError::ManifestMissing(id)) } } } #[derive(Debug, Default)] struct FileStats { count: u64, size: u64, } fn filestats(dirname: &Path) -> Result { let mut stats = FileStats::default(); for e in WalkDir::new(dirname) { let e = e.map_err(|err| SuiteError::FileCount(dirname.to_path_buf(), err))?; stats.count += 1; stats.size += e .metadata() .map_err(|err| SuiteError::FileMeta(e.path().to_path_buf(), err))? .len(); } Ok(stats) }