From 42bd070d7b71c3f9360f88ee50291eb66e54ccd9 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 30 Dec 2021 14:09:04 +0200 Subject: feat: actually run client and server Also, switch Subplot step functions to Python: it was too hard to debug failures in the Rust ones. Sponsored-by: author --- build.rs | 26 -------------- check | 6 ++-- obnam-benchmark.md | 10 +++--- src/bin/obnam-benchmark.rs | 2 ++ src/client.rs | 84 ++++++++++++++++++++++++++++++++++++++++++++++ src/daemon.rs | 66 ++++++++++++++++++++++++++++++------ src/lib.rs | 1 + src/server.rs | 28 ++++++++++++++-- src/suite.rs | 47 +++++++++++++------------- subplot/benchmark.py | 28 ++++++++++++++++ subplot/benchmark.rs | 2 +- subplot/benchmark.yaml | 8 ++--- tests/obnam-benchmark.rs | 1 - 13 files changed, 235 insertions(+), 74 deletions(-) delete mode 100644 build.rs create mode 100644 src/client.rs create mode 100644 subplot/benchmark.py delete mode 100644 tests/obnam-benchmark.rs diff --git a/build.rs b/build.rs deleted file mode 100644 index 1d7729e..0000000 --- a/build.rs +++ /dev/null @@ -1,26 +0,0 @@ -use glob::glob; -use std::path::Path; - -fn main() { - if let Err(err) = real_main() { - eprintln!("ERROR: {}", err); - std::process::exit(1); - } -} - -fn real_main() -> anyhow::Result<()> { - println!("cargo:rerun-if-env-changed=DEB_BUILD_OPTIONS"); - let subplots = glob("[a-z]*.md").expect("failed to find subplots"); - let tests = Path::new("tests"); - for entry in subplots { - let entry = entry?; - let mut inc = tests.join(&entry); - inc.set_extension("rs"); - if !inc.exists() { - panic!("missing include file: {}", inc.display()); - } - println!("cargo:rerun-if-changed={}", inc.display()); - subplot_build::codegen(Path::new(&entry))?; - } - Ok(()) -} diff --git a/check b/check index 2a22fd3..c928b06 100755 --- a/check +++ b/check @@ -3,9 +3,11 @@ set -euo pipefail cargo clippy -q --all-targets -- -D clippy::all -subplot docgen obnam-benchmark.md -o obnam-benchmark.html -subplot docgen obnam-benchmark.md -o obnam-benchmark.pdf cargo build --all-targets -q chronic cargo test -q +subplot docgen obnam-benchmark.md -o obnam-benchmark.html +subplot docgen obnam-benchmark.md -o obnam-benchmark.pdf +subplot codegen obnam-benchmark.md -o test.py +python3 test.py --log test.log "$@" cargo fmt -- --check echo A-OK diff --git a/obnam-benchmark.md b/obnam-benchmark.md index 2fc1452..4440b57 100644 --- a/obnam-benchmark.md +++ b/obnam-benchmark.md @@ -2,13 +2,15 @@ title: "`obnam-benchmark`---tool to run benchmarks" author: "The Obnam project" documentclass: report -template: rust bindings: - subplot/benchmark.yaml - lib/files.yaml - lib/runcmd.yaml -functions: -- subplot/benchmark.rs +impls: + python: + - subplot/benchmark.py + - lib/files.py + - lib/runcmd.py ... @@ -109,7 +111,7 @@ given an installed Rust program obnam-benchmark given file spec.yaml given file expected.json when I run obnam-benchmark spec spec.yaml --output spec.json -then JSON files spec.yaml and expected.json match +then JSON files spec.json and expected.json match ``` ```{#spec.yaml .yaml .file .numberLines} diff --git a/src/bin/obnam-benchmark.rs b/src/bin/obnam-benchmark.rs index 1a26ae0..e6a476c 100644 --- a/src/bin/obnam-benchmark.rs +++ b/src/bin/obnam-benchmark.rs @@ -10,6 +10,7 @@ use structopt::StructOpt; fn main() { pretty_env_logger::init_custom_env("OBNAM_BENCHMARK_LOG"); + println!("START"); info!("obnam-benchmark starts"); if let Err(err) = real_main() { eprintln!("ERROR: {}", err); @@ -17,6 +18,7 @@ fn main() { exit(1); } info!("obnam-benchmark ends successfully"); + println!("END"); } fn real_main() -> anyhow::Result<()> { diff --git a/src/client.rs b/src/client.rs new file mode 100644 index 0000000..fb25009 --- /dev/null +++ b/src/client.rs @@ -0,0 +1,84 @@ +use log::debug; +use serde::Serialize; +use std::path::{Path, PathBuf}; +use std::process::Command; +use tempfile::{tempdir, TempDir}; + +#[derive(Debug, thiserror::Error)] +pub enum ObnamClientError { + #[error("failed to create temporary directory for server: {0}")] + TempDir(std::io::Error), + + #[error("failed to write client configuration to {0}: {1}")] + WriteConfig(PathBuf, std::io::Error), + + #[error("failed to run obnam: {0}")] + Run(std::io::Error), +} + +#[derive(Debug)] +pub struct ObnamClient { + #[allow(dead_code)] + tempdir: TempDir, + #[allow(dead_code)] + config: PathBuf, +} + +impl ObnamClient { + pub fn new(server_url: String, root: PathBuf) -> Result { + debug!("creating ObnamClient"); + let tempdir = tempdir().map_err(ObnamClientError::TempDir)?; + let config_filename = tempdir.path().join("client.yaml"); + + let config = ClientConfig::new(server_url, root); + config.write(&config_filename)?; + + Ok(Self { + tempdir, + config: config_filename, + }) + } + + pub fn run(&self, args: &[&str]) -> Result { + let output = Command::new("obnam") + .arg("--config") + .arg(&self.config) + .args(args) + .output() + .map_err(ObnamClientError::Run)?; + if output.status.code() != Some(0) { + eprintln!("{}", String::from_utf8_lossy(&output.stdout)); + eprintln!("{}", String::from_utf8_lossy(&output.stderr)); + std::process::exit(1); + } + + Ok(String::from_utf8_lossy(&output.stdout) + .to_owned() + .to_string()) + } +} + +#[derive(Debug, Serialize)] +pub struct ClientConfig { + server_url: String, + verify_tls_cert: bool, + roots: Vec, + log: PathBuf, +} + +impl ClientConfig { + fn new(server_url: String, root: PathBuf) -> Self { + Self { + server_url, + verify_tls_cert: false, + roots: vec![root], + log: PathBuf::from("obnam.log"), + } + } + + fn write(&self, filename: &Path) -> Result<(), ObnamClientError> { + std::fs::write(filename, serde_yaml::to_string(self).unwrap()) + .map_err(|err| ObnamClientError::WriteConfig(filename.to_path_buf(), err))?; + Ok(()) + } +} diff --git a/src/daemon.rs b/src/daemon.rs index fa8f287..8bf8adb 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -1,7 +1,11 @@ +use log::{debug, error, info}; use nix::sys::signal::kill; use nix::sys::signal::Signal; use nix::unistd::Pid; +use std::ffi::OsStr; use std::fs::read; +use std::os::unix::ffi::OsStrExt; +use std::path::{Path, PathBuf}; use std::process::Command; use std::thread::sleep; use std::time::{Duration, Instant}; @@ -19,6 +23,10 @@ pub enum DaemonError { #[error(transparent)] TempFile(#[from] std::io::Error), + /// Something went wrong read standard output of daemon. + #[error("failed to read daemon stdout: {0}")] + Stdout(std::io::Error), + /// Something went wrong read error output of daemon. #[error("failed to read daemon stderr: {0}")] Stderr(std::io::Error), @@ -67,18 +75,22 @@ impl DaemonManager { /// it reads a configuration file and that has errors. This /// function won't wait for that to happen: it only cares about /// the PID. - pub fn start(&self, argv: &[&str]) -> Result { - let stdout = NamedTempFile::new()?; - let stderr = NamedTempFile::new()?; + pub fn start( + &self, + argv: &[&OsStr], + stdout: &Path, + stderr: &Path, + ) -> Result { + info!("start daemon: {:?}", argv); let pid = NamedTempFile::new()?; let output = Command::new("daemonize") .args(&[ "-c", "/", "-e", - &stderr.path().display().to_string(), + &stderr.display().to_string(), "-o", - &stdout.path().display().to_string(), + &stdout.display().to_string(), "-p", &pid.path().display().to_string(), ]) @@ -91,6 +103,7 @@ impl DaemonManager { std::process::exit(1); } + debug!("waiting for daemon to write PID file"); let time = Instant::now(); while time.elapsed() < self.timeout { // Do we have the pid file? @@ -102,7 +115,8 @@ impl DaemonManager { // Parse as an integer, if possible. if let Ok(pid) = pid.parse() { // We have a pid, stop waiting. - return Ok(Daemon::new(pid)); + info!("got pid for daemon: pid"); + return Ok(Daemon::new(pid, stdout, stderr)); } } sleep_briefly(); @@ -111,8 +125,22 @@ impl DaemonManager { } } - let cmd = argv.join(" "); - let err = read(stderr.path()).map_err(DaemonError::Stderr)?; + error!( + "no PID file within {} ms, giving up", + self.timeout.as_millis() + ); + let mut cmd = String::new(); + for arg in argv { + if !cmd.is_empty() { + cmd.push(' '); + } + cmd.push_str( + &String::from_utf8_lossy(arg.as_bytes()) + .to_owned() + .to_string(), + ); + } + let err = read(&stderr).map_err(DaemonError::Stderr)?; let err = String::from_utf8_lossy(&err).into_owned(); Err(DaemonError::Timeout(self.timeout.as_millis(), cmd, err)) } @@ -124,11 +152,18 @@ impl DaemonManager { #[derive(Debug)] pub struct Daemon { pid: Option, + stdout: PathBuf, + stderr: PathBuf, } impl Daemon { - fn new(pid: i32) -> Self { - Self { pid: Some(pid) } + fn new(pid: i32, stdout: &Path, stderr: &Path) -> Self { + info!("started daemon with PID {}", pid); + Self { + pid: Some(pid), + stdout: stdout.to_path_buf(), + stderr: stderr.to_path_buf(), + } } /// Explicitly stop a daemon. @@ -137,11 +172,22 @@ impl Daemon { /// errors. It can only be called once. pub fn stop(&mut self) -> Result<(), DaemonError> { if let Some(raw_pid) = self.pid.take() { + info!("stopping daemon with PID {}", raw_pid); let pid = Pid::from_raw(raw_pid); kill(pid, Some(Signal::SIGKILL)).map_err(|e| DaemonError::Kill(raw_pid, e))?; } Ok(()) } + + /// Return what the daemon has written to its stderr so far. + pub fn stdout(&self) -> Result, DaemonError> { + std::fs::read(&self.stdout).map_err(DaemonError::Stdout) + } + + /// Return what the daemon has written to its stderr so far. + pub fn stderr(&self) -> Result, DaemonError> { + std::fs::read(&self.stderr).map_err(DaemonError::Stderr) + } } impl Drop for Daemon { diff --git a/src/lib.rs b/src/lib.rs index e1e2b81..e8c6d82 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,6 +28,7 @@ //! This crate only collects data from a set of benchmarks. It does //! not analyze the data. The data can be stored for later analysis. +pub mod client; pub mod daemon; pub mod junk; pub mod obnam; diff --git a/src/server.rs b/src/server.rs index 9a809a5..26f418b 100644 --- a/src/server.rs +++ b/src/server.rs @@ -3,6 +3,7 @@ use crate::tlsgen::{Tls, TlsError}; use log::debug; use rand::random; use serde::Serialize; +use std::ffi::OsStr; use std::ops::Range; use std::path::{Path, PathBuf}; use std::time::Instant; @@ -18,6 +19,9 @@ pub enum ObnamServerError { #[error("took too long to pick a random port for server")] Port, + #[error("error creating chunks directory: {0}")] + ChunksDir(std::io::Error), + #[error("failed to create temporary directory for server: {0}")] TempDir(std::io::Error), @@ -40,11 +44,12 @@ pub struct ObnamServer { tempdir: TempDir, chunks: PathBuf, daemon: Option, + url: String, } impl ObnamServer { pub fn new(manager: &DaemonManager) -> Result { - debug!("creating ObamServer"); + debug!("creating ObnamServer"); let tempdir = tempdir().map_err(ObnamServerError::TempDir)?; let config_filename = tempdir.path().join("server.yaml"); let chunks = tempdir.path().join("chunks"); @@ -55,22 +60,37 @@ impl ObnamServer { write(&tls_key, tls.key())?; write(&tls_cert, tls.cert())?; + std::fs::create_dir(&chunks).map_err(ObnamServerError::ChunksDir)?; + let port = pick_port()?; + debug!("server listens on port {}", port); let config = ServerConfig::new(port, chunks.clone(), tls_key, tls_cert); config.write(&config_filename)?; let daemon = manager - .start(&["/bin/sleep", "1000"]) + .start( + &[ + OsStr::new("/usr/bin/obnam-server"), + OsStr::new(&config_filename), + ], + Path::new("server.out"), + Path::new("server.log"), + ) .map_err(ObnamServerError::Daemon)?; Ok(Self { tempdir, chunks, daemon: Some(daemon), + url: format!("https://localhost:{}", port), }) } + pub fn url(&self) -> String { + self.url.clone() + } + pub fn stop(&mut self) { self.daemon.take(); } @@ -115,7 +135,9 @@ impl ServerConfig { } fn write(&self, filename: &Path) -> Result<(), ObnamServerError> { - std::fs::write(filename, serde_yaml::to_string(self).unwrap()) + let config = serde_yaml::to_string(self).unwrap(); + debug!("server config:\n{}\n(end config)", config); + std::fs::write(filename, &config) .map_err(|err| ObnamServerError::WriteConfig(filename.to_path_buf(), err))?; Ok(()) } diff --git a/src/suite.rs b/src/suite.rs index 90ecf37..e900330 100644 --- a/src/suite.rs +++ b/src/suite.rs @@ -1,6 +1,6 @@ +use crate::client::{ObnamClient, ObnamClientError}; use crate::daemon::DaemonManager; use crate::junk::junk; -use crate::obnam::{Obnam, ObnamError}; use crate::result::{Measurement, OpMeasurements, Operation}; use crate::server::{ObnamServer, ObnamServerError}; use crate::specification::{Create, FileCount}; @@ -9,6 +9,7 @@ use log::{debug, info}; use std::fs::File; use std::path::{Path, PathBuf}; use std::time::Instant; +use tempfile::{tempdir, TempDir}; use walkdir::WalkDir; /// A running benchmark suite. @@ -39,9 +40,9 @@ pub enum SuiteError { #[error("Error looking up file metadata: {0}: {1}")] FileMeta(PathBuf, walkdir::Error), - /// Error managing an Obnam system. + /// Error using an Obnam client. #[error(transparent)] - Obnam(#[from] ObnamError), + Client(#[from] ObnamClientError), /// Error managing an Obnam server. #[error(transparent)] @@ -60,8 +61,8 @@ impl Suite { /// /// Return a measurement of the step. pub fn execute(&mut self, step: &Step) -> Result { - debug!("executing step {:?}", step); let time = Instant::now(); + eprintln!("step: {:?}", step); let mut om = match step { Step::Start(name) => { assert!(self.benchmark.is_none()); @@ -111,19 +112,21 @@ impl Suite { struct Benchmark { name: String, - // We store an Obnam in an Option so that we can destroy the - // Obnam, and thereby delete any temporary files. We want to do - // that intentionally, so that it can be measured. - obnam: Option, - server: Option, + client: ObnamClient, + server: ObnamServer, + live: TempDir, } impl Benchmark { fn new(name: &str, manager: &DaemonManager) -> Result { + let server = ObnamServer::new(manager)?; + let live = tempdir().map_err(SuiteError::TempDir)?; + let client = ObnamClient::new(server.url(), live.path().to_path_buf())?; Ok(Self { name: name.to_string(), - obnam: Some(Obnam::new()?), - server: Some(ObnamServer::new(manager)?), + client, + server, + live, }) } @@ -131,29 +134,27 @@ impl Benchmark { &self.name } - fn obnam(&mut self) -> &mut Obnam { - self.obnam.as_mut().unwrap() + fn live(&self) -> PathBuf { + self.live.path().to_path_buf() } fn start(&mut self) -> Result { info!("starting benchmark {}", self.name()); - self.obnam().start_server()?; + self.client + .run(&["init", "--insecure-passphrase=hunter2"])?; Ok(OpMeasurements::new(self.name(), Operation::Start)) } fn stop(&mut self) -> Result { info!("ending benchmark {}", self.name); - self.obnam().stop_server()?; - self.obnam.take().unwrap(); // This destroys the Obnam - self.server.as_mut().unwrap().stop(); + self.server.stop(); Ok(OpMeasurements::new(self.name(), Operation::Stop)) } fn create(&mut self, create: &Create) -> Result { - info!("creating {} test data files", create.files); - let root = self.obnam().root(); - debug!( - "creating {} files of {} bytes in {}", + let root = self.live(); + info!( + "creating {} files of {} bytes each in {}", create.files, create.file_size, root.display() @@ -161,7 +162,6 @@ impl Benchmark { for i in 0..create.files { let filename = root.join(format!("{}", i)); - debug!("creating {}", filename.display()); let mut f = File::create(&filename).map_err(|err| SuiteError::CreateFile(filename, err))?; junk(&mut f, create.file_size)?; @@ -182,8 +182,9 @@ impl Benchmark { fn backup(&mut self, n: usize) -> Result { info!("making backup {} in benchmark {}", n, self.name()); + self.client.run(&["backup"])?; let mut om = OpMeasurements::new(self.name(), Operation::Backup); - let stats = filestats(self.obnam().root())?; + let stats = filestats(&self.live())?; om.push(Measurement::TotalFiles(stats.count)); om.push(Measurement::TotalData(stats.size)); Ok(om) diff --git a/subplot/benchmark.py b/subplot/benchmark.py new file mode 100644 index 0000000..5400692 --- /dev/null +++ b/subplot/benchmark.py @@ -0,0 +1,28 @@ +import json +import os + + +def install_rust_program(ctx): + runcmd_prepend_to_path = globals()["runcmd_prepend_to_path"] + srcdir = globals()["srcdir"] + + # Add the directory with built Rust binaries to the path. + default_target = os.path.join(srcdir, "target") + target = os.environ.get("CARGO_TARGET_DIR", default_target) + runcmd_prepend_to_path(ctx, dirname=os.path.join(target, "debug")) + + +def file_is_at_least_this_long(ctx, filename=None, number=None): + st = os.lstat(filename) + assert st.st_size >= int(number) + + +def json_files_match(ctx, first=None, second=None): + assert_eq = globals()["assert_eq"] + first = json.load(open(first)) + second = json.load(open(second)) + assert_eq(first, second) + + +def file_is_valid_json(ctx, filename=None): + json.load(open(filename)) diff --git a/subplot/benchmark.rs b/subplot/benchmark.rs index 082e02b..d7eda37 100644 --- a/subplot/benchmark.rs +++ b/subplot/benchmark.rs @@ -51,7 +51,7 @@ fn json_files_match(context: &ScenarioContext, first: &str, second: &str) { Ok(()) }, false, - ); + )?; } #[step] diff --git a/subplot/benchmark.yaml b/subplot/benchmark.yaml index dc3a136..f745b23 100644 --- a/subplot/benchmark.yaml +++ b/subplot/benchmark.yaml @@ -1,19 +1,19 @@ - given: an installed Rust program obnam-benchmark impl: - rust: + python: function: install_rust_program - then: JSON files {first} and {second} match impl: - rust: + python: function: json_files_match - then: file {filename} is valid JSON impl: - rust: + python: function: file_is_valid_json - then: file {filename} is at least {number:int} bytes long impl: - rust: + python: function: file_is_at_least_this_long diff --git a/tests/obnam-benchmark.rs b/tests/obnam-benchmark.rs deleted file mode 100644 index b6200da..0000000 --- a/tests/obnam-benchmark.rs +++ /dev/null @@ -1 +0,0 @@ -include!(concat!(env!("OUT_DIR"), "/obnam-benchmark.rs")); -- cgit v1.2.1