summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLars Wirzenius <liw@liw.fi>2021-12-30 14:09:04 +0200
committerLars Wirzenius <liw@liw.fi>2022-01-04 17:49:08 +0200
commit42bd070d7b71c3f9360f88ee50291eb66e54ccd9 (patch)
tree82c238f8b0e8bf8330e3eb84c0b14c99d668e571
parent010480a957c52e14172049aea3e692447be8cdfe (diff)
downloadobnam-benchmark-42bd070d7b71c3f9360f88ee50291eb66e54ccd9.tar.gz
feat: actually run client and server
Also, switch Subplot step functions to Python: it was too hard to debug failures in the Rust ones. Sponsored-by: author
-rw-r--r--build.rs26
-rwxr-xr-xcheck6
-rw-r--r--obnam-benchmark.md10
-rw-r--r--src/bin/obnam-benchmark.rs2
-rw-r--r--src/client.rs84
-rw-r--r--src/daemon.rs66
-rw-r--r--src/lib.rs1
-rw-r--r--src/server.rs28
-rw-r--r--src/suite.rs47
-rw-r--r--subplot/benchmark.py28
-rw-r--r--subplot/benchmark.rs2
-rw-r--r--subplot/benchmark.yaml8
-rw-r--r--tests/obnam-benchmark.rs1
13 files changed, 235 insertions, 74 deletions
diff --git a/build.rs b/build.rs
deleted file mode 100644
index 1d7729e..0000000
--- a/build.rs
+++ /dev/null
@@ -1,26 +0,0 @@
-use glob::glob;
-use std::path::Path;
-
-fn main() {
- if let Err(err) = real_main() {
- eprintln!("ERROR: {}", err);
- std::process::exit(1);
- }
-}
-
-fn real_main() -> anyhow::Result<()> {
- println!("cargo:rerun-if-env-changed=DEB_BUILD_OPTIONS");
- let subplots = glob("[a-z]*.md").expect("failed to find subplots");
- let tests = Path::new("tests");
- for entry in subplots {
- let entry = entry?;
- let mut inc = tests.join(&entry);
- inc.set_extension("rs");
- if !inc.exists() {
- panic!("missing include file: {}", inc.display());
- }
- println!("cargo:rerun-if-changed={}", inc.display());
- subplot_build::codegen(Path::new(&entry))?;
- }
- Ok(())
-}
diff --git a/check b/check
index 2a22fd3..c928b06 100755
--- a/check
+++ b/check
@@ -3,9 +3,11 @@
set -euo pipefail
cargo clippy -q --all-targets -- -D clippy::all
-subplot docgen obnam-benchmark.md -o obnam-benchmark.html
-subplot docgen obnam-benchmark.md -o obnam-benchmark.pdf
cargo build --all-targets -q
chronic cargo test -q
+subplot docgen obnam-benchmark.md -o obnam-benchmark.html
+subplot docgen obnam-benchmark.md -o obnam-benchmark.pdf
+subplot codegen obnam-benchmark.md -o test.py
+python3 test.py --log test.log "$@"
cargo fmt -- --check
echo A-OK
diff --git a/obnam-benchmark.md b/obnam-benchmark.md
index 2fc1452..4440b57 100644
--- a/obnam-benchmark.md
+++ b/obnam-benchmark.md
@@ -2,13 +2,15 @@
title: "`obnam-benchmark`---tool to run benchmarks"
author: "The Obnam project"
documentclass: report
-template: rust
bindings:
- subplot/benchmark.yaml
- lib/files.yaml
- lib/runcmd.yaml
-functions:
-- subplot/benchmark.rs
+impls:
+ python:
+ - subplot/benchmark.py
+ - lib/files.py
+ - lib/runcmd.py
...
@@ -109,7 +111,7 @@ given an installed Rust program obnam-benchmark
given file spec.yaml
given file expected.json
when I run obnam-benchmark spec spec.yaml --output spec.json
-then JSON files spec.yaml and expected.json match
+then JSON files spec.json and expected.json match
```
```{#spec.yaml .yaml .file .numberLines}
diff --git a/src/bin/obnam-benchmark.rs b/src/bin/obnam-benchmark.rs
index 1a26ae0..e6a476c 100644
--- a/src/bin/obnam-benchmark.rs
+++ b/src/bin/obnam-benchmark.rs
@@ -10,6 +10,7 @@ use structopt::StructOpt;
fn main() {
pretty_env_logger::init_custom_env("OBNAM_BENCHMARK_LOG");
+ println!("START");
info!("obnam-benchmark starts");
if let Err(err) = real_main() {
eprintln!("ERROR: {}", err);
@@ -17,6 +18,7 @@ fn main() {
exit(1);
}
info!("obnam-benchmark ends successfully");
+ println!("END");
}
fn real_main() -> anyhow::Result<()> {
diff --git a/src/client.rs b/src/client.rs
new file mode 100644
index 0000000..fb25009
--- /dev/null
+++ b/src/client.rs
@@ -0,0 +1,84 @@
+use log::debug;
+use serde::Serialize;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use tempfile::{tempdir, TempDir};
+
+#[derive(Debug, thiserror::Error)]
+pub enum ObnamClientError {
+ #[error("failed to create temporary directory for server: {0}")]
+ TempDir(std::io::Error),
+
+ #[error("failed to write client configuration to {0}: {1}")]
+ WriteConfig(PathBuf, std::io::Error),
+
+ #[error("failed to run obnam: {0}")]
+ Run(std::io::Error),
+}
+
+#[derive(Debug)]
+pub struct ObnamClient {
+ #[allow(dead_code)]
+ tempdir: TempDir,
+ #[allow(dead_code)]
+ config: PathBuf,
+}
+
+impl ObnamClient {
+ pub fn new(server_url: String, root: PathBuf) -> Result<Self, ObnamClientError> {
+ debug!("creating ObnamClient");
+ let tempdir = tempdir().map_err(ObnamClientError::TempDir)?;
+ let config_filename = tempdir.path().join("client.yaml");
+
+ let config = ClientConfig::new(server_url, root);
+ config.write(&config_filename)?;
+
+ Ok(Self {
+ tempdir,
+ config: config_filename,
+ })
+ }
+
+ pub fn run(&self, args: &[&str]) -> Result<String, ObnamClientError> {
+ let output = Command::new("obnam")
+ .arg("--config")
+ .arg(&self.config)
+ .args(args)
+ .output()
+ .map_err(ObnamClientError::Run)?;
+ if output.status.code() != Some(0) {
+ eprintln!("{}", String::from_utf8_lossy(&output.stdout));
+ eprintln!("{}", String::from_utf8_lossy(&output.stderr));
+ std::process::exit(1);
+ }
+
+ Ok(String::from_utf8_lossy(&output.stdout)
+ .to_owned()
+ .to_string())
+ }
+}
+
+#[derive(Debug, Serialize)]
+pub struct ClientConfig {
+ server_url: String,
+ verify_tls_cert: bool,
+ roots: Vec<PathBuf>,
+ log: PathBuf,
+}
+
+impl ClientConfig {
+ fn new(server_url: String, root: PathBuf) -> Self {
+ Self {
+ server_url,
+ verify_tls_cert: false,
+ roots: vec![root],
+ log: PathBuf::from("obnam.log"),
+ }
+ }
+
+ fn write(&self, filename: &Path) -> Result<(), ObnamClientError> {
+ std::fs::write(filename, serde_yaml::to_string(self).unwrap())
+ .map_err(|err| ObnamClientError::WriteConfig(filename.to_path_buf(), err))?;
+ Ok(())
+ }
+}
diff --git a/src/daemon.rs b/src/daemon.rs
index fa8f287..8bf8adb 100644
--- a/src/daemon.rs
+++ b/src/daemon.rs
@@ -1,7 +1,11 @@
+use log::{debug, error, info};
use nix::sys::signal::kill;
use nix::sys::signal::Signal;
use nix::unistd::Pid;
+use std::ffi::OsStr;
use std::fs::read;
+use std::os::unix::ffi::OsStrExt;
+use std::path::{Path, PathBuf};
use std::process::Command;
use std::thread::sleep;
use std::time::{Duration, Instant};
@@ -19,6 +23,10 @@ pub enum DaemonError {
#[error(transparent)]
TempFile(#[from] std::io::Error),
+ /// Something went wrong read standard output of daemon.
+ #[error("failed to read daemon stdout: {0}")]
+ Stdout(std::io::Error),
+
/// Something went wrong read error output of daemon.
#[error("failed to read daemon stderr: {0}")]
Stderr(std::io::Error),
@@ -67,18 +75,22 @@ impl DaemonManager {
/// it reads a configuration file and that has errors. This
/// function won't wait for that to happen: it only cares about
/// the PID.
- pub fn start(&self, argv: &[&str]) -> Result<Daemon, DaemonError> {
- let stdout = NamedTempFile::new()?;
- let stderr = NamedTempFile::new()?;
+ pub fn start(
+ &self,
+ argv: &[&OsStr],
+ stdout: &Path,
+ stderr: &Path,
+ ) -> Result<Daemon, DaemonError> {
+ info!("start daemon: {:?}", argv);
let pid = NamedTempFile::new()?;
let output = Command::new("daemonize")
.args(&[
"-c",
"/",
"-e",
- &stderr.path().display().to_string(),
+ &stderr.display().to_string(),
"-o",
- &stdout.path().display().to_string(),
+ &stdout.display().to_string(),
"-p",
&pid.path().display().to_string(),
])
@@ -91,6 +103,7 @@ impl DaemonManager {
std::process::exit(1);
}
+ debug!("waiting for daemon to write PID file");
let time = Instant::now();
while time.elapsed() < self.timeout {
// Do we have the pid file?
@@ -102,7 +115,8 @@ impl DaemonManager {
// Parse as an integer, if possible.
if let Ok(pid) = pid.parse() {
// We have a pid, stop waiting.
- return Ok(Daemon::new(pid));
+ info!("got pid for daemon: pid");
+ return Ok(Daemon::new(pid, stdout, stderr));
}
}
sleep_briefly();
@@ -111,8 +125,22 @@ impl DaemonManager {
}
}
- let cmd = argv.join(" ");
- let err = read(stderr.path()).map_err(DaemonError::Stderr)?;
+ error!(
+ "no PID file within {} ms, giving up",
+ self.timeout.as_millis()
+ );
+ let mut cmd = String::new();
+ for arg in argv {
+ if !cmd.is_empty() {
+ cmd.push(' ');
+ }
+ cmd.push_str(
+ &String::from_utf8_lossy(arg.as_bytes())
+ .to_owned()
+ .to_string(),
+ );
+ }
+ let err = read(&stderr).map_err(DaemonError::Stderr)?;
let err = String::from_utf8_lossy(&err).into_owned();
Err(DaemonError::Timeout(self.timeout.as_millis(), cmd, err))
}
@@ -124,11 +152,18 @@ impl DaemonManager {
#[derive(Debug)]
pub struct Daemon {
pid: Option<i32>,
+ stdout: PathBuf,
+ stderr: PathBuf,
}
impl Daemon {
- fn new(pid: i32) -> Self {
- Self { pid: Some(pid) }
+ fn new(pid: i32, stdout: &Path, stderr: &Path) -> Self {
+ info!("started daemon with PID {}", pid);
+ Self {
+ pid: Some(pid),
+ stdout: stdout.to_path_buf(),
+ stderr: stderr.to_path_buf(),
+ }
}
/// Explicitly stop a daemon.
@@ -137,11 +172,22 @@ impl Daemon {
/// errors. It can only be called once.
pub fn stop(&mut self) -> Result<(), DaemonError> {
if let Some(raw_pid) = self.pid.take() {
+ info!("stopping daemon with PID {}", raw_pid);
let pid = Pid::from_raw(raw_pid);
kill(pid, Some(Signal::SIGKILL)).map_err(|e| DaemonError::Kill(raw_pid, e))?;
}
Ok(())
}
+
+ /// Return what the daemon has written to its stderr so far.
+ pub fn stdout(&self) -> Result<Vec<u8>, DaemonError> {
+ std::fs::read(&self.stdout).map_err(DaemonError::Stdout)
+ }
+
+ /// Return what the daemon has written to its stderr so far.
+ pub fn stderr(&self) -> Result<Vec<u8>, DaemonError> {
+ std::fs::read(&self.stderr).map_err(DaemonError::Stderr)
+ }
}
impl Drop for Daemon {
diff --git a/src/lib.rs b/src/lib.rs
index e1e2b81..e8c6d82 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -28,6 +28,7 @@
//! This crate only collects data from a set of benchmarks. It does
//! not analyze the data. The data can be stored for later analysis.
+pub mod client;
pub mod daemon;
pub mod junk;
pub mod obnam;
diff --git a/src/server.rs b/src/server.rs
index 9a809a5..26f418b 100644
--- a/src/server.rs
+++ b/src/server.rs
@@ -3,6 +3,7 @@ use crate::tlsgen::{Tls, TlsError};
use log::debug;
use rand::random;
use serde::Serialize;
+use std::ffi::OsStr;
use std::ops::Range;
use std::path::{Path, PathBuf};
use std::time::Instant;
@@ -18,6 +19,9 @@ pub enum ObnamServerError {
#[error("took too long to pick a random port for server")]
Port,
+ #[error("error creating chunks directory: {0}")]
+ ChunksDir(std::io::Error),
+
#[error("failed to create temporary directory for server: {0}")]
TempDir(std::io::Error),
@@ -40,11 +44,12 @@ pub struct ObnamServer {
tempdir: TempDir,
chunks: PathBuf,
daemon: Option<Daemon>,
+ url: String,
}
impl ObnamServer {
pub fn new(manager: &DaemonManager) -> Result<Self, ObnamServerError> {
- debug!("creating ObamServer");
+ debug!("creating ObnamServer");
let tempdir = tempdir().map_err(ObnamServerError::TempDir)?;
let config_filename = tempdir.path().join("server.yaml");
let chunks = tempdir.path().join("chunks");
@@ -55,22 +60,37 @@ impl ObnamServer {
write(&tls_key, tls.key())?;
write(&tls_cert, tls.cert())?;
+ std::fs::create_dir(&chunks).map_err(ObnamServerError::ChunksDir)?;
+
let port = pick_port()?;
+ debug!("server listens on port {}", port);
let config = ServerConfig::new(port, chunks.clone(), tls_key, tls_cert);
config.write(&config_filename)?;
let daemon = manager
- .start(&["/bin/sleep", "1000"])
+ .start(
+ &[
+ OsStr::new("/usr/bin/obnam-server"),
+ OsStr::new(&config_filename),
+ ],
+ Path::new("server.out"),
+ Path::new("server.log"),
+ )
.map_err(ObnamServerError::Daemon)?;
Ok(Self {
tempdir,
chunks,
daemon: Some(daemon),
+ url: format!("https://localhost:{}", port),
})
}
+ pub fn url(&self) -> String {
+ self.url.clone()
+ }
+
pub fn stop(&mut self) {
self.daemon.take();
}
@@ -115,7 +135,9 @@ impl ServerConfig {
}
fn write(&self, filename: &Path) -> Result<(), ObnamServerError> {
- std::fs::write(filename, serde_yaml::to_string(self).unwrap())
+ let config = serde_yaml::to_string(self).unwrap();
+ debug!("server config:\n{}\n(end config)", config);
+ std::fs::write(filename, &config)
.map_err(|err| ObnamServerError::WriteConfig(filename.to_path_buf(), err))?;
Ok(())
}
diff --git a/src/suite.rs b/src/suite.rs
index 90ecf37..e900330 100644
--- a/src/suite.rs
+++ b/src/suite.rs
@@ -1,6 +1,6 @@
+use crate::client::{ObnamClient, ObnamClientError};
use crate::daemon::DaemonManager;
use crate::junk::junk;
-use crate::obnam::{Obnam, ObnamError};
use crate::result::{Measurement, OpMeasurements, Operation};
use crate::server::{ObnamServer, ObnamServerError};
use crate::specification::{Create, FileCount};
@@ -9,6 +9,7 @@ use log::{debug, info};
use std::fs::File;
use std::path::{Path, PathBuf};
use std::time::Instant;
+use tempfile::{tempdir, TempDir};
use walkdir::WalkDir;
/// A running benchmark suite.
@@ -39,9 +40,9 @@ pub enum SuiteError {
#[error("Error looking up file metadata: {0}: {1}")]
FileMeta(PathBuf, walkdir::Error),
- /// Error managing an Obnam system.
+ /// Error using an Obnam client.
#[error(transparent)]
- Obnam(#[from] ObnamError),
+ Client(#[from] ObnamClientError),
/// Error managing an Obnam server.
#[error(transparent)]
@@ -60,8 +61,8 @@ impl Suite {
///
/// Return a measurement of the step.
pub fn execute(&mut self, step: &Step) -> Result<OpMeasurements, SuiteError> {
- debug!("executing step {:?}", step);
let time = Instant::now();
+ eprintln!("step: {:?}", step);
let mut om = match step {
Step::Start(name) => {
assert!(self.benchmark.is_none());
@@ -111,19 +112,21 @@ impl Suite {
struct Benchmark {
name: String,
- // We store an Obnam in an Option so that we can destroy the
- // Obnam, and thereby delete any temporary files. We want to do
- // that intentionally, so that it can be measured.
- obnam: Option<Obnam>,
- server: Option<ObnamServer>,
+ client: ObnamClient,
+ server: ObnamServer,
+ live: TempDir,
}
impl Benchmark {
fn new(name: &str, manager: &DaemonManager) -> Result<Self, SuiteError> {
+ let server = ObnamServer::new(manager)?;
+ let live = tempdir().map_err(SuiteError::TempDir)?;
+ let client = ObnamClient::new(server.url(), live.path().to_path_buf())?;
Ok(Self {
name: name.to_string(),
- obnam: Some(Obnam::new()?),
- server: Some(ObnamServer::new(manager)?),
+ client,
+ server,
+ live,
})
}
@@ -131,29 +134,27 @@ impl Benchmark {
&self.name
}
- fn obnam(&mut self) -> &mut Obnam {
- self.obnam.as_mut().unwrap()
+ fn live(&self) -> PathBuf {
+ self.live.path().to_path_buf()
}
fn start(&mut self) -> Result<OpMeasurements, SuiteError> {
info!("starting benchmark {}", self.name());
- self.obnam().start_server()?;
+ self.client
+ .run(&["init", "--insecure-passphrase=hunter2"])?;
Ok(OpMeasurements::new(self.name(), Operation::Start))
}
fn stop(&mut self) -> Result<OpMeasurements, SuiteError> {
info!("ending benchmark {}", self.name);
- self.obnam().stop_server()?;
- self.obnam.take().unwrap(); // This destroys the Obnam
- self.server.as_mut().unwrap().stop();
+ self.server.stop();
Ok(OpMeasurements::new(self.name(), Operation::Stop))
}
fn create(&mut self, create: &Create) -> Result<OpMeasurements, SuiteError> {
- info!("creating {} test data files", create.files);
- let root = self.obnam().root();
- debug!(
- "creating {} files of {} bytes in {}",
+ let root = self.live();
+ info!(
+ "creating {} files of {} bytes each in {}",
create.files,
create.file_size,
root.display()
@@ -161,7 +162,6 @@ impl Benchmark {
for i in 0..create.files {
let filename = root.join(format!("{}", i));
- debug!("creating {}", filename.display());
let mut f =
File::create(&filename).map_err(|err| SuiteError::CreateFile(filename, err))?;
junk(&mut f, create.file_size)?;
@@ -182,8 +182,9 @@ impl Benchmark {
fn backup(&mut self, n: usize) -> Result<OpMeasurements, SuiteError> {
info!("making backup {} in benchmark {}", n, self.name());
+ self.client.run(&["backup"])?;
let mut om = OpMeasurements::new(self.name(), Operation::Backup);
- let stats = filestats(self.obnam().root())?;
+ let stats = filestats(&self.live())?;
om.push(Measurement::TotalFiles(stats.count));
om.push(Measurement::TotalData(stats.size));
Ok(om)
diff --git a/subplot/benchmark.py b/subplot/benchmark.py
new file mode 100644
index 0000000..5400692
--- /dev/null
+++ b/subplot/benchmark.py
@@ -0,0 +1,28 @@
+import json
+import os
+
+
+def install_rust_program(ctx):
+ runcmd_prepend_to_path = globals()["runcmd_prepend_to_path"]
+ srcdir = globals()["srcdir"]
+
+ # Add the directory with built Rust binaries to the path.
+ default_target = os.path.join(srcdir, "target")
+ target = os.environ.get("CARGO_TARGET_DIR", default_target)
+ runcmd_prepend_to_path(ctx, dirname=os.path.join(target, "debug"))
+
+
+def file_is_at_least_this_long(ctx, filename=None, number=None):
+ st = os.lstat(filename)
+ assert st.st_size >= int(number)
+
+
+def json_files_match(ctx, first=None, second=None):
+ assert_eq = globals()["assert_eq"]
+ first = json.load(open(first))
+ second = json.load(open(second))
+ assert_eq(first, second)
+
+
+def file_is_valid_json(ctx, filename=None):
+ json.load(open(filename))
diff --git a/subplot/benchmark.rs b/subplot/benchmark.rs
index 082e02b..d7eda37 100644
--- a/subplot/benchmark.rs
+++ b/subplot/benchmark.rs
@@ -51,7 +51,7 @@ fn json_files_match(context: &ScenarioContext, first: &str, second: &str) {
Ok(())
},
false,
- );
+ )?;
}
#[step]
diff --git a/subplot/benchmark.yaml b/subplot/benchmark.yaml
index dc3a136..f745b23 100644
--- a/subplot/benchmark.yaml
+++ b/subplot/benchmark.yaml
@@ -1,19 +1,19 @@
- given: an installed Rust program obnam-benchmark
impl:
- rust:
+ python:
function: install_rust_program
- then: JSON files {first} and {second} match
impl:
- rust:
+ python:
function: json_files_match
- then: file {filename} is valid JSON
impl:
- rust:
+ python:
function: file_is_valid_json
- then: file {filename} is at least {number:int} bytes long
impl:
- rust:
+ python:
function: file_is_at_least_this_long
diff --git a/tests/obnam-benchmark.rs b/tests/obnam-benchmark.rs
deleted file mode 100644
index b6200da..0000000
--- a/tests/obnam-benchmark.rs
+++ /dev/null
@@ -1 +0,0 @@
-include!(concat!(env!("OUT_DIR"), "/obnam-benchmark.rs"));