diff options
Diffstat (limited to 'subplotlib')
| -rw-r--r-- | subplotlib/Cargo.toml | 7 | ||||
| -rw-r--r-- | subplotlib/build.rs | 5 | ||||
| -rw-r--r-- | subplotlib/files.md | 102 | ||||
| -rw-r--r-- | subplotlib/runcmd.md | 187 | ||||
| -rw-r--r-- | subplotlib/src/scenario.rs | 28 | ||||
| -rw-r--r-- | subplotlib/src/steplibrary/datadir.rs | 11 | ||||
| -rw-r--r-- | subplotlib/src/steplibrary/runcmd.rs | 15 | ||||
| -rw-r--r-- | subplotlib/subplot-rust-support.rs | 219 | ||||
| -rw-r--r-- | subplotlib/tests/subplot.rs | 1 |
9 files changed, 282 insertions, 293 deletions
diff --git a/subplotlib/Cargo.toml b/subplotlib/Cargo.toml index 781a017..ed3fb8a 100644 --- a/subplotlib/Cargo.toml +++ b/subplotlib/Cargo.toml @@ -16,7 +16,7 @@ repository = "https://gitlab.com/subplot/subplot" [dependencies] fehler = "1" -subplotlib-derive = { version="0.1", path = "../subplotlib-derive" } +subplotlib-derive = { version = "0.1", path = "../subplotlib-derive" } lazy_static = "1" base64 = "0.13" state = "0.5" @@ -32,4 +32,7 @@ remove_dir_all = "0.7" [build-dependencies] glob = "0.3" -subplot-build = { version="0.1", path = "../subplot-build" } +subplot-build = { version = "0.1", path = "../subplot-build" } + +[dev-dependencies] +serde_json = "1.0" diff --git a/subplotlib/build.rs b/subplotlib/build.rs index 5f94883..d1eeefd 100644 --- a/subplotlib/build.rs +++ b/subplotlib/build.rs @@ -17,9 +17,12 @@ use std::path::Path; fn main() { let subplots = glob("*.md").expect("failed to find subplots in subplotlib"); let tests = Path::new("tests"); + let subplots = subplots.chain(Some(Ok("../subplot.md".into()))); + let subplots = subplots + .chain(glob("../tests/subplots/common/*.md").expect("failed to find common subplots")); for entry in subplots { let entry = entry.expect("failed to get subplot dir entry in subplotlib"); - let mut inc = tests.join(&entry); + let mut inc = tests.join(&entry.file_name().unwrap()); inc.set_extension("rs"); if !inc.exists() { panic!("missing include file: {}", inc.display()); diff --git a/subplotlib/files.md b/subplotlib/files.md deleted file mode 100644 index 22e8f3c..0000000 --- a/subplotlib/files.md +++ /dev/null @@ -1,102 +0,0 @@ ---- -title: Acceptance criteria for the files subplotlib step library -author: The Subplot project -bindings: - - lib/files.yaml -impls: - rust: [] -... - -# Introduction - -The [Subplot][] library `files` provides scenario steps and their -implementations for managing files on the file system during tests. -The library consists of a bindings file `subplotlib/steplibrary/files.yaml` and -implementations in Rust as part of `subplotlib`. - -[subplot]: https://subplot.liw.fi/ - -This document explains the acceptance criteria for the library and how -they're verified. It uses the steps and functions from the `files` -step library. - -# Create on-disk files from embedded files - -Subplot allows the source document to embed test files, and the -`files` library provides steps to create real, on-disk files from -the embedded files. - -```scenario -given file hello.txt -then file hello.txt exists -and file hello.txt contains "hello, world" -and file other.txt does not exist -given file other.txt from hello.txt -then file other.txt exists -and files hello.txt and other.txt match -and only files hello.txt, other.txt exist -``` - -```{#hello.txt .file .numberLines} -hello, world -``` - -# File metadata - -These steps create files and manage their metadata. - -```scenario -given file hello.txt -when I remember metadata for file hello.txt -then file hello.txt has same metadata as before - -when I write "yo" to file hello.txt -then file hello.txt has different metadata from before -``` - -# File modification time - -These steps manipulate and test file modification times. - -```scenario -given file foo.dat has modification time 1970-01-02 03:04:05 -then file foo.dat has a very old modification time - -when I touch file foo.dat -then file foo.dat has a very recent modification time -``` - -# File contents - -These steps verify contents of files. - -```scenario -given file hello.txt -then file hello.txt contains "hello, world" -and file hello.txt matches regex "hello, .*" -and file hello.txt matches regex /hello, .*/ -``` - -# Directories - -There are also a large number of directory based steps and some directory -based behaviour available in creating files which are available in the files -library. - -```scenario -given a directory first -then directory first exists -and directory first is empty -and directory second does not exist -when I remove directory first -then directory first does not exist -when I create directory second -then directory second exists -and directory second is empty -given file second/third/hello.txt from hello.txt -then directory second is not empty -and directory second/third exists -and directory second/third is not empty -when I remove directory second -then directory second does not exist -``` diff --git a/subplotlib/runcmd.md b/subplotlib/runcmd.md deleted file mode 100644 index f6b9d85..0000000 --- a/subplotlib/runcmd.md +++ /dev/null @@ -1,187 +0,0 @@ ---- -title: Acceptance criteria for the runcmd step library for subplotlib. -author: The Subplot project -bindings: - - lib/runcmd.yaml - - lib/files.yaml -impls: - rust: [] -... - -# Introduction - -The [Subplot][] step library `runcmd` for Rust provides scenario steps -and their implementations for running Unix commands and examining the -results. The library consists of a bindings file `steplibrary/runcmd.yaml` and -implementations inside the `subplotlib` crate itself. - -[subplot]: https://subplot.liw.fi/ - -This document explains the acceptance criteria for the library and how -they're verified. It uses the steps and functions from the -`steplibrary/runcmd` library. The scenarios all have the same structure: run a -command, then examine the exit code, standard output (stdout for -short), or standard error output (stderr) of the command. - -The scenarios use the Unix commands `true` and `false` to -generate exit codes, and `echo` to produce stdout. To generate -stderr, they use the little helper script below. - -```{#err.sh .file .sh .numberLines} -#!/bin/sh -echo "$@" 1>&2 -``` - -# Check exit code - -These scenarios verify the exit code. To make it easier to write -scenarios in language that flows more naturally, there are a couple of -variations. - -## Successful execution - -```scenario -when I run true -then exit code is 0 -and command is successful -``` - -## Successful execution in a sub-directory - -```scenario -given a directory xyzzy -when I run, in xyzzy, pwd -then exit code is 0 -then command is successful -then stdout contains "/xyzzy" -``` - -## Failed execution - -```scenario -when I try to run false -then exit code is not 0 -and command fails -``` - -## Failed execution in a sub-directory - -```scenario -given a directory xyzzy -when I try to run, in xyzzy, false -then exit code is not 0 -and command fails -``` - -# Check output has what we want - -These scenarios verify that stdout or stderr do have something we want -to have. - -## Check stdout is exactly as wanted - -Note that the string is surrounded by double quotes to make it clear -to the reader what's inside. Also, C-style string escapes are -understood. - -```scenario -when I run echo hello, world -then stdout is exactly "hello, world\n" -``` - -## Check stderr is exactly as wanted - -```scenario -given helper script err.sh for runcmd -when I run sh err.sh hello, world -then stderr is exactly "hello, world\n" -``` - -## Check stdout using sub-string search - -Exact string comparisons are not always enough, so we can verify a -sub-string is in output. - -```scenario -when I run echo hello, world -then stdout contains "world\n" -and exit code is 0 -``` - -## Check stderr using sub-string search - -```scenario -given helper script err.sh for runcmd -when I run sh err.sh hello, world -then stderr contains "world\n" -``` - -## Check stdout using regular expressions - -Fixed strings are not always enough, so we can verify output matches a -regular expression. Note that the regular expression is not delimited -and does not get any C-style string escaped decoded. - -```scenario -when I run echo hello, world -then stdout matches regex world$ -``` - -## Check stderr using regular expressions - -```scenario -given helper script err.sh for runcmd -when I run sh err.sh hello, world -then stderr matches regex world$ -``` - -# Check output doesn't have what we want to avoid - -These scenarios verify that the stdout or stderr do not -have something we want to avoid. - -## Check stdout is not exactly something - -```scenario -when I run echo hi -then stdout isn't exactly "hello, world\n" -``` - -## Check stderr is not exactly something - -```scenario -given helper script err.sh for runcmd -when I run sh err.sh hi -then stderr isn't exactly "hello, world\n" -``` - -## Check stdout doesn't contain sub-string - -```scenario -when I run echo hi -then stdout doesn't contain "world" -``` - -## Check stderr doesn't contain sub-string - -```scenario -given helper script err.sh for runcmd -when I run sh err.sh hi -then stderr doesn't contain "world" -``` - -## Check stdout doesn't match regular expression - -```scenario -when I run echo hi -then stdout doesn't match regex world$ - -``` - -## Check stderr doesn't match regular expressions - -```scenario -given helper script err.sh for runcmd -when I run sh err.sh hi -then stderr doesn't match regex world$ -``` diff --git a/subplotlib/src/scenario.rs b/subplotlib/src/scenario.rs index b239883..c0f3e87 100644 --- a/subplotlib/src/scenario.rs +++ b/subplotlib/src/scenario.rs @@ -284,6 +284,7 @@ impl Scenario { // Firstly, we start all the contexts let mut ret = Ok(()); let mut highest_start = None; + println!("Scenario Start: {}", self.contexts.title()); for (i, hook) in self.contexts.hooks.borrow().iter().enumerate() { let res = hook.scenario_starts(&self.contexts); if res.is_err() { @@ -292,10 +293,14 @@ impl Scenario { } highest_start = Some(i); } - + println!( + "*** Context hooks returned {}", + if ret.is_ok() { "OK" } else { "Failure" } + ); if ret.is_ok() { let mut highest = None; for (i, step) in self.steps.iter().map(|(step, _)| step).enumerate() { + println!(" !!! Step {}", step.name()); let mut highest_prep = None; for (i, prep) in self.contexts.hooks.borrow().iter().enumerate() { let res = prep.step_starts(&self.contexts, step.name()); @@ -305,8 +310,17 @@ impl Scenario { } highest_prep = Some(i); } + println!( + " *** Context hooks returned {}", + if ret.is_ok() { "OK" } else { "Failure" } + ); if ret.is_ok() { + println!(" >>> Run step function"); let res = step.call(&self.contexts, false); + println!( + " Step returned {}", + if res.is_ok() { "OK" } else { "Failure" } + ); if res.is_err() { ret = res; break; @@ -314,6 +328,7 @@ impl Scenario { highest = Some(i); } if let Some(n) = highest_prep { + println!(" *** Unwinding step contexts"); for hookn in (0..=n).rev() { let res = self.contexts.hooks.borrow()[hookn].step_stops(&self.contexts); ret = ret.and(res) @@ -321,9 +336,15 @@ impl Scenario { } } if let Some(n) = highest { + println!(" *** Running cleanup functions"); for stepn in (0..=n).rev() { if let (_, Some(cleanup)) = &self.steps[stepn] { + println!(" >>> Cleanup {}", cleanup.name()); let res = cleanup.call(&self.contexts, true); + println!( + " Cleanup returned {}", + if res.is_ok() { "OK" } else { "Failure" } + ); ret = ret.and(res); } } @@ -331,11 +352,16 @@ impl Scenario { } if let Some(n) = highest_start { + println!("*** Running scenario closedown"); for hookn in (0..=n).rev() { let res = self.contexts.hooks.borrow()[hookn].scenario_stops(&self.contexts); ret = ret.and(res); } } + println!( + "<<< Scenario returns {}", + if ret.is_ok() { "OK" } else { "Failure" } + ); ret } } diff --git a/subplotlib/src/steplibrary/datadir.rs b/subplotlib/src/steplibrary/datadir.rs index 8aa6f00..88e6375 100644 --- a/subplotlib/src/steplibrary/datadir.rs +++ b/subplotlib/src/steplibrary/datadir.rs @@ -91,6 +91,17 @@ impl Datadir { .open(full_path)? } + /// Open a file for reading + #[throws(StepError)] + pub fn open_read<S: AsRef<Path>>(&self, subpath: S) -> File { + let full_path = self.canonicalise_filename(subpath)?; + OpenOptions::new() + .create(false) + .write(false) + .read(true) + .open(full_path)? + } + #[throws(StepError)] pub fn create_dir_all<S: AsRef<Path>>(&self, subpath: S) { let full_path = self.canonicalise_filename(subpath)?; diff --git a/subplotlib/src/steplibrary/runcmd.rs b/subplotlib/src/steplibrary/runcmd.rs index f42df6e..5fc12eb 100644 --- a/subplotlib/src/steplibrary/runcmd.rs +++ b/subplotlib/src/steplibrary/runcmd.rs @@ -67,9 +67,24 @@ impl ContextElement for Runcmd { } impl Runcmd { + /// Prepend the given location to the run path pub fn prepend_to_path<S: Into<OsString>>(&mut self, element: S) { self.paths.push(element.into()); } + + /// Retrieve the last run command's stdout as a string. + /// + /// This does a lossy conversion from utf8 so should always succeed. + pub fn stdout_as_string(&self) -> String { + String::from_utf8_lossy(&self.stdout).into_owned() + } + + /// Retrieve the last run command's stderr as a string. + /// + /// This does a lossy conversion from utf8 so should always succeed. + pub fn stderr_as_string(&self) -> String { + String::from_utf8_lossy(&self.stderr).into_owned() + } } #[step] diff --git a/subplotlib/subplot-rust-support.rs b/subplotlib/subplot-rust-support.rs new file mode 100644 index 0000000..19a4ff7 --- /dev/null +++ b/subplotlib/subplot-rust-support.rs @@ -0,0 +1,219 @@ +// Rust support for running subplot-rust.md + +use subplotlib::steplibrary::datadir::Datadir; +use subplotlib::steplibrary::runcmd::{self, Runcmd}; + +use tempfile::TempDir; + +use std::io::{Read, Seek, SeekFrom}; + +#[derive(Default)] +struct SubplotContext { + bin_dir: Option<TempDir>, +} + +impl ContextElement for SubplotContext {} + +#[step] +fn do_nothing(context: &ScenarioContext) { + // Nothing to do here +} + +#[step] +#[context(SubplotContext)] +#[context(Runcmd)] +fn install_subplot(context: &ScenarioContext) { + if let Some(bindir) = std::env::var_os("SUBPLOT_DIR") { + println!("Found SUBPLOT_DIR environment variable, using that"); + context.with_mut( + |rc: &mut Runcmd| { + rc.prepend_to_path(bindir); + Ok(()) + }, + false, + )?; + } else { + let bin_dir = TempDir::new()?; + println!("Creating temporary rundir at {}", bin_dir.path().display()); + + // Since we don't get CARGO_BIN_EXE_subplot when building a subcrate + // we retrieve the path to `subplot` via the assumption that integration + // tests are always located one dir down from the outer crate binaries. + let target_path = std::fs::canonicalize( + std::env::current_exe() + .expect("Cannot determine test exe path") + .parent() + .unwrap() + .join(".."), + ) + .expect("Cannot canonicalise path to binaries"); + + let src_dir = env!("CARGO_MANIFEST_DIR"); + for bin_name in &["subplot"] { + let file_path = bin_dir.path().join(bin_name); + std::fs::write( + &file_path, + format!( + r#" +#!/bin/sh +set -eu +exec '{target_path}/{bin_name}' --resources '{src_dir}/share' "$@" +"#, + target_path = target_path.display(), + bin_name = bin_name, + src_dir = src_dir, + ), + )?; + { + let mut perms = std::fs::metadata(&file_path)?.permissions(); + use std::os::unix::fs::PermissionsExt; + perms.set_mode(perms.mode() | 0o111); // Set executable bit + std::fs::set_permissions(&file_path, perms)?; + } + } + + context.with_mut( + |context: &mut Runcmd| { + context.prepend_to_path(bin_dir.path()); + context.prepend_to_path(target_path); + Ok(()) + }, + false, + )?; + } +} + +#[step] +fn uninstall_subplot(context: &mut SubplotContext) { + context.bin_dir.take(); +} + +#[step] +#[context(Runcmd)] +fn scenario_was_run(context: &ScenarioContext, name: &str) { + let text = format!("\nscenario: {}\n", name); + runcmd::stdout_contains::call(context, &text)?; +} + +#[step] +#[context(Runcmd)] +fn scenario_was_not_run(context: &ScenarioContext, name: &str) { + let text = format!("\nscenario: {}\n", name); + runcmd::stdout_doesnt_contain::call(context, &text)?; +} + +#[step] +#[context(Runcmd)] +fn step_was_run(context: &ScenarioContext, keyword: &str, name: &str) { + let text = format!("\n step: {} {}\n", keyword, name); + runcmd::stdout_contains::call(context, &text)?; +} + +#[step] +#[context(Runcmd)] +fn step_was_run_and_then( + context: &ScenarioContext, + keyword1: &str, + name1: &str, + keyword2: &str, + name2: &str, +) { + let text = format!( + "\n step: {} {}\n step: {} {}", + keyword1, name1, keyword2, name2 + ); + runcmd::stdout_contains::call(context, &text)?; +} + +#[step] +#[context(Runcmd)] +fn cleanup_was_run( + context: &ScenarioContext, + keyword1: &str, + name1: &str, + keyword2: &str, + name2: &str, +) { + let text = format!( + "\n cleanup: {} {}\n cleanup: {} {}\n", + keyword1, name1, keyword2, name2 + ); + runcmd::stdout_contains::call(context, &text)?; +} + +#[step] +#[context(Runcmd)] +fn cleanup_was_not_run(context: &ScenarioContext, keyword: &str, name: &str) { + let text = format!("\n cleanup: {} {}\n", keyword, name); + runcmd::stdout_doesnt_contain::call(context, &text)?; +} + +#[throws(StepError)] +fn end_of_file(context: &Datadir, filename: &str, nbytes: usize) -> Vec<u8> { + let mut fh = context.open_read(filename)?; + fh.seek(SeekFrom::End(-(nbytes as i64)))?; + let mut b = vec![0; nbytes]; + fh.read_exact(&mut b[0..nbytes])?; + b +} + +#[step] +fn file_ends_in_zero_newlines(context: &Datadir, filename: &str) { + let b = end_of_file(context, filename, 1)?; + if b[0] == b'\n' { + throw!(format!("File {} ends in unexpected newline", filename)); + } +} + +#[step] +fn file_ends_in_one_newline(context: &Datadir, filename: &str) { + let b = end_of_file(context, filename, 2)?; + if !(b[0] != b'\n' && b[1] == b'\n') { + throw!(format!( + "File {} does not end in exactly one newline", + filename + )); + } +} + +#[step] +fn file_ends_in_two_newlines(context: &Datadir, filename: &str) { + let b = end_of_file(context, filename, 2)?; + if b[0] != b'\n' || b[1] != b'\n' { + throw!(format!( + "File {} does not end in exactly two newlines", + filename + )); + } +} + +#[step] +fn sleep_seconds(context: &Datadir, delay: u64) { + std::thread::sleep(std::time::Duration::from_secs(delay)); +} + +#[step] +#[context(Datadir)] +#[context(Runcmd)] +fn json_output_matches_file(context: &ScenarioContext, filename: &str) { + let output = context.with(|rc: &Runcmd| Ok(rc.stdout_as_string()), false)?; + let fcontent = context.with( + |dd: &Datadir| { + Ok(std::fs::read_to_string( + dd.canonicalise_filename(filename)?, + )?) + }, + false, + )?; + let output: serde_json::Value = serde_json::from_str(&output)?; + let fcontent: serde_json::Value = serde_json::from_str(&fcontent)?; + println!("########"); + println!("Output:\n{:#}", output); + println!("File:\n{:#}", fcontent); + println!("########"); + assert_eq!( + output, fcontent, + "Command output does not match the content of {}", + filename + ); +} diff --git a/subplotlib/tests/subplot.rs b/subplotlib/tests/subplot.rs new file mode 100644 index 0000000..00accd7 --- /dev/null +++ b/subplotlib/tests/subplot.rs @@ -0,0 +1 @@ +include!(concat!(env!("OUT_DIR"), "/subplot.rs")); |