From b9de4d5e810d07b679ad9ab9e46d58e4c76213cc Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Sat, 1 Jan 2022 18:09:35 +0000 Subject: tests: Make runcmd.md a common test set now Signed-off-by: Daniel Silverstone --- subplotlib/runcmd.md | 187 ---------------------------- tests/subplots/common/runcmd.md | 216 +++++++++++++++++++++++++++++++++ tests/subplots/common/runcmd_test.py | 15 +++ tests/subplots/common/runcmd_test.rs | 25 ++++ tests/subplots/common/runcmd_test.yaml | 13 ++ tests/subplots/python/runcmd.md | 214 -------------------------------- tests/subplots/python/runcmd_test.py | 15 --- tests/subplots/python/runcmd_test.yaml | 9 -- 8 files changed, 269 insertions(+), 425 deletions(-) delete mode 100644 subplotlib/runcmd.md create mode 100644 tests/subplots/common/runcmd.md create mode 100644 tests/subplots/common/runcmd_test.py create mode 100644 tests/subplots/common/runcmd_test.rs create mode 100644 tests/subplots/common/runcmd_test.yaml delete mode 100644 tests/subplots/python/runcmd.md delete mode 100644 tests/subplots/python/runcmd_test.py delete mode 100644 tests/subplots/python/runcmd_test.yaml diff --git a/subplotlib/runcmd.md b/subplotlib/runcmd.md deleted file mode 100644 index f6b9d85..0000000 --- a/subplotlib/runcmd.md +++ /dev/null @@ -1,187 +0,0 @@ ---- -title: Acceptance criteria for the runcmd step library for subplotlib. -author: The Subplot project -bindings: - - lib/runcmd.yaml - - lib/files.yaml -impls: - rust: [] -... - -# Introduction - -The [Subplot][] step library `runcmd` for Rust provides scenario steps -and their implementations for running Unix commands and examining the -results. The library consists of a bindings file `steplibrary/runcmd.yaml` and -implementations inside the `subplotlib` crate itself. - -[subplot]: https://subplot.liw.fi/ - -This document explains the acceptance criteria for the library and how -they're verified. It uses the steps and functions from the -`steplibrary/runcmd` library. The scenarios all have the same structure: run a -command, then examine the exit code, standard output (stdout for -short), or standard error output (stderr) of the command. - -The scenarios use the Unix commands `true` and `false` to -generate exit codes, and `echo` to produce stdout. To generate -stderr, they use the little helper script below. - -```{#err.sh .file .sh .numberLines} -#!/bin/sh -echo "$@" 1>&2 -``` - -# Check exit code - -These scenarios verify the exit code. To make it easier to write -scenarios in language that flows more naturally, there are a couple of -variations. - -## Successful execution - -```scenario -when I run true -then exit code is 0 -and command is successful -``` - -## Successful execution in a sub-directory - -```scenario -given a directory xyzzy -when I run, in xyzzy, pwd -then exit code is 0 -then command is successful -then stdout contains "/xyzzy" -``` - -## Failed execution - -```scenario -when I try to run false -then exit code is not 0 -and command fails -``` - -## Failed execution in a sub-directory - -```scenario -given a directory xyzzy -when I try to run, in xyzzy, false -then exit code is not 0 -and command fails -``` - -# Check output has what we want - -These scenarios verify that stdout or stderr do have something we want -to have. - -## Check stdout is exactly as wanted - -Note that the string is surrounded by double quotes to make it clear -to the reader what's inside. Also, C-style string escapes are -understood. - -```scenario -when I run echo hello, world -then stdout is exactly "hello, world\n" -``` - -## Check stderr is exactly as wanted - -```scenario -given helper script err.sh for runcmd -when I run sh err.sh hello, world -then stderr is exactly "hello, world\n" -``` - -## Check stdout using sub-string search - -Exact string comparisons are not always enough, so we can verify a -sub-string is in output. - -```scenario -when I run echo hello, world -then stdout contains "world\n" -and exit code is 0 -``` - -## Check stderr using sub-string search - -```scenario -given helper script err.sh for runcmd -when I run sh err.sh hello, world -then stderr contains "world\n" -``` - -## Check stdout using regular expressions - -Fixed strings are not always enough, so we can verify output matches a -regular expression. Note that the regular expression is not delimited -and does not get any C-style string escaped decoded. - -```scenario -when I run echo hello, world -then stdout matches regex world$ -``` - -## Check stderr using regular expressions - -```scenario -given helper script err.sh for runcmd -when I run sh err.sh hello, world -then stderr matches regex world$ -``` - -# Check output doesn't have what we want to avoid - -These scenarios verify that the stdout or stderr do not -have something we want to avoid. - -## Check stdout is not exactly something - -```scenario -when I run echo hi -then stdout isn't exactly "hello, world\n" -``` - -## Check stderr is not exactly something - -```scenario -given helper script err.sh for runcmd -when I run sh err.sh hi -then stderr isn't exactly "hello, world\n" -``` - -## Check stdout doesn't contain sub-string - -```scenario -when I run echo hi -then stdout doesn't contain "world" -``` - -## Check stderr doesn't contain sub-string - -```scenario -given helper script err.sh for runcmd -when I run sh err.sh hi -then stderr doesn't contain "world" -``` - -## Check stdout doesn't match regular expression - -```scenario -when I run echo hi -then stdout doesn't match regex world$ - -``` - -## Check stderr doesn't match regular expressions - -```scenario -given helper script err.sh for runcmd -when I run sh err.sh hi -then stderr doesn't match regex world$ -``` diff --git a/tests/subplots/common/runcmd.md b/tests/subplots/common/runcmd.md new file mode 100644 index 0000000..4f66685 --- /dev/null +++ b/tests/subplots/common/runcmd.md @@ -0,0 +1,216 @@ +# Introduction + +The [Subplot][] library `runcmd` for Python provides scenario steps +and their implementations for running Unix commands and examining the +results. The library consists of a bindings file `lib/runcmd.yaml` and +implementations in Python in `lib/runcmd.py` or in the Rust subplotlib +step library. There is no Bash version. + +[Subplot]: https://subplot.liw.fi/ + +This document explains the acceptance criteria for the library and how +they're verified. It uses the steps and functions from the +`lib/runcmd` library. The scenarios all have the same structure: run a +command, then examine the exit code, standard output (stdout for +short), or standard error output (stderr) of the command. + +The scenarios use the Unix commands `true` and `false` to +generate exit codes, and `echo` to produce stdout. To generate +stderr, they use the little helper script below. + +~~~{#err.sh .file .sh .numberLines} +#!/bin/sh +echo "$@" 1>&2 +~~~ + +# Check exit code + +These scenarios verify the exit code. To make it easier to write +scenarios in language that flows more naturally, there are a couple of +variations. + +## Successful execution + +~~~scenario +when I run true +then exit code is 0 +and command is successful +~~~ + +## Successful execution in a sub-directory + +~~~scenario +given a directory xyzzy +when I run, in xyzzy, pwd +then exit code is 0 +then command is successful +then stdout contains "/xyzzy" +~~~ + +## Failed execution + +~~~scenario +when I try to run false +then exit code is not 0 +and command fails +~~~ + +## Failed execution in a sub-directory + +~~~scenario +given a directory xyzzy +when I try to run, in xyzzy, false +then exit code is not 0 +and command fails +~~~ + +# Check we can prepend to $PATH + +This scenario verifies that we can add a directory to the beginning of +the PATH environment variable, so that we can have `runcmd` invoke a +binary from our build tree rather than from system directories. This +is especially useful for testing new versions of software that's +already installed on the system. + +~~~scenario +given executable script ls from ls.sh +when I prepend . to PATH +when I run ls +then command is successful +then stdout contains "custom ls, not system ls" +~~~ + +~~~{#ls.sh .file .sh .numberLines} +#!/bin/sh +echo "custom ls, not system ls" +~~~ + +# Check output has what we want + +These scenarios verify that stdout or stderr do have something we want +to have. + +## Check stdout is exactly as wanted + +Note that the string is surrounded by double quotes to make it clear +to the reader what's inside. Also, C-style string escapes are +understood. + +~~~scenario +when I run echo hello, world +then stdout is exactly "hello, world\n" +~~~ + +## Check stderr is exactly as wanted + +~~~scenario +given helper script err.sh for runcmd +when I run sh err.sh hello, world +then stderr is exactly "hello, world\n" +~~~ + +## Check stdout using sub-string search + +Exact string comparisons are not always enough, so we can verify a +sub-string is in output. + +~~~scenario +when I run echo hello, world +then stdout contains "world\n" +and exit code is 0 +~~~ + +## Check stderr using sub-string search + +~~~scenario +given helper script err.sh for runcmd +when I run sh err.sh hello, world +then stderr contains "world\n" +~~~ + +## Check stdout using regular expressions + +Fixed strings are not always enough, so we can verify output matches a +regular expression. Note that the regular expression is not delimited +and does not get any C-style string escaped decoded. + +~~~scenario +when I run echo hello, world +then stdout matches regex world$ +~~~ + +## Check stderr using regular expressions + +~~~scenario +given helper script err.sh for runcmd +when I run sh err.sh hello, world +then stderr matches regex world$ +~~~ + +# Check output doesn't have what we want to avoid + +These scenarios verify that the stdout or stderr do not +have something we want to avoid. + +## Check stdout is not exactly something + +~~~scenario +when I run echo hi +then stdout isn't exactly "hello, world\n" +~~~ + +## Check stderr is not exactly something + +~~~scenario +given helper script err.sh for runcmd +when I run sh err.sh hi +then stderr isn't exactly "hello, world\n" +~~~ + +## Check stdout doesn't contain sub-string + +~~~scenario +when I run echo hi +then stdout doesn't contain "world" +~~~ + +## Check stderr doesn't contain sub-string + +~~~scenario +given helper script err.sh for runcmd +when I run sh err.sh hi +then stderr doesn't contain "world" +~~~ + +## Check stdout doesn't match regular expression + +~~~scenario +when I run echo hi +then stdout doesn't match regex world$ + +~~~ + +## Check stderr doesn't match regular expressions + +~~~scenario +given helper script err.sh for runcmd +when I run sh err.sh hi +then stderr doesn't match regex world$ +~~~ + + +--- +title: Acceptance criteria for the lib/runcmd Subplot library +author: The Subplot project +bindings: +- lib/runcmd.yaml +- runcmd_test.yaml +- lib/files.yaml +impls: + python: + - lib/runcmd.py + - runcmd_test.py + - lib/files.py + rust: + - runcmd_test.rs +... diff --git a/tests/subplots/common/runcmd_test.py b/tests/subplots/common/runcmd_test.py new file mode 100644 index 0000000..4aa5f49 --- /dev/null +++ b/tests/subplots/common/runcmd_test.py @@ -0,0 +1,15 @@ +import os + + +def create_script_from_embedded(ctx, filename=None, embedded=None): + files_create_from_embedded_with_other_name = globals()[ + "files_create_from_embedded_with_other_name" + ] + + # Create the file. + files_create_from_embedded_with_other_name( + ctx, filename_on_disk=filename, embedded_file=embedded + ) + + # Make the new file executable. + os.chmod(filename, 0o755) diff --git a/tests/subplots/common/runcmd_test.rs b/tests/subplots/common/runcmd_test.rs new file mode 100644 index 0000000..7759e5f --- /dev/null +++ b/tests/subplots/common/runcmd_test.rs @@ -0,0 +1,25 @@ +use subplotlib::steplibrary::files::{self, Datadir}; +use subplotlib::steplibrary::runcmd::Runcmd; + +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; + +#[step] +#[context(Datadir)] +fn create_script_from_embedded( + context: &ScenarioContext, + filename: &str, + embedded: SubplotDataFile, +) { + files::create_from_embedded_with_other_name::call(context, filename, embedded)?; + let filename = context.with(|dd: &Datadir| dd.canonicalise_filename(filename), false)?; + let mut perms = std::fs::symlink_metadata(&filename)?.permissions(); + #[cfg(unix)] + perms.set_mode(perms.mode() | 0o111); + std::fs::set_permissions(&filename, perms)?; +} + +#[step] +fn prepend_to_path(context: &mut Runcmd, dirname: &str) { + context.prepend_to_path(dirname); +} diff --git a/tests/subplots/common/runcmd_test.yaml b/tests/subplots/common/runcmd_test.yaml new file mode 100644 index 0000000..daab202 --- /dev/null +++ b/tests/subplots/common/runcmd_test.yaml @@ -0,0 +1,13 @@ +- given: "executable script {filename} from {embedded:file}" + impl: + python: + function: create_script_from_embedded + rust: + function: create_script_from_embedded + +- when: "I prepend {dirname} to PATH" + impl: + python: + function: runcmd_prepend_to_path + rust: + function: prepend_to_path diff --git a/tests/subplots/python/runcmd.md b/tests/subplots/python/runcmd.md deleted file mode 100644 index 01e6904..0000000 --- a/tests/subplots/python/runcmd.md +++ /dev/null @@ -1,214 +0,0 @@ -# Introduction - -The [Subplot][] library `runcmd` for Python provides scenario steps -and their implementations for running Unix commands and examining the -results. The library consists of a bindings file `lib/runcmd.yaml` and -implementations in Python in `lib/runcmd.py`. There is no Bash -version. - -[Subplot]: https://subplot.liw.fi/ - -This document explains the acceptance criteria for the library and how -they're verified. It uses the steps and functions from the -`lib/runcmd` library. The scenarios all have the same structure: run a -command, then examine the exit code, standard output (stdout for -short), or standard error output (stderr) of the command. - -The scenarios use the Unix commands `true` and `false` to -generate exit codes, and `echo` to produce stdout. To generate -stderr, they use the little helper script below. - -~~~{#err.sh .file .sh .numberLines} -#!/bin/sh -echo "$@" 1>&2 -~~~ - -# Check exit code - -These scenarios verify the exit code. To make it easier to write -scenarios in language that flows more naturally, there are a couple of -variations. - -## Successful execution - -~~~scenario -when I run true -then exit code is 0 -and command is successful -~~~ - -## Successful execution in a sub-directory - -~~~scenario -given a directory xyzzy -when I run, in xyzzy, pwd -then exit code is 0 -then command is successful -then stdout contains "/xyzzy" -~~~ - -## Failed execution - -~~~scenario -when I try to run false -then exit code is not 0 -and command fails -~~~ - -## Failed execution in a sub-directory - -~~~scenario -given a directory xyzzy -when I try to run, in xyzzy, false -then exit code is not 0 -and command fails -~~~ - -# Check we can prepend to $PATH - -This scenario verifies that we can add a directory to the beginning of -the PATH environment variable, so that we can have `runcmd` invoke a -binary from our build tree rather than from system directories. This -is especially useful for testing new versions of software that's -already installed on the system. - -~~~scenario -given executable script ls from ls.sh -when I prepend . to PATH -when I run ls -then command is successful -then stdout contains "custom ls, not system ls" -~~~ - -~~~{#ls.sh .file .sh .numberLines} -#!/bin/sh -echo "custom ls, not system ls" -~~~ - -# Check output has what we want - -These scenarios verify that stdout or stderr do have something we want -to have. - -## Check stdout is exactly as wanted - -Note that the string is surrounded by double quotes to make it clear -to the reader what's inside. Also, C-style string escapes are -understood. - -~~~scenario -when I run echo hello, world -then stdout is exactly "hello, world\n" -~~~ - -## Check stderr is exactly as wanted - -~~~scenario -given helper script err.sh for runcmd -when I run sh err.sh hello, world -then stderr is exactly "hello, world\n" -~~~ - -## Check stdout using sub-string search - -Exact string comparisons are not always enough, so we can verify a -sub-string is in output. - -~~~scenario -when I run echo hello, world -then stdout contains "world\n" -and exit code is 0 -~~~ - -## Check stderr using sub-string search - -~~~scenario -given helper script err.sh for runcmd -when I run sh err.sh hello, world -then stderr contains "world\n" -~~~ - -## Check stdout using regular expressions - -Fixed strings are not always enough, so we can verify output matches a -regular expression. Note that the regular expression is not delimited -and does not get any C-style string escaped decoded. - -~~~scenario -when I run echo hello, world -then stdout matches regex world$ -~~~ - -## Check stderr using regular expressions - -~~~scenario -given helper script err.sh for runcmd -when I run sh err.sh hello, world -then stderr matches regex world$ -~~~ - -# Check output doesn't have what we want to avoid - -These scenarios verify that the stdout or stderr do not -have something we want to avoid. - -## Check stdout is not exactly something - -~~~scenario -when I run echo hi -then stdout isn't exactly "hello, world\n" -~~~ - -## Check stderr is not exactly something - -~~~scenario -given helper script err.sh for runcmd -when I run sh err.sh hi -then stderr isn't exactly "hello, world\n" -~~~ - -## Check stdout doesn't contain sub-string - -~~~scenario -when I run echo hi -then stdout doesn't contain "world" -~~~ - -## Check stderr doesn't contain sub-string - -~~~scenario -given helper script err.sh for runcmd -when I run sh err.sh hi -then stderr doesn't contain "world" -~~~ - -## Check stdout doesn't match regular expression - -~~~scenario -when I run echo hi -then stdout doesn't match regex world$ - -~~~ - -## Check stderr doesn't match regular expressions - -~~~scenario -given helper script err.sh for runcmd -when I run sh err.sh hi -then stderr doesn't match regex world$ -~~~ - - ---- -title: Acceptance criteria for the lib/runcmd Subplot library -author: The Subplot project -bindings: -- lib/runcmd.yaml -- runcmd_test.yaml -- lib/files.yaml -impls: - python: - - lib/runcmd.py - - runcmd_test.py - - lib/files.py -... diff --git a/tests/subplots/python/runcmd_test.py b/tests/subplots/python/runcmd_test.py deleted file mode 100644 index 4aa5f49..0000000 --- a/tests/subplots/python/runcmd_test.py +++ /dev/null @@ -1,15 +0,0 @@ -import os - - -def create_script_from_embedded(ctx, filename=None, embedded=None): - files_create_from_embedded_with_other_name = globals()[ - "files_create_from_embedded_with_other_name" - ] - - # Create the file. - files_create_from_embedded_with_other_name( - ctx, filename_on_disk=filename, embedded_file=embedded - ) - - # Make the new file executable. - os.chmod(filename, 0o755) diff --git a/tests/subplots/python/runcmd_test.yaml b/tests/subplots/python/runcmd_test.yaml deleted file mode 100644 index 2ad981e..0000000 --- a/tests/subplots/python/runcmd_test.yaml +++ /dev/null @@ -1,9 +0,0 @@ -- given: "executable script {filename} from {embedded}" - impl: - python: - function: create_script_from_embedded - -- when: "I prepend {dirname} to PATH" - impl: - python: - function: runcmd_prepend_to_path -- cgit v1.2.1