subplotlib: Add files step library, with tests

Signed-off-by: Daniel Silverstone <dsilvers@digital-scurf.org>

6 files changed, 693 insertions, 1 deletions

diff --git a/subplotlib/Cargo.toml b/subplotlib/Cargo.toml
index e99c7e0..e19705a 100644
--- a/subplotlib/Cargo.toml
+++ b/subplotlib/Cargo.toml
@@ -12,4 +12,7 @@ lazy_static = "1"
 base64 = "0.13"
 state = "0.4"
 tempfile = "3.1"
-fs2 = "0.4"
\ No newline at end of file
+fs2 = "0.4"
+chrono = "0.4"
+filetime = "0.2"
+regex = "1.4"
\ No newline at end of file
diff --git a/subplotlib/files.md b/subplotlib/files.md
new file mode 100644
index 0000000..dc5c9b5
--- /dev/null
+++ b/subplotlib/files.md
@@ -0,0 +1,77 @@
+---
+title: Acceptance criteria for the files subplotlib step library
+author: The Subplot project
+template: rust
+bindings:
+  - steplibrary/files.yaml
+---
+
+# Introduction
+
+The [Subplot][] library `files` provides scenario steps and their
+implementations for managing files on the file system during tests.
+The library consists of a bindings file `subplotlib/steplibrary/files.yaml` and
+implementations in Rust as part of `subplotlib`.
+
+[subplot]: https://subplot.liw.fi/
+
+This document explains the acceptance criteria for the library and how
+they're verified. It uses the steps and functions from the `files`
+step library.
+
+# Create on-disk files from embedded files
+
+Subplot allows the source document to embed test files, and the
+`files` library provides steps to create real, on-disk files from
+the embedded files.
+
+```scenario
+given file hello.txt
+then file hello.txt exists
+and file hello.txt contains "hello, world"
+and file other.txt does not exist
+given file other.txt from hello.txt
+then file other.txt exists
+and files hello.txt and other.txt match
+and only files hello.txt, other.txt exist
+```
+
+```{#hello.txt .file .numberLines}
+hello, world
+```
+
+# File metadata
+
+These steps create files and manage their metadata.
+
+```scenario
+given file hello.txt
+when I remember metadata for file hello.txt
+then file hello.txt has same metadata as before
+
+when I write "yo" to file hello.txt
+then file hello.txt has different metadata from before
+```
+
+# File modification time
+
+These steps manipulate and test file modification times.
+
+```scenario
+given file foo.dat has modification time 1970-01-02 03:04:05
+then file foo.dat has a very old modification time
+
+when I touch file foo.dat
+then file foo.dat has a very recent modification time
+```
+
+# File contents
+
+These steps verify contents of files.
+
+```scenario
+given file hello.txt
+then file hello.txt contains "hello, world"
+and file hello.txt matches regex "hello, .*"
+and file hello.txt matches regex /hello, .*/
+```
diff --git a/subplotlib/src/steplibrary.rs b/subplotlib/src/steplibrary.rs
index 3466bd8..94bfa2b 100644
--- a/subplotlib/src/steplibrary.rs
+++ b/subplotlib/src/steplibrary.rs
@@ -18,3 +18,4 @@
 //! requisite yaml files.
 
 pub mod datadir;
+pub mod files;
diff --git a/subplotlib/src/steplibrary/files.rs b/subplotlib/src/steplibrary/files.rs
new file mode 100644
index 0000000..2a451a5
--- /dev/null
+++ b/subplotlib/src/steplibrary/files.rs
@@ -0,0 +1,268 @@
+//! Library of steps for handling files in the data dir.
+//!
+//! The files step library is intended to help with standard operations which
+//! people might need when writing subplot scenarios which use embedded files.
+
+use std::collections::{HashMap, HashSet};
+use std::ffi::OsString;
+use std::fs::{self, Metadata, OpenOptions};
+use std::io::{self, Write};
+use std::path::{Component, Path, PathBuf};
+use std::time::{Duration, SystemTime};
+
+use chrono::{TimeZone, Utc};
+use filetime::FileTime;
+use regex::Regex;
+
+pub use crate::prelude::*;
+
+pub use super::datadir::Datadir;
+
+#[derive(Default)]
+pub struct Files {
+    metadata: HashMap<String, Metadata>,
+}
+
+impl ContextElement for Files {
+    fn created(&mut self, scenario: &Scenario) {
+        scenario.register_context_type::<Datadir>();
+    }
+}
+
+#[throws(StepError)]
+fn canonicalise_filename<S: AsRef<Path>>(base: &Path, subpath: S) -> PathBuf {
+    let mut ret = base.to_path_buf();
+    for component in subpath.as_ref().components() {
+        match component {
+            Component::CurDir => {}
+            Component::ParentDir => {
+                throw!("embedded filenames may not contain ..");
+            }
+            Component::RootDir | Component::Prefix(_) => {
+                throw!("embedded filenames must be relative");
+            }
+            c => ret.push(c),
+        }
+    }
+    ret
+}
+
+#[step]
+#[context(Datadir)]
+pub fn create_from_embedded(context: &ScenarioContext, embedded_file: SubplotDataFile) {
+    let filename_on_disk = format!("{}", embedded_file.name().display());
+    create_from_embedded_with_other_name::call(context, &filename_on_disk, embedded_file)
+}
+
+#[step]
+pub fn create_from_embedded_with_other_name(
+    context: &Datadir,
+    filename_on_disk: &str,
+    embedded_file: SubplotDataFile,
+) {
+    let full_path = canonicalise_filename(context.base_path(), filename_on_disk)?;
+    let mut f = OpenOptions::new()
+        .create(true)
+        .write(true)
+        .truncate(true)
+        .open(full_path)?;
+    f.write_all(embedded_file.data())?;
+}
+
+#[step]
+pub fn touch_with_timestamp(context: &Datadir, filename: &str, mtime: &str) {
+    let ts = Utc.datetime_from_str(mtime, "%Y-%m-%d %H:%M:%S")?;
+    let (secs, nanos) = (ts.timestamp(), ts.timestamp_subsec_nanos());
+    let mtime = FileTime::from_unix_time(secs, nanos);
+    let full_path = canonicalise_filename(context.base_path(), filename)?;
+    // If the file doesn't exist, create it
+    drop(
+        OpenOptions::new()
+            .create(true)
+            .write(true)
+            .open(&full_path)?,
+    );
+    // And set its mtime
+    filetime::set_file_mtime(full_path, mtime)?;
+}
+
+#[step]
+pub fn create_from_text(context: &Datadir, text: &str, filename: &str) {
+    let full_path = canonicalise_filename(context.base_path(), filename)?;
+    let mut f = OpenOptions::new()
+        .create(true)
+        .write(true)
+        .truncate(true)
+        .open(full_path)?;
+    f.write_all(text.as_bytes())?;
+}
+
+#[step]
+#[context(Datadir)]
+#[context(Files)]
+pub fn remember_metadata(context: &ScenarioContext, filename: &str) {
+    let full_path = context.with(
+        |context: &Datadir| canonicalise_filename(context.base_path(), filename),
+        false,
+    )?;
+    let metadata = fs::metadata(&full_path)?;
+    context.with_mut(
+        |context: &mut Files| {
+            context.metadata.insert(filename.to_owned(), metadata);
+            Ok(())
+        },
+        false,
+    )
+}
+
+#[step]
+pub fn touch(context: &Datadir, filename: &str) {
+    let full_path = canonicalise_filename(context.base_path(), filename)?;
+    let now = FileTime::now();
+    filetime::set_file_mtime(full_path, now)?;
+}
+
+#[step]
+pub fn file_exists(context: &Datadir, filename: &str) {
+    let full_path = canonicalise_filename(context.base_path(), filename)?;
+    match fs::metadata(full_path) {
+        Ok(_) => (),
+        Err(e) => {
+            if matches!(e.kind(), io::ErrorKind::NotFound) {
+                throw!(format!("file '{}' was not found", filename))
+            } else {
+                throw!(e);
+            }
+        }
+    }
+}
+
+#[step]
+pub fn file_does_not_exist(context: &Datadir, filename: &str) {
+    let full_path = canonicalise_filename(context.base_path(), filename)?;
+    match fs::metadata(full_path) {
+        Ok(_) => {
+            throw!(format!("file '{}' was unexpectedly found", filename))
+        }
+        Err(e) => {
+            if !matches!(e.kind(), io::ErrorKind::NotFound) {
+                throw!(e);
+            }
+        }
+    }
+}
+
+#[step]
+pub fn only_these_exist(context: &Datadir, filenames: &str) {
+    let filenames: HashSet<OsString> = filenames
+        .replace(',', "")
+        .split_ascii_whitespace()
+        .map(|s| s.into())
+        .collect();
+    let fnames: HashSet<OsString> = fs::read_dir(context.base_path())?
+        .map(|entry| entry.map(|entry| entry.file_name()))
+        .collect::<Result<_, _>>()?;
+    assert_eq!(filenames, fnames);
+}
+
+#[step]
+pub fn file_contains(context: &Datadir, filename: &str, data: &str) {
+    let full_path = canonicalise_filename(context.base_path(), filename)?;
+    let body = fs::read_to_string(full_path)?;
+    if !body.contains(data) {
+        throw!("expected file content not found");
+    }
+}
+
+#[step]
+pub fn file_matches_regex(context: &Datadir, filename: &str, regex: &str) {
+    let full_path = canonicalise_filename(context.base_path(), filename)?;
+    let regex = Regex::new(regex)?;
+    let body = fs::read_to_string(full_path)?;
+    if !regex.is_match(&body) {
+        throw!("file content does not match given regex");
+    }
+}
+
+#[step]
+pub fn file_match(context: &Datadir, filename1: &str, filename2: &str) {
+    let full_path1 = canonicalise_filename(context.base_path(), filename1)?;
+    let full_path2 = canonicalise_filename(context.base_path(), filename2)?;
+    let body1 = fs::read(full_path1)?;
+    let body2 = fs::read(full_path2)?;
+    if body1 != body2 {
+        throw!("file contents do not match each other");
+    }
+}
+
+#[step]
+#[context(Datadir)]
+#[context(Files)]
+pub fn has_remembered_metadata(context: &ScenarioContext, filename: &str) {
+    let full_path = context.with(
+        |context: &Datadir| canonicalise_filename(context.base_path(), filename),
+        false,
+    )?;
+    let metadata = fs::metadata(&full_path)?;
+    if let Some(remembered) = context.with(
+        |context: &Files| Ok(context.metadata.get(filename).cloned()),
+        false,
+    )? {
+        if metadata.permissions() != remembered.permissions()
+            || metadata.modified()? != remembered.modified()?
+            || metadata.len() != remembered.len()
+            || metadata.is_file() != remembered.is_file()
+        {
+            throw!(format!("metadata change detected for {}", filename));
+        }
+    } else {
+        throw!(format!("no remembered metadata for {}", filename));
+    }
+}
+
+#[step]
+#[context(Datadir)]
+#[context(Files)]
+pub fn has_different_metadata(context: &ScenarioContext, filename: &str) {
+    let full_path = context.with(
+        |context: &Datadir| canonicalise_filename(context.base_path(), filename),
+        false,
+    )?;
+    let metadata = fs::metadata(&full_path)?;
+    if let Some(remembered) = context.with(
+        |context: &Files| Ok(context.metadata.get(filename).cloned()),
+        false,
+    )? {
+        if metadata.permissions() == remembered.permissions()
+            && metadata.modified()? == remembered.modified()?
+            && metadata.len() == remembered.len()
+            && metadata.is_file() == remembered.is_file()
+        {
+            throw!(format!("metadata change not detected for {}", filename));
+        }
+    } else {
+        throw!(format!("no remembered metadata for {}", filename));
+    }
+}
+
+#[step]
+pub fn mtime_is_recent(context: &Datadir, filename: &str) {
+    let full_path = canonicalise_filename(context.base_path(), filename)?;
+    let metadata = fs::metadata(full_path)?;
+    let mtime = metadata.modified()?;
+    let diff = SystemTime::now().duration_since(mtime)?;
+    if diff > (Duration::from_secs(5)) {
+        throw!(format!("{} is older than 5 seconds", filename));
+    }
+}
+
+#[step]
+pub fn mtime_is_ancient(context: &Datadir, filename: &str) {
+    let full_path = canonicalise_filename(context.base_path(), filename)?;
+    let metadata = fs::metadata(full_path)?;
+    let mtime = metadata.modified()?;
+    let diff = SystemTime::now().duration_since(mtime)?;
+    if diff < (Duration::from_secs(39 * 365 * 24 * 3600)) {
+        throw!(format!("{} is younger than 39 years", filename));
+    }
+}
diff --git a/subplotlib/steplibrary/files.yaml b/subplotlib/steplibrary/files.yaml
new file mode 100644
index 0000000..339e7cf
--- /dev/null
+++ b/subplotlib/steplibrary/files.yaml
@@ -0,0 +1,68 @@
+# Bindings for the files steps
+# These bind the files step library for subplotlib
+
+- given: file {embedded_file}
+  function: subplotlib::steplibrary::files::create_from_embedded
+  types:
+    embedded_file: file
+
+- given: file {filename_on_disk} from {embedded_file}
+  function: subplotlib::steplibrary::files::create_from_embedded_with_other_name
+  types:
+    embedded_file: file
+
+- given: file (?P<filename>\S+) has modification time (?P<mtime>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})
+  regex: true
+  function: subplotlib::steplibrary::files::touch_with_timestamp
+  types:
+    mtime: text
+
+- when: I write "(?P<text>.*)" to file (?P<filename>\S+)
+  regex: true
+  function: subplotlib::steplibrary::files::create_from_text
+
+- when: I remember metadata for file {filename}
+  function: subplotlib::steplibrary::files::remember_metadata
+
+- when: I touch file {filename}
+  function: subplotlib::steplibrary::files::touch
+
+- then: file {filename} exists
+  function: subplotlib::steplibrary::files::file_exists
+
+- then: file {filename} does not exist
+  function: subplotlib::steplibrary::files::file_does_not_exist
+
+- then: only files (?P<filenames>.+) exist
+  function: subplotlib::steplibrary::files::only_these_exist
+  regex: true
+
+- then: file (?P<filename>\S+) contains "(?P<data>.*)"
+  regex: true
+  function: subplotlib::steplibrary::files::file_contains
+
+- then: file (?P<filename>\S+) matches regex /(?P<regex>.*)/
+  regex: true
+  function: subplotlib::steplibrary::files::file_matches_regex
+
+- then: file (?P<filename>\S+) matches regex "(?P<regex>.*)"
+  regex: true
+  function: subplotlib::steplibrary::files::file_matches_regex
+
+- then: files {filename1} and {filename2} match
+  function: subplotlib::steplibrary::files::file_match
+
+- then: file {filename} has same metadata as before
+  function: subplotlib::steplibrary::files::has_remembered_metadata
+
+- then: file {filename} has different metadata from before
+  function: subplotlib::steplibrary::files::has_different_metadata
+
+- then: file {filename} has changed from before
+  function: subplotlib::steplibrary::files::has_different_metadata
+
+- then: file {filename} has a very recent modification time
+  function: subplotlib::steplibrary::files::mtime_is_recent
+
+- then: file {filename} has a very old modification time
+  function: subplotlib::steplibrary::files::mtime_is_ancient
diff --git a/subplotlib/tests/files.rs b/subplotlib/tests/files.rs
new file mode 100644
index 0000000..8b3ba41
--- /dev/null
+++ b/subplotlib/tests/files.rs
@@ -0,0 +1,275 @@
+use subplotlib::prelude::*;
+
+// --------------------------------
+
+lazy_static! {
+    static ref SUBPLOT_EMBEDDED_FILES: Vec<SubplotDataFile> =
+        vec![SubplotDataFile::new("aGVsbG8udHh0", "aGVsbG8sIHdvcmxkCg=="),];
+}
+
+// ---------------------------------
+
+// Create on-disk files from embedded files
+#[test]
+fn create_on_disk_files_from_embedded_files() {
+    let mut scenario = Scenario::new(&base64_decode(
+        "Q3JlYXRlIG9uLWRpc2sgZmlsZXMgZnJvbSBlbWJlZGRlZCBmaWxlcw==",
+    ));
+
+    let step = subplotlib::steplibrary::files::create_from_embedded::Builder::default()
+        .embedded_file({
+            use std::path::PathBuf;
+            // hello.txt
+            let target_name: PathBuf = base64_decode("aGVsbG8udHh0").into();
+            SUBPLOT_EMBEDDED_FILES
+                .iter()
+                .find(|df| df.name() == target_name)
+                .expect("Unable to find file at runtime")
+                .clone()
+        })
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::file_exists::Builder::default()
+        .filename(
+            // "hello.txt"
+            &base64_decode("aGVsbG8udHh0"),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::file_contains::Builder::default()
+        .filename(
+            // "hello.txt"
+            &base64_decode("aGVsbG8udHh0"),
+        )
+        .data(
+            // "hello, world"
+            &base64_decode("aGVsbG8sIHdvcmxk"),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::file_does_not_exist::Builder::default()
+        .filename(
+            // "other.txt"
+            &base64_decode("b3RoZXIudHh0"),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step =
+        subplotlib::steplibrary::files::create_from_embedded_with_other_name::Builder::default()
+            .filename_on_disk(
+                // "other.txt"
+                &base64_decode("b3RoZXIudHh0"),
+            )
+            .embedded_file({
+                use std::path::PathBuf;
+                // hello.txt
+                let target_name: PathBuf = base64_decode("aGVsbG8udHh0").into();
+                SUBPLOT_EMBEDDED_FILES
+                    .iter()
+                    .find(|df| df.name() == target_name)
+                    .expect("Unable to find file at runtime")
+                    .clone()
+            })
+            .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::file_exists::Builder::default()
+        .filename(
+            // "other.txt"
+            &base64_decode("b3RoZXIudHh0"),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::file_match::Builder::default()
+        .filename1(
+            // "hello.txt"
+            &base64_decode("aGVsbG8udHh0"),
+        )
+        .filename2(
+            // "other.txt"
+            &base64_decode("b3RoZXIudHh0"),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::only_these_exist::Builder::default()
+        .filenames(
+            // "hello.txt, other.txt"
+            &base64_decode("aGVsbG8udHh0LCBvdGhlci50eHQ="),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    scenario.run().unwrap();
+}
+
+// ---------------------------------
+
+// File metadata
+#[test]
+fn file_metadata() {
+    let mut scenario = Scenario::new(&base64_decode("RmlsZSBtZXRhZGF0YQ=="));
+
+    let step = subplotlib::steplibrary::files::create_from_embedded::Builder::default()
+        .embedded_file({
+            use std::path::PathBuf;
+            // hello.txt
+            let target_name: PathBuf = base64_decode("aGVsbG8udHh0").into();
+            SUBPLOT_EMBEDDED_FILES
+                .iter()
+                .find(|df| df.name() == target_name)
+                .expect("Unable to find file at runtime")
+                .clone()
+        })
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::remember_metadata::Builder::default()
+        .filename(
+            // "hello.txt"
+            &base64_decode("aGVsbG8udHh0"),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::has_remembered_metadata::Builder::default()
+        .filename(
+            // "hello.txt"
+            &base64_decode("aGVsbG8udHh0"),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::create_from_text::Builder::default()
+        .text(
+            // "yo"
+            &base64_decode("eW8="),
+        )
+        .filename(
+            // "hello.txt"
+            &base64_decode("aGVsbG8udHh0"),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::has_different_metadata::Builder::default()
+        .filename(
+            // "hello.txt"
+            &base64_decode("aGVsbG8udHh0"),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    scenario.run().unwrap();
+}
+
+// ---------------------------------
+
+// File modification time
+#[test]
+fn file_modification_time() {
+    let mut scenario = Scenario::new(&base64_decode("RmlsZSBtb2RpZmljYXRpb24gdGltZQ=="));
+
+    let step = subplotlib::steplibrary::files::touch_with_timestamp::Builder::default()
+        .filename(
+            // "foo.dat"
+            &base64_decode("Zm9vLmRhdA=="),
+        )
+        .mtime(
+            // "1970-01-02 03:04:05"
+            &base64_decode("MTk3MC0wMS0wMiAwMzowNDowNQ=="),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::mtime_is_ancient::Builder::default()
+        .filename(
+            // "foo.dat"
+            &base64_decode("Zm9vLmRhdA=="),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::touch::Builder::default()
+        .filename(
+            // "foo.dat"
+            &base64_decode("Zm9vLmRhdA=="),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::mtime_is_recent::Builder::default()
+        .filename(
+            // "foo.dat"
+            &base64_decode("Zm9vLmRhdA=="),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    scenario.run().unwrap();
+}
+
+// ---------------------------------
+
+// File contents
+#[test]
+fn file_contents() {
+    let mut scenario = Scenario::new(&base64_decode("RmlsZSBjb250ZW50cw=="));
+
+    let step = subplotlib::steplibrary::files::create_from_embedded::Builder::default()
+        .embedded_file({
+            use std::path::PathBuf;
+            // hello.txt
+            let target_name: PathBuf = base64_decode("aGVsbG8udHh0").into();
+            SUBPLOT_EMBEDDED_FILES
+                .iter()
+                .find(|df| df.name() == target_name)
+                .expect("Unable to find file at runtime")
+                .clone()
+        })
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::file_contains::Builder::default()
+        .filename(
+            // "hello.txt"
+            &base64_decode("aGVsbG8udHh0"),
+        )
+        .data(
+            // "hello, world"
+            &base64_decode("aGVsbG8sIHdvcmxk"),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::file_matches_regex::Builder::default()
+        .filename(
+            // "hello.txt"
+            &base64_decode("aGVsbG8udHh0"),
+        )
+        .regex(
+            // "hello, .*"
+            &base64_decode("aGVsbG8sIC4q"),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::file_matches_regex::Builder::default()
+        .filename(
+            // "hello.txt"
+            &base64_decode("aGVsbG8udHh0"),
+        )
+        .regex(
+            // "hello, .*"
+            &base64_decode("aGVsbG8sIC4q"),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    scenario.run().unwrap();
+}