Merge branch 'fix-158' into 'main'

Add directory steps to subplotlib files library

Closes #158

See merge request larswirzenius/subplot!132

8 files changed, 360 insertions, 3 deletions

diff --git a/Cargo.lock b/Cargo.lock
index d225749..224ef7c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -152,6 +152,47 @@ dependencies = [
 ]
 
 [[package]]
+name = "const_fn"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28b9d6de7f49e22cf97ad17fc4036ece69300032f45f78f30b4a4482cdc3f4a6"
+
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775"
+dependencies = [
+ "cfg-if 1.0.0",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
+dependencies = [
+ "cfg-if 1.0.0",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d"
+dependencies = [
+ "cfg-if 1.0.0",
+ "const_fn",
+ "crossbeam-utils",
+ "lazy_static",
+ "memoffset",
+ "scopeguard",
+]
+
+[[package]]
 name = "crossbeam-utils"
 version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -401,6 +442,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
 
 [[package]]
+name = "memoffset"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
 name = "num-integer"
 version = "0.1.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -420,6 +470,16 @@ dependencies = [
 ]
 
 [[package]]
+name = "num_cpus"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
+dependencies = [
+ "hermit-abi",
+ "libc",
+]
+
+[[package]]
 name = "opaque-debug"
 version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -643,6 +703,31 @@ dependencies = [
 ]
 
 [[package]]
+name = "rayon"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674"
+dependencies = [
+ "autocfg",
+ "crossbeam-deque",
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-utils",
+ "lazy_static",
+ "num_cpus",
+]
+
+[[package]]
 name = "redox_syscall"
 version = "0.1.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -676,6 +761,18 @@ dependencies = [
 ]
 
 [[package]]
+name = "remove_dir_all"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7b19f5c2df95a07275e7224924cc62f76f04525f4fda801473f85e325e81977"
+dependencies = [
+ "log",
+ "num_cpus",
+ "rayon",
+ "winapi",
+]
+
+[[package]]
 name = "roadmap"
 version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -704,6 +801,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "scopeguard"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
+
+[[package]]
 name = "serde"
 version = "1.0.118"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -857,6 +960,7 @@ dependencies = [
  "fs2",
  "lazy_static",
  "regex",
+ "remove_dir_all 0.6.1",
  "shell-words",
  "state",
  "subplotlib-derive",
@@ -895,7 +999,7 @@ dependencies = [
  "libc",
  "rand 0.7.3",
  "redox_syscall",
- "remove_dir_all",
+ "remove_dir_all 0.5.3",
  "winapi",
 ]
 
diff --git a/share/rust/lib/files.yaml b/share/rust/lib/files.yaml
index 339e7cf..5fa4f45 100644
--- a/share/rust/lib/files.yaml
+++ b/share/rust/lib/files.yaml
@@ -66,3 +66,24 @@
 
 - then: file {filename} has a very old modification time
   function: subplotlib::steplibrary::files::mtime_is_ancient
+
+- given: a directory {path}
+  function: subplotlib::steplibrary::files::make_directory
+
+- when: I create directory {path}
+  function: subplotlib::steplibrary::files::make_directory
+
+- when: I remove directory {path}
+  function: subplotlib::steplibrary::files::remove_directory
+
+- then: directory {path} exists
+  function: subplotlib::steplibrary::files::path_exists
+
+- then: directory {path} does not exist
+  function: subplotlib::steplibrary::files::path_does_not_exist
+
+- then: directory {path} is empty
+  function: subplotlib::steplibrary::files::path_is_empty
+
+- then: directory {path} is not empty
+  function: subplotlib::steplibrary::files::path_is_not_empty
diff --git a/subplotlib/Cargo.toml b/subplotlib/Cargo.toml
index df1c6d7..a9e8f85 100644
--- a/subplotlib/Cargo.toml
+++ b/subplotlib/Cargo.toml
@@ -17,4 +17,5 @@ chrono = "0.4"
 filetime = "0.2"
 regex = "1.4"
 shell-words = "1.0"
-unescape = "0.1"
\ No newline at end of file
+unescape = "0.1"
+remove_dir_all = "0.6"
diff --git a/subplotlib/files.md b/subplotlib/files.md
index d005be5..56bea41 100644
--- a/subplotlib/files.md
+++ b/subplotlib/files.md
@@ -75,3 +75,27 @@ then file hello.txt contains "hello, world"
 and file hello.txt matches regex "hello, .*"
 and file hello.txt matches regex /hello, .*/
 ```
+
+# Directories
+
+There are also a large number of directory based steps and some directory
+based behaviour available in creating files which are available in the files
+library.
+
+```scenario
+given a directory first
+then directory first exists
+and directory first is empty
+and directory second does not exist
+when I remove directory first
+then directory first does not exist
+when I create directory second
+then directory second exists
+and directory second is empty
+given file second/third/hello.txt from hello.txt
+then directory second is not empty
+and directory second/third exists
+and directory second/third is not empty
+when I remove directory second
+then directory second does not exist
+```
diff --git a/subplotlib/src/steplibrary/datadir.rs b/subplotlib/src/steplibrary/datadir.rs
index 1ef485e..ab80d25 100644
--- a/subplotlib/src/steplibrary/datadir.rs
+++ b/subplotlib/src/steplibrary/datadir.rs
@@ -95,6 +95,12 @@ impl Datadir {
             .truncate(true)
             .open(full_path)?
     }
+
+    #[throws(StepError)]
+    pub fn create_dir_all<S: AsRef<Path>>(&self, subpath: S) {
+        let full_path = self.canonicalise_filename(subpath)?;
+        std::fs::create_dir_all(full_path)?;
+    }
 }
 
 /// A simple check for enough disk space in the data dir
diff --git a/subplotlib/src/steplibrary/files.rs b/subplotlib/src/steplibrary/files.rs
index f940f08..8ba27cf 100644
--- a/subplotlib/src/steplibrary/files.rs
+++ b/subplotlib/src/steplibrary/files.rs
@@ -7,6 +7,7 @@ use std::collections::{HashMap, HashSet};
 use std::ffi::OsString;
 use std::fs::{self, Metadata, OpenOptions};
 use std::io::{self, Write};
+use std::path::PathBuf;
 use std::time::{Duration, SystemTime};
 
 use chrono::{TimeZone, Utc};
@@ -41,6 +42,14 @@ pub fn create_from_embedded_with_other_name(
     filename_on_disk: &str,
     embedded_file: SubplotDataFile,
 ) {
+    let filename_on_disk = PathBuf::from(filename_on_disk);
+    let parentpath = filename_on_disk.parent().ok_or_else(|| {
+        format!(
+            "No parent directory found for {}",
+            filename_on_disk.display()
+        )
+    })?;
+    context.create_dir_all(parentpath)?;
     context
         .open_write(filename_on_disk)?
         .write_all(embedded_file.data())?;
@@ -245,3 +254,55 @@ pub fn mtime_is_ancient(context: &Datadir, filename: &str) {
         throw!(format!("{} is younger than 39 years", filename));
     }
 }
+
+#[step]
+pub fn make_directory(context: &Datadir, path: &str) {
+    context.create_dir_all(path)?;
+}
+
+#[step]
+pub fn remove_directory(context: &Datadir, path: &str) {
+    let full_path = context.canonicalise_filename(path)?;
+    remove_dir_all::remove_dir_all(full_path)?;
+}
+
+#[step]
+pub fn path_exists(context: &Datadir, path: &str) {
+    let full_path = context.canonicalise_filename(path)?;
+    fs::metadata(full_path)?;
+}
+
+#[step]
+pub fn path_does_not_exist(context: &Datadir, path: &str) {
+    let full_path = context.canonicalise_filename(path)?;
+    match fs::metadata(&full_path) {
+        Ok(_) => throw!(format!("{} exists", full_path.display())),
+        Err(e) => {
+            if !matches!(e.kind(), io::ErrorKind::NotFound) {
+                throw!(e);
+            }
+        }
+    };
+}
+
+#[step]
+pub fn path_is_empty(context: &Datadir, path: &str) {
+    let full_path = context.canonicalise_filename(path)?;
+    let mut iter = fs::read_dir(&full_path)?;
+    match iter.next() {
+        None => {}
+        Some(Ok(_)) => throw!(format!("{} is not empty", full_path.display())),
+        Some(Err(e)) => throw!(e),
+    }
+}
+
+#[step]
+pub fn path_is_not_empty(context: &Datadir, path: &str) {
+    let full_path = context.canonicalise_filename(path)?;
+    let mut iter = fs::read_dir(&full_path)?;
+    match iter.next() {
+        None => throw!(format!("{} is empty", full_path.display())),
+        Some(Ok(_)) => {}
+        Some(Err(e)) => throw!(e),
+    }
+}
diff --git a/subplotlib/src/steplibrary/runcmd.rs b/subplotlib/src/steplibrary/runcmd.rs
index c215f60..383f9e3 100644
--- a/subplotlib/src/steplibrary/runcmd.rs
+++ b/subplotlib/src/steplibrary/runcmd.rs
@@ -1,6 +1,5 @@
 //! Step library for running subprocesses as part of scenarios
 
-use env::vars_os;
 use regex::RegexBuilder;
 
 pub use super::datadir::Datadir;
diff --git a/subplotlib/tests/files.rs b/subplotlib/tests/files.rs
index 8b3ba41..a8fce3d 100644
--- a/subplotlib/tests/files.rs
+++ b/subplotlib/tests/files.rs
@@ -273,3 +273,144 @@ fn file_contents() {
 
     scenario.run().unwrap();
 }
+
+// ---------------------------------
+
+// Directories
+#[test]
+fn directories() {
+    let mut scenario = Scenario::new(&base64_decode("RGlyZWN0b3JpZXM="));
+
+    let step = subplotlib::steplibrary::files::make_directory::Builder::default()
+        .path(
+            // "first"
+            &base64_decode("Zmlyc3Q="),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::path_exists::Builder::default()
+        .path(
+            // "first"
+            &base64_decode("Zmlyc3Q="),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::path_is_empty::Builder::default()
+        .path(
+            // "first"
+            &base64_decode("Zmlyc3Q="),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::path_does_not_exist::Builder::default()
+        .path(
+            // "second"
+            &base64_decode("c2Vjb25k"),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::remove_directory::Builder::default()
+        .path(
+            // "first"
+            &base64_decode("Zmlyc3Q="),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::path_does_not_exist::Builder::default()
+        .path(
+            // "first"
+            &base64_decode("Zmlyc3Q="),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::make_directory::Builder::default()
+        .path(
+            // "second"
+            &base64_decode("c2Vjb25k"),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::path_exists::Builder::default()
+        .path(
+            // "second"
+            &base64_decode("c2Vjb25k"),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::path_is_empty::Builder::default()
+        .path(
+            // "second"
+            &base64_decode("c2Vjb25k"),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step =
+        subplotlib::steplibrary::files::create_from_embedded_with_other_name::Builder::default()
+            .filename_on_disk(
+                // "second/third/hello.txt"
+                &base64_decode("c2Vjb25kL3RoaXJkL2hlbGxvLnR4dA=="),
+            )
+            .embedded_file({
+                use std::path::PathBuf;
+                // hello.txt
+                let target_name: PathBuf = base64_decode("aGVsbG8udHh0").into();
+                SUBPLOT_EMBEDDED_FILES
+                    .iter()
+                    .find(|df| df.name() == target_name)
+                    .expect("Unable to find file at runtime")
+                    .clone()
+            })
+            .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::path_is_not_empty::Builder::default()
+        .path(
+            // "second"
+            &base64_decode("c2Vjb25k"),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::path_exists::Builder::default()
+        .path(
+            // "second/third"
+            &base64_decode("c2Vjb25kL3RoaXJk"),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::path_is_not_empty::Builder::default()
+        .path(
+            // "second/third"
+            &base64_decode("c2Vjb25kL3RoaXJk"),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::remove_directory::Builder::default()
+        .path(
+            // "second"
+            &base64_decode("c2Vjb25k"),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    let step = subplotlib::steplibrary::files::path_does_not_exist::Builder::default()
+        .path(
+            // "second"
+            &base64_decode("c2Vjb25k"),
+        )
+        .build();
+    scenario.add_step(step, None);
+
+    scenario.run().unwrap();
+}