Merge branch 'liw/check-libdocgen' into 'main'

Improve binding docs and generate docs from them

Closes #346

See merge request subplot/subplot!365

7 files changed, 174 insertions, 6 deletions

diff --git a/check b/check
index d1edc07..12196a5 100755
--- a/check
+++ b/check
@@ -162,6 +162,21 @@ class Runcmd:
             **kwargs,
         )
 
+    def libdocgen(self, bindings, output):
+        self.cargo(
+            [
+                "run",
+                "--package=subplot",
+                "--bin=subplot",
+                "--",
+                f"--resources={os.path.abspath('share')}",
+                "libdocgen",
+                "--output",
+                output,
+                bindings,
+            ]
+        )
+
     def get_templates(self, filename):
         metadata = self.cargo(
             [
@@ -358,6 +373,26 @@ def check_tooling(r):
         )
 
 
+def check_doc(r):
+    docs = os.path.join("test-outputs", "libdocs")
+    if not os.path.exists(docs):
+        os.mkdir(docs)
+
+    bindings = []
+    for dirname, _, basenames in os.walk("share"):
+        dirname = dirname[len("share/") :]
+        bindings += [
+            os.path.join(dirname, x)
+            for x in basenames
+            if x != "template.yaml" and x.endswith(".yaml")
+        ]
+
+    for filename in bindings:
+        md = os.path.splitext(os.path.basename(filename))[0] + ".md"
+        md = os.path.join(docs, md)
+        r.libdocgen(filename, md)
+
+
 def parse_args():
     """Parse command line arguments to this script"""
     p = argparse.ArgumentParser()
@@ -375,7 +410,7 @@ def parse_args():
         "--offline", action="store_true", help="only run tests that can be run offline"
     )
 
-    all_whats = ["tooling", "python", "shell", "rust", "subplots"]
+    all_whats = ["tooling", "python", "shell", "rust", "subplots", "doc"]
     p.add_argument(
         "what", nargs="*", default=all_whats, help=f"what to test: {all_whats}"
     )
@@ -410,6 +445,8 @@ def main():
             check_subplots(r)
         elif what == "tooling":
             check_tooling(r)
+        elif what == "doc":
+            check_doc(r)
         else:
             sys.exit(f"Unknown test {what}")
 
diff --git a/share/common/lib/files.yaml b/share/common/lib/files.yaml
index cb39b96..8f3f932 100644
--- a/share/common/lib/files.yaml
+++ b/share/common/lib/files.yaml
@@ -10,6 +10,9 @@
       function: files_create_from_embedded
   types:
     embedded_file: file
+  doc: |
+    Create a file on disk from an embedded file in the subplot
+    document. The created file has the same name as the embedded file.
 
 - given: file {filename_on_disk} from {embedded_file}
   impl:
@@ -20,6 +23,9 @@
   types:
     filename_on_disk: path
     embedded_file: file
+  doc: |
+    Create a file on disk from an embedded file in the subplot
+    document. Set the name of the created file.
 
 - when: I write "{text}" to file {filename}
   impl:
@@ -30,6 +36,8 @@
   types:
     filename: path
     text: text
+  doc: |
+    Create a file on disk with the given content.
 
 # Manage directories (distinct from files).
 
@@ -41,6 +49,8 @@
       function: files_make_directory
   types:
     path: path
+  doc: |
+    Create a directory on disk.
 
 - when: I create directory {path}
   impl:
@@ -50,6 +60,8 @@
       function: files_make_directory
   types:
     path: path
+  doc: |
+    Create a directory on disk.
 
 - when: I remove directory {path}
   impl:
@@ -59,6 +71,8 @@
       function: files_remove_directory
   types:
     path: path
+  doc: |
+    Remove a directory on disk.
 
 - then: directory {path} exists
   impl:
@@ -68,6 +82,8 @@
       function: files_directory_exists
   types:
     path: path
+  doc: |
+    Check that a directory exists.
 
 - then: directory {path} does not exist
   impl:
@@ -77,6 +93,8 @@
       function: files_directory_does_not_exist
   types:
     path: path
+  doc: |
+    Check that a directory does not exist.
 
 - then: directory {path} is empty
   impl:
@@ -86,6 +104,8 @@
       function: files_directory_is_empty
   types:
     path: path
+  doc: |
+    Check that a directory exists and does not contain anything.
 
 - then: directory {path} is not empty
   impl:
@@ -95,6 +115,8 @@
       function: files_directory_is_not_empty
   types:
     path: path
+  doc: |
+    Check that a directory exists and contains something.
 
 # File metadata management and testing.
 
@@ -108,6 +130,8 @@
   types:
     filename: path
     mtime: text
+  doc: |
+    Create a file with specific modification time.
 
 - when: I remember metadata for file {filename}
   impl:
@@ -117,6 +141,8 @@
       function: files_remember_metadata
   types:
     filename: path
+  doc: |
+    Remember the metadata of a file.
 
 - when: I touch file {filename}
   impl:
@@ -126,6 +152,8 @@
       function: files_touch
   types:
     filename: path
+  doc: |
+    Update the modification time of a file to be current time.
 
 - then: file {filename} has same metadata as before
   impl:
@@ -135,6 +163,9 @@
       function: files_has_remembered_metadata
   types:
     filename: path
+  doc: |
+    Check that a file has the same metadata as remembered from
+    earlier.
 
 - then: file {filename} has different metadata from before
   impl:
@@ -144,6 +175,8 @@
       function: files_has_different_metadata
   types:
     filename: path
+  doc: |
+    Check that a file metadata has changed from earlier.
 
 - then: file {filename} has changed from before
   impl:
@@ -153,6 +186,8 @@
       function: files_has_different_metadata
   types:
     filename: path
+  doc: |
+    Check that file metadata has changed from before.
 
 - then: file {filename} has a very recent modification time
   impl:
@@ -162,6 +197,8 @@
       function: files_mtime_is_recent
   types:
     filename: path
+  doc: |
+    Check that file modification time is recent.
 
 - then: file {filename} has a very old modification time
   impl:
@@ -171,6 +208,8 @@
       function: files_mtime_is_ancient
   types:
     filename: path
+  doc: |
+    Check that file modification is far in the past.
 
 # Testing file existence.
 
@@ -182,6 +221,8 @@
       function: files_file_exists
   types:
     filename: path
+  doc: |
+    Check that a file exist.
 
 - then: file {filename} does not exist
   impl:
@@ -191,6 +232,8 @@
       function: files_file_does_not_exist
   types:
     filename: path
+  doc: |
+    Check that a file does not exist.
 
 - then: only files (?P<filenames>.+) exist
   impl:
@@ -199,6 +242,8 @@
     python:
       function: files_only_these_exist
   regex: true
+  doc: |
+    Check that the test directory only contains specific files.
 
 # Tests on file content.
 
@@ -211,6 +256,8 @@
   types:
     filename: path
     data: text
+  doc: |
+    Check that a file contains a string.
 
 - then: file {filename} doesn't contain "{data}"
   impl:
@@ -221,6 +268,8 @@
   types:
     filename: path
     data: text
+  doc: |
+    Check that a file does not contain a string.
 
 - then: file {filename} matches regex /{regex}/
   impl:
@@ -231,6 +280,8 @@
   types:
     filename: path
     regex: text
+  doc: |
+    Check that file content matches a regular expression.
 
 - then: file {filename} matches regex "{regex}"
   impl:
@@ -241,6 +292,8 @@
   types:
     filename: path
     regex: text
+  doc: |
+    Check that file content matches a regular expression.
 
 - then: files {filename1} and {filename2} match
   impl:
@@ -251,4 +304,5 @@
   types:
     filename1: path
     filename2: path
-
+  doc: |
+    Check that two files have the same content.
diff --git a/share/common/lib/runcmd.yaml b/share/common/lib/runcmd.yaml
index 0a387ed..b20eac8 100644
--- a/share/common/lib/runcmd.yaml
+++ b/share/common/lib/runcmd.yaml
@@ -9,7 +9,7 @@
   types:
     script: file
   doc: |
-    FIXME.
+    Install a helper script from an embedded file.
 
 - given: srcdir is in the PATH
   impl:
diff --git a/share/python/lib/daemon.yaml b/share/python/lib/daemon.yaml
index acca151..25b6007 100644
--- a/share/python/lib/daemon.yaml
+++ b/share/python/lib/daemon.yaml
@@ -2,6 +2,8 @@
   impl:
     python:
       function: daemon_no_such_process
+  doc: |
+    Ensure a given process is not running.
 
 - given: a daemon helper shell script {filename}
   impl:
@@ -9,23 +11,35 @@
       function: _daemon_shell_script
   types:
     filename: file
+  doc: |
+    Install a helper script from an embedded file.
 
 - when: I start "{path}{args:text}" as a background process as {name}, on port {port}
   impl:
     python:
       function: daemon_start_on_port
+  doc: |
+    Start a process in the background (as a daemon) and wait until it
+    listens on its assigned port.
 
 - when: I start "(?P<path>[^ "]+)(?P<args>[^"]*)" as a background process as (?P<name>[^,]+), on port (?P<port>\d+), with environment (?P<env>.*)
   regex: true
   impl:
     python:
       function: daemon_start_on_port
+  doc: |
+    Start a process in the background (as a daemon) and wait until it
+    listens on its assigned port. Remember the process under the given
+    name.
 
 - when: I try to start "{path}{args:text}" as {name}, on port {port}
   impl:
     python:
       function: _daemon_start_soonish
       cleanup: _daemon_stop_soonish
+  doc: |
+    Try to start a background process (as a daemon), but don't fail if
+    starting it fails.
 
 - when: I try to start "(?P<path>[^ "]+)(?P<args>[^"]*)" as (?P<name>[^,]+), on port (?P<port>\d+), with environment (?P<env>.*)
   regex: true
@@ -33,64 +47,102 @@
     python:
       function: _daemon_start_soonish
       cleanup: _daemon_stop_soonish
+  doc: |
+    Start a process in the background (as a daemon) and wait until it
+    listens on its assigned port. Remember the process under the given
+    name. Don't fail if this fails.
 
 - when: I start "{path}{args:text}" as a background process as {name}
   impl:
     python:
       function: _daemon_start
+  doc: |
+    Start a process in the background (as a daemon). Remember the
+    process under the given name. Don't fail if this fails.
 
 - when: I start "(?P<path>[^ "]+)(?P<args>[^"]*)" as a background process as (?P<name>[^,]+), with environment (?P<env>.*)
   regex: true
   impl:
     python:
       function: _daemon_start
+  doc: |
+    Start a process in the background (as a daemon), with specific
+    environment variables set. Remember the process under the given
+    name. Don't fail if this fails.
 
 - when: I stop background process {name}
   impl:
     python:
       function: daemon_stop
+  doc: |
+    Stop a background process that was started earlier with the given
+    name.
 
 - when: daemon {name} has produced output
   impl:
     python:
       function: daemon_has_produced_output
+  doc: |
+    Wait until the named daemon has produced output to its stdout or
+    stderr.
 
 - then: a process "{args:text}" is running
   impl:
     python:
       function: daemon_process_exists
+  doc: |
+    Check that a given process is running.
 
 - then: there is no "{args:text}" process
   impl:
     python:
       function: daemon_no_such_process
+  doc: |
+    Check that a given process is not running.
 
 - then: starting daemon fails with "{message:text}"
   impl:
     python:
       function: daemon_start_fails_with
+  doc: |
+    Check that starting a daemon previously failed, and the error
+    message contains the given text.
 
 - then: starting the daemon succeeds
   impl:
     python:
       function: daemon_start_succeeds
+  doc: |
+    Check that staring a daemon previous succeeded.
 
 - then: daemon {name} stdout is "{text:text}"
   impl:
     python:
       function: daemon_stdout_is
+  doc: |
+    Check that the named daemon has written exactly the given text to
+    its stdout.
 
 - then: daemon {name} stdout contains "{text:text}"
   impl:
     python:
       function: daemon_stdout_contains
+  doc: |
+    Check that the named daemon has written the given text to its
+    stdout, possibly among other text.
 
 - then: daemon {name} stdout doesn't contain "{text:text}"
   impl:
     python:
       function: daemon_stdout_doesnt_contain
+  doc: |
+    Check that the named daemon has not written the given text to its
+    stdout.
 
 - then: daemon {name} stderr is "{text:text}"
   impl:
     python:
       function: daemon_stderr_is
+  doc: |
+    Check that the named daemon has written exactly the given text to
+    its stderr.
diff --git a/share/rust/lib/datadir.yaml b/share/rust/lib/datadir.yaml
index f4c313b..b77e5ef 100644
--- a/share/rust/lib/datadir.yaml
+++ b/share/rust/lib/datadir.yaml
@@ -9,9 +9,16 @@
       function: subplotlib::steplibrary::datadir::datadir_has_enough_space
   types:
     bytes: uint
+  doc: |
+    Check the test data directory has at least the given amount of
+    free space expressed as bytes.
+
 - given: datadir has at least {megabytes}M of space
   impl:
     rust:
       function: subplotlib::steplibrary::datadir::datadir_has_enough_space_megabytes
   types:
     megabytes: uint
+  doc: |
+    Check the test data directory has at least the given amount of
+    free space expressed as megabytes.
diff --git a/src/bin/subplot.rs b/src/bin/subplot.rs
index 40f0f55..37545a5 100644
--- a/src/bin/subplot.rs
+++ b/src/bin/subplot.rs
@@ -393,6 +393,10 @@ struct Libdocgen {
     /// If not specified, subplot will try and find a unique template name from the document
     #[clap(name = "TEMPLATE", long = "template", short = 't')]
     template: Option<String>,
+
+    /// Be merciful by allowing bindings to not have documentation.
+    #[clap(long)]
+    merciful: bool,
 }
 
 impl Libdocgen {
@@ -413,7 +417,7 @@ impl Libdocgen {
             doc.push_binding(b);
         }
 
-        std::fs::write(&self.output, doc.to_markdown())?;
+        std::fs::write(&self.output, doc.to_markdown(self.merciful)?)?;
 
         debug!("libdogen ends successfully");
         Ok(())
@@ -437,13 +441,20 @@ impl LibDoc {
         self.bindings.push(binding.clone());
     }
 
-    fn to_markdown(&self) -> String {
+    fn to_markdown(&self, merciful: bool) -> Result<String> {
         let mut md = String::new();
         md.push_str(&format!("# Library `{}`\n\n", self.filename.display()));
         for b in self.bindings.iter() {
             md.push_str(&format!("\n## {} `{}`\n", b.kind(), b.pattern()));
             if let Some(doc) = b.doc() {
                 md.push_str(&format!("\n{}\n", doc));
+            } else if !merciful {
+                return Err(SubplotError::NoBindingDoc(
+                    self.filename.clone(),
+                    b.kind(),
+                    b.pattern().into(),
+                )
+                .into());
             }
             if b.types().count() > 0 {
                 md.push_str("\nCaptures:\n\n");
@@ -452,7 +463,7 @@ impl LibDoc {
                 }
             }
         }
-        md
+        Ok(md)
     }
 }
 
diff --git a/src/error.rs b/src/error.rs
index 6859aa4..5c72879 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -50,6 +50,13 @@ pub enum SubplotError {
     #[error("binding file failed to parse: {0}")]
     BindingFileParseError(PathBuf, #[source] Box<SubplotError>),
 
+    /// Binding lacks documentation.
+    ///
+    /// Add a `doc` field to the binding with text the documents the
+    /// binding.
+    #[error("binding lacks documentation: {0}: {1} {2}")]
+    NoBindingDoc(PathBuf, crate::StepKind, String),
+
     /// Scenario step does not match a known binding
     ///
     /// This may be due to the binding missing entirely, or that the