feat! read document metadata from a YAML file

This is a huge change all in one commit, sorry. However, as it changes
a fundamental part of the command line interface (namely, what
constitutes as the input file), there doesn't seem a way to break this
into a tidy series of small commits.

Most of the diff is in subplot.md, where every scenario that invokes
Subplot needs multiple changes, thus touching much of the file.

The overall change is that where we previously had document metadata
in embedded YAML in the Markdown file, we now have it in a separate
YAML file. The Markdown file is named in the YAML file.

We still parse the Markdown with Pandoc for everything, except
codegen. Switching from Pandoc to pulldown_cmark for parsing will be
another big change that I didn't want to include in this huge change
set.

Sponsored-by: author

5 files changed, 74 insertions, 47 deletions

diff --git a/src/ast.rs b/src/ast.rs
index f02fdd1..7efe836 100644
--- a/src/ast.rs
+++ b/src/ast.rs
@@ -279,7 +279,7 @@ pub enum Error {
 /// block we can work with, in any input file. By being strict here we
 /// make it easier to tell the user when a metadata block has, say, a
 /// misspelled field.
-#[derive(Debug, Default, Deserialize)]
+#[derive(Debug, Default, Clone, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub struct YamlMetadata {
     title: String,
@@ -288,6 +288,7 @@ pub struct YamlMetadata {
     date: Option<String>,
     classes: Option<Vec<String>>,
     bibliography: Option<Vec<PathBuf>>,
+    markdowns: Vec<PathBuf>,
     bindings: Option<Vec<PathBuf>>,
     documentclass: Option<String>,
     #[serde(default)]
@@ -301,7 +302,13 @@ impl YamlMetadata {
         Ok(meta)
     }
 
-    fn to_map(&self) -> Map<String, MetaValue> {
+    /// Name of file with the Markdown for the subplot document.
+    pub fn markdown(&self) -> &Path {
+        &self.markdowns[0]
+    }
+
+    /// Convert into a pandoc_ast::Map.
+    pub fn to_map(&self) -> Map<String, MetaValue> {
         trace!("Creating metadata map from parsed YAML");
         let mut map: Map<String, MetaValue> = Map::new();
 
@@ -367,8 +374,8 @@ fn meta_path_bufs(v: &[PathBuf]) -> MetaValue {
 
 #[cfg(test)]
 mod test {
-    use super::{extract_metadata, parse_code_block_attrs, AbstractSyntaxTree, YamlMetadata};
-    use std::path::PathBuf;
+    use super::{parse_code_block_attrs, YamlMetadata};
+    use std::path::{Path, PathBuf};
 
     #[test]
     fn code_block_attrs() {
@@ -392,26 +399,6 @@ mod test {
     }
 
     #[test]
-    fn parses_leading_meta() {
-        let markdown = "\n\n---\ntitle: Foo Bar\n...\nfoobar\n";
-        let (meta, markdown) = extract_metadata(markdown).unwrap();
-        let ast = AbstractSyntaxTree::new(meta, markdown);
-        let doc = ast.to_pandoc();
-        let keys: Vec<String> = doc.meta.keys().cloned().collect();
-        assert_eq!(keys, ["title"]);
-    }
-
-    #[test]
-    fn parses_trailing_meta() {
-        let markdown = "foobar\n---\ntitle: Foo Bar\n...\n\n\n";
-        let (meta, markdown) = extract_metadata(markdown).unwrap();
-        let ast = AbstractSyntaxTree::new(meta, markdown);
-        let doc = ast.to_pandoc();
-        let keys: Vec<String> = doc.meta.keys().cloned().collect();
-        assert_eq!(keys, ["title"]);
-    }
-
-    #[test]
     fn full_meta() {
         let meta = YamlMetadata::new(
             "\
@@ -425,6 +412,8 @@ impls:
 bibliography:
 - foo.bib
 - bar.bib
+markdowns:
+- test.md
 bindings:
 - foo.yaml
 - bar.yaml
@@ -438,6 +427,7 @@ bindings:
             meta.bibliography.unwrap(),
             &[path("foo.bib"), path("bar.bib")]
         );
+        assert_eq!(meta.markdowns, vec![Path::new("test.md")]);
         assert_eq!(
             meta.bindings.unwrap(),
             &[path("foo.yaml"), path("bar.yaml")]
diff --git a/src/bin/subplot.rs b/src/bin/subplot.rs
index b618ac9..907e616 100644
--- a/src/bin/subplot.rs
+++ b/src/bin/subplot.rs
@@ -278,7 +278,7 @@ impl Docgen {
         } else if let Some(date) = doc.meta().date() {
             date.to_string()
         } else {
-            Self::mtime_formatted(Self::mtime(&self.input)?)
+            Self::mtime_formatted(Self::mtime(doc.meta().markdown_filename())?)
         };
         pandoc.add_option(pandoc::PandocOption::Meta("date".to_string(), Some(date)));
         pandoc.add_option(pandoc::PandocOption::TableOfContents);
diff --git a/src/doc.rs b/src/doc.rs
index 38e14b5..2ed3ef5 100644
--- a/src/doc.rs
+++ b/src/doc.rs
@@ -12,12 +12,14 @@ use crate::Scenario;
 use crate::ScenarioStep;
 use crate::Style;
 use crate::SubplotError;
+use crate::YamlMetadata;
 use crate::{bindings::CaptureType, parser::parse_scenario_snippet};
 use crate::{Warning, Warnings};
 
 use std::collections::HashSet;
 use std::default::Default;
 use std::fmt::Debug;
+use std::fs::read;
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
 
@@ -58,7 +60,7 @@ static KNOWN_PANDOC_CLASSES: &[&str] = &["numberLines", "noNumberLines"];
 /// # Example
 ///
 /// fix this example;
-/// ~~~~
+/// ~~~~ignored
 /// let markdown = "\
 /// ---
 /// title: Test Title
@@ -114,6 +116,7 @@ impl Document {
     fn from_ast<P>(
         basedir: P,
         markdowns: Vec<PathBuf>,
+        yamlmeta: &ast::YamlMetadata,
         mut ast: Pandoc,
         style: Style,
         template: Option<&str>,
@@ -121,7 +124,7 @@ impl Document {
     where
         P: AsRef<Path> + Debug,
     {
-        let meta = Metadata::new(basedir, &ast, template)?;
+        let meta = Metadata::new(basedir, yamlmeta, template)?;
         let mut linter = LintingVisitor::default();
         trace!("Walking AST for linting...");
         linter.walk_pandoc(&mut ast);
@@ -151,10 +154,15 @@ impl Document {
             basedir.display(),
             filename.display()
         );
-        let markdowns = vec![filename.to_path_buf()];
+
+        let meta = load_metadata_from_yaml_file(filename)?;
+
+        let mdfile = meta.markdown();
+        let mdfile = basedir.join(mdfile);
+        let markdowns = vec![mdfile.clone()];
 
         let mut pandoc = pandoc::new();
-        pandoc.add_input(&filename);
+        pandoc.add_input(&mdfile);
         pandoc.set_input_format(
             pandoc::InputFormat::Markdown,
             vec![pandoc::MarkdownExtension::Citations],
@@ -167,7 +175,7 @@ impl Document {
 
         trace!(
             "Invoking Pandoc to parse document {:?} into AST as JSON",
-            filename
+            mdfile,
         );
         let json = match pandoc.execute().map_err(SubplotError::Pandoc)? {
             pandoc::PandocOutput::ToBuffer(o) => o,
@@ -176,8 +184,9 @@ impl Document {
         trace!("Pandoc was happy");
 
         trace!("Parsing document AST as JSON...");
-        let ast: Pandoc = serde_json::from_str(&json).map_err(SubplotError::AstJson)?;
-        let doc = Self::from_ast(basedir, markdowns, ast, style, template)?;
+        let mut ast: Pandoc = serde_json::from_str(&json).map_err(SubplotError::AstJson)?;
+        ast.meta = meta.to_map();
+        let doc = Self::from_ast(basedir, markdowns, &meta, ast, style, template)?;
 
         trace!("Loaded document OK");
         Ok(doc)
@@ -195,14 +204,22 @@ impl Document {
         template: Option<&str>,
     ) -> Result<Document, SubplotError> {
         trace!("Parsing document with pullmark-cmark from {:?}", filename);
-        let filename = filename.to_path_buf();
-        let markdown = std::fs::read_to_string(&filename)
-            .map_err(|err| SubplotError::ReadFile(filename.clone(), err))?;
-        let (meta, markdown) = ast::extract_metadata(&markdown)?;
-        let ast = ast::AbstractSyntaxTree::new(meta, markdown);
+        let meta = load_metadata_from_yaml_file(filename)?;
+        let mdfile = meta.markdown();
+        let mdfile = basedir.join(mdfile);
+        let markdown = std::fs::read_to_string(&mdfile)
+            .map_err(|err| SubplotError::ReadFile(mdfile.clone(), err))?;
+        let ast = ast::AbstractSyntaxTree::new(meta.clone(), &markdown);
 
         trace!("Parsed document OK");
-        Self::from_ast(basedir, vec![filename], ast.to_pandoc(), style, template)
+        Self::from_ast(
+            basedir,
+            vec![filename.into()],
+            &meta,
+            ast.to_pandoc(),
+            style,
+            template,
+        )
     }
 
     /// Return the AST of a Document, serialized as JSON.
@@ -481,6 +498,14 @@ impl Document {
     }
 }
 
+fn load_metadata_from_yaml_file(filename: &Path) -> Result<YamlMetadata, SubplotError> {
+    let yaml = read(filename).map_err(|e| SubplotError::ReadFile(filename.into(), e))?;
+    trace!("Parsing YAML metadata from {}", filename.display());
+    let meta: ast::YamlMetadata = serde_yaml::from_slice(&yaml)
+        .map_err(|e| SubplotError::MetadataFile(filename.into(), e))?;
+    Ok(meta)
+}
+
 /// Load a `Document` from a file.
 ///
 /// This version uses Pandoc to parse the Markdown.
diff --git a/src/error.rs b/src/error.rs
index 17870c5..53eccad 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -311,6 +311,10 @@ pub enum SubplotError {
     Ast(#[from] crate::ast::Error),
 
     /// UTF8 conversion error.
+    #[error("failed to parse UTF8 in file {0}")]
+    FileUtf8(PathBuf, #[source] std::string::FromUtf8Error),
+
+    /// UTF8 conversion error.
     #[error(transparent)]
     Utf8Error(#[from] std::str::Utf8Error),
 
diff --git a/src/metadata.rs b/src/metadata.rs
index 5f5e183..dee0b50 100644
--- a/src/metadata.rs
+++ b/src/metadata.rs
@@ -1,11 +1,11 @@
-use crate::{Bindings, SubplotError, TemplateSpec};
+use crate::{Bindings, SubplotError, TemplateSpec, YamlMetadata};
 
 use std::collections::HashMap;
 use std::fmt::Debug;
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
 
-use pandoc_ast::{Inline, Map, MetaValue, Pandoc};
+use pandoc_ast::{Inline, Map, MetaValue};
 
 use log::trace;
 
@@ -14,6 +14,7 @@ use log::trace;
 pub struct Metadata {
     title: String,
     date: Option<String>,
+    markdown_filename: PathBuf,
     bindings_filenames: Vec<PathBuf>,
     bindings: Bindings,
     impls: HashMap<String, DocumentImpl>,
@@ -32,22 +33,23 @@ impl Metadata {
     /// Construct a Metadata from a Document, if possible.
     pub fn new<P>(
         basedir: P,
-        doc: &Pandoc,
+        meta: &YamlMetadata,
         template: Option<&str>,
     ) -> Result<Metadata, SubplotError>
     where
         P: AsRef<Path> + Debug,
     {
-        let title = get_title(&doc.meta);
-        let date = get_date(&doc.meta);
-        let bindings_filenames = get_bindings_filenames(&doc.meta);
-        let bibliographies = get_bibliographies(basedir.as_ref(), &doc.meta);
-        let classes = get_classes(&doc.meta);
+        let map = meta.to_map();
+        let title = get_title(&map);
+        let date = get_date(&map);
+        let bindings_filenames = get_bindings_filenames(&map);
+        let bibliographies = get_bibliographies(basedir.as_ref(), &map);
+        let classes = get_classes(&map);
         trace!("Loaded basic metadata");
 
         let mut impls = HashMap::new();
 
-        if let Some(raw_impls) = doc.meta.get("impls") {
+        if let Some(raw_impls) = map.get("impls") {
             match raw_impls {
                 MetaValue::MetaMap(raw_impls) => {
                     for (impl_name, functions_filenames) in raw_impls.iter() {
@@ -74,6 +76,7 @@ impl Metadata {
         Ok(Metadata {
             title,
             date,
+            markdown_filename: meta.markdown().into(),
             bindings_filenames,
             bindings,
             impls,
@@ -92,6 +95,11 @@ impl Metadata {
         self.date.as_deref()
     }
 
+    /// Return filename of the markdown file.
+    pub fn markdown_filename(&self) -> &Path {
+        &self.markdown_filename
+    }
+
     /// Return filename where bindings are specified.
     pub fn bindings_filenames(&self) -> Vec<&Path> {
         self.bindings_filenames.iter().map(|f| f.as_ref()).collect()