From c7b37b35c4530dc9ee79a4b79a46041ed682b183 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 5 Aug 2022 07:21:56 +0300 Subject: refactor: use new type for names This is a big change that was not feasible to turn into a tidy series of small steps. Sorry. Re-work the processing queue so that wikitext pages are not loaded as soon as the files are round in Site::scan, but later, though still in ::scan. This is a little cleaner. Fix: don't copy markdown files to the destination tree, and verify that it doesn't happen in the subplot. Also, un-publish Site::add_wikitextpage and Site::add_other_file. They're not needed from outside site.rs. Also, drop Sites::included_files, as it was the same as Site::files. Sponsored-by: author --- riki.md | 12 ++++ src/bin/riki.rs | 17 +++--- src/name.rs | 8 +++ src/page.rs | 64 +++++++------------- src/site.rs | 181 ++++++++++++++++++++++++++++++++------------------------ 5 files changed, 156 insertions(+), 126 deletions(-) diff --git a/riki.md b/riki.md index 7adb3ff..c5dc4c0 100644 --- a/riki.md +++ b/riki.md @@ -532,6 +532,18 @@ then stdout doesn't contain "#index.mdwn#" ## Output directory tree +### No markdown files in output tree + +_Requirement: Markdown files are not copied to the output tree._ + +~~~scenario +given an installed riki +given file site/index.mdwn from empty +when I run riki build site output +then file output/index.html exists +then file output/index.mdwn does not exist +~~~ + ### Output files have source file modification times _Requirement: Files in the output directory have the same time stamp diff --git a/src/bin/riki.rs b/src/bin/riki.rs index 83d1a9c..70bf71c 100644 --- a/src/bin/riki.rs +++ b/src/bin/riki.rs @@ -2,6 +2,7 @@ use clap::{CommandFactory, FromArgMatches, Parser}; use git_testament::{git_testament, render_testament, GitModification}; use log::{debug, error, info}; use riki::error::SiteError; +use riki::name::Name; use riki::site::Site; use riki::util::{canonicalize, copy_file_from_source, mkdir, set_mtime}; use std::error::Error; @@ -145,16 +146,14 @@ impl Build { } else { page.to_html()? }; - let output = page.meta().destination_filename(&destdir); + let output = page.meta().destination_filename(); debug!("writing: {}", output.display()); htmlpage.write(&output)?; set_mtime(&output, page.meta().mtime())?; } - for file in site.files() { - let input = site.input_filename(file)?; - let output = site.output_filename(file)?; - copy_file_from_source(&input, &output)?; + for file in site.files_only() { + copy_file_from_source(file.source_path(), file.destination_path())?; } Ok(()) @@ -173,10 +172,10 @@ impl List { let srcdir = canonicalize(&self.srcdir)?; let mut site = Site::new(&srcdir, &srcdir); site.scan()?; - let mut filenames = site.included_files().to_vec(); - filenames.sort_unstable(); - for filename in filenames { - println!("{}", filename.display()); + let mut names: Vec<&Name> = site.pages_and_files().collect(); + names.sort_by_cached_key(|name| name.page_path()); + for name in names { + println!("{}", name); } Ok(()) } diff --git a/src/name.rs b/src/name.rs index e0b4882..9cfca6b 100644 --- a/src/name.rs +++ b/src/name.rs @@ -73,6 +73,14 @@ impl NameBuilder { } } + pub fn srcdir(&self) -> &Path { + &self.srcdir + } + + pub fn destdir(&self) -> &Path { + &self.destdir + } + fn name(&self, path: &Path, ext: Option<&str>) -> Name { assert!(path.starts_with(&self.srcdir)); let src = path.into(); diff --git a/src/page.rs b/src/page.rs index e6e43d3..d3bc8e7 100644 --- a/src/page.rs +++ b/src/page.rs @@ -1,8 +1,9 @@ use crate::error::SiteError; use crate::html::{parse, Content, Element, ElementTag, HtmlPage}; +use crate::name::Name; use crate::parser::WikitextParser; use crate::site::Site; -use crate::util::{get_mtime, join_subpath, make_path_relative_to}; +use crate::util::get_mtime; use crate::wikitext::Snippet; use log::{info, trace}; use std::path::{Path, PathBuf}; @@ -19,21 +20,16 @@ impl WikitextPage { Self { meta, wikitext } } - pub fn read(srcdir: &Path, filename: &Path) -> Result { - info!("input file: {}", filename.display()); - let relative = make_path_relative_to(srcdir, filename).with_extension(""); - let absolute = Path::new("/").join(&relative); - let name = relative - .file_name() - .unwrap_or_else(|| panic!("get filename from {}", relative.display())) - .to_string_lossy() - .to_string(); - let data = std::fs::read(filename).map_err(|e| SiteError::FileRead(filename.into(), e))?; - let wikitext = String::from_utf8(data).map_err(|e| SiteError::Utf8(filename.into(), e))?; - let mtime = get_mtime(filename)?; + pub fn read(name: &Name) -> Result { + info!("input file: {}", name); + + let src = name.source_path(); + let data = std::fs::read(&src).map_err(|e| SiteError::FileRead(src.into(), e))?; + let wikitext = String::from_utf8(data).map_err(|e| SiteError::Utf8(src.into(), e))?; + let mtime = get_mtime(src)?; + let meta = MetaBuilder::default() - .name(name) - .path(absolute) + .name(name.clone()) .mtime(mtime) .build(); Ok(Self::new(meta, wikitext)) @@ -125,31 +121,24 @@ impl MarkdownPage { #[derive(Debug, Clone, Eq, PartialEq)] pub struct PageMeta { - name: String, + name: Name, title: Option, - path: PathBuf, mtime: SystemTime, } impl PageMeta { - fn new(name: String, title: Option, path: PathBuf, mtime: SystemTime) -> Self { + fn new(name: Name, title: Option, mtime: SystemTime) -> Self { trace!( - "PageMeta: name={:?} title={:?} path={:?} mtime={:?}", + "PageMeta: name={:?} title={:?} mtime={:?}", name, title, - path, mtime, ); - Self { - name, - title, - path, - mtime, - } + Self { name, title, mtime } } - pub fn destination_filename(&self, destdir: &Path) -> PathBuf { - join_subpath(destdir, &self.path).with_extension("html") + pub fn destination_filename(&self) -> PathBuf { + self.name.destination_path().into() } pub fn set_title(&mut self, title: String) { @@ -161,12 +150,12 @@ impl PageMeta { if let Some(title) = &self.title { title } else { - &self.name + self.name.page_name() } } pub fn path(&self) -> &Path { - &self.path + self.name.page_path() } pub fn mtime(&self) -> SystemTime { @@ -176,24 +165,22 @@ impl PageMeta { #[derive(Debug, Default)] pub struct MetaBuilder { - name: String, + name: Option, title: Option, - path: Option, mtime: Option, } impl MetaBuilder { pub fn build(self) -> PageMeta { PageMeta::new( - self.name, + self.name.expect("name set on MetaBuilder"), self.title, - self.path.expect("path set on MetaBuilder"), self.mtime.expect("mtime set on MetaBuilder"), ) } - pub fn name(mut self, name: String) -> Self { - self.name = name; + pub fn name(mut self, name: Name) -> Self { + self.name = Some(name); self } @@ -202,11 +189,6 @@ impl MetaBuilder { self } - pub fn path(mut self, path: PathBuf) -> Self { - self.path = Some(path); - self - } - pub fn mtime(mut self, mtime: SystemTime) -> Self { self.mtime = Some(mtime); self diff --git a/src/site.rs b/src/site.rs index fd28a2b..2d1368d 100644 --- a/src/site.rs +++ b/src/site.rs @@ -1,23 +1,23 @@ use crate::error::SiteError; +use crate::name::{Name, NameBuilder, Names}; use crate::page::{MarkdownPage, UnprocessedPage, WikitextPage}; use crate::parser::WikitextParser; use crate::token::TokenPatterns; -use crate::util::{join_subpath, make_path_absolute, make_path_relative_to, make_relative_link}; +use crate::util::make_relative_link; use log::{debug, info, trace}; use std::collections::HashMap; use std::path::{Path, PathBuf}; use walkdir::WalkDir; pub struct Site { + builder: NameBuilder, wikitext_pages: Vec, unprocessed_pages: Vec, markdown_pages: Vec, - files: Vec, - included_files: Vec, + files: Names, patterns: TokenPatterns, - srcdir: PathBuf, - destdir: PathBuf, - pages: PageSet, + name_queue: Vec, + page_queue: PageSet, } impl Site { @@ -29,55 +29,73 @@ impl Site { P: AsRef, { Self { + builder: NameBuilder::new(srcdir.as_ref(), destdir.as_ref()), wikitext_pages: vec![], unprocessed_pages: vec![], markdown_pages: vec![], - files: vec![], - included_files: vec![], + files: Names::default(), patterns: TokenPatterns::default(), - srcdir: srcdir.as_ref().into(), - destdir: destdir.as_ref().into(), - pages: PageSet::default(), + name_queue: vec![], + page_queue: PageSet::default(), } } pub fn scan(&mut self) -> Result<(), SiteError> { - for filename in Self::all_files(&self.srcdir)? { - self.included_files - .push(make_path_relative_to(&self.srcdir, &filename)); - if Self::is_markdown(&filename) { - let page = WikitextPage::read(&self.srcdir, &filename)?; - self.add_wikitextpage(page); - } else if filename.is_file() || filename.is_symlink() { - self.add_other_file(filename); + for name in self.all_files()? { + trace!("scan: name={}", name); + if name.is_wikitext_page() { + trace!("scan: it's a page"); + self.name_queue.push(name); + } else { + trace!("scan: it's a non-page file"); + let filename = name.source_path(); + if filename.is_file() || filename.is_symlink() { + self.add_other_file(name); + } } } Ok(()) } - pub fn add_wikitextpage(&mut self, page: WikitextPage) { + fn add_wikitextpage(&mut self, page: WikitextPage) { info!("add wikitext page {}", page.meta().path().display()); - self.pages.insert(&page); + self.page_queue.insert(&page); self.wikitext_pages.push(page); } - pub fn add_other_file(&mut self, filename: PathBuf) { - info!("add other file {}", filename.display()); - let filename = make_path_relative_to(&self.srcdir, &filename); - let filename = make_path_absolute(&filename); - self.files.push(filename); + fn add_other_file(&mut self, name: Name) { + info!("add other file {}", name); + self.files.insert(name); } pub fn process(&mut self) -> Result<(), SiteError> { + trace!("processing queues"); loop { - if !self.process_wikipage()? && !self.process_unrocessed_page()? { + if !self.process_name()? + && !self.process_wikipage()? + && !self.process_unrocessed_page()? + { + trace!("processing queues done"); break; } } Ok(()) } - pub fn process_wikipage(&mut self) -> Result { + fn process_name(&mut self) -> Result { + if let Some(name) = self.name_queue.pop() { + debug!("loading wikitext page {}", name.source_path().display()); + let page = WikitextPage::read(&name)?; + self.files.insert(name); + self.add_wikitextpage(page); + Ok(true) + } else { + trace!("name_queue was empty"); + Ok(false) + } + } + + fn process_wikipage(&mut self) -> Result { if let Some(page) = self.wikitext_pages.pop() { debug!("processing wikitext page {}", page.meta().path().display()); let mut parser = WikitextParser::new(page.wikitext(), &self.patterns); @@ -85,11 +103,12 @@ impl Site { self.unprocessed_pages.push(page); Ok(true) } else { + trace!("wikitext_ages was empty"); Ok(false) } } - pub fn process_unrocessed_page(&mut self) -> Result { + fn process_unrocessed_page(&mut self) -> Result { if let Some(page) = self.unprocessed_pages.pop() { debug!( "processing unprocessed page {}", @@ -99,6 +118,7 @@ impl Site { self.markdown_pages.push(page); Ok(true) } else { + trace!("unprocessed_ages was empty"); Ok(false) } } @@ -107,35 +127,38 @@ impl Site { &self.markdown_pages } - pub fn files(&self) -> &[PathBuf] { - &self.files - } - - pub fn included_files(&self) -> &[PathBuf] { - &self.included_files - } - - pub fn input_filename(&self, filename: &Path) -> Result { - Ok(join_subpath(&self.srcdir, filename)) + pub fn files_only(&self) -> impl Iterator { + self.files.files() } - pub fn output_filename(&self, filename: &Path) -> Result { - Ok(join_subpath(&self.destdir, filename)) + pub fn pages_and_files(&self) -> impl Iterator { + self.files.iter().chain(self.name_queue.iter()) } - fn all_files(root: &Path) -> Result, SiteError> { - let mut files = vec![]; + fn all_files(&self) -> Result, SiteError> { + let mut names = vec![]; + let root = self.builder.srcdir(); + trace!("all_files: root={}", root.display()); for e in WalkDir::new(root) { let e = e.map_err(|err| SiteError::WalkDir(root.to_path_buf(), err))?; let path = e.path(); + trace!("all_files: path={}", path.display()); if Self::is_excluded(path) { debug!("exclude {}", path.display()); } else { debug!("include {}", path.display()); - files.push(path.to_path_buf()); + if Self::is_markdown(path) { + trace!("it's markdown"); + names.push(self.builder.page(path)); + } else if path.is_file() { + trace!("it's not markdown"); + names.push(self.builder.file(path)); + } else { + trace!("it's not a file"); + } } } - Ok(files) + Ok(names) } fn is_excluded(path: &Path) -> bool { @@ -182,7 +205,7 @@ impl Site { // Is target absolute? if target.starts_with("/") { - if let Some(path) = self.pages.get(target) { + if let Some(path) = self.page_queue.get(target) { trace!("absolute target exists"); return Ok(path.into()); } else { @@ -194,7 +217,7 @@ impl Site { // Does a sub-page exist? let path = page.join(target); trace!("checking for subpage {}", path.display()); - if let Some(path) = self.pages.get(&path) { + if let Some(path) = self.page_queue.get(&path) { trace!("subpage exists: {}", path.display()); return Ok(path.into()); } @@ -207,7 +230,7 @@ impl Site { parent.display(), path.display() ); - if let Some(path) = self.pages.get(path.as_path()) { + if let Some(path) = self.page_queue.get(path.as_path()) { trace!("sibling page exists: {}", path.display()); return Ok(path.into()); } @@ -224,8 +247,8 @@ impl Site { } fn file_exists(&self, filename: &Path) -> bool { - for existing in self.files.iter() { - if filename == existing { + for existing in self.files.files() { + if filename == existing.page_path() { return true; } } @@ -256,32 +279,37 @@ impl PageSet { #[cfg(test)] mod test { - use super::{Site, SiteError, WikitextPage}; + use super::{NameBuilder, Site, SiteError, WikitextPage}; use crate::page::MetaBuilder; use std::{ path::{Path, PathBuf}, time::SystemTime, }; + fn site() -> Site { + Site::new("/src", "/dest") + } + + fn builder() -> NameBuilder { + NameBuilder::new(Path::new("/src"), Path::new("/dest")) + } + fn page(path: &str) -> WikitextPage { + let name = builder().page(Path::new(path)); let mtime = SystemTime::now(); - let meta = MetaBuilder::default() - .path(PathBuf::from(path)) - .mtime(mtime) - .build(); + let meta = MetaBuilder::default().name(name).mtime(mtime).build(); WikitextPage::new(meta, "".into()) } #[test] fn has_no_pages_initially() { - let site = Site::new(".", "."); - assert_eq!(site.markdown_pages().to_vec(), vec![]); + assert_eq!(site().markdown_pages().to_vec(), vec![]); } #[test] fn absolute_link_resolves_to_link_relative_root_of_site() { - let mut site = Site::new(".", "."); - site.add_wikitextpage(page("/yo/yoyo")); + let mut site = site(); + site.add_wikitextpage(page("/src/yo/yoyo")); assert_eq!( site.resolve("/foo/bar", "/yo/yoyo").unwrap(), Path::new("../yo/yoyo") @@ -290,11 +318,11 @@ mod test { #[test] fn link_to_missing_is_an_error() { - let site = Site::new(".", "."); - match site.resolve("/foo/bar", "yo") { + let site = site(); + match site.resolve("/src/foo/bar", "yo") { Err(SiteError::PageMissing(page, target)) => { assert_eq!(target, PathBuf::from("yo")); - assert_eq!(page, PathBuf::from("/foo/bar")); + assert_eq!(page, PathBuf::from("/src/foo/bar")); } _ => panic!("unexpected success"), } @@ -302,40 +330,40 @@ mod test { #[test] fn link_to_sibling_resolves_to_it() { - let mut site = Site::new(".", "."); - site.add_wikitextpage(page("/foo/yo")); + let mut site = site(); + site.add_wikitextpage(page("/src/foo/yo")); site.process().unwrap(); assert_eq!(site.resolve("/foo/bar", "yo").unwrap(), Path::new("yo")); } #[test] fn link_using_other_casing_is_resolved() { - let mut site = Site::new(".", "."); - site.add_wikitextpage(page("/foo/yo")); + let mut site = site(); + site.add_wikitextpage(page("/src/foo/yo")); site.process().unwrap(); assert_eq!(site.resolve("/foo/bar", "YO").unwrap(), Path::new("yo")); } #[test] fn link_to_sublpage_resolves_to_it() { - let mut site = Site::new(".", "."); - site.add_wikitextpage(page("/foo/bar/yo")); + let mut site = site(); + site.add_wikitextpage(page("/src/foo/bar/yo")); site.process().unwrap(); assert_eq!(site.resolve("/foo/bar", "yo").unwrap(), Path::new("bar/yo")); } #[test] fn link_to_sublpage_resolves_to_it_and_not_sibling() { - let mut site = Site::new(".", "."); - site.add_wikitextpage(page("/foo/bar/yo")); - site.add_wikitextpage(page("/foo/yo")); + let mut site = site(); + site.add_wikitextpage(page("/src/foo/bar/yo")); + site.add_wikitextpage(page("/src/foo/yo")); site.process().unwrap(); assert_eq!(site.resolve("/foo/bar", "yo").unwrap(), Path::new("bar/yo")); } #[test] fn link_to_unrelated_subpage_is_an_error() { - let mut site = Site::new(".", "."); + let mut site = site(); site.process().unwrap(); match site.resolve("/foo/bar", "yo/yoyo") { Err(SiteError::PageMissing(page, target)) => { @@ -348,8 +376,8 @@ mod test { #[test] fn link_to_subsubpage_resolves_to_it() { - let mut site = Site::new(".", "."); - site.add_wikitextpage(page("/foo/bar/yo/yoyo")); + let mut site = site(); + site.add_wikitextpage(page("/src/foo/bar/yo/yoyo")); site.process().unwrap(); assert_eq!( site.resolve("/foo/bar", "yo/yoyo").unwrap(), @@ -359,8 +387,9 @@ mod test { #[test] fn link_to_sibling_file_resolves_to_it() { - let mut site = Site::new("/src", "/dest"); - site.add_other_file(PathBuf::from("/src/foo/bar.jpg")); + let mut site = site(); + let name = builder().file(Path::new("/src/foo/bar.jpg")); + site.add_other_file(name); site.process().unwrap(); assert_eq!( site.resolve("/foo/bar", "bar.jpg").unwrap(), -- cgit v1.2.1