summaryrefslogtreecommitdiff
path: root/src/parser.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/parser.rs')
-rw-r--r--src/parser.rs103
1 files changed, 59 insertions, 44 deletions
diff --git a/src/parser.rs b/src/parser.rs
index 8058aeb..973d0b2 100644
--- a/src/parser.rs
+++ b/src/parser.rs
@@ -1,24 +1,28 @@
use crate::error::SiteError;
-use crate::token::{Token, TokenParser, TokenPatterns};
+use crate::token::{TokenKind, TokenParser, TokenPatterns};
use crate::wikitext::{ParsedDirective, Snippet, WikiLink};
-use log::trace;
+use line_col::LineColLookup;
+use log::{debug, trace};
use std::collections::HashMap;
#[derive(Debug)]
pub struct WikitextParser {
- tokens: Vec<Token>,
+ tokens: Vec<(TokenKind, usize, usize)>,
}
impl WikitextParser {
pub fn new(input: &str, patterns: &TokenPatterns) -> Self {
+ let linecol = LineColLookup::new(input);
let mut p = TokenParser::new(input, patterns);
let mut tokens = vec![];
loop {
let token = p.parse();
- if token == Token::End {
+ debug!("token {:?}", token);
+ if token.token == TokenKind::End {
break;
}
- tokens.push(token);
+ let (line, col) = linecol.get(token.pos);
+ tokens.push((token.token, line, col));
}
Self { tokens }
@@ -29,58 +33,67 @@ impl WikitextParser {
return Ok(None);
}
+ let (_, line, col) = self.tokens[0];
+ debug!("token at {}:{}", line, col);
let snippet = match &self.tokens[..] {
- [Token::OpenBrackets, Token::Word(target), Token::CloseBrackets, ..] => {
+ [(TokenKind::OpenBrackets, _, _), (TokenKind::Word(target), _, _), (TokenKind::CloseBrackets, _, _), ..] => {
let wikilink = WikiLink::new(target, target);
let snippet = Snippet::WikiLink(wikilink);
self.tokens.drain(..3);
snippet
}
- [Token::OpenBrackets, Token::Word(word), ..] => {
+ [(TokenKind::OpenBrackets, _, _), (TokenKind::Word(word), _, _), ..] => {
trace!("match [[{:?}", word);
let mut link_text = word.to_string();
let mut target = None;
self.tokens.drain(..2);
loop {
+ let (_, line, col) = self.tokens[0];
match &self.tokens[..] {
- [Token::Spaces(_), ..] => {
+ [(TokenKind::Spaces(_), _, _), ..] => {
trace!("match space");
self.tokens.drain(..1);
link_text.push(' ');
}
- [Token::Markdown(s), ..] => {
+ [(TokenKind::Markdown(s), _, _), ..] => {
trace!("match markdown {:?}", s);
link_text.push_str(s);
self.tokens.drain(..1);
}
- [Token::OpenParens, Token::Word(word), ..] => {
+ [(TokenKind::OpenParens, _, _), (TokenKind::Word(word), _, _), ..] => {
trace!("match ({:?}", word);
link_text.push('(');
link_text.push_str(word);
self.tokens.drain(..2);
}
- [Token::Word(word), ..] => {
+ [(TokenKind::Word(word), _, _), ..] => {
trace!("match {:?}", word);
link_text.push_str(word);
self.tokens.drain(..1);
}
- [Token::ClosedParens, ..] => {
+ [(TokenKind::ClosedParens, _, _), ..] => {
trace!("match )");
link_text.push(')');
self.tokens.drain(..1);
}
- [Token::CloseBrackets, ..] => {
+ [(TokenKind::CloseBrackets, _, _), ..] => {
trace!("match ]]");
self.tokens.drain(..1);
break;
}
- [Token::Pipe, Token::Word(word), Token::CloseBrackets, ..] => {
+ [(TokenKind::Pipe, _, _), (TokenKind::Word(word), _, _), (TokenKind::CloseBrackets, _, _), ..] => {
trace!("match |{:?}]]", word);
target = Some(word.to_string());
self.tokens.drain(..3);
break;
}
- _ => panic!("can't parse: {:?}", &self.tokens[..5]),
+ [(TokenKind::Pipe, _, _), (TokenKind::Spaces(_), _, _), (TokenKind::Word(word), _, _), (TokenKind::CloseBrackets, _, _), ..] => {
+ trace!("match |{:?}]]", word);
+ target = Some(word.to_string());
+ self.tokens.drain(..3);
+ break;
+ }
+ _ => panic!("a can't parse line {} column {}: {:?}", line, col, &self.tokens[..5]),
}
}
if target.is_none() {
@@ -89,141 +102,143 @@ impl WikitextParser {
let wikilink = WikiLink::new(&link_text, &target.unwrap());
Snippet::WikiLink(wikilink)
}
- [Token::OpenBrackets, Token::Bang, Token::Word(name), ..] => {
+ [(TokenKind::OpenBrackets, _, _), (TokenKind::Bang, _, _), (TokenKind::Word(name), _, _), ..] => {
trace!("match [[!{:?}", name);
let name = name.to_string();
let mut args = HashMap::new();
self.tokens.drain(..3);
loop {
+ let (_, line, col) = self.tokens[0];
match &self.tokens[..] {
- [Token::Spaces(_), ..] => {
+ [(TokenKind::Spaces(_), _, _), ..] => {
trace!("match spaces");
self.tokens.drain(..1);
}
- [Token::CloseBrackets, ..] => {
+ [(TokenKind::CloseBrackets, _, _), ..] => {
trace!("match ]]");
self.tokens.drain(..1);
break;
}
- [Token::Word(word), Token::Spaces(_), ..] => {
+ [(TokenKind::Word(word), _, _), (TokenKind::Spaces(_), _, _), ..] => {
trace!("match {:?} spaces", word);
args.insert(word.to_string(), "".to_string());
self.tokens.drain(..2);
}
- [Token::Word(word), Token::CloseBrackets, ..] => {
+ [(TokenKind::Word(word), _, _), (TokenKind::CloseBrackets, _, _), ..] => {
trace!("match {:?}]]", word);
args.insert(word.to_string(), "".to_string());
self.tokens.drain(..2);
break;
}
- [Token::Word(name), Token::Equals, Token::Word(value), ..] => {
+ [(TokenKind::Word(name), _, _), (TokenKind::Equals, _, _), (TokenKind::Word(value), _, _), ..] => {
trace!("match {:?}={:?}", name, value);
args.insert(name.to_string(), value.to_string());
self.tokens.drain(..3);
}
- [Token::Word(name), Token::Equals, Token::QuotedValue(value), ..] => {
+ [(TokenKind::Word(name), _, _), (TokenKind::Equals, _, _), (TokenKind::QuotedValue(value), _, _), ..] => {
trace!("match {:?}={:?}", name, value);
args.insert(name.to_string(), value.to_string());
self.tokens.drain(..3);
}
- [Token::QuotedValue(value), ..] => {
+ [(TokenKind::QuotedValue(value), _, _), ..] => {
trace!("match {:?}", value);
args.insert(value.to_string(), "".to_string());
self.tokens.drain(..1);
}
- _ => panic!("can't parse: {:?}", &self.tokens[..5]),
+ _ => panic!("b can't parse line {} column {}: {:?}", line, col, &self.tokens[..5]),
}
}
Snippet::Directive(ParsedDirective::new(&name, args)?)
}
- [Token::Bang, Token::OpenBracket, ..] => {
+ [(TokenKind::Bang, _, _), (TokenKind::OpenBracket, _, _), ..] => {
let mut link_text = String::new();
#[allow(unused_assignments)]
let mut target = None;
self.tokens.drain(..2);
loop {
+ let (_, line, col) = self.tokens[0];
match &self.tokens[..] {
- [Token::Word(word), ..] => {
+ [(TokenKind::Word(word), _, _), ..] => {
link_text.push_str(word);
self.tokens.drain(..1);
}
- [Token::Spaces(_), ..] => {
+ [(TokenKind::Spaces(_), _, _), ..] => {
link_text.push(' ');
self.tokens.drain(..1);
}
- [Token::ClosedBracket, Token::OpenParens, Token::Word(word), Token::ClosedParens, ..] =>
+ [(TokenKind::ClosedBracket, _, _), (TokenKind::OpenParens, _, _), (TokenKind::Word(word), _, _), (TokenKind::ClosedParens, _, _), ..] =>
{
target = Some(word.to_string());
self.tokens.drain(..4);
break;
}
- _ => panic!("can't parse: {:?}", &self.tokens[..5]),
+ _ => panic!("c can't parse line {} column {}: {:?}", line, col, &self.tokens[..5]),
}
}
Snippet::Markdown(format!("![{}]({})", link_text, target.unwrap()))
}
- [Token::Markdown(text), ..] => {
+ [(TokenKind::Markdown(text), _, _), ..] => {
let snippet = Snippet::Markdown(text.to_string());
self.tokens.drain(..1);
snippet
}
- [Token::Spaces(s), ..] => {
+ [(TokenKind::Spaces(s), _, _), ..] => {
let snippet = Snippet::Markdown(s.to_string());
self.tokens.drain(..1);
snippet
}
- [Token::Word(text), ..] => {
+ [(TokenKind::Word(text), _, _), ..] => {
let snippet = Snippet::Markdown(text.to_string());
self.tokens.drain(..1);
snippet
}
- [Token::Equals, ..] => {
+ [(TokenKind::Equals, _, _), ..] => {
self.tokens.drain(..1);
Snippet::Markdown("=".into())
}
- [Token::Bang, ..] => {
+ [(TokenKind::Bang, _, _), ..] => {
self.tokens.drain(..1);
Snippet::Markdown("!".into())
}
- [Token::Pipe, ..] => {
+ [(TokenKind::Pipe, _, _), ..] => {
self.tokens.drain(..1);
Snippet::Markdown("|".into())
}
- [Token::PageName(s), ..] => {
+ [(TokenKind::PageName(s), _, _), ..] => {
let snippet = Snippet::Markdown(s.to_string());
self.tokens.drain(..1);
snippet
}
- [Token::QuotedValue(s), ..] => {
+ [(TokenKind::QuotedValue(s), _, _), ..] => {
let snippet = Snippet::Markdown(format!("\"{}\"", s));
self.tokens.drain(..1);
snippet
}
- [Token::OpenParens, ..] => {
+ [(TokenKind::OpenParens, _, _), ..] => {
self.tokens.drain(..1);
Snippet::Markdown("(".into())
}
- [Token::ClosedParens, ..] => {
+ [(TokenKind::ClosedParens, _, _), ..] => {
self.tokens.drain(..1);
Snippet::Markdown(")".into())
}
- [Token::OpenBracket, ..] => {
+ [(TokenKind::OpenBracket, _, _), ..] => {
self.tokens.drain(..1);
Snippet::Markdown("[".into())
}
- [Token::ClosedBracket, ..] => {
+ [(TokenKind::ClosedBracket, _, _), ..] => {
self.tokens.drain(..1);
Snippet::Markdown("]".into())
}
- [Token::OpenBrackets, ..] => {
+ [(TokenKind::OpenBrackets, _, _), ..] => {
self.tokens.drain(..1);
Snippet::Markdown("[[".into())
}
- [Token::CloseBrackets, ..] => {
+ [(TokenKind::CloseBrackets, _, _), ..] => {
self.tokens.drain(..1);
Snippet::Markdown("]]".into())
}
- _ => panic!("eeek: {:?}", self.tokens),
+ _ => panic!("d can't parse line {} column {}: {:?}", line, col, self.tokens),
};
Ok(Some(snippet))
}
generated by cgit v1.2.1 (git 2.18.0) at 2024-05-17 06:32:33 +0000