diff options
Diffstat (limited to 'rprt-engine/src/token.rs')
| -rw-r--r-- | rprt-engine/src/token.rs | 279 |
1 files changed, 279 insertions, 0 deletions
diff --git a/rprt-engine/src/token.rs b/rprt-engine/src/token.rs new file mode 100644 index 0000000..cde8164 --- /dev/null +++ b/rprt-engine/src/token.rs @@ -0,0 +1,279 @@ +use logos::{Lexer, Logos}; + +fn lex_delimited( + lex: &mut Lexer<Token>, + delimiter: char, + ignore_inside: Option<&[(char, char)]>, +) -> Option<String> { + let remainder = lex.remainder(); + let mut chars = remainder.chars(); + let mut escaped = false; + let mut pos = 0; + let mut looking_for: Option<char> = None; + + while let Some(c) = chars.next() { + pos += c.len_utf8(); + + // escaping + if escaped { + escaped = false; + continue; + } + if c == '\\' { + escaped = true; + continue; + } + + // capture in + if let Some(end) = looking_for { + if c == end { + looking_for = None; + } + continue; + } + if let Some(ignore_inside) = ignore_inside { + for &(start, end) in ignore_inside { + if c == start { + looking_for = Some(end); + continue; + } + } + } + + if c == delimiter { + let content = &remainder[..pos - 1]; + lex.bump(pos); + return Some(content.to_string()); + } + } + + None +} + +fn lex_regex(lex: &mut Lexer<Token>) -> Option<String> { + let character_class = vec![('[', ']')]; + lex_delimited(lex, '/', Some(&character_class)) +} + +fn lex_text(lex: &mut Lexer<Token>) -> Option<String> { + lex_delimited(lex, '"', None) +} + +#[derive(Debug, Clone, PartialEq, Logos)] +#[logos(skip r"\s+")] +pub enum Token { + // Literals + #[token("\"", lex_text)] + TextLiteral(String), + + #[regex(r"[0-9]+", |lex| lex.slice().parse().ok())] + Number(usize), + + #[token("/", lex_regex)] + Regex(String), + + // Selection Functions + #[token("e")] + Empty, + + #[token("$")] + EndOfBuffer, + + #[token("#")] + CharacterOffset, + + #[token("-")] + Span, + + #[token("x")] + AllMatches, + + #[token("+")] + PositiveLineOffset, + + #[token("B")] + BufferMatch, + + // Text Functions + #[token(".")] + Dot, + + #[token("c")] + Change, + + #[token("i")] + Insert, + + #[token("a")] + Append, + + #[token("d")] + Delete, + + #[token("w")] + Write, + + #[token("|")] + Pipe, + + // Selection Function Operators + #[token("'")] + Reverse, + + #[token(";")] + Sequential, + + #[token("~")] + Complement, + + #[token("?")] + Question, + + // Text Function Operators + #[token("@")] + Swap, + + // Composition/Combinators + #[token(">")] + Before, + + #[token("<")] + After, + + // Grouping + #[token("(")] + LParen, + + #[token(")")] + RParen, + + #[token("{")] + LBrace, + + #[token("}")] + RBrace, + + // Statement/Buffer control + #[token("b")] + BufferSwitch, + + // Separators + #[token(",")] + Comma, +} + +pub fn tokenize(input: &str) -> Result<Vec<Token>, String> { + let mut tokens = Vec::new(); + let mut lexer = Token::lexer(input); + + while let Some(result) = lexer.next() { + match result { + Ok(token) => tokens.push(token), + Err(()) => { + return Err(format!( + "Unexpected character at position {}: '{}'", + lexer.span().start, + lexer.slice() + )); + } + } + } + + Ok(tokens) +} + +#[cfg(test)] +mod tests { + use crate::token::*; + + #[test] + fn test_delimited_literals_with_escapes() { + assert_eq!( + tokenize(r#""" "hello" "escaped\"quote" "backslash\\" "mixed\n\t""#), + Ok(vec![ + Token::TextLiteral("".to_string()), + Token::TextLiteral("hello".to_string()), + Token::TextLiteral(r#"escaped\"quote"#.to_string()), + Token::TextLiteral(r#"backslash\\"#.to_string()), + Token::TextLiteral(r#"mixed\n\t"#.to_string()), + ]) + ); + + assert_eq!( + tokenize(r#"// /[a-z]+/ /foo\/bar/ /[/]/ /(a|b)\d+/ /\\/"#), + Ok(vec![ + Token::Regex("".to_string()), + Token::Regex("[a-z]+".to_string()), + Token::Regex(r#"foo\/bar"#.to_string()), + Token::Regex("[/]".to_string()), + Token::Regex(r#"(a|b)\d+"#.to_string()), + Token::Regex(r#"\\"#.to_string()), + ]) + ); + } + + #[test] + fn test_realistic_expressions() { + assert_eq!( + tokenize(r#"/pattern/ c "replacement""#), + Ok(vec![ + Token::Regex("pattern".to_string()), + Token::Change, + Token::TextLiteral("replacement".to_string()) + ]) + ); + + assert_eq!( + tokenize(r#"B/*.txt/ ?'/\w+/ ;#42"#), + Ok(vec![ + Token::BufferMatch, + Token::Regex(r#"*.txt"#.to_string()), + Token::Question, + Token::Reverse, + Token::Regex(r#"\w+"#.to_string()), + Token::Sequential, + Token::CharacterOffset, + Token::Number(42) + ]) + ); + + assert_eq!( + tokenize(r#"{ /a/ @i, /b/ c<(|<"sort") }"#), + Ok(vec![ + Token::LBrace, + Token::Regex("a".to_string()), + Token::Swap, + Token::Insert, + Token::Comma, + Token::Regex("b".to_string()), + Token::Change, + Token::After, + Token::LParen, + Token::Pipe, + Token::After, + Token::TextLiteral("sort".to_string()), + Token::RParen, + Token::RBrace + ]) + ); + + assert_eq!( + tokenize(r#"~?';x/pattern/"#), + Ok(vec![ + Token::Complement, + Token::Question, + Token::Reverse, + Token::Sequential, + Token::AllMatches, + Token::Regex("pattern".to_string()) + ]) + ); + } + + #[test] + fn test_errors() { + assert!(tokenize(r#""unterminated"#).is_err()); + assert!(tokenize(r#"/unterminated"#).is_err()); + assert!(tokenize(r#""escaped at end\"#).is_err()); + } +} |