use logos::{Lexer, Logos}; fn lex_delimited( lex: &mut Lexer, delimiter: char, ignore_inside: Option<&[(char, char)]>, ) -> Option { let remainder = lex.remainder(); let mut chars = remainder.chars(); let mut escaped = false; let mut pos = 0; let mut looking_for: Option = None; while let Some(c) = chars.next() { pos += c.len_utf8(); // escaping if escaped { escaped = false; continue; } if c == '\\' { escaped = true; continue; } // capture in if let Some(end) = looking_for { if c == end { looking_for = None; } continue; } if let Some(ignore_inside) = ignore_inside { for &(start, end) in ignore_inside { if c == start { looking_for = Some(end); continue; } } } if c == delimiter { let content = &remainder[..pos - 1]; lex.bump(pos); return Some(content.to_string()); } } None } fn lex_regex(lex: &mut Lexer) -> Option { let character_class = vec![('[', ']')]; lex_delimited(lex, '/', Some(&character_class)) } fn lex_text(lex: &mut Lexer) -> Option { lex_delimited(lex, '"', None) } #[derive(Debug, Clone, PartialEq, Logos)] #[logos(skip r"\s+")] pub enum Token { // Literals #[token("\"", lex_text)] TextLiteral(String), #[regex(r"[0-9]+", |lex| lex.slice().parse().ok())] Number(usize), #[token("/", lex_regex)] Regex(String), // Selection Functions #[token("e")] Empty, #[token("$")] EndOfBuffer, #[token("#")] CharacterOffset, #[token("-")] Span, #[token("x")] AllMatches, #[token("+")] LineOffset, #[token("B")] BufferMatch, // Text Functions #[token(".")] Dot, #[token("c")] Change, #[token("i")] Insert, #[token("a")] Append, #[token("d")] Delete, #[token("w")] Write, #[token("|")] Pipe, // Selection Function Operators #[token("'")] Reverse, #[token(";")] Sequential, #[token("~")] Complement, #[token("?")] Conditional, // Text Function Operators #[token("@")] Swap, // Composition/Combinators #[token(">")] Before, #[token("<")] After, // Grouping #[token("(")] LParen, #[token(")")] RParen, #[token("{")] LBrace, #[token("}")] RBrace, // Statement/Buffer control #[token("b")] BufferSwitch, // Separators #[token(",")] Comma, } pub fn tokenise(input: &str) -> Result, String> { let mut tokens = Vec::new(); let mut lexer = Token::lexer(input); while let Some(result) = lexer.next() { match result { Ok(token) => tokens.push(token), Err(()) => { return Err(format!( "Unexpected character at position {}: '{}'", lexer.span().start, lexer.slice() )); } } } Ok(tokens) } #[cfg(test)] mod tests { use crate::token::*; #[test] fn test_delimited_literals_with_escapes() { assert_eq!( tokenise(r#""" "hello" "escaped\"quote" "backslash\\" "mixed\n\t""#), Ok(vec![ Token::TextLiteral("".to_string()), Token::TextLiteral("hello".to_string()), Token::TextLiteral(r#"escaped\"quote"#.to_string()), Token::TextLiteral(r#"backslash\\"#.to_string()), Token::TextLiteral(r#"mixed\n\t"#.to_string()), ]) ); assert_eq!( tokenise(r#"// /[a-z]+/ /foo\/bar/ /[/]/ /(a|b)\d+/ /\\/"#), Ok(vec![ Token::Regex("".to_string()), Token::Regex("[a-z]+".to_string()), Token::Regex(r#"foo\/bar"#.to_string()), Token::Regex("[/]".to_string()), Token::Regex(r#"(a|b)\d+"#.to_string()), Token::Regex(r#"\\"#.to_string()), ]) ); } #[test] fn test_realistic_expressions() { assert_eq!( tokenise(r#"/pattern/ c "replacement""#), Ok(vec![ Token::Regex("pattern".to_string()), Token::Change, Token::TextLiteral("replacement".to_string()) ]) ); assert_eq!( tokenise(r#"B/*.txt/ ?'/\w+/ ;#42"#), Ok(vec![ Token::BufferMatch, Token::Regex(r#"*.txt"#.to_string()), Token::Conditional, Token::Reverse, Token::Regex(r#"\w+"#.to_string()), Token::Sequential, Token::CharacterOffset, Token::Number(42) ]) ); assert_eq!( tokenise(r#"{ /a/ @i, /b/ c<(|<"sort") }"#), Ok(vec![ Token::LBrace, Token::Regex("a".to_string()), Token::Swap, Token::Insert, Token::Comma, Token::Regex("b".to_string()), Token::Change, Token::After, Token::LParen, Token::Pipe, Token::After, Token::TextLiteral("sort".to_string()), Token::RParen, Token::RBrace ]) ); assert_eq!( tokenise(r#"~?';x/pattern/"#), Ok(vec![ Token::Complement, Token::Conditional, Token::Reverse, Token::Sequential, Token::AllMatches, Token::Regex("pattern".to_string()) ]) ); } #[test] fn test_errors() { assert!(tokenise(r#""unterminated"#).is_err()); assert!(tokenise(r#"/unterminated"#).is_err()); assert!(tokenise(r#""escaped at end\"#).is_err()); } }