aboutsummaryrefslogtreecommitdiff
path: root/rprt-engine/src/token.rs
diff options
context:
space:
mode:
Diffstat (limited to 'rprt-engine/src/token.rs')
-rw-r--r--rprt-engine/src/token.rs28
1 files changed, 14 insertions, 14 deletions
diff --git a/rprt-engine/src/token.rs b/rprt-engine/src/token.rs
index cde8164..d16434b 100644
--- a/rprt-engine/src/token.rs
+++ b/rprt-engine/src/token.rs
@@ -89,7 +89,7 @@ pub enum Token {
AllMatches,
#[token("+")]
- PositiveLineOffset,
+ LineOffset,
#[token("B")]
BufferMatch,
@@ -127,7 +127,7 @@ pub enum Token {
Complement,
#[token("?")]
- Question,
+ Conditional,
// Text Function Operators
#[token("@")]
@@ -162,7 +162,7 @@ pub enum Token {
Comma,
}
-pub fn tokenize(input: &str) -> Result<Vec<Token>, String> {
+pub fn tokenise(input: &str) -> Result<Vec<Token>, String> {
let mut tokens = Vec::new();
let mut lexer = Token::lexer(input);
@@ -189,7 +189,7 @@ mod tests {
#[test]
fn test_delimited_literals_with_escapes() {
assert_eq!(
- tokenize(r#""" "hello" "escaped\"quote" "backslash\\" "mixed\n\t""#),
+ tokenise(r#""" "hello" "escaped\"quote" "backslash\\" "mixed\n\t""#),
Ok(vec![
Token::TextLiteral("".to_string()),
Token::TextLiteral("hello".to_string()),
@@ -200,7 +200,7 @@ mod tests {
);
assert_eq!(
- tokenize(r#"// /[a-z]+/ /foo\/bar/ /[/]/ /(a|b)\d+/ /\\/"#),
+ tokenise(r#"// /[a-z]+/ /foo\/bar/ /[/]/ /(a|b)\d+/ /\\/"#),
Ok(vec![
Token::Regex("".to_string()),
Token::Regex("[a-z]+".to_string()),
@@ -215,7 +215,7 @@ mod tests {
#[test]
fn test_realistic_expressions() {
assert_eq!(
- tokenize(r#"/pattern/ c "replacement""#),
+ tokenise(r#"/pattern/ c "replacement""#),
Ok(vec![
Token::Regex("pattern".to_string()),
Token::Change,
@@ -224,11 +224,11 @@ mod tests {
);
assert_eq!(
- tokenize(r#"B/*.txt/ ?'/\w+/ ;#42"#),
+ tokenise(r#"B/*.txt/ ?'/\w+/ ;#42"#),
Ok(vec![
Token::BufferMatch,
Token::Regex(r#"*.txt"#.to_string()),
- Token::Question,
+ Token::Conditional,
Token::Reverse,
Token::Regex(r#"\w+"#.to_string()),
Token::Sequential,
@@ -238,7 +238,7 @@ mod tests {
);
assert_eq!(
- tokenize(r#"{ /a/ @i, /b/ c<(|<"sort") }"#),
+ tokenise(r#"{ /a/ @i, /b/ c<(|<"sort") }"#),
Ok(vec![
Token::LBrace,
Token::Regex("a".to_string()),
@@ -258,10 +258,10 @@ mod tests {
);
assert_eq!(
- tokenize(r#"~?';x/pattern/"#),
+ tokenise(r#"~?';x/pattern/"#),
Ok(vec![
Token::Complement,
- Token::Question,
+ Token::Conditional,
Token::Reverse,
Token::Sequential,
Token::AllMatches,
@@ -272,8 +272,8 @@ mod tests {
#[test]
fn test_errors() {
- assert!(tokenize(r#""unterminated"#).is_err());
- assert!(tokenize(r#"/unterminated"#).is_err());
- assert!(tokenize(r#""escaped at end\"#).is_err());
+ assert!(tokenise(r#""unterminated"#).is_err());
+ assert!(tokenise(r#"/unterminated"#).is_err());
+ assert!(tokenise(r#""escaped at end\"#).is_err());
}
}