Some basic train and parenthesis parsing, things are beginning to become exciting

4 files changed, 447 insertions, 87 deletions

diff --git a/rprt-engine/src/expression.rs b/rprt-engine/src/expression.rs
index 68849cb..1ee898e 100644
--- a/rprt-engine/src/expression.rs
+++ b/rprt-engine/src/expression.rs
@@ -1,55 +1,32 @@
 #[derive(Debug, Clone, PartialEq)]
-pub enum SelectionValuedExpr {
-    // Nullary selection function application
-    NullaryApplicationS(SelectionFn),
-    NullaryApplicationT(TextFn),
-
-    // Monadic selection function application: α F
-    MonadicApplicationS {
-        arg: Box<SelectionValuedExpr>,
-        func: SelectionFn,
-    },
-    // Monadic selection function application: F τ
-    MonadicApplicationT {
-        arg: Box<TextValuedExpr>,
-        func: TextFn,
-    },
-
-    // Dyadic selection function application: α F β
-    DyadicApplicationS {
-        left: Box<SelectionValuedExpr>,
-        right: Box<SelectionValuedExpr>,
-        func: SelectionFn,
-    },
-    // Dyadic text function application: α F τ
-    DyadicApplicationT {
-        left: Box<SelectionValuedExpr>,
-        right: Box<SelectionValuedExpr>,
-        func: SelectionFn,
-    },
-}
-
-#[derive(Debug, Clone, PartialEq)]
-pub enum TextValuedExpr {
-    // Selection coerced to text (implicit coercion)
-    FromSelection(Box<SelectionValuedExpr>),
-}
-
-#[derive(Debug, Clone, PartialEq)]
-pub enum SelectionFn {
-    Function {
+pub enum Composite {
+    SelectionFunction {
         func: BuiltinSelectionFn,
         search_mod: Option<SearchModifier>,
         result_transform: Option<ResultTransform>,
     },
 
+    TextFunction {
+        func: BuiltinTextFn,
+        swapped: bool,
+    },
+
     Hook {
         kind: HookKind,
-        left: Box<SelectionFn>,
-        right: Box<SelectionFn>,
+        left: Box<Composite>,
+        right: Box<Composite>,
     },
 
-    Train(Vec<SelectionFn>),
+    Train2 {
+        f: Box<Composite>,
+        g: Box<Composite>,
+    },
+
+    Train3 {
+        f: Box<Composite>,
+        g: Box<Composite>,
+        h: Box<Composite>,
+    },
 }
 
 #[derive(Debug, Clone, PartialEq)]
@@ -65,7 +42,16 @@ pub enum TextFn {
         right: Box<TextFn>,
     },
 
-    Train(Vec<TextFn>),
+    Train2 {
+        f: Box<TextFn>,
+        g: Box<TextFn>,
+    },
+
+    Train3 {
+        f: Box<TextFn>,
+        g: Box<TextFn>,
+        h: Box<TextFn>,
+    },
 }
 
 #[derive(Debug, Clone, PartialEq)]
diff --git a/rprt-engine/src/parser.rs b/rprt-engine/src/parser.rs
index 44feda0..c98d977 100644
--- a/rprt-engine/src/parser.rs
+++ b/rprt-engine/src/parser.rs
@@ -18,9 +18,9 @@ struct ParseContext {
 }
 
 impl ParseContext {
-    fn advance(&mut self) -> Result<(), ParseError> {
-        self.pos += 1;
-        if self.pos >= self.tokens.len() {
+    fn advance(&mut self, how_much: usize, need_more: bool) -> Result<(), ParseError> {
+        self.pos += how_much;
+        if need_more && self.pos >= self.tokens.len() {
             Err(ParseError::UnexpectedEndOfInput {
                 expecting: "more tokens",
             })
@@ -35,48 +35,196 @@ impl ParseContext {
             .cloned()
             .ok_or(ParseError::UnexpectedEndOfInput { expecting: "token" })
     }
+
+    fn peek(&self, offset: usize) -> Option<&Token> {
+        self.tokens.get(self.pos + offset)
+    }
 }
 
-pub fn parse(tokens: Vec<Token>) -> Result<SelectionValuedExpr, ParseError> {
+pub fn parse(tokens: Vec<Token>) -> Result<Composite, ParseError> {
     let mut context = ParseContext {
         tokens: tokens,
         pos: 0,
     };
-    let func = parse_function(&mut context)?;
-    Ok(SelectionValuedExpr::NullaryApplicationS(func))
+    parse_function_in_application(&mut context)
+}
+
+fn parse_function_in_application(context: &mut ParseContext) -> Result<Composite, ParseError> {
+    let mut functions = Vec::new();
+    while can_start_selection_function(context, 0) {
+        functions.push(parse_function_atom(context)?);
+    }
+    if functions.is_empty() {
+        return Err(ParseError::UnexpectedToken {
+            found: context.current()?,
+            expecting: "function",
+        });
+    }
+    Ok(build_train(functions))
+}
+
+fn build_train(mut functions: Vec<Composite>) -> Composite {
+    // Rust is insane and has no easy way to destructure a vector and own the elements
+    // so this function is littered with unwraps for no reason, we already own the vector!
+    match functions.len() {
+        1 => functions.pop().unwrap(),
+        2 => {
+            let [f, g] = <[_; 2]>::try_from(functions).unwrap();
+            Composite::Train2 {
+                f: Box::new(f),
+                g: Box::new(g),
+            }
+        }
+        3 => {
+            let [f, g, h] = <[_; 3]>::try_from(functions).unwrap();
+            Composite::Train3 {
+                f: Box::new(f),
+                g: Box::new(g),
+                h: Box::new(h),
+            }
+        }
+        n if n % 2 == 0 => {
+            let rest = functions.split_off(1);
+            let [f] = <[_; 1]>::try_from(functions).unwrap();
+            Composite::Train2 {
+                f: Box::new(f),
+                g: Box::new(build_train(rest)),
+            }
+        }
+        _ => {
+            let rest = functions.split_off(2);
+            let [f, g] = <[_; 2]>::try_from(functions).unwrap();
+            Composite::Train3 {
+                f: Box::new(f),
+                g: Box::new(g),
+                h: Box::new(build_train(rest)),
+            }
+        }
+    }
+}
+
+fn can_start_selection_function(context: &ParseContext, offset: usize) -> bool {
+    matches!(
+        context.peek(offset),
+        Some(Token::Empty)
+            | Some(Token::EndOfBuffer)
+            | Some(Token::Span)
+            | Some(Token::CharacterOffset)
+            | Some(Token::Number(_))
+            | Some(Token::Regex(_))
+            | Some(Token::AllMatches)
+            | Some(Token::LineOffset)
+            | Some(Token::BufferMatch)
+            | Some(Token::Reverse)
+            | Some(Token::Sequential)
+            | Some(Token::Complement)
+            | Some(Token::Conditional)
+            | Some(Token::LParen)
+    )
 }
 
-fn parse_function(context: &mut ParseContext) -> Result<SelectionFn, ParseError> {
+fn parse_function_atom(context: &mut ParseContext) -> Result<Composite, ParseError> {
+    if matches!(context.current()?, Token::LParen) {
+        context.advance(1, true)?;
+        let func = parse_function_in_application(context)?;
+        if !matches!(context.current()?, Token::RParen) {
+            return Err(ParseError::UnexpectedToken {
+                found: context.current()?,
+                expecting: ")",
+            });
+        }
+        context.advance(1, false)?;
+        return Ok(func);
+    }
+    let result_trans = parse_result_trans(context)?;
+    let search_mod = parse_search_mod(context)?;
     let func = parse_builtin_function_s(context)?;
-    Ok(SelectionFn::Function {
+    Ok(Composite::SelectionFunction {
         func: func,
-        search_mod: None,
-        result_transform: None,
+        search_mod: search_mod,
+        result_transform: result_trans,
     })
 }
 
+fn parse_search_mod(context: &mut ParseContext) -> Result<Option<SearchModifier>, ParseError> {
+    let maybe_mod = |token| match token {
+        Token::Sequential => Ok(Some(SearchModifier::Sequential)),
+        Token::Reverse => Ok(Some(SearchModifier::Reverse)),
+        _ => Ok(None),
+    };
+    let token = context.current()?;
+    if let Some(m1) = maybe_mod(token)? {
+        context.advance(1, true)?;
+        if let Some(m2) = maybe_mod(context.current()?)? {
+            context.advance(1, true)?;
+            return Ok(Some(match (m1, m2) {
+                (SearchModifier::Sequential, SearchModifier::Reverse) => {
+                    SearchModifier::SequentialReverse
+                }
+                (SearchModifier::Reverse, SearchModifier::Sequential) => {
+                    SearchModifier::ReverseSequential
+                }
+                (_, _) => unreachable!(),
+            }));
+        }
+        return Ok(Some(m1));
+    };
+    Ok(None)
+}
+
+fn parse_result_trans(context: &mut ParseContext) -> Result<Option<ResultTransform>, ParseError> {
+    let maybe_mod = |token| match token {
+        Token::Complement => Ok(Some(ResultTransform::Complement)),
+        Token::Conditional => Ok(Some(ResultTransform::Conditional)),
+        _ => Ok(None),
+    };
+    let token = context.current()?;
+    if let Some(m1) = maybe_mod(token)? {
+        context.advance(1, true)?;
+        if let Some(m2) = maybe_mod(context.current()?)? {
+            context.advance(1, true)?;
+            return Ok(Some(match (m1, m2) {
+                (ResultTransform::Complement, ResultTransform::Conditional) => {
+                    ResultTransform::ComplementConditional
+                }
+                (ResultTransform::Conditional, ResultTransform::Complement) => {
+                    ResultTransform::ConditionalComplement
+                }
+                (_, _) => unreachable!(),
+            }));
+        }
+        return Ok(Some(m1));
+    };
+    Ok(None)
+}
+
 fn parse_builtin_function_s(context: &mut ParseContext) -> Result<BuiltinSelectionFn, ParseError> {
     let token = context.current()?;
-    match token {
-        Token::Empty => Ok(BuiltinSelectionFn::Empty),
-        Token::EndOfBuffer => Ok(BuiltinSelectionFn::EndOfBuffer),
-        Token::Span => Ok(BuiltinSelectionFn::Span),
-        Token::CharacterOffset => parse_char_offset(context),
-        Token::Number(n) => Ok(BuiltinSelectionFn::Line(n)),
-        Token::Regex(pattern) => Ok(BuiltinSelectionFn::Regex(pattern)),
-        Token::AllMatches => parse_all_matches(context),
-        Token::PositiveLineOffset => parse_relative_line(context),
-        Token::BufferMatch => parse_buffer_match(context),
-        _ => Err(ParseError::UnexpectedToken {
-            found: token,
-            expecting: "selection function",
-        }),
-    }
+    let result = match token {
+        Token::Empty => BuiltinSelectionFn::Empty,
+        Token::EndOfBuffer => BuiltinSelectionFn::EndOfBuffer,
+        Token::Span => BuiltinSelectionFn::Span,
+        Token::CharacterOffset => return parse_char_offset(context),
+        Token::Number(n) => BuiltinSelectionFn::Line(n),
+        Token::Regex(pattern) => BuiltinSelectionFn::Regex(pattern),
+        Token::AllMatches => return parse_all_matches(context),
+        Token::LineOffset => return parse_relative_line(context),
+        Token::BufferMatch => return parse_buffer_match(context),
+        _ => {
+            return Err(ParseError::UnexpectedToken {
+                found: token,
+                expecting: "selection function",
+            });
+        }
+    };
+    context.advance(1, false)?;
+    Ok(result)
 }
 
 fn parse_char_offset(context: &mut ParseContext) -> Result<BuiltinSelectionFn, ParseError> {
-    context.advance()?;
+    context.advance(1, true)?;
     let token = context.current()?;
+    context.advance(1, false)?;
     match token {
         Token::Number(n) => Ok(BuiltinSelectionFn::CharOffset(n)),
         _ => Err(ParseError::UnexpectedToken {
@@ -87,8 +235,9 @@ fn parse_char_offset(context: &mut ParseContext) -> Result<BuiltinSelectionFn, P
 }
 
 fn parse_all_matches(context: &mut ParseContext) -> Result<BuiltinSelectionFn, ParseError> {
-    context.advance()?;
+    context.advance(1, true)?;
     let token = context.current()?;
+    context.advance(1, false)?;
     match token {
         Token::Regex(pattern) => Ok(BuiltinSelectionFn::AllMatches(pattern)),
         _ => Err(ParseError::UnexpectedToken {
@@ -99,8 +248,9 @@ fn parse_all_matches(context: &mut ParseContext) -> Result<BuiltinSelectionFn, P
 }
 
 fn parse_relative_line(context: &mut ParseContext) -> Result<BuiltinSelectionFn, ParseError> {
-    context.advance()?;
+    context.advance(1, true)?;
     let token = context.current()?;
+    context.advance(1, false)?;
     match token {
         Token::Number(n) => Ok(BuiltinSelectionFn::RelativeLine(n)),
         _ => Err(ParseError::UnexpectedToken {
@@ -111,8 +261,9 @@ fn parse_relative_line(context: &mut ParseContext) -> Result<BuiltinSelectionFn,
 }
 
 fn parse_buffer_match(context: &mut ParseContext) -> Result<BuiltinSelectionFn, ParseError> {
-    context.advance()?;
+    context.advance(1, true)?;
     let token = context.current()?;
+    context.advance(1, false)?;
     match token {
         Token::Regex(pattern) => Ok(BuiltinSelectionFn::BufferMatch(pattern)),
         _ => Err(ParseError::UnexpectedToken {
@@ -121,3 +272,226 @@ fn parse_buffer_match(context: &mut ParseContext) -> Result<BuiltinSelectionFn,
         }),
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::token::tokenise;
+
+    #[test]
+    fn test_parse_nullary() {
+        let input = "/foo/";
+        let tokens = tokenise(input).unwrap();
+        let result = parse(tokens).unwrap();
+        assert!(matches!(
+            result,
+            Composite::SelectionFunction {
+                func: BuiltinSelectionFn::Regex(_),
+                search_mod: None,
+                result_transform: None,
+            }
+        ));
+    }
+
+    #[test]
+    fn test_parse_train_with_modifiers() {
+        let input = "e ?'/foo/";
+        let tokens = tokenise(input).unwrap();
+        let result = parse(tokens).unwrap();
+        println!("{:?}", result);
+        match result {
+            Composite::Train2 { f, g } => {
+                assert!(matches!(
+                    *f,
+                    Composite::SelectionFunction {
+                        func: BuiltinSelectionFn::Empty,
+                        search_mod: None,
+                        result_transform: None,
+                    }
+                ));
+                assert!(matches!(
+                    *g,
+                    Composite::SelectionFunction {
+                        func: BuiltinSelectionFn::Regex(_),
+                        search_mod: Some(SearchModifier::Reverse),
+                        result_transform: Some(ResultTransform::Conditional),
+                    }
+                ));
+            }
+            _ => panic!(),
+        }
+    }
+
+    #[test]
+    fn test_parse_nullary_with_modifier() {
+        let input = "'e";
+        let tokens = tokenise(input).unwrap();
+        let result = parse(tokens).unwrap();
+        println!("{:?}", result);
+        match result {
+            Composite::SelectionFunction {
+                func: BuiltinSelectionFn::Empty,
+                search_mod: Some(SearchModifier::Reverse),
+                result_transform: None,
+            } => {}
+            _ => panic!(),
+        }
+    }
+
+    #[test]
+    fn test_parse_2train() {
+        let input = "- $";
+        let tokens = tokenise(input).unwrap();
+        let result = parse(tokens).unwrap();
+        println!("{:?}", result);
+        match result {
+            Composite::Train2 { f, g } => {
+                assert!(matches!(
+                    *f,
+                    Composite::SelectionFunction {
+                        func: BuiltinSelectionFn::Span,
+                        search_mod: None,
+                        result_transform: None,
+                    }
+                ));
+                assert!(matches!(
+                    *g,
+                    Composite::SelectionFunction {
+                        func: BuiltinSelectionFn::EndOfBuffer,
+                        search_mod: None,
+                        result_transform: None,
+                    }
+                ));
+            }
+            _ => panic!(),
+        }
+    }
+
+    #[test]
+    fn test_parse_3train() {
+        let input = "x/start/ - x/end/";
+        let tokens = tokenise(input).unwrap();
+        let result = parse(tokens).unwrap();
+        println!("{:?}", result);
+        match result {
+            Composite::Train3 { f, g, h } => {
+                match *f {
+                    Composite::SelectionFunction {
+                        func: BuiltinSelectionFn::AllMatches(pattern),
+                        search_mod: None,
+                        result_transform: None,
+                    } => assert_eq!(pattern, "start"),
+                    _ => panic!(),
+                }
+                assert!(matches!(
+                    *g,
+                    Composite::SelectionFunction {
+                        func: BuiltinSelectionFn::Span,
+                        search_mod: None,
+                        result_transform: None,
+                    }
+                ));
+                match *h {
+                    Composite::SelectionFunction {
+                        func: BuiltinSelectionFn::AllMatches(pattern),
+                        search_mod: None,
+                        result_transform: None,
+                    } => assert_eq!(pattern, "end"),
+                    _ => panic!(),
+                }
+            }
+            _ => panic!(),
+        }
+    }
+
+    #[test]
+    fn test_parse_parenthesized() {
+        let input = "(- $)";
+        let tokens = tokenise(input).unwrap();
+        let result = parse(tokens).unwrap();
+        println!("{:?}", result);
+        match result {
+            Composite::Train2 { f, g } => {
+                assert!(matches!(
+                    *f,
+                    Composite::SelectionFunction {
+                        func: BuiltinSelectionFn::Span,
+                        search_mod: None,
+                        result_transform: None,
+                    }
+                ));
+                assert!(matches!(
+                    *g,
+                    Composite::SelectionFunction {
+                        func: BuiltinSelectionFn::EndOfBuffer,
+                        search_mod: None,
+                        result_transform: None,
+                    }
+                ));
+            }
+            _ => panic!(),
+        }
+    }
+
+    #[test]
+    fn test_parse_5train() {
+        let input = "#0 - #10 - #20";
+        let tokens = tokenise(input).unwrap();
+        let result = parse(tokens).unwrap();
+        println!("{:?}", result);
+        match result {
+            Composite::Train3 { f, g, h } => {
+                match *f {
+                    Composite::SelectionFunction {
+                        func: BuiltinSelectionFn::CharOffset(n),
+                        search_mod: None,
+                        result_transform: None,
+                    } => assert_eq!(n, 0),
+                    _ => panic!(),
+                }
+                assert!(matches!(
+                    *g,
+                    Composite::SelectionFunction {
+                        func: BuiltinSelectionFn::Span,
+                        search_mod: None,
+                        result_transform: None,
+                    }
+                ));
+                match *h {
+                    Composite::Train3 {
+                        ref f,
+                        ref g,
+                        ref h,
+                    } => {
+                        match **f {
+                            Composite::SelectionFunction {
+                                func: BuiltinSelectionFn::CharOffset(n),
+                                search_mod: None,
+                                result_transform: None,
+                            } => assert_eq!(n, 10),
+                            _ => panic!(),
+                        }
+                        assert!(matches!(
+                            **g,
+                            Composite::SelectionFunction {
+                                func: BuiltinSelectionFn::Span,
+                                search_mod: None,
+                                result_transform: None,
+                            }
+                        ));
+                        match **h {
+                            Composite::SelectionFunction {
+                                func: BuiltinSelectionFn::CharOffset(n),
+                                search_mod: None,
+                                result_transform: None,
+                            } => assert_eq!(n, 20),
+                            _ => panic!(),
+                        }
+                    }
+                    _ => panic!(),
+                }
+            }
+            _ => panic!(),
+        }
+    }
+}
diff --git a/rprt-engine/src/selection.rs b/rprt-engine/src/selection.rs
index 9494544..968d859 100644
--- a/rprt-engine/src/selection.rs
+++ b/rprt-engine/src/selection.rs
@@ -256,7 +256,7 @@ impl Selection {
         }
     }
 
-    pub fn broadcast<E>(
+    pub fn vectorise<E>(
         &self,
         fn_rank_zero: &impl Fn(&BufferID, &usize) -> Result<Self, E>,
         fn_rank_one: &impl Fn(&BufferID, &Interval) -> Result<Self, E>,
diff --git a/rprt-engine/src/token.rs b/rprt-engine/src/token.rs
index cde8164..d16434b 100644
--- a/rprt-engine/src/token.rs
+++ b/rprt-engine/src/token.rs
@@ -89,7 +89,7 @@ pub enum Token {
     AllMatches,
 
     #[token("+")]
-    PositiveLineOffset,
+    LineOffset,
 
     #[token("B")]
     BufferMatch,
@@ -127,7 +127,7 @@ pub enum Token {
     Complement,
 
     #[token("?")]
-    Question,
+    Conditional,
 
     // Text Function Operators
     #[token("@")]
@@ -162,7 +162,7 @@ pub enum Token {
     Comma,
 }
 
-pub fn tokenize(input: &str) -> Result<Vec<Token>, String> {
+pub fn tokenise(input: &str) -> Result<Vec<Token>, String> {
     let mut tokens = Vec::new();
     let mut lexer = Token::lexer(input);
 
@@ -189,7 +189,7 @@ mod tests {
     #[test]
     fn test_delimited_literals_with_escapes() {
         assert_eq!(
-            tokenize(r#""" "hello" "escaped\"quote" "backslash\\" "mixed\n\t""#),
+            tokenise(r#""" "hello" "escaped\"quote" "backslash\\" "mixed\n\t""#),
             Ok(vec![
                 Token::TextLiteral("".to_string()),
                 Token::TextLiteral("hello".to_string()),
@@ -200,7 +200,7 @@ mod tests {
         );
 
         assert_eq!(
-            tokenize(r#"// /[a-z]+/ /foo\/bar/ /[/]/ /(a|b)\d+/ /\\/"#),
+            tokenise(r#"// /[a-z]+/ /foo\/bar/ /[/]/ /(a|b)\d+/ /\\/"#),
             Ok(vec![
                 Token::Regex("".to_string()),
                 Token::Regex("[a-z]+".to_string()),
@@ -215,7 +215,7 @@ mod tests {
     #[test]
     fn test_realistic_expressions() {
         assert_eq!(
-            tokenize(r#"/pattern/ c "replacement""#),
+            tokenise(r#"/pattern/ c "replacement""#),
             Ok(vec![
                 Token::Regex("pattern".to_string()),
                 Token::Change,
@@ -224,11 +224,11 @@ mod tests {
         );
 
         assert_eq!(
-            tokenize(r#"B/*.txt/ ?'/\w+/ ;#42"#),
+            tokenise(r#"B/*.txt/ ?'/\w+/ ;#42"#),
             Ok(vec![
                 Token::BufferMatch,
                 Token::Regex(r#"*.txt"#.to_string()),
-                Token::Question,
+                Token::Conditional,
                 Token::Reverse,
                 Token::Regex(r#"\w+"#.to_string()),
                 Token::Sequential,
@@ -238,7 +238,7 @@ mod tests {
         );
 
         assert_eq!(
-            tokenize(r#"{ /a/ @i, /b/ c<(|<"sort") }"#),
+            tokenise(r#"{ /a/ @i, /b/ c<(|<"sort") }"#),
             Ok(vec![
                 Token::LBrace,
                 Token::Regex("a".to_string()),
@@ -258,10 +258,10 @@ mod tests {
         );
 
         assert_eq!(
-            tokenize(r#"~?';x/pattern/"#),
+            tokenise(r#"~?';x/pattern/"#),
             Ok(vec![
                 Token::Complement,
-                Token::Question,
+                Token::Conditional,
                 Token::Reverse,
                 Token::Sequential,
                 Token::AllMatches,
@@ -272,8 +272,8 @@ mod tests {
 
     #[test]
     fn test_errors() {
-        assert!(tokenize(r#""unterminated"#).is_err());
-        assert!(tokenize(r#"/unterminated"#).is_err());
-        assert!(tokenize(r#""escaped at end\"#).is_err());
+        assert!(tokenise(r#""unterminated"#).is_err());
+        assert!(tokenise(r#"/unterminated"#).is_err());
+        assert!(tokenise(r#""escaped at end\"#).is_err());
     }
 }