use crate::{expression::*, token::*}; #[derive(Debug, Clone)] pub enum ParseError { UnexpectedToken { found: Token, expecting: &'static str, }, UnexpectedEndOfInput { expecting: &'static str, }, } struct ParseContext { tokens: Vec, pos: usize, } impl ParseContext { fn advance(&mut self, how_much: usize, need_more: bool) -> Result<(), ParseError> { self.pos += how_much; if need_more && self.pos >= self.tokens.len() { Err(ParseError::UnexpectedEndOfInput { expecting: "more tokens", }) } else { Ok(()) } } fn current(&self) -> Result { self.tokens .get(self.pos) .cloned() .ok_or(ParseError::UnexpectedEndOfInput { expecting: "token" }) } fn peek(&self, offset: usize) -> Option<&Token> { self.tokens.get(self.pos + offset) } } pub fn parse(tokens: Vec) -> Result { let mut context = ParseContext { tokens: tokens, pos: 0, }; parse_composition(&mut context) } fn parse_composition(context: &mut ParseContext) -> Result { let mut functions = Vec::new(); while can_start_composition(context, 0) { functions.push(parse_function_atom(context)?); } if functions.is_empty() { return Err(ParseError::UnexpectedToken { found: context.current()?, expecting: "function", }); } Ok(build_train(functions)) } fn build_train(mut functions: Vec) -> Composite { // It would appear that there is no easy way to destructure a vector and own // the elements so this function is littered with unwraps for no reason, we // already own the vector! match functions.len() { 1 => functions.pop().unwrap(), 2 => { let [f, g] = <[_; 2]>::try_from(functions).unwrap(); Composite::Train2 { f: Box::new(f), g: Box::new(g), } } 3 => { let [f, g, h] = <[_; 3]>::try_from(functions).unwrap(); Composite::Train3 { f: Box::new(f), g: Box::new(g), h: Box::new(h), } } n if n % 2 == 0 => { let g = Box::new(functions.pop().unwrap()); Composite::Train2 { f: Box::new(build_train(functions)), g, } } _ => { let h = Box::new(functions.pop().unwrap()); let g = Box::new(functions.pop().unwrap()); Composite::Train3 { f: Box::new(build_train(functions)), g, h, } } } } fn can_start_composition(context: &ParseContext, offset: usize) -> bool { matches!( context.peek(offset), Some(Token::Empty) | Some(Token::EndOfBuffer) | Some(Token::Span) | Some(Token::CharacterOffset) | Some(Token::Number(_)) | Some(Token::Regex(_)) | Some(Token::AllMatches) | Some(Token::LineOffset) | Some(Token::BufferMatch) | Some(Token::Reverse) | Some(Token::Sequential) | Some(Token::Complement) | Some(Token::Conditional) | Some(Token::LParen) | Some(Token::LBrace) | Some(Token::Dot) | Some(Token::Change) | Some(Token::Insert) | Some(Token::Append) | Some(Token::Delete) | Some(Token::Write) | Some(Token::Pipe) | Some(Token::TextLiteral(_)) | Some(Token::Swap) ) } fn parse_base_function(context: &mut ParseContext) -> Result { if matches!(context.current()?, Token::LParen) { context.advance(1, true)?; let func = parse_composition(context)?; if !matches!(context.current()?, Token::RParen) { return Err(ParseError::UnexpectedToken { found: context.current()?, expecting: ")", }); } context.advance(1, false)?; Ok(func) } else if matches!(context.current()?, Token::LBrace) { context.advance(1, true)?; let mut operations = vec![parse_composition(context)?]; while matches!(context.current(), Ok(Token::Comma)) { context.advance(1, true)?; operations.push(parse_composition(context)?); } if !matches!(context.current()?, Token::RBrace) { return Err(ParseError::UnexpectedToken { found: context.current()?, expecting: "}", }); } context.advance(1, false)?; Ok(Composite::Group { operations }) } else if is_function_t(context)? { parse_builtin_function_t(context) } else if is_function_s(context)? { parse_builtin_function_s(context) } else { Err(ParseError::UnexpectedToken { found: context.current()?, expecting: "function", }) } } fn parse_function_atom(context: &mut ParseContext) -> Result { let result = parse_base_function(context)?; match context.current() { Ok(Token::Before) => { context.advance(1, true)?; let rest = parse_composition(context)?; return Ok(Composite::Hook { kind: HookKind::Before, left: Box::new(result), right: Box::new(rest), }); } Ok(Token::After) => { context.advance(1, true)?; let rest = parse_composition(context)?; return Ok(Composite::Hook { kind: HookKind::After, left: Box::new(result), right: Box::new(rest), }); } _ => return Ok(result), } } fn is_function_t(context: &ParseContext) -> Result { Ok(matches!( context.current()?, Token::Dot | Token::Change | Token::Insert | Token::Append | Token::Delete | Token::Write | Token::Pipe | Token::TextLiteral(_) | Token::Swap )) } fn parse_builtin_function_t(context: &mut ParseContext) -> Result { let mut swapped = false; if context.current()? == Token::Swap { swapped = true; context.advance(1, true)?; } let token = context.current()?; let result = match token { Token::Dot => BuiltinTextFn::Dot, Token::Change => BuiltinTextFn::Change, Token::Insert => BuiltinTextFn::Insert, Token::Append => BuiltinTextFn::Append, Token::Delete => BuiltinTextFn::Delete, Token::Write => BuiltinTextFn::Write, Token::Pipe => BuiltinTextFn::Pipe, Token::TextLiteral(s) => BuiltinTextFn::Literal(s), _ => { return Err(ParseError::UnexpectedToken { found: token, expecting: "text function", }); } }; context.advance(1, false)?; Ok(Composite::TextFunction { func: result, swapped: swapped, }) } fn is_function_s(context: &ParseContext) -> Result { Ok(matches!( context.current()?, Token::Empty | Token::EndOfBuffer | Token::Span | Token::CharacterOffset | Token::Number(_) | Token::Regex(_) | Token::AllMatches | Token::LineOffset | Token::BufferMatch | Token::Reverse | Token::Sequential | Token::Complement | Token::Conditional | Token::LParen )) } fn parse_builtin_function_s(context: &mut ParseContext) -> Result { fn parse_modifier_pair( context: &mut ParseContext, parse_single: impl Fn(Token) -> Option, combine: impl Fn(T, T) -> T, ) -> Result, ParseError> { let token = context.current()?; if let Some(m1) = parse_single(token) { context.advance(1, true)?; if let Some(m2) = parse_single(context.current()?) { context.advance(1, true)?; return Ok(Some(combine(m1, m2))); } return Ok(Some(m1)); } Ok(None) } fn parse_prefixed_builtin( context: &mut ParseContext, expected: &'static str, matcher: impl FnOnce(Token) -> Option, ) -> Result { context.advance(1, true)?; let token = context.current()?; context.advance(1, false)?; matcher(token.clone()).ok_or(ParseError::UnexpectedToken { found: token, expecting: expected, }) } let result_trans = parse_modifier_pair( context, |token| match token { Token::Complement => Some(ResultTransform::Complement), Token::Conditional => Some(ResultTransform::Conditional), _ => None, }, |m1, m2| match (m1, m2) { (ResultTransform::Complement, ResultTransform::Conditional) => { ResultTransform::ComplementConditional } (ResultTransform::Conditional, ResultTransform::Complement) => { ResultTransform::ConditionalComplement } _ => unreachable!(), }, )?; let search_mod = parse_modifier_pair( context, |token| match token { Token::Sequential => Some(SearchModifier::Sequential), Token::Reverse => Some(SearchModifier::Reverse), _ => None, }, |m1, m2| match (m1, m2) { (SearchModifier::Sequential, SearchModifier::Reverse) => { SearchModifier::SequentialReverse } (SearchModifier::Reverse, SearchModifier::Sequential) => { SearchModifier::ReverseSequential } _ => unreachable!(), }, )?; let token = context.current()?; let func = match token { Token::Empty => { context.advance(1, false)?; BuiltinSelectionFn::Empty } Token::EndOfBuffer => { context.advance(1, false)?; BuiltinSelectionFn::EndOfBuffer } Token::Span => { context.advance(1, false)?; BuiltinSelectionFn::Span } Token::CharacterOffset => parse_prefixed_builtin(context, "number", |t| match t { Token::Number(n) => Some(BuiltinSelectionFn::CharOffset(n)), _ => None, })?, Token::Number(n) => { context.advance(1, false)?; BuiltinSelectionFn::Line(n) } Token::Regex(pattern) => { context.advance(1, false)?; BuiltinSelectionFn::Regex(pattern) } Token::AllMatches => parse_prefixed_builtin(context, "regex", |t| match t { Token::Regex(pattern) => Some(BuiltinSelectionFn::AllMatches(pattern)), _ => None, })?, Token::LineOffset => parse_prefixed_builtin(context, "number", |t| match t { Token::Number(n) => Some(BuiltinSelectionFn::RelativeLine(n)), _ => None, })?, Token::BufferMatch => parse_prefixed_builtin(context, "regex", |t| match t { Token::Regex(pattern) => Some(BuiltinSelectionFn::BufferMatch(pattern)), _ => None, })?, _ => { return Err(ParseError::UnexpectedToken { found: token, expecting: "selection function", }); } }; return Ok(Composite::SelectionFunction { func: func, search_mod: search_mod, result_transform: result_trans, }); } #[cfg(test)] mod tests { use super::*; use crate::token::tokenise; #[test] fn test_parse_nullary() { let input = "/foo/"; let tokens = tokenise(input).unwrap(); let result = parse(tokens).unwrap(); assert!(matches!( result, Composite::SelectionFunction { func: BuiltinSelectionFn::Regex(_), search_mod: None, result_transform: None, } )); } #[test] fn test_parse_train_with_modifiers() { let input = "e ?'/foo/"; let tokens = tokenise(input).unwrap(); let result = parse(tokens).unwrap(); println!("{:?}", result); match result { Composite::Train2 { f, g } => { assert!(matches!( *f, Composite::SelectionFunction { func: BuiltinSelectionFn::Empty, search_mod: None, result_transform: None, } )); assert!(matches!( *g, Composite::SelectionFunction { func: BuiltinSelectionFn::Regex(_), search_mod: Some(SearchModifier::Reverse), result_transform: Some(ResultTransform::Conditional), } )); } _ => panic!(), } } #[test] fn test_parse_nullary_with_modifier() { let input = "'e"; let tokens = tokenise(input).unwrap(); let result = parse(tokens).unwrap(); println!("{:?}", result); match result { Composite::SelectionFunction { func: BuiltinSelectionFn::Empty, search_mod: Some(SearchModifier::Reverse), result_transform: None, } => {} _ => panic!(), } } #[test] fn test_parse_2train() { let input = "- $"; let tokens = tokenise(input).unwrap(); let result = parse(tokens).unwrap(); println!("{:?}", result); match result { Composite::Train2 { f, g } => { assert!(matches!( *f, Composite::SelectionFunction { func: BuiltinSelectionFn::Span, search_mod: None, result_transform: None, } )); assert!(matches!( *g, Composite::SelectionFunction { func: BuiltinSelectionFn::EndOfBuffer, search_mod: None, result_transform: None, } )); } _ => panic!(), } } #[test] fn test_parse_3train() { let input = "x/start/ - x/end/"; let tokens = tokenise(input).unwrap(); let result = parse(tokens).unwrap(); println!("{:?}", result); match result { Composite::Train3 { f, g, h } => { match *f { Composite::SelectionFunction { func: BuiltinSelectionFn::AllMatches(pattern), search_mod: None, result_transform: None, } => assert_eq!(pattern, "start"), _ => panic!(), } assert!(matches!( *g, Composite::SelectionFunction { func: BuiltinSelectionFn::Span, search_mod: None, result_transform: None, } )); match *h { Composite::SelectionFunction { func: BuiltinSelectionFn::AllMatches(pattern), search_mod: None, result_transform: None, } => assert_eq!(pattern, "end"), _ => panic!(), } } _ => panic!(), } } #[test] fn test_parse_parenthesized() { let input = "(- $)"; let tokens = tokenise(input).unwrap(); let result = parse(tokens).unwrap(); println!("{:?}", result); match result { Composite::Train2 { f, g } => { assert!(matches!( *f, Composite::SelectionFunction { func: BuiltinSelectionFn::Span, search_mod: None, result_transform: None, } )); assert!(matches!( *g, Composite::SelectionFunction { func: BuiltinSelectionFn::EndOfBuffer, search_mod: None, result_transform: None, } )); } _ => panic!(), } } #[test] fn test_parse_5train() { let input = "#0 - #10 - #20"; let tokens = tokenise(input).unwrap(); let result = parse(tokens).unwrap(); println!("{:?}", result); match result { Composite::Train3 { f, g, h } => { match *f { Composite::Train3 { ref f, ref g, ref h, } => { match **f { Composite::SelectionFunction { func: BuiltinSelectionFn::CharOffset(n), search_mod: None, result_transform: None, } => assert_eq!(n, 0), _ => panic!(), } assert!(matches!( **g, Composite::SelectionFunction { func: BuiltinSelectionFn::Span, search_mod: None, result_transform: None, } )); match **h { Composite::SelectionFunction { func: BuiltinSelectionFn::CharOffset(n), search_mod: None, result_transform: None, } => assert_eq!(n, 10), _ => panic!(), } } _ => panic!(), } assert!(matches!( *g, Composite::SelectionFunction { func: BuiltinSelectionFn::Span, search_mod: None, result_transform: None, } )); match *h { Composite::SelectionFunction { func: BuiltinSelectionFn::CharOffset(n), search_mod: None, result_transform: None, } => assert_eq!(n, 20), _ => panic!(), } } _ => panic!(), } } #[test] fn test_parse_train_selection_and_text() { let input = r#""replacement" @c /pattern/"#; let tokens = tokenise(input).unwrap(); let result = parse(tokens).unwrap(); println!("{:?}", result); match result { Composite::Train3 { f, g, h } => { match *f { Composite::TextFunction { func: BuiltinTextFn::Literal(ref s), swapped: false, } => assert_eq!(s, "replacement"), _ => panic!(), } assert!(matches!( *g, Composite::TextFunction { func: BuiltinTextFn::Change, swapped: true, } )); match *h { Composite::SelectionFunction { func: BuiltinSelectionFn::Regex(ref pattern), search_mod: None, result_transform: None, } => assert_eq!(pattern, "pattern"), _ => panic!(), } } _ => panic!(), } } #[test] fn test_parse_train_with_pipe() { let input = r#"| "date""#; let tokens = tokenise(input).unwrap(); let result = parse(tokens).unwrap(); match result { Composite::Train2 { f, g } => { assert!(matches!( *f, Composite::TextFunction { func: BuiltinTextFn::Pipe, swapped: false, } )); match *g { Composite::TextFunction { func: BuiltinTextFn::Literal(ref s), swapped: false, } => assert_eq!(s, "date"), _ => panic!(), } } _ => panic!(), } } #[test] fn test_parse_before_hook() { let input = "e>d"; let tokens = tokenise(input).unwrap(); let result = parse(tokens).unwrap(); println!("{:?}", result); match result { Composite::Hook { kind, left, right } => { assert_eq!(kind, HookKind::Before); assert!(matches!( *left, Composite::SelectionFunction { func: BuiltinSelectionFn::Empty, search_mod: None, result_transform: None, } )); assert!(matches!( *right, Composite::TextFunction { func: BuiltinTextFn::Delete, swapped: false, } )); } _ => panic!(), } } #[test] fn test_parse_nested_hooks_left_associative() { let input = r#"(c<|)<"sort""#; let tokens = tokenise(input).unwrap(); let result = parse(tokens).unwrap(); println!("{:?}", result); match result { Composite::Hook { kind: outer_kind, left: outer_left, right: outer_right, } => { assert_eq!(outer_kind, HookKind::After); match *outer_left { Composite::Hook { kind: inner_kind, left: inner_left, right: inner_right, } => { assert_eq!(inner_kind, HookKind::After); assert!(matches!( *inner_left, Composite::TextFunction { func: BuiltinTextFn::Change, swapped: false, } )); assert!(matches!( *inner_right, Composite::TextFunction { func: BuiltinTextFn::Pipe, swapped: false, } )); } _ => panic!(), } match *outer_right { Composite::TextFunction { func: BuiltinTextFn::Literal(ref s), swapped: false, } => assert_eq!(s, "sort"), _ => panic!(), } } _ => panic!(), } } #[test] fn test_parse_hooks_without_parens() { let input = r#"c<|<"sort""#; let tokens = tokenise(input).unwrap(); let result = parse(tokens).unwrap(); println!("{:?}", result); match result { Composite::Hook { kind: outer_kind, left: outer_left, right: outer_right, } => { assert_eq!(outer_kind, HookKind::After); assert!(matches!( *outer_left, Composite::TextFunction { func: BuiltinTextFn::Change, swapped: false, } )); match *outer_right { Composite::Hook { kind: inner_kind, left: inner_left, right: inner_right, } => { assert_eq!(inner_kind, HookKind::After); assert!(matches!( *inner_left, Composite::TextFunction { func: BuiltinTextFn::Pipe, swapped: false, } )); match *inner_right { Composite::TextFunction { func: BuiltinTextFn::Literal(ref s), swapped: false, } => assert_eq!(s, "sort"), _ => panic!(), } } _ => panic!(), } } _ => panic!(), } } #[test] fn test_parse_triple_nested_hooks() { let input = "e>di"; let tokens = tokenise(input).unwrap(); let result = parse(tokens).unwrap(); println!("{:?}", result); match result { Composite::Hook { kind: k1, left: l1, right: r1, } => { assert_eq!(k1, HookKind::Before); assert!(matches!( *l1, Composite::SelectionFunction { func: BuiltinSelectionFn::Empty, search_mod: None, result_transform: None, } )); match *r1 { Composite::Hook { kind: k2, left: l2, right: r2, } => { assert_eq!(k2, HookKind::After); assert!(matches!( *l2, Composite::TextFunction { func: BuiltinTextFn::Delete, swapped: false, } )); match *r2 { Composite::Hook { kind: k3, left: l3, right: r3, } => { assert_eq!(k3, HookKind::Before); assert!(matches!( *l3, Composite::TextFunction { func: BuiltinTextFn::Change, swapped: false, } )); assert!(matches!( *r3, Composite::TextFunction { func: BuiltinTextFn::Insert, swapped: false, } )); } _ => panic!(), } } _ => panic!(), } } _ => panic!(), } } #[test] fn test_parse_hook_in_train() { let input = "e>d $"; let tokens = tokenise(input).unwrap(); let result = parse(tokens).unwrap(); println!("{:?}", result); match result { Composite::Hook { kind, left, right } => { assert_eq!(kind, HookKind::Before); assert!(matches!( *left, Composite::SelectionFunction { func: BuiltinSelectionFn::Empty, search_mod: None, result_transform: None, } )); match *right { Composite::Train2 { f, g } => { assert!(matches!( *f, Composite::TextFunction { func: BuiltinTextFn::Delete, swapped: false, } )); assert!(matches!( *g, Composite::SelectionFunction { func: BuiltinSelectionFn::EndOfBuffer, search_mod: None, result_transform: None, } )); } _ => panic!(), } } _ => panic!(), } } #[test] fn test_parse_group() { let input = r#"{e, #5 -, "tr"}"#; let tokens = tokenise(input).unwrap(); let result = parse(tokens).unwrap(); match result { Composite::Group { operations } => { assert_eq!(operations.len(), 3); match &operations[0] { Composite::SelectionFunction { func: BuiltinSelectionFn::Empty, search_mod: None, result_transform: None, } => {} _ => panic!(), } match &operations[1] { Composite::Train2 { f, g } => { match &**f { Composite::SelectionFunction { func: BuiltinSelectionFn::CharOffset(5), search_mod: None, result_transform: None, } => {} _ => panic!(), } match &**g { Composite::SelectionFunction { func: BuiltinSelectionFn::Span, search_mod: None, result_transform: None, } => {} _ => panic!(), } } _ => panic!(), } match &operations[2] { Composite::TextFunction { func: BuiltinTextFn::Literal(s), swapped: false, } => { assert_eq!(s, "tr"); } _ => panic!(), } } _ => panic!(), } } #[test] fn test_parse_nested_groups_in_train() { let input = r#"{e<#5, ;$ i} (~e { match f.as_ref() { Composite::Group { operations } => { assert_eq!(operations.len(), 2); match &operations[0] { Composite::Hook { kind: HookKind::After, left, right, } => { match left.as_ref() { Composite::SelectionFunction { func: BuiltinSelectionFn::Empty, search_mod: None, result_transform: None, } => {} _ => panic!(), } match right.as_ref() { Composite::SelectionFunction { func: BuiltinSelectionFn::CharOffset(5), search_mod: None, result_transform: None, } => {} _ => panic!(), } } _ => panic!(), } match &operations[1] { Composite::Train2 { f: f2, g: g2 } => { match f2.as_ref() { Composite::SelectionFunction { func: BuiltinSelectionFn::EndOfBuffer, search_mod: Some(SearchModifier::Sequential), result_transform: None, } => {} _ => panic!(), } match g2.as_ref() { Composite::TextFunction { func: BuiltinTextFn::Insert, swapped: false, } => {} _ => panic!(), } } _ => panic!(), } } _ => panic!(), } match g.as_ref() { Composite::Hook { kind: HookKind::After, left, right, } => { match left.as_ref() { Composite::SelectionFunction { func: BuiltinSelectionFn::Empty, search_mod: None, result_transform: Some(ResultTransform::Complement), } => {} _ => panic!(), } match right.as_ref() { Composite::TextFunction { func: BuiltinTextFn::Delete, swapped: false, } => {} _ => panic!(), } } _ => panic!(), } match h.as_ref() { Composite::SelectionFunction { func: BuiltinSelectionFn::EndOfBuffer, search_mod: None, result_transform: None, } => {} _ => panic!(), } } _ => panic!(), } } }