From fed8c900080107540a2fa0d908b8a16692ddda06 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 11 Nov 2024 03:10:26 +0400 Subject: [PATCH 01/26] feat: implement Pratt parser --- src/combinator/mod.rs | 2 + src/combinator/precedence.rs | 481 +++++++++++++++++++++++++++++++++++ 2 files changed, 483 insertions(+) create mode 100644 src/combinator/precedence.rs diff --git a/src/combinator/mod.rs b/src/combinator/mod.rs index df791adaa..b8ee8737b 100644 --- a/src/combinator/mod.rs +++ b/src/combinator/mod.rs @@ -164,6 +164,7 @@ mod core; mod debug; mod multi; mod parser; +mod precedence; mod sequence; #[cfg(test)] @@ -174,6 +175,7 @@ pub use self::core::*; pub use self::debug::*; pub use self::multi::*; pub use self::parser::*; +pub use self::precedence::*; pub use self::sequence::*; #[allow(unused_imports)] diff --git a/src/combinator/precedence.rs b/src/combinator/precedence.rs new file mode 100644 index 000000000..0d99031e8 --- /dev/null +++ b/src/combinator/precedence.rs @@ -0,0 +1,481 @@ +use core::cell::RefCell; + +use crate::{ + combinator::{alt, fail, opt, trace}, + error::{ErrMode, ParserError}, + stream::{Stream, StreamIsPartial}, + PResult, Parser, +}; + +/// An adapter for the [`Parser`] trait to enable its use in the [`precedence`] parser. +pub trait PrecedenceParserExt { + /// Specifies that the parser is a `unary` `prefix` operator within a [`precedence`] parser. + /// + /// In most languages, operators such negation: `-`, `not` or `!`, dereferencing: `*`, etc. are prefix unary operators. + /// + /// The argument `fold` is a fold function that defines how to combine the operator and operand into a new expression. + /// It must have the following signature: + /// ```ignore + /// impl Fn(O) -> O + /// ``` + #[inline(always)] + fn prefix(self, fold: F) -> Prefix> + where + F: UnaryOp, + Self: Sized, + { + Prefix(Operator::new(self, fold)) + } + /// Specifies that the parser is a `unary` `postfix` operator within a [`precedence`] parser. + /// + /// Operators like the factorial `!` are postfix unary operators. + /// + /// The argument `fold` is a fold function that defines how to combine the operator and operand into a new + /// expression. It must have the following signature: + /// ```ignore + /// impl Fn(O) -> O + /// ``` + #[inline(always)] + fn postfix(self, fold: F) -> Postfix> + where + F: UnaryOp, + Self: Sized, + { + Postfix(Operator::new(self, fold)) + } + /// Specifies that the parser is a `binary` `infix` operator within a [`precedence`] parser. + /// + /// Operators like factorial `+`, `-`, `*`, `/` are infix binary operators. + /// + /// The argument is a fold function that defines how to combine the operator and two operands into a new + /// expression. It must have the following signature: + /// ```ignore + /// impl Fn(O, O) -> O + /// ``` + #[inline(always)] + fn infix(self, fold: F) -> Infix> + where + F: BinaryOp, + Self: Sized, + { + Infix(Operator::new(self, fold)) + } +} +impl> PrecedenceParserExt for T where I: Stream {} + +/// `NewType` that indicates this type is a prefix operator a [`precedence`] parser. +/// See: [`PrecedenceParserExt::prefix`] +/// +/// Can hold and arbitrary parser, such as a tuple of multiple operator parsers: `(Operator<...>, Operator<...>)` +pub struct Prefix(T); + +/// `NewType` that indicates this type is a postfix operator a [`precedence`] parser. +/// See: [`PrecedenceParserExt::postfix`] +pub struct Postfix(T); + +/// `NewType` that indicates this type is a infix operator within a [`precedence`] parser. +/// See: [`PrecedenceParserExt::infix`] +pub struct Infix(T); + +/// Implementation of the operator parser for the [`precedence`] parser. +pub struct Operator { + // We use two different `ReffCell`s to enable mutable borrowing within the recursion + // while holding a reference to the predicate `op`: + // ``` + // let lhs = ...; + // let op: &ReffCell> = operator.parse_next(i); // calls `operator.parser.borrow_mut().parse_next(i)` + // let rhs = recursion(&operator); + // let result = op.borrow_mut().fold_binary(lhs, rhs); + // ``` + op: RefCell, + parser: RefCell, +} + +impl Operator { + /// Creates a new [`Operator`] from a parser and a predicate + #[inline(always)] + pub fn new(parser: OperatorParser, predicate: OperatorFunc) -> Self { + Self { + op: RefCell::new(predicate), + parser: RefCell::new(parser), + } + } +} + +/// Type-erased unary predicate that folds an expression into a new expression. +/// Useful for supporting not only closures but also arbitrary types as operator predicates within the [`precedence`] parser. +pub trait UnaryOp { + /// Invokes the [`UnaryOp`] predicate. + fn fold_unary(&mut self, o: O) -> O; +} +/// Type-erased binary predicate that folds two expressions into a new expression similar to +/// [`UnaryOp`] within the [`precedence`] parser. +pub trait BinaryOp { + /// Invokes the [`BinaryOp`] predicate. + fn fold_binary(&mut self, lhs: O, rhs: O) -> O; +} + +impl UnaryOp for F +where + F: Fn(O) -> O, +{ + #[inline(always)] + fn fold_unary(&mut self, o: O) -> O { + (self)(o) + } +} +impl BinaryOp for F +where + F: Fn(O, O) -> O, +{ + #[inline(always)] + fn fold_binary(&mut self, lhs: O, rhs: O) -> O { + (self)(lhs, rhs) + } +} + +impl<'s, UO, O, I, P, E> Parser>, usize), E> for &'s Operator +where + UO: UnaryOp + 'static, + I: Stream + StreamIsPartial, + P: Parser, + E: ParserError, +{ + #[inline(always)] + fn parse_next( + &mut self, + input: &mut I, + ) -> PResult<(&'s RefCell + 'static>, usize), E> { + let power = self.parser.borrow_mut().parse_next(input)?; + Ok((&self.op, power)) + } +} +impl<'s, BO, O, I, P, E> Parser>, usize), E> for &'s Operator +where + BO: BinaryOp + 'static, + I: Stream + StreamIsPartial, + P: Parser, + E: ParserError, +{ + #[inline(always)] + fn parse_next( + &mut self, + input: &mut I, + ) -> PResult<(&'s RefCell + 'static>, usize), E> { + let power = self.parser.borrow_mut().parse_next(input)?; + Ok((&self.op, power)) + } +} + +/// Ability to request a parser of the specified affix from the [`impl Parser`](Parser) object. +pub trait AsPrecedence> { + /// Interprets a parser as a [`PrecedenceParserExt::prefix`] parser that returns an `unary + /// predicate` [`UnaryOp`] and a `binding power` as its parsing result. + #[inline(always)] + fn as_prefix(&self) -> impl Parser>, usize), E> { + fail + } + /// Interprets a parser as a [`PrecedenceParserExt::postfix`] parser that returns an `unary + /// predicate` [`UnaryOp`] and a `binding power` as its parsing result. + #[inline(always)] + fn as_postfix(&self) -> impl Parser>, usize), E> { + fail + } + /// Interprets a parser as a [`PrecedenceParserExt::infix`] parser that returns a `binary + /// predicate` [`BinaryOp`] and a `binding power` as its parsing result. + #[inline(always)] + fn as_infix(&self) -> impl Parser>, usize), E> { + fail + } +} + +impl<'s, F, O, I, P, E> Parser>, usize), E> + for &'s Prefix> +where + F: UnaryOp + 'static, + I: Stream, + P: Parser, + E: ParserError, +{ + #[inline(always)] + fn parse_next( + &mut self, + input: &mut I, + ) -> PResult<(&'s RefCell + 'static>, usize), E> { + let power = self.0.parser.borrow_mut().parse_next(input)?; + Ok((&self.0.op, power)) + } +} + +impl AsPrecedence for Prefix> +where + F: UnaryOp + 'static, + I: Stream + StreamIsPartial, + P: Parser, + E: ParserError, +{ + #[inline(always)] + fn as_prefix(&self) -> impl Parser>, usize), E> { + &self.0 + } +} + +impl AsPrecedence for Postfix> +where + F: UnaryOp + 'static, + I: Stream + StreamIsPartial, + P: Parser, + E: ParserError, +{ + #[inline(always)] + fn as_postfix(&self) -> impl Parser>, usize), E> { + &self.0 + } +} +impl AsPrecedence for Infix> +where + F: BinaryOp + 'static, + I: Stream + StreamIsPartial, + P: Parser, + E: ParserError, +{ + #[inline(always)] + fn as_infix(&self) -> impl Parser>, usize), E> { + &self.0 + } +} + +macro_rules! impl_parser_for_tuple { + () => {}; + ($head:ident $($X:ident)*) => { + impl_parser_for_tuple!($($X)*); + impl_parser_for_tuple!(~ $head $($X)*); + }; + (~ $($X:ident)+) => { + + #[allow(unused_variables, non_snake_case)] + impl AsPrecedence for ($($X,)*) + where + I: Stream + StreamIsPartial, + E: ParserError, + $($X: AsPrecedence),* + { + #[inline(always)] + fn as_prefix<'s>( + &'s self, + ) -> impl Parser>, usize), E> { + Prefix(self) + } + #[inline(always)] + fn as_infix<'s>( + &'s self, + ) -> impl Parser>, usize), E> { + Infix(self) + } + #[inline(always)] + fn as_postfix<'s>( + &'s self, + ) -> impl Parser>, usize), E> { + Postfix(self) + } + } + + #[allow(unused_variables, non_snake_case)] + impl<'s, I, O: 'static, E, $($X),*> Parser>, usize), E> + for Prefix<&'s ($($X,)*)> + where + I: Stream + StreamIsPartial, + E: ParserError, + $($X: AsPrecedence),* + + { + #[inline(always)] + fn parse_next(&mut self, input: &mut I) -> PResult<(&'s RefCell>, usize), E> { + let ($($X,)*) = self.0; + alt(($($X.as_prefix(),)*)).parse_next(input) + } + } + #[allow(unused_variables, non_snake_case)] + impl<'s, I, O: 'static, E, $($X),*> Parser>, usize), E> + for Postfix<&'s ($($X,)*)> + where + I: Stream + StreamIsPartial, + E: ParserError, + $($X: AsPrecedence),* + { + #[inline(always)] + fn parse_next(&mut self, input: &mut I) -> PResult<(&'s RefCell>, usize), E> { + let ($($X,)*) = self.0; + alt(($($X.as_postfix(),)*)).parse_next(input) + } + } + #[allow(unused_variables, non_snake_case)] + impl<'s, I, O: 'static, E, $($X),*> Parser>, usize), E> + for Infix<&'s ($($X,)*)> + where + I: Stream + StreamIsPartial, + E: ParserError, + $($X: AsPrecedence),* + { + #[inline(always)] + fn parse_next(&mut self, input: &mut I) -> PResult<(&'s RefCell>, usize), E> { + let ($($X,)*) = self.0; + alt(($($X.as_infix(),)*)).parse_next(input) + } + } + + }; +} + +impl_parser_for_tuple!(P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 P16 P17 P18 P19 P20 P21); + +/// Constructs an expression parser from an operand parser and operator parsers to parse an +/// arbitrary expression separated by `prefix`, `postfix`, and `infix` operators of various precedence. +/// +/// This technique is powerful and recommended for parsing expressions. +/// +/// The implementation uses [Pratt parsing](https://en.wikipedia.org/wiki/Operator-precedence_parser#Pratt_parsing). +/// This algorithm is similar to the [Shunting Yard](https://en.wikipedia.org/wiki/Shunting_yard_algorithm) algorithm +/// in that both are linear, both use precedence and binding power, and both serve the same purpose. +/// However, the `Shunting Yard` algorithm additionally uses `left` and `right` associativity, +/// while `Pratt` parsing only relies on binding power. +#[doc(alias = "pratt")] +#[doc(alias = "separated")] +#[doc(alias = "shunting yard")] +#[doc(alias = "precedence climbing")] +#[inline(always)] +pub fn precedence( + mut parse_operand: ParseOperand, + ops: Operators, +) -> impl Parser +where + Operators: AsPrecedence, + ParseOperand: Parser, + I: Stream + StreamIsPartial, + E: ParserError, +{ + trace("precedence", move |i: &mut I| { + let result = precedence_impl(i, &mut parse_operand, &ops, 0); + result + }) +} + +// recursive function +fn precedence_impl( + i: &mut I, + parse_operand: &mut ParseOperand, + ops: &Operators, + start_power: usize, +) -> PResult +where + I: Stream + StreamIsPartial, + Operators: AsPrecedence, + ParseOperand: Parser, + E: ParserError, +{ + let operand = trace("operand", opt(parse_operand.by_ref())).parse_next(i)?; + let mut operand = if let Some(operand) = operand { + operand + } else { + // Prefix unary operators + let len = i.eof_offset(); + let (fold_prefix, power) = trace("prefix", ops.as_prefix()).parse_next(i)?; + // infinite loop check: the parser must always consume + if i.eof_offset() == len { + return Err(ErrMode::assert(i, "`prefix` parsers must always consume")); + } + let operand = precedence_impl(i, parse_operand, ops, power)?; + fold_prefix.borrow_mut().fold_unary(operand) + }; + + 'parse: while i.eof_offset() > 0 { + // Postfix unary operators + let start = i.checkpoint(); + let len = i.eof_offset(); + if let Some((fold_postfix, power)) = + trace("postfix", opt(ops.as_postfix())).parse_next(i)? + { + // infinite loop check: the parser must always consume + if i.eof_offset() == len { + return Err(ErrMode::assert(i, "`postfix` parsers must always consume")); + } + if power < start_power { + i.reset(&start); + break; + } + operand = fold_postfix.borrow_mut().fold_unary(operand); + + continue 'parse; + } + + // Infix binary operators + let start = i.checkpoint(); + let len = i.eof_offset(); + if let Some((fold_infix, power)) = trace("infix", opt(ops.as_infix())).parse_next(i)? { + // infinite loop check: the parser must always consume + if i.eof_offset() == len { + return Err(ErrMode::assert(i, "`infix` parsers must always consume")); + } + if power < start_power { + i.reset(&start); + break; + } + let rhs = precedence_impl(i, parse_operand, ops, power)?; + operand = fold_infix.borrow_mut().fold_binary(operand, rhs); + + continue 'parse; + } + + break 'parse; + } + + Ok(operand) +} + +#[cfg(test)] +mod tests { + use crate::ascii::{digit1, space0}; + use crate::combinator::delimited; + use crate::error::ContextError; + + use super::*; + + fn factorial(x: i32) -> i32 { + if x == 0 { + 1 + } else { + x * factorial(x - 1) + } + } + fn parser<'i>() -> impl Parser<&'i str, i32, ContextError> { + move |i: &mut &str| { + precedence( + delimited(space0, digit1.try_map(|d: &str| d.parse::()), space0), + ( + "-".value(2).prefix(|a: i32| -a), + "+".value(2).prefix(|a| a), + "!".value(2).postfix(factorial), + "+".value(0).infix(|a, b| a + b), + "-".value(0).infix(|a, b| a + b), + "*".value(1).infix(|a, b| a * b), + "/".value(1).infix(|a, b| a / b), + ), + ) + .parse_next(i) + } + } + + #[test] + fn test_precedence() { + assert_eq!(parser().parse("-3!+-3 * 4"), Ok(-18)); + assert_eq!(parser().parse("+2 + 3 * 4"), Ok(14)); + assert_eq!(parser().parse("2 * 3+4"), Ok(10)); + } + #[test] + fn test_unary() { + assert_eq!(parser().parse("-2"), Ok(-2)); + assert_eq!(parser().parse("4!"), Ok(24)); + assert_eq!(parser().parse("2 + 4!"), Ok(26)); + assert_eq!(parser().parse("-2 + 2"), Ok(0)); + } +} From ee4459db9a99d1e3fa6b5a62cf11972c74accce3 Mon Sep 17 00:00:00 2001 From: Igor Date: Sat, 16 Nov 2024 12:34:57 +0400 Subject: [PATCH 02/26] commit suggestion Co-authored-by: Ed Page --- src/combinator/precedence.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/combinator/precedence.rs b/src/combinator/precedence.rs index 0d99031e8..1562f2850 100644 --- a/src/combinator/precedence.rs +++ b/src/combinator/precedence.rs @@ -61,6 +61,7 @@ pub trait PrecedenceParserExt { Infix(Operator::new(self, fold)) } } + impl> PrecedenceParserExt for T where I: Stream {} /// `NewType` that indicates this type is a prefix operator a [`precedence`] parser. From 4b1499df9c1d84f7a55d9dbbadc64b463b86fa38 Mon Sep 17 00:00:00 2001 From: Igor Date: Sat, 16 Nov 2024 12:33:26 +0400 Subject: [PATCH 03/26] remove spaces from #[doc(alias = "...")] --- src/combinator/precedence.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/combinator/precedence.rs b/src/combinator/precedence.rs index 1562f2850..733d31712 100644 --- a/src/combinator/precedence.rs +++ b/src/combinator/precedence.rs @@ -342,8 +342,8 @@ impl_parser_for_tuple!(P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 P16 P1 /// while `Pratt` parsing only relies on binding power. #[doc(alias = "pratt")] #[doc(alias = "separated")] -#[doc(alias = "shunting yard")] -#[doc(alias = "precedence climbing")] +#[doc(alias = "shunting_yard")] +#[doc(alias = "precedence_climbing")] #[inline(always)] pub fn precedence( mut parse_operand: ParseOperand, From acf45777240b2555fabce5a312567154499bb9a6 Mon Sep 17 00:00:00 2001 From: Igor Date: Sat, 16 Nov 2024 12:42:10 +0400 Subject: [PATCH 04/26] remove `UnaryOp` and `BinaryOp` in favor of `Fn` This feature was an overengineering based on suggestion "Why make our own trait" in https://github.com/winnow-rs/winnow/pull/614#discussion_r1838261641 --- src/combinator/precedence.rs | 101 +++++++++++++---------------------- 1 file changed, 36 insertions(+), 65 deletions(-) diff --git a/src/combinator/precedence.rs b/src/combinator/precedence.rs index 733d31712..9bdc84080 100644 --- a/src/combinator/precedence.rs +++ b/src/combinator/precedence.rs @@ -21,7 +21,7 @@ pub trait PrecedenceParserExt { #[inline(always)] fn prefix(self, fold: F) -> Prefix> where - F: UnaryOp, + F: Fn(O) -> O, Self: Sized, { Prefix(Operator::new(self, fold)) @@ -38,7 +38,7 @@ pub trait PrecedenceParserExt { #[inline(always)] fn postfix(self, fold: F) -> Postfix> where - F: UnaryOp, + F: Fn(O) -> O, Self: Sized, { Postfix(Operator::new(self, fold)) @@ -55,7 +55,7 @@ pub trait PrecedenceParserExt { #[inline(always)] fn infix(self, fold: F) -> Infix> where - F: BinaryOp, + F: Fn(O, O) -> O, Self: Sized, { Infix(Operator::new(self, fold)) @@ -103,41 +103,9 @@ impl Operator { } } -/// Type-erased unary predicate that folds an expression into a new expression. -/// Useful for supporting not only closures but also arbitrary types as operator predicates within the [`precedence`] parser. -pub trait UnaryOp { - /// Invokes the [`UnaryOp`] predicate. - fn fold_unary(&mut self, o: O) -> O; -} -/// Type-erased binary predicate that folds two expressions into a new expression similar to -/// [`UnaryOp`] within the [`precedence`] parser. -pub trait BinaryOp { - /// Invokes the [`BinaryOp`] predicate. - fn fold_binary(&mut self, lhs: O, rhs: O) -> O; -} - -impl UnaryOp for F -where - F: Fn(O) -> O, -{ - #[inline(always)] - fn fold_unary(&mut self, o: O) -> O { - (self)(o) - } -} -impl BinaryOp for F -where - F: Fn(O, O) -> O, -{ - #[inline(always)] - fn fold_binary(&mut self, lhs: O, rhs: O) -> O { - (self)(lhs, rhs) - } -} - -impl<'s, UO, O, I, P, E> Parser>, usize), E> for &'s Operator +impl<'s, UO, O, I, P, E> Parser O>, usize), E> for &'s Operator where - UO: UnaryOp + 'static, + UO: Fn(O) -> O + 'static, I: Stream + StreamIsPartial, P: Parser, E: ParserError, @@ -146,14 +114,15 @@ where fn parse_next( &mut self, input: &mut I, - ) -> PResult<(&'s RefCell + 'static>, usize), E> { + ) -> PResult<(&'s RefCell O + 'static>, usize), E> { let power = self.parser.borrow_mut().parse_next(input)?; Ok((&self.op, power)) } } -impl<'s, BO, O, I, P, E> Parser>, usize), E> for &'s Operator +impl<'s, BO, O, I, P, E> Parser O>, usize), E> + for &'s Operator where - BO: BinaryOp + 'static, + BO: Fn(O, O) -> O + 'static, I: Stream + StreamIsPartial, P: Parser, E: ParserError, @@ -162,7 +131,7 @@ where fn parse_next( &mut self, input: &mut I, - ) -> PResult<(&'s RefCell + 'static>, usize), E> { + ) -> PResult<(&'s RefCell O + 'static>, usize), E> { let power = self.parser.borrow_mut().parse_next(input)?; Ok((&self.op, power)) } @@ -173,27 +142,29 @@ pub trait AsPrecedence> { /// Interprets a parser as a [`PrecedenceParserExt::prefix`] parser that returns an `unary /// predicate` [`UnaryOp`] and a `binding power` as its parsing result. #[inline(always)] - fn as_prefix(&self) -> impl Parser>, usize), E> { + fn as_prefix(&self) -> impl Parser Operand>, usize), E> { fail } /// Interprets a parser as a [`PrecedenceParserExt::postfix`] parser that returns an `unary /// predicate` [`UnaryOp`] and a `binding power` as its parsing result. #[inline(always)] - fn as_postfix(&self) -> impl Parser>, usize), E> { + fn as_postfix(&self) -> impl Parser Operand>, usize), E> { fail } /// Interprets a parser as a [`PrecedenceParserExt::infix`] parser that returns a `binary /// predicate` [`BinaryOp`] and a `binding power` as its parsing result. #[inline(always)] - fn as_infix(&self) -> impl Parser>, usize), E> { + fn as_infix( + &self, + ) -> impl Parser Operand>, usize), E> { fail } } -impl<'s, F, O, I, P, E> Parser>, usize), E> +impl<'s, F, O, I, P, E> Parser O>, usize), E> for &'s Prefix> where - F: UnaryOp + 'static, + F: Fn(O) -> O + 'static, I: Stream, P: Parser, E: ParserError, @@ -202,7 +173,7 @@ where fn parse_next( &mut self, input: &mut I, - ) -> PResult<(&'s RefCell + 'static>, usize), E> { + ) -> PResult<(&'s RefCell O + 'static>, usize), E> { let power = self.0.parser.borrow_mut().parse_next(input)?; Ok((&self.0.op, power)) } @@ -210,38 +181,38 @@ where impl AsPrecedence for Prefix> where - F: UnaryOp + 'static, + F: Fn(O) -> O + 'static, I: Stream + StreamIsPartial, P: Parser, E: ParserError, { #[inline(always)] - fn as_prefix(&self) -> impl Parser>, usize), E> { + fn as_prefix(&self) -> impl Parser O>, usize), E> { &self.0 } } impl AsPrecedence for Postfix> where - F: UnaryOp + 'static, + F: Fn(O) -> O + 'static, I: Stream + StreamIsPartial, P: Parser, E: ParserError, { #[inline(always)] - fn as_postfix(&self) -> impl Parser>, usize), E> { + fn as_postfix(&self) -> impl Parser O>, usize), E> { &self.0 } } impl AsPrecedence for Infix> where - F: BinaryOp + 'static, + F: Fn(O, O) -> O + 'static, I: Stream + StreamIsPartial, P: Parser, E: ParserError, { #[inline(always)] - fn as_infix(&self) -> impl Parser>, usize), E> { + fn as_infix(&self) -> impl Parser O>, usize), E> { &self.0 } } @@ -264,25 +235,25 @@ macro_rules! impl_parser_for_tuple { #[inline(always)] fn as_prefix<'s>( &'s self, - ) -> impl Parser>, usize), E> { + ) -> impl Parser O>, usize), E> { Prefix(self) } #[inline(always)] fn as_infix<'s>( &'s self, - ) -> impl Parser>, usize), E> { + ) -> impl Parser O>, usize), E> { Infix(self) } #[inline(always)] fn as_postfix<'s>( &'s self, - ) -> impl Parser>, usize), E> { + ) -> impl Parser O>, usize), E> { Postfix(self) } } #[allow(unused_variables, non_snake_case)] - impl<'s, I, O: 'static, E, $($X),*> Parser>, usize), E> + impl<'s, I, O: 'static, E, $($X),*> Parser O>, usize), E> for Prefix<&'s ($($X,)*)> where I: Stream + StreamIsPartial, @@ -291,13 +262,13 @@ macro_rules! impl_parser_for_tuple { { #[inline(always)] - fn parse_next(&mut self, input: &mut I) -> PResult<(&'s RefCell>, usize), E> { + fn parse_next(&mut self, input: &mut I) -> PResult<(&'s RefCell O>, usize), E> { let ($($X,)*) = self.0; alt(($($X.as_prefix(),)*)).parse_next(input) } } #[allow(unused_variables, non_snake_case)] - impl<'s, I, O: 'static, E, $($X),*> Parser>, usize), E> + impl<'s, I, O: 'static, E, $($X),*> Parser O>, usize), E> for Postfix<&'s ($($X,)*)> where I: Stream + StreamIsPartial, @@ -305,13 +276,13 @@ macro_rules! impl_parser_for_tuple { $($X: AsPrecedence),* { #[inline(always)] - fn parse_next(&mut self, input: &mut I) -> PResult<(&'s RefCell>, usize), E> { + fn parse_next(&mut self, input: &mut I) -> PResult<(&'s RefCell O>, usize), E> { let ($($X,)*) = self.0; alt(($($X.as_postfix(),)*)).parse_next(input) } } #[allow(unused_variables, non_snake_case)] - impl<'s, I, O: 'static, E, $($X),*> Parser>, usize), E> + impl<'s, I, O: 'static, E, $($X),*> Parser O>, usize), E> for Infix<&'s ($($X,)*)> where I: Stream + StreamIsPartial, @@ -319,7 +290,7 @@ macro_rules! impl_parser_for_tuple { $($X: AsPrecedence),* { #[inline(always)] - fn parse_next(&mut self, input: &mut I) -> PResult<(&'s RefCell>, usize), E> { + fn parse_next(&mut self, input: &mut I) -> PResult<(&'s RefCell O>, usize), E> { let ($($X,)*) = self.0; alt(($($X.as_infix(),)*)).parse_next(input) } @@ -386,7 +357,7 @@ where return Err(ErrMode::assert(i, "`prefix` parsers must always consume")); } let operand = precedence_impl(i, parse_operand, ops, power)?; - fold_prefix.borrow_mut().fold_unary(operand) + fold_prefix.borrow_mut()(operand) }; 'parse: while i.eof_offset() > 0 { @@ -404,7 +375,7 @@ where i.reset(&start); break; } - operand = fold_postfix.borrow_mut().fold_unary(operand); + operand = fold_postfix.borrow_mut()(operand); continue 'parse; } @@ -422,7 +393,7 @@ where break; } let rhs = precedence_impl(i, parse_operand, ops, power)?; - operand = fold_infix.borrow_mut().fold_binary(operand, rhs); + operand = fold_infix.borrow_mut()(operand, rhs); continue 'parse; } From a816a1c54c4cd1cd79f17a155545e073532268a1 Mon Sep 17 00:00:00 2001 From: Igor Date: Sat, 16 Nov 2024 12:47:44 +0400 Subject: [PATCH 05/26] remove redundant trait impl works without it --- src/combinator/precedence.rs | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/combinator/precedence.rs b/src/combinator/precedence.rs index 9bdc84080..72f0c072e 100644 --- a/src/combinator/precedence.rs +++ b/src/combinator/precedence.rs @@ -161,24 +161,6 @@ pub trait AsPrecedence> { } } -impl<'s, F, O, I, P, E> Parser O>, usize), E> - for &'s Prefix> -where - F: Fn(O) -> O + 'static, - I: Stream, - P: Parser, - E: ParserError, -{ - #[inline(always)] - fn parse_next( - &mut self, - input: &mut I, - ) -> PResult<(&'s RefCell O + 'static>, usize), E> { - let power = self.0.parser.borrow_mut().parse_next(input)?; - Ok((&self.0.op, power)) - } -} - impl AsPrecedence for Prefix> where F: Fn(O) -> O + 'static, From 2a80e657ba7bf30bf8403e6b18b4cf556c332780 Mon Sep 17 00:00:00 2001 From: Igor Date: Sat, 16 Nov 2024 12:49:57 +0400 Subject: [PATCH 06/26] remove `allow_unused`, move `allow(non_snake_case)` to where it should be - based on review "Why allow non_snake_case?" in https://github.com/winnow-rs/winnow/pull/614#discussion_r1838551812 - remove `allow_unused` based on "Whats getting unused?" https://github.com/winnow-rs/winnow/pull/614#discussion_r1838553734 --- src/combinator/precedence.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/combinator/precedence.rs b/src/combinator/precedence.rs index 72f0c072e..a924129a4 100644 --- a/src/combinator/precedence.rs +++ b/src/combinator/precedence.rs @@ -207,7 +207,6 @@ macro_rules! impl_parser_for_tuple { }; (~ $($X:ident)+) => { - #[allow(unused_variables, non_snake_case)] impl AsPrecedence for ($($X,)*) where I: Stream + StreamIsPartial, @@ -234,7 +233,6 @@ macro_rules! impl_parser_for_tuple { } } - #[allow(unused_variables, non_snake_case)] impl<'s, I, O: 'static, E, $($X),*> Parser O>, usize), E> for Prefix<&'s ($($X,)*)> where @@ -245,6 +243,7 @@ macro_rules! impl_parser_for_tuple { { #[inline(always)] fn parse_next(&mut self, input: &mut I) -> PResult<(&'s RefCell O>, usize), E> { + #[allow(non_snake_case)] let ($($X,)*) = self.0; alt(($($X.as_prefix(),)*)).parse_next(input) } From 29fe18dd50c438e87cb1ff34833c4de767cfc908 Mon Sep 17 00:00:00 2001 From: Igor Date: Sat, 16 Nov 2024 12:56:47 +0400 Subject: [PATCH 07/26] stop dumping pratt into `combinator` namespace until we find a satisfactory api based on https://github.com/winnow-rs/winnow/pull/614#discussion_r1838607344 > "We are dumping a lot of stray types into combinator. The single-line summaries should make it very easy to tell they are related to precedence" --- src/combinator/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/combinator/mod.rs b/src/combinator/mod.rs index b8ee8737b..574c906bf 100644 --- a/src/combinator/mod.rs +++ b/src/combinator/mod.rs @@ -164,9 +164,10 @@ mod core; mod debug; mod multi; mod parser; -mod precedence; mod sequence; +pub mod precedence; + #[cfg(test)] mod tests; From 5a4f4b497254d27b4395bd65b3dca44de34c4246 Mon Sep 17 00:00:00 2001 From: Igor Date: Sat, 16 Nov 2024 12:59:25 +0400 Subject: [PATCH 08/26] move important things to go first based on "Organizationally, O prefer the "top level" thing going first and then branching out from there. In this case, precedence is core." https://github.com/winnow-rs/winnow/pull/614#discussion_r1838617660 --- src/combinator/precedence.rs | 206 ++++++++++++++++++----------------- 1 file changed, 104 insertions(+), 102 deletions(-) diff --git a/src/combinator/precedence.rs b/src/combinator/precedence.rs index a924129a4..20eabeb33 100644 --- a/src/combinator/precedence.rs +++ b/src/combinator/precedence.rs @@ -7,6 +7,110 @@ use crate::{ PResult, Parser, }; +/// Constructs an expression parser from an operand parser and operator parsers to parse an +/// arbitrary expression separated by `prefix`, `postfix`, and `infix` operators of various precedence. +/// +/// This technique is powerful and recommended for parsing expressions. +/// +/// The implementation uses [Pratt parsing](https://en.wikipedia.org/wiki/Operator-precedence_parser#Pratt_parsing). +/// This algorithm is similar to the [Shunting Yard](https://en.wikipedia.org/wiki/Shunting_yard_algorithm) algorithm +/// in that both are linear, both use precedence and binding power, and both serve the same purpose. +/// However, the `Shunting Yard` algorithm additionally uses `left` and `right` associativity, +/// while `Pratt` parsing only relies on binding power. +#[doc(alias = "pratt")] +#[doc(alias = "separated")] +#[doc(alias = "shunting_yard")] +#[doc(alias = "precedence_climbing")] +#[inline(always)] +pub fn precedence( + mut parse_operand: ParseOperand, + ops: Operators, +) -> impl Parser +where + Operators: AsPrecedence, + ParseOperand: Parser, + I: Stream + StreamIsPartial, + E: ParserError, +{ + trace("precedence", move |i: &mut I| { + let result = precedence_impl(i, &mut parse_operand, &ops, 0); + result + }) +} + +// recursive function +fn precedence_impl( + i: &mut I, + parse_operand: &mut ParseOperand, + ops: &Operators, + start_power: usize, +) -> PResult +where + I: Stream + StreamIsPartial, + Operators: AsPrecedence, + ParseOperand: Parser, + E: ParserError, +{ + let operand = trace("operand", opt(parse_operand.by_ref())).parse_next(i)?; + let mut operand = if let Some(operand) = operand { + operand + } else { + // Prefix unary operators + let len = i.eof_offset(); + let (fold_prefix, power) = trace("prefix", ops.as_prefix()).parse_next(i)?; + // infinite loop check: the parser must always consume + if i.eof_offset() == len { + return Err(ErrMode::assert(i, "`prefix` parsers must always consume")); + } + let operand = precedence_impl(i, parse_operand, ops, power)?; + fold_prefix.borrow_mut()(operand) + }; + + 'parse: while i.eof_offset() > 0 { + // Postfix unary operators + let start = i.checkpoint(); + let len = i.eof_offset(); + if let Some((fold_postfix, power)) = + trace("postfix", opt(ops.as_postfix())).parse_next(i)? + { + // infinite loop check: the parser must always consume + if i.eof_offset() == len { + return Err(ErrMode::assert(i, "`postfix` parsers must always consume")); + } + if power < start_power { + i.reset(&start); + break; + } + operand = fold_postfix.borrow_mut()(operand); + + continue 'parse; + } + + // Infix binary operators + let start = i.checkpoint(); + let len = i.eof_offset(); + if let Some((fold_infix, power)) = trace("infix", opt(ops.as_infix())).parse_next(i)? { + // infinite loop check: the parser must always consume + if i.eof_offset() == len { + return Err(ErrMode::assert(i, "`infix` parsers must always consume")); + } + if power < start_power { + i.reset(&start); + break; + } + let rhs = precedence_impl(i, parse_operand, ops, power)?; + operand = fold_infix.borrow_mut()(operand, rhs); + + continue 'parse; + } + + break 'parse; + } + + Ok(operand) +} + + /// An adapter for the [`Parser`] trait to enable its use in the [`precedence`] parser. pub trait PrecedenceParserExt { /// Specifies that the parser is a `unary` `prefix` operator within a [`precedence`] parser. @@ -282,108 +386,6 @@ macro_rules! impl_parser_for_tuple { impl_parser_for_tuple!(P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 P16 P17 P18 P19 P20 P21); -/// Constructs an expression parser from an operand parser and operator parsers to parse an -/// arbitrary expression separated by `prefix`, `postfix`, and `infix` operators of various precedence. -/// -/// This technique is powerful and recommended for parsing expressions. -/// -/// The implementation uses [Pratt parsing](https://en.wikipedia.org/wiki/Operator-precedence_parser#Pratt_parsing). -/// This algorithm is similar to the [Shunting Yard](https://en.wikipedia.org/wiki/Shunting_yard_algorithm) algorithm -/// in that both are linear, both use precedence and binding power, and both serve the same purpose. -/// However, the `Shunting Yard` algorithm additionally uses `left` and `right` associativity, -/// while `Pratt` parsing only relies on binding power. -#[doc(alias = "pratt")] -#[doc(alias = "separated")] -#[doc(alias = "shunting_yard")] -#[doc(alias = "precedence_climbing")] -#[inline(always)] -pub fn precedence( - mut parse_operand: ParseOperand, - ops: Operators, -) -> impl Parser -where - Operators: AsPrecedence, - ParseOperand: Parser, - I: Stream + StreamIsPartial, - E: ParserError, -{ - trace("precedence", move |i: &mut I| { - let result = precedence_impl(i, &mut parse_operand, &ops, 0); - result - }) -} - -// recursive function -fn precedence_impl( - i: &mut I, - parse_operand: &mut ParseOperand, - ops: &Operators, - start_power: usize, -) -> PResult -where - I: Stream + StreamIsPartial, - Operators: AsPrecedence, - ParseOperand: Parser, - E: ParserError, -{ - let operand = trace("operand", opt(parse_operand.by_ref())).parse_next(i)?; - let mut operand = if let Some(operand) = operand { - operand - } else { - // Prefix unary operators - let len = i.eof_offset(); - let (fold_prefix, power) = trace("prefix", ops.as_prefix()).parse_next(i)?; - // infinite loop check: the parser must always consume - if i.eof_offset() == len { - return Err(ErrMode::assert(i, "`prefix` parsers must always consume")); - } - let operand = precedence_impl(i, parse_operand, ops, power)?; - fold_prefix.borrow_mut()(operand) - }; - - 'parse: while i.eof_offset() > 0 { - // Postfix unary operators - let start = i.checkpoint(); - let len = i.eof_offset(); - if let Some((fold_postfix, power)) = - trace("postfix", opt(ops.as_postfix())).parse_next(i)? - { - // infinite loop check: the parser must always consume - if i.eof_offset() == len { - return Err(ErrMode::assert(i, "`postfix` parsers must always consume")); - } - if power < start_power { - i.reset(&start); - break; - } - operand = fold_postfix.borrow_mut()(operand); - - continue 'parse; - } - - // Infix binary operators - let start = i.checkpoint(); - let len = i.eof_offset(); - if let Some((fold_infix, power)) = trace("infix", opt(ops.as_infix())).parse_next(i)? { - // infinite loop check: the parser must always consume - if i.eof_offset() == len { - return Err(ErrMode::assert(i, "`infix` parsers must always consume")); - } - if power < start_power { - i.reset(&start); - break; - } - let rhs = precedence_impl(i, parse_operand, ops, power)?; - operand = fold_infix.borrow_mut()(operand, rhs); - - continue 'parse; - } - - break 'parse; - } - - Ok(operand) -} #[cfg(test)] mod tests { From 919a1cb070427bd2b35e69c50f7bd0513479715f Mon Sep 17 00:00:00 2001 From: Igor Date: Sat, 16 Nov 2024 13:05:55 +0400 Subject: [PATCH 09/26] strip fancy api for now the api has an unsound problem. The `Parser` trait is implemented on the `&Operator` but inside `parse_next` a mutable ref and `ReffCell::borrow_mut` are used which can lead to potential problems. We can return to the API later. But for now lets keep only the essential algorithm and pass affix parsers as 3 separate entities Also add left_binding_power and right_binding_power to the operators based on https://github.com/winnow-rs/winnow/pull/614#discussion_r1839963227 --- src/combinator/precedence.rs | 391 +++++++---------------------------- 1 file changed, 69 insertions(+), 322 deletions(-) diff --git a/src/combinator/precedence.rs b/src/combinator/precedence.rs index 20eabeb33..8a16dfb5e 100644 --- a/src/combinator/precedence.rs +++ b/src/combinator/precedence.rs @@ -1,7 +1,5 @@ -use core::cell::RefCell; - use crate::{ - combinator::{alt, fail, opt, trace}, + combinator::{opt, trace}, error::{ErrMode, ParserError}, stream::{Stream, StreamIsPartial}, PResult, Parser, @@ -22,33 +20,50 @@ use crate::{ #[doc(alias = "shunting_yard")] #[doc(alias = "precedence_climbing")] #[inline(always)] -pub fn precedence( - mut parse_operand: ParseOperand, - ops: Operators, +pub fn precedence<'i, I, ParseOperand, ParseInfix, ParsePrefix, ParsePostfix, Operand: 'static, E>( + mut operand: ParseOperand, + mut prefix: ParsePrefix, + mut postfix: ParsePostfix, + mut infix: ParseInfix, ) -> impl Parser where - Operators: AsPrecedence, - ParseOperand: Parser, I: Stream + StreamIsPartial, + ParseOperand: Parser, + ParseInfix: Parser Operand), E>, + ParsePrefix: Parser Operand), E>, + ParsePostfix: Parser Operand), E>, E: ParserError, { trace("precedence", move |i: &mut I| { - let result = precedence_impl(i, &mut parse_operand, &ops, 0); - result + let result = precedence_impl(i, &mut operand, &mut prefix, &mut postfix, &mut infix, 0)?; + Ok(result) }) } // recursive function -fn precedence_impl( +fn precedence_impl< + 'i, + I, + ParseOperand, + ParseInfix, + ParsePrefix, + ParsePostfix, + Operand: 'static, + E, +>( i: &mut I, parse_operand: &mut ParseOperand, - ops: &Operators, - start_power: usize, + prefix: &mut ParsePrefix, + postfix: &mut ParsePostfix, + infix: &mut ParseInfix, + min_power: usize, ) -> PResult where I: Stream + StreamIsPartial, - Operators: AsPrecedence, ParseOperand: Parser, + ParseInfix: Parser Operand), E>, + ParsePrefix: Parser Operand), E>, + ParsePostfix: Parser Operand), E>, E: ParserError, { let operand = trace("operand", opt(parse_operand.by_ref())).parse_next(i)?; @@ -57,49 +72,40 @@ where } else { // Prefix unary operators let len = i.eof_offset(); - let (fold_prefix, power) = trace("prefix", ops.as_prefix()).parse_next(i)?; + let (power, fold_prefix) = prefix.parse_next(i)?; // infinite loop check: the parser must always consume if i.eof_offset() == len { return Err(ErrMode::assert(i, "`prefix` parsers must always consume")); } - let operand = precedence_impl(i, parse_operand, ops, power)?; - fold_prefix.borrow_mut()(operand) + let operand = precedence_impl(i, parse_operand, prefix, postfix, infix, power)?; + fold_prefix(operand) }; 'parse: while i.eof_offset() > 0 { // Postfix unary operators let start = i.checkpoint(); - let len = i.eof_offset(); - if let Some((fold_postfix, power)) = - trace("postfix", opt(ops.as_postfix())).parse_next(i)? + if let Some((power, fold_postfix)) = + trace("postfix", opt(postfix.by_ref())).parse_next(i)? { - // infinite loop check: the parser must always consume - if i.eof_offset() == len { - return Err(ErrMode::assert(i, "`postfix` parsers must always consume")); - } - if power < start_power { + if power < min_power { i.reset(&start); break; } - operand = fold_postfix.borrow_mut()(operand); + operand = fold_postfix(operand); continue 'parse; } // Infix binary operators let start = i.checkpoint(); - let len = i.eof_offset(); - if let Some((fold_infix, power)) = trace("infix", opt(ops.as_infix())).parse_next(i)? { - // infinite loop check: the parser must always consume - if i.eof_offset() == len { - return Err(ErrMode::assert(i, "`infix` parsers must always consume")); - } - if power < start_power { + let parse_result = opt(infix.by_ref()).parse_next(i)?; + if let Some((lpower, rpower, fold_infix)) = parse_result { + if rpower < min_power { i.reset(&start); break; } - let rhs = precedence_impl(i, parse_operand, ops, power)?; - operand = fold_infix.borrow_mut()(operand, rhs); + let rhs = precedence_impl(i, parse_operand, prefix, postfix, infix, lpower)?; + operand = fold_infix(operand, rhs); continue 'parse; } @@ -110,288 +116,13 @@ where Ok(operand) } - -/// An adapter for the [`Parser`] trait to enable its use in the [`precedence`] parser. -pub trait PrecedenceParserExt { - /// Specifies that the parser is a `unary` `prefix` operator within a [`precedence`] parser. - /// - /// In most languages, operators such negation: `-`, `not` or `!`, dereferencing: `*`, etc. are prefix unary operators. - /// - /// The argument `fold` is a fold function that defines how to combine the operator and operand into a new expression. - /// It must have the following signature: - /// ```ignore - /// impl Fn(O) -> O - /// ``` - #[inline(always)] - fn prefix(self, fold: F) -> Prefix> - where - F: Fn(O) -> O, - Self: Sized, - { - Prefix(Operator::new(self, fold)) - } - /// Specifies that the parser is a `unary` `postfix` operator within a [`precedence`] parser. - /// - /// Operators like the factorial `!` are postfix unary operators. - /// - /// The argument `fold` is a fold function that defines how to combine the operator and operand into a new - /// expression. It must have the following signature: - /// ```ignore - /// impl Fn(O) -> O - /// ``` - #[inline(always)] - fn postfix(self, fold: F) -> Postfix> - where - F: Fn(O) -> O, - Self: Sized, - { - Postfix(Operator::new(self, fold)) - } - /// Specifies that the parser is a `binary` `infix` operator within a [`precedence`] parser. - /// - /// Operators like factorial `+`, `-`, `*`, `/` are infix binary operators. - /// - /// The argument is a fold function that defines how to combine the operator and two operands into a new - /// expression. It must have the following signature: - /// ```ignore - /// impl Fn(O, O) -> O - /// ``` - #[inline(always)] - fn infix(self, fold: F) -> Infix> - where - F: Fn(O, O) -> O, - Self: Sized, - { - Infix(Operator::new(self, fold)) - } -} - -impl> PrecedenceParserExt for T where I: Stream {} - -/// `NewType` that indicates this type is a prefix operator a [`precedence`] parser. -/// See: [`PrecedenceParserExt::prefix`] -/// -/// Can hold and arbitrary parser, such as a tuple of multiple operator parsers: `(Operator<...>, Operator<...>)` -pub struct Prefix(T); - -/// `NewType` that indicates this type is a postfix operator a [`precedence`] parser. -/// See: [`PrecedenceParserExt::postfix`] -pub struct Postfix(T); - -/// `NewType` that indicates this type is a infix operator within a [`precedence`] parser. -/// See: [`PrecedenceParserExt::infix`] -pub struct Infix(T); - -/// Implementation of the operator parser for the [`precedence`] parser. -pub struct Operator { - // We use two different `ReffCell`s to enable mutable borrowing within the recursion - // while holding a reference to the predicate `op`: - // ``` - // let lhs = ...; - // let op: &ReffCell> = operator.parse_next(i); // calls `operator.parser.borrow_mut().parse_next(i)` - // let rhs = recursion(&operator); - // let result = op.borrow_mut().fold_binary(lhs, rhs); - // ``` - op: RefCell, - parser: RefCell, -} - -impl Operator { - /// Creates a new [`Operator`] from a parser and a predicate - #[inline(always)] - pub fn new(parser: OperatorParser, predicate: OperatorFunc) -> Self { - Self { - op: RefCell::new(predicate), - parser: RefCell::new(parser), - } - } -} - -impl<'s, UO, O, I, P, E> Parser O>, usize), E> for &'s Operator -where - UO: Fn(O) -> O + 'static, - I: Stream + StreamIsPartial, - P: Parser, - E: ParserError, -{ - #[inline(always)] - fn parse_next( - &mut self, - input: &mut I, - ) -> PResult<(&'s RefCell O + 'static>, usize), E> { - let power = self.parser.borrow_mut().parse_next(input)?; - Ok((&self.op, power)) - } -} -impl<'s, BO, O, I, P, E> Parser O>, usize), E> - for &'s Operator -where - BO: Fn(O, O) -> O + 'static, - I: Stream + StreamIsPartial, - P: Parser, - E: ParserError, -{ - #[inline(always)] - fn parse_next( - &mut self, - input: &mut I, - ) -> PResult<(&'s RefCell O + 'static>, usize), E> { - let power = self.parser.borrow_mut().parse_next(input)?; - Ok((&self.op, power)) - } -} - -/// Ability to request a parser of the specified affix from the [`impl Parser`](Parser) object. -pub trait AsPrecedence> { - /// Interprets a parser as a [`PrecedenceParserExt::prefix`] parser that returns an `unary - /// predicate` [`UnaryOp`] and a `binding power` as its parsing result. - #[inline(always)] - fn as_prefix(&self) -> impl Parser Operand>, usize), E> { - fail - } - /// Interprets a parser as a [`PrecedenceParserExt::postfix`] parser that returns an `unary - /// predicate` [`UnaryOp`] and a `binding power` as its parsing result. - #[inline(always)] - fn as_postfix(&self) -> impl Parser Operand>, usize), E> { - fail - } - /// Interprets a parser as a [`PrecedenceParserExt::infix`] parser that returns a `binary - /// predicate` [`BinaryOp`] and a `binding power` as its parsing result. - #[inline(always)] - fn as_infix( - &self, - ) -> impl Parser Operand>, usize), E> { - fail - } -} - -impl AsPrecedence for Prefix> -where - F: Fn(O) -> O + 'static, - I: Stream + StreamIsPartial, - P: Parser, - E: ParserError, -{ - #[inline(always)] - fn as_prefix(&self) -> impl Parser O>, usize), E> { - &self.0 - } -} - -impl AsPrecedence for Postfix> -where - F: Fn(O) -> O + 'static, - I: Stream + StreamIsPartial, - P: Parser, - E: ParserError, -{ - #[inline(always)] - fn as_postfix(&self) -> impl Parser O>, usize), E> { - &self.0 - } -} -impl AsPrecedence for Infix> -where - F: Fn(O, O) -> O + 'static, - I: Stream + StreamIsPartial, - P: Parser, - E: ParserError, -{ - #[inline(always)] - fn as_infix(&self) -> impl Parser O>, usize), E> { - &self.0 - } -} - -macro_rules! impl_parser_for_tuple { - () => {}; - ($head:ident $($X:ident)*) => { - impl_parser_for_tuple!($($X)*); - impl_parser_for_tuple!(~ $head $($X)*); - }; - (~ $($X:ident)+) => { - - impl AsPrecedence for ($($X,)*) - where - I: Stream + StreamIsPartial, - E: ParserError, - $($X: AsPrecedence),* - { - #[inline(always)] - fn as_prefix<'s>( - &'s self, - ) -> impl Parser O>, usize), E> { - Prefix(self) - } - #[inline(always)] - fn as_infix<'s>( - &'s self, - ) -> impl Parser O>, usize), E> { - Infix(self) - } - #[inline(always)] - fn as_postfix<'s>( - &'s self, - ) -> impl Parser O>, usize), E> { - Postfix(self) - } - } - - impl<'s, I, O: 'static, E, $($X),*> Parser O>, usize), E> - for Prefix<&'s ($($X,)*)> - where - I: Stream + StreamIsPartial, - E: ParserError, - $($X: AsPrecedence),* - - { - #[inline(always)] - fn parse_next(&mut self, input: &mut I) -> PResult<(&'s RefCell O>, usize), E> { - #[allow(non_snake_case)] - let ($($X,)*) = self.0; - alt(($($X.as_prefix(),)*)).parse_next(input) - } - } - #[allow(unused_variables, non_snake_case)] - impl<'s, I, O: 'static, E, $($X),*> Parser O>, usize), E> - for Postfix<&'s ($($X,)*)> - where - I: Stream + StreamIsPartial, - E: ParserError, - $($X: AsPrecedence),* - { - #[inline(always)] - fn parse_next(&mut self, input: &mut I) -> PResult<(&'s RefCell O>, usize), E> { - let ($($X,)*) = self.0; - alt(($($X.as_postfix(),)*)).parse_next(input) - } - } - #[allow(unused_variables, non_snake_case)] - impl<'s, I, O: 'static, E, $($X),*> Parser O>, usize), E> - for Infix<&'s ($($X,)*)> - where - I: Stream + StreamIsPartial, - E: ParserError, - $($X: AsPrecedence),* - { - #[inline(always)] - fn parse_next(&mut self, input: &mut I) -> PResult<(&'s RefCell O>, usize), E> { - let ($($X,)*) = self.0; - alt(($($X.as_infix(),)*)).parse_next(input) - } - } - - }; -} - -impl_parser_for_tuple!(P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 P16 P17 P18 P19 P20 P21); - - #[cfg(test)] mod tests { use crate::ascii::{digit1, space0}; - use crate::combinator::delimited; + use crate::combinator::{delimited, empty, fail, peek}; + use crate::dispatch; use crate::error::ContextError; + use crate::token::any; use super::*; @@ -405,16 +136,32 @@ mod tests { fn parser<'i>() -> impl Parser<&'i str, i32, ContextError> { move |i: &mut &str| { precedence( - delimited(space0, digit1.try_map(|d: &str| d.parse::()), space0), - ( - "-".value(2).prefix(|a: i32| -a), - "+".value(2).prefix(|a| a), - "!".value(2).postfix(factorial), - "+".value(0).infix(|a, b| a + b), - "-".value(0).infix(|a, b| a + b), - "*".value(1).infix(|a, b| a * b), - "/".value(1).infix(|a, b| a / b), + delimited( + space0, + dispatch! {peek(any); + '(' => delimited('(', parser(), ')'), + _ => digit1.parse_to::() + }, + space0, ), + dispatch! {any; + '+' => empty.value((9, (&|a| a) as _)), + '-' => empty.value((9, (&|a: i32| -a) as _)), + _ => fail + }, + dispatch! {any; + '!' => empty.value((9, &factorial as _)), + _ => fail + }, + dispatch! {any; + '+' => empty.value((5, 6, (&|a, b| a + b) as _)), + '-' => empty.value((5, 6, (&|a, b| a - b) as _)), + '*' => empty.value((7, 8, (&|a, b| a * b) as _)), + '/' => empty.value((7, 8, (&|a, b| a / b) as _)), + '%' => empty.value((7, 8, (&|a, b| a % b) as _)), + '^' => empty.value((9, 10, (&|a, b| a ^ b) as _)), + _ => fail + }, ) .parse_next(i) } From 0273a29db517e2987f6a8142b174e675b108f533 Mon Sep 17 00:00:00 2001 From: Igor Date: Sat, 16 Nov 2024 13:17:33 +0400 Subject: [PATCH 10/26] remove wrong and long doc for now I will write the documentation later --- src/combinator/precedence.rs | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/combinator/precedence.rs b/src/combinator/precedence.rs index 8a16dfb5e..f39047cbf 100644 --- a/src/combinator/precedence.rs +++ b/src/combinator/precedence.rs @@ -5,16 +5,7 @@ use crate::{ PResult, Parser, }; -/// Constructs an expression parser from an operand parser and operator parsers to parse an -/// arbitrary expression separated by `prefix`, `postfix`, and `infix` operators of various precedence. -/// -/// This technique is powerful and recommended for parsing expressions. -/// -/// The implementation uses [Pratt parsing](https://en.wikipedia.org/wiki/Operator-precedence_parser#Pratt_parsing). -/// This algorithm is similar to the [Shunting Yard](https://en.wikipedia.org/wiki/Shunting_yard_algorithm) algorithm -/// in that both are linear, both use precedence and binding power, and both serve the same purpose. -/// However, the `Shunting Yard` algorithm additionally uses `left` and `right` associativity, -/// while `Pratt` parsing only relies on binding power. +/// Parses an expression based on operator precedence. #[doc(alias = "pratt")] #[doc(alias = "separated")] #[doc(alias = "shunting_yard")] From f218911f7549692b73ad5417195e303dac250cb0 Mon Sep 17 00:00:00 2001 From: Igor Date: Sun, 17 Nov 2024 01:11:59 +0400 Subject: [PATCH 11/26] fix: precedence for associativity, remove `trace()` - require explicit `trace` for operators - fix associativity handling for infix operators: `1 + 2 + 3` should be `(1 + 2) + 3` and not `1 + (2 + 3)` --- src/combinator/precedence.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/combinator/precedence.rs b/src/combinator/precedence.rs index f39047cbf..6ccadb0f3 100644 --- a/src/combinator/precedence.rs +++ b/src/combinator/precedence.rs @@ -57,7 +57,7 @@ where ParsePostfix: Parser Operand), E>, E: ParserError, { - let operand = trace("operand", opt(parse_operand.by_ref())).parse_next(i)?; + let operand = opt(parse_operand.by_ref()).parse_next(i)?; let mut operand = if let Some(operand) = operand { operand } else { @@ -75,9 +75,9 @@ where 'parse: while i.eof_offset() > 0 { // Postfix unary operators let start = i.checkpoint(); - if let Some((power, fold_postfix)) = - trace("postfix", opt(postfix.by_ref())).parse_next(i)? - { + if let Some((power, fold_postfix)) = opt(postfix.by_ref()).parse_next(i)? { + // control precedence over the prefix e.g.: + // `--(i++)` or `(--i)++` if power < min_power { i.reset(&start); break; @@ -91,11 +91,11 @@ where let start = i.checkpoint(); let parse_result = opt(infix.by_ref()).parse_next(i)?; if let Some((lpower, rpower, fold_infix)) = parse_result { - if rpower < min_power { + if lpower < min_power { i.reset(&start); break; } - let rhs = precedence_impl(i, parse_operand, prefix, postfix, infix, lpower)?; + let rhs = precedence_impl(i, parse_operand, prefix, postfix, infix, rpower)?; operand = fold_infix(operand, rhs); continue 'parse; From 3d7ef41df6bb11e506ed855e94d223981574e336 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 18 Nov 2024 01:32:52 +0400 Subject: [PATCH 12/26] switch from `&dyn Fn(O) -> O` to `fn(O) -> O` --- src/combinator/precedence.rs | 43 ++++++++++++++---------------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/src/combinator/precedence.rs b/src/combinator/precedence.rs index 6ccadb0f3..8fa1091e7 100644 --- a/src/combinator/precedence.rs +++ b/src/combinator/precedence.rs @@ -11,7 +11,7 @@ use crate::{ #[doc(alias = "shunting_yard")] #[doc(alias = "precedence_climbing")] #[inline(always)] -pub fn precedence<'i, I, ParseOperand, ParseInfix, ParsePrefix, ParsePostfix, Operand: 'static, E>( +pub fn precedence( mut operand: ParseOperand, mut prefix: ParsePrefix, mut postfix: ParsePostfix, @@ -20,9 +20,9 @@ pub fn precedence<'i, I, ParseOperand, ParseInfix, ParsePrefix, ParsePostfix, Op where I: Stream + StreamIsPartial, ParseOperand: Parser, - ParseInfix: Parser Operand), E>, - ParsePrefix: Parser Operand), E>, - ParsePostfix: Parser Operand), E>, + ParseInfix: Parser Operand), E>, + ParsePrefix: Parser Operand), E>, + ParsePostfix: Parser Operand), E>, E: ParserError, { trace("precedence", move |i: &mut I| { @@ -32,16 +32,7 @@ where } // recursive function -fn precedence_impl< - 'i, - I, - ParseOperand, - ParseInfix, - ParsePrefix, - ParsePostfix, - Operand: 'static, - E, ->( +fn precedence_impl( i: &mut I, parse_operand: &mut ParseOperand, prefix: &mut ParsePrefix, @@ -52,9 +43,9 @@ fn precedence_impl< where I: Stream + StreamIsPartial, ParseOperand: Parser, - ParseInfix: Parser Operand), E>, - ParsePrefix: Parser Operand), E>, - ParsePostfix: Parser Operand), E>, + ParseInfix: Parser Operand), E>, + ParsePrefix: Parser Operand), E>, + ParsePostfix: Parser Operand), E>, E: ParserError, { let operand = opt(parse_operand.by_ref()).parse_next(i)?; @@ -136,21 +127,21 @@ mod tests { space0, ), dispatch! {any; - '+' => empty.value((9, (&|a| a) as _)), - '-' => empty.value((9, (&|a: i32| -a) as _)), + '+' => empty.value((9, (|a| a) as _)), + '-' => empty.value((9, (|a: i32| -a) as _)), _ => fail }, dispatch! {any; - '!' => empty.value((9, &factorial as _)), + '!' => empty.value((9, factorial as _)), _ => fail }, dispatch! {any; - '+' => empty.value((5, 6, (&|a, b| a + b) as _)), - '-' => empty.value((5, 6, (&|a, b| a - b) as _)), - '*' => empty.value((7, 8, (&|a, b| a * b) as _)), - '/' => empty.value((7, 8, (&|a, b| a / b) as _)), - '%' => empty.value((7, 8, (&|a, b| a % b) as _)), - '^' => empty.value((9, 10, (&|a, b| a ^ b) as _)), + '+' => empty.value((5, 6, (|a, b| a + b) as _ )), + '-' => empty.value((5, 6, (|a, b| a - b) as _)), + '*' => empty.value((7, 8, (|a, b| a * b) as _)), + '/' => empty.value((7, 8, (|a, b| a / b) as _)), + '%' => empty.value((7, 8, (|a, b| a % b) as _)), + '^' => empty.value((9, 10, (|a, b| a ^ b) as _)), _ => fail }, ) From a6cbc1a1fbb751f0a20151ee6da9fd89b35b6ba0 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 18 Nov 2024 01:37:41 +0400 Subject: [PATCH 13/26] feat: pass Input into operator closures --- src/combinator/precedence.rs | 38 ++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/combinator/precedence.rs b/src/combinator/precedence.rs index 8fa1091e7..4276272a5 100644 --- a/src/combinator/precedence.rs +++ b/src/combinator/precedence.rs @@ -20,9 +20,9 @@ pub fn precedence, - ParseInfix: Parser Operand), E>, - ParsePrefix: Parser Operand), E>, - ParsePostfix: Parser Operand), E>, + ParseInfix: Parser Operand), E>, + ParsePrefix: Parser Operand), E>, + ParsePostfix: Parser Operand), E>, E: ParserError, { trace("precedence", move |i: &mut I| { @@ -43,9 +43,9 @@ fn precedence_impl, - ParseInfix: Parser Operand), E>, - ParsePrefix: Parser Operand), E>, - ParsePostfix: Parser Operand), E>, + ParseInfix: Parser Operand), E>, + ParsePrefix: Parser Operand), E>, + ParsePostfix: Parser Operand), E>, E: ParserError, { let operand = opt(parse_operand.by_ref()).parse_next(i)?; @@ -60,7 +60,7 @@ where return Err(ErrMode::assert(i, "`prefix` parsers must always consume")); } let operand = precedence_impl(i, parse_operand, prefix, postfix, infix, power)?; - fold_prefix(operand) + fold_prefix(i, operand) }; 'parse: while i.eof_offset() > 0 { @@ -73,7 +73,7 @@ where i.reset(&start); break; } - operand = fold_postfix(operand); + operand = fold_postfix(i, operand); continue 'parse; } @@ -87,7 +87,7 @@ where break; } let rhs = precedence_impl(i, parse_operand, prefix, postfix, infix, rpower)?; - operand = fold_infix(operand, rhs); + operand = fold_infix(i, operand, rhs); continue 'parse; } @@ -108,11 +108,11 @@ mod tests { use super::*; - fn factorial(x: i32) -> i32 { + fn factorial(i: &mut &str, x: i32) -> i32 { if x == 0 { 1 } else { - x * factorial(x - 1) + x * factorial(i, x - 1) } } fn parser<'i>() -> impl Parser<&'i str, i32, ContextError> { @@ -127,8 +127,8 @@ mod tests { space0, ), dispatch! {any; - '+' => empty.value((9, (|a| a) as _)), - '-' => empty.value((9, (|a: i32| -a) as _)), + '+' => empty.value((9, (|_: &mut _, a| a) as _)), + '-' => empty.value((9, (|_: &mut _, a: i32| -a) as _)), _ => fail }, dispatch! {any; @@ -136,12 +136,12 @@ mod tests { _ => fail }, dispatch! {any; - '+' => empty.value((5, 6, (|a, b| a + b) as _ )), - '-' => empty.value((5, 6, (|a, b| a - b) as _)), - '*' => empty.value((7, 8, (|a, b| a * b) as _)), - '/' => empty.value((7, 8, (|a, b| a / b) as _)), - '%' => empty.value((7, 8, (|a, b| a % b) as _)), - '^' => empty.value((9, 10, (|a, b| a ^ b) as _)), + '+' => empty.value((5, 6, (|_: &mut _, a, b| a + b) as _ )), + '-' => empty.value((5, 6, (|_: &mut _, a, b| a - b) as _)), + '*' => empty.value((7, 8, (|_: &mut _, a, b| a * b) as _)), + '/' => empty.value((7, 8, (|_: &mut _, a, b| a / b) as _)), + '%' => empty.value((7, 8, (|_: &mut _, a, b| a % b) as _)), + '^' => empty.value((9, 10, (|_: &mut _, a, b| a ^ b) as _)), _ => fail }, ) From 29b64faf513bb9498e2f461f0ee7e58075bb598e Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 18 Nov 2024 01:43:15 +0400 Subject: [PATCH 14/26] add `trace` for `tests` parser --- src/combinator/precedence.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/combinator/precedence.rs b/src/combinator/precedence.rs index 4276272a5..6ae723e94 100644 --- a/src/combinator/precedence.rs +++ b/src/combinator/precedence.rs @@ -118,24 +118,24 @@ mod tests { fn parser<'i>() -> impl Parser<&'i str, i32, ContextError> { move |i: &mut &str| { precedence( - delimited( + trace("operand", delimited( space0, dispatch! {peek(any); '(' => delimited('(', parser(), ')'), _ => digit1.parse_to::() }, space0, - ), - dispatch! {any; + )), + trace("prefix", dispatch! {any; '+' => empty.value((9, (|_: &mut _, a| a) as _)), '-' => empty.value((9, (|_: &mut _, a: i32| -a) as _)), _ => fail - }, - dispatch! {any; + }), + trace("postfix", dispatch! {any; '!' => empty.value((9, factorial as _)), _ => fail - }, - dispatch! {any; + }), + trace("infix", dispatch! {any; '+' => empty.value((5, 6, (|_: &mut _, a, b| a + b) as _ )), '-' => empty.value((5, 6, (|_: &mut _, a, b| a - b) as _)), '*' => empty.value((7, 8, (|_: &mut _, a, b| a * b) as _)), @@ -143,7 +143,7 @@ mod tests { '%' => empty.value((7, 8, (|_: &mut _, a, b| a % b) as _)), '^' => empty.value((9, 10, (|_: &mut _, a, b| a ^ b) as _)), _ => fail - }, + }), ) .parse_next(i) } From b31a3a3d86b8d4486030e1e30ea895c17dfe5be1 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 18 Nov 2024 20:34:55 +0400 Subject: [PATCH 15/26] feat: operator closures must return PResult --- src/combinator/precedence.rs | 100 ++++++++++++++++++++++------------- 1 file changed, 64 insertions(+), 36 deletions(-) diff --git a/src/combinator/precedence.rs b/src/combinator/precedence.rs index 6ae723e94..a04e5d5df 100644 --- a/src/combinator/precedence.rs +++ b/src/combinator/precedence.rs @@ -20,9 +20,17 @@ pub fn precedence, - ParseInfix: Parser Operand), E>, - ParsePrefix: Parser Operand), E>, - ParsePostfix: Parser Operand), E>, + ParseInfix: Parser< + I, + ( + usize, + usize, + fn(&mut I, Operand, Operand) -> PResult, + ), + E, + >, + ParsePrefix: Parser PResult), E>, + ParsePostfix: Parser PResult), E>, E: ParserError, { trace("precedence", move |i: &mut I| { @@ -43,9 +51,17 @@ fn precedence_impl, - ParseInfix: Parser Operand), E>, - ParsePrefix: Parser Operand), E>, - ParsePostfix: Parser Operand), E>, + ParseInfix: Parser< + I, + ( + usize, + usize, + fn(&mut I, Operand, Operand) -> PResult, + ), + E, + >, + ParsePrefix: Parser PResult), E>, + ParsePostfix: Parser PResult), E>, E: ParserError, { let operand = opt(parse_operand.by_ref()).parse_next(i)?; @@ -60,7 +76,7 @@ where return Err(ErrMode::assert(i, "`prefix` parsers must always consume")); } let operand = precedence_impl(i, parse_operand, prefix, postfix, infix, power)?; - fold_prefix(i, operand) + fold_prefix(i, operand)? }; 'parse: while i.eof_offset() > 0 { @@ -73,7 +89,7 @@ where i.reset(&start); break; } - operand = fold_postfix(i, operand); + operand = fold_postfix(i, operand)?; continue 'parse; } @@ -87,7 +103,7 @@ where break; } let rhs = precedence_impl(i, parse_operand, prefix, postfix, infix, rpower)?; - operand = fold_infix(i, operand, rhs); + operand = fold_infix(i, operand, rhs)?; continue 'parse; } @@ -108,42 +124,54 @@ mod tests { use super::*; - fn factorial(i: &mut &str, x: i32) -> i32 { + fn factorial(x: i32) -> i32 { if x == 0 { 1 } else { - x * factorial(i, x - 1) + x * factorial(x - 1) } } fn parser<'i>() -> impl Parser<&'i str, i32, ContextError> { move |i: &mut &str| { precedence( - trace("operand", delimited( - space0, - dispatch! {peek(any); - '(' => delimited('(', parser(), ')'), - _ => digit1.parse_to::() + trace( + "operand", + delimited( + space0, + dispatch! {peek(any); + '(' => delimited('(', parser(), ')'), + _ => digit1.parse_to::() + }, + space0, + ), + ), + trace( + "prefix", + dispatch! {any; + '+' => empty.value((9, (|_: &mut _, a| Ok(a)) as _)), + '-' => empty.value((9, (|_: &mut _, a: i32| Ok(-a)) as _)), + _ => fail }, - space0, - )), - trace("prefix", dispatch! {any; - '+' => empty.value((9, (|_: &mut _, a| a) as _)), - '-' => empty.value((9, (|_: &mut _, a: i32| -a) as _)), - _ => fail - }), - trace("postfix", dispatch! {any; - '!' => empty.value((9, factorial as _)), - _ => fail - }), - trace("infix", dispatch! {any; - '+' => empty.value((5, 6, (|_: &mut _, a, b| a + b) as _ )), - '-' => empty.value((5, 6, (|_: &mut _, a, b| a - b) as _)), - '*' => empty.value((7, 8, (|_: &mut _, a, b| a * b) as _)), - '/' => empty.value((7, 8, (|_: &mut _, a, b| a / b) as _)), - '%' => empty.value((7, 8, (|_: &mut _, a, b| a % b) as _)), - '^' => empty.value((9, 10, (|_: &mut _, a, b| a ^ b) as _)), - _ => fail - }), + ), + trace( + "postfix", + dispatch! {any; + '!' => empty.value((9, (|_: &mut _, a| {Ok(factorial(a))}) as _)), + _ => fail + }, + ), + trace( + "infix", + dispatch! {any; + '+' => empty.value((5, 6, (|_: &mut _, a, b| Ok(a + b)) as _ )), + '-' => empty.value((5, 6, (|_: &mut _, a, b| Ok(a - b)) as _)), + '*' => empty.value((7, 8, (|_: &mut _, a, b| Ok(a * b)) as _)), + '/' => empty.value((7, 8, (|_: &mut _, a, b| Ok(a / b)) as _)), + '%' => empty.value((7, 8, (|_: &mut _, a, b| Ok(a % b)) as _)), + '^' => empty.value((9, 10, (|_: &mut _, a, b| Ok(a ^ b)) as _)), + _ => fail + }, + ), ) .parse_next(i) } From 33c82f35c07928b044387bf84bd8f397e064bcd3 Mon Sep 17 00:00:00 2001 From: Igor Date: Tue, 19 Nov 2024 00:19:07 +0400 Subject: [PATCH 16/26] feat: allow the user to specify starting power --- src/combinator/precedence.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/combinator/precedence.rs b/src/combinator/precedence.rs index a04e5d5df..4f7c993a8 100644 --- a/src/combinator/precedence.rs +++ b/src/combinator/precedence.rs @@ -12,6 +12,7 @@ use crate::{ #[doc(alias = "precedence_climbing")] #[inline(always)] pub fn precedence( + start_power: usize, mut operand: ParseOperand, mut prefix: ParsePrefix, mut postfix: ParsePostfix, @@ -34,7 +35,7 @@ where E: ParserError, { trace("precedence", move |i: &mut I| { - let result = precedence_impl(i, &mut operand, &mut prefix, &mut postfix, &mut infix, 0)?; + let result = precedence_impl(i, &mut operand, &mut prefix, &mut postfix, &mut infix, start_power)?; Ok(result) }) } @@ -133,7 +134,7 @@ mod tests { } fn parser<'i>() -> impl Parser<&'i str, i32, ContextError> { move |i: &mut &str| { - precedence( + precedence(0, trace( "operand", delimited( From 040dd8503aecf1046a046792193769a18671a244 Mon Sep 17 00:00:00 2001 From: Igor Date: Tue, 19 Nov 2024 12:29:15 +0400 Subject: [PATCH 17/26] feat: enum `Assoc` for infix operators. Add `Neither` associativity --- src/combinator/precedence.rs | 90 +++++++++++++++++++++--------------- 1 file changed, 52 insertions(+), 38 deletions(-) diff --git a/src/combinator/precedence.rs b/src/combinator/precedence.rs index 4f7c993a8..72419fedb 100644 --- a/src/combinator/precedence.rs +++ b/src/combinator/precedence.rs @@ -12,7 +12,7 @@ use crate::{ #[doc(alias = "precedence_climbing")] #[inline(always)] pub fn precedence( - start_power: usize, + start_power: isize, mut operand: ParseOperand, mut prefix: ParsePrefix, mut postfix: ParsePostfix, @@ -21,25 +21,31 @@ pub fn precedence, - ParseInfix: Parser< - I, - ( - usize, - usize, - fn(&mut I, Operand, Operand) -> PResult, - ), - E, - >, - ParsePrefix: Parser PResult), E>, - ParsePostfix: Parser PResult), E>, + ParseInfix: Parser PResult), E>, + ParsePrefix: Parser PResult), E>, + ParsePostfix: Parser PResult), E>, E: ParserError, { trace("precedence", move |i: &mut I| { - let result = precedence_impl(i, &mut operand, &mut prefix, &mut postfix, &mut infix, start_power)?; + let result = precedence_impl( + i, + &mut operand, + &mut prefix, + &mut postfix, + &mut infix, + start_power, + )?; Ok(result) }) } +#[derive(Debug, Clone, Copy)] +pub enum Assoc { + Left(isize), + Right(isize), + Neither(isize), +} + // recursive function fn precedence_impl( i: &mut I, @@ -47,22 +53,14 @@ fn precedence_impl PResult where I: Stream + StreamIsPartial, ParseOperand: Parser, - ParseInfix: Parser< - I, - ( - usize, - usize, - fn(&mut I, Operand, Operand) -> PResult, - ), - E, - >, - ParsePrefix: Parser PResult), E>, - ParsePostfix: Parser PResult), E>, + ParseInfix: Parser PResult), E>, + ParsePrefix: Parser PResult), E>, + ParsePostfix: Parser PResult), E>, E: ParserError, { let operand = opt(parse_operand.by_ref()).parse_next(i)?; @@ -80,6 +78,11 @@ where fold_prefix(i, operand)? }; + // A variable to stop the `'parse` loop when `Assoc::Neither` with the same + // precedence is encountered e.g. `a == b == c`. `Assoc::Neither` has similar + // associativity rules as `Assoc::Left`, but we stop parsing when the next operator + // is the same as the current one. + let mut prev_op_is_neither = None; 'parse: while i.eof_offset() > 0 { // Postfix unary operators let start = i.checkpoint(); @@ -88,7 +91,7 @@ where // `--(i++)` or `(--i)++` if power < min_power { i.reset(&start); - break; + break 'parse; } operand = fold_postfix(i, operand)?; @@ -98,11 +101,21 @@ where // Infix binary operators let start = i.checkpoint(); let parse_result = opt(infix.by_ref()).parse_next(i)?; - if let Some((lpower, rpower, fold_infix)) = parse_result { - if lpower < min_power { + if let Some((assoc, fold_infix)) = parse_result { + let mut is_neither = None; + let (lpower, rpower) = match assoc { + Assoc::Right(p) => (p, p - 1), + Assoc::Left(p) => (p, p), + Assoc::Neither(p) => { + is_neither = Some(p); + (p, p) + } + }; + if lpower <= min_power || prev_op_is_neither.is_some_and(|p| lpower == p) { i.reset(&start); - break; + break 'parse; } + prev_op_is_neither = is_neither; let rhs = precedence_impl(i, parse_operand, prefix, postfix, infix, rpower)?; operand = fold_infix(i, operand, rhs)?; @@ -134,7 +147,8 @@ mod tests { } fn parser<'i>() -> impl Parser<&'i str, i32, ContextError> { move |i: &mut &str| { - precedence(0, + precedence( + 0, trace( "operand", delimited( @@ -164,12 +178,12 @@ mod tests { trace( "infix", dispatch! {any; - '+' => empty.value((5, 6, (|_: &mut _, a, b| Ok(a + b)) as _ )), - '-' => empty.value((5, 6, (|_: &mut _, a, b| Ok(a - b)) as _)), - '*' => empty.value((7, 8, (|_: &mut _, a, b| Ok(a * b)) as _)), - '/' => empty.value((7, 8, (|_: &mut _, a, b| Ok(a / b)) as _)), - '%' => empty.value((7, 8, (|_: &mut _, a, b| Ok(a % b)) as _)), - '^' => empty.value((9, 10, (|_: &mut _, a, b| Ok(a ^ b)) as _)), + '+' => empty.value((Assoc::Left(5), (|_: &mut _, a, b| Ok(a + b)) as _ )), + '-' => empty.value((Assoc::Left(5), (|_: &mut _, a, b| Ok(a - b)) as _)), + '*' => empty.value((Assoc::Left(7), (|_: &mut _, a, b| Ok(a * b)) as _)), + '/' => empty.value((Assoc::Left(7), (|_: &mut _, a, b| Ok(a / b)) as _)), + '%' => empty.value((Assoc::Left(7), (|_: &mut _, a, b| Ok(a % b)) as _)), + '^' => empty.value((Assoc::Right(9), (|_: &mut _, a, b| Ok(a ^ b)) as _)), _ => fail }, ), @@ -180,8 +194,8 @@ mod tests { #[test] fn test_precedence() { - assert_eq!(parser().parse("-3!+-3 * 4"), Ok(-18)); - assert_eq!(parser().parse("+2 + 3 * 4"), Ok(14)); + // assert_eq!(parser().parse("-3!+-3 * 4"), Ok(-18)); + // assert_eq!(parser().parse("+2 + 3 * 4"), Ok(14)); assert_eq!(parser().parse("2 * 3+4"), Ok(10)); } #[test] From 6d88dffb2b228c8b6ac41945bf8d854ec1128aa0 Mon Sep 17 00:00:00 2001 From: Igor Date: Tue, 19 Nov 2024 12:49:34 +0400 Subject: [PATCH 18/26] fix: switch to i64, fix precedence checking --- src/combinator/precedence.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/combinator/precedence.rs b/src/combinator/precedence.rs index 72419fedb..4566694d4 100644 --- a/src/combinator/precedence.rs +++ b/src/combinator/precedence.rs @@ -12,7 +12,7 @@ use crate::{ #[doc(alias = "precedence_climbing")] #[inline(always)] pub fn precedence( - start_power: isize, + start_power: i64, mut operand: ParseOperand, mut prefix: ParsePrefix, mut postfix: ParsePostfix, @@ -22,8 +22,8 @@ where I: Stream + StreamIsPartial, ParseOperand: Parser, ParseInfix: Parser PResult), E>, - ParsePrefix: Parser PResult), E>, - ParsePostfix: Parser PResult), E>, + ParsePrefix: Parser PResult), E>, + ParsePostfix: Parser PResult), E>, E: ParserError, { trace("precedence", move |i: &mut I| { @@ -41,9 +41,9 @@ where #[derive(Debug, Clone, Copy)] pub enum Assoc { - Left(isize), - Right(isize), - Neither(isize), + Left(i64), + Right(i64), + Neither(i64), } // recursive function @@ -53,14 +53,14 @@ fn precedence_impl PResult where I: Stream + StreamIsPartial, ParseOperand: Parser, ParseInfix: Parser PResult), E>, - ParsePrefix: Parser PResult), E>, - ParsePostfix: Parser PResult), E>, + ParsePrefix: Parser PResult), E>, + ParsePostfix: Parser PResult), E>, E: ParserError, { let operand = opt(parse_operand.by_ref()).parse_next(i)?; @@ -105,13 +105,13 @@ where let mut is_neither = None; let (lpower, rpower) = match assoc { Assoc::Right(p) => (p, p - 1), - Assoc::Left(p) => (p, p), + Assoc::Left(p) => (p, p + 1), Assoc::Neither(p) => { is_neither = Some(p); - (p, p) + (p, p + 1) } }; - if lpower <= min_power || prev_op_is_neither.is_some_and(|p| lpower == p) { + if lpower < min_power || prev_op_is_neither.is_some_and(|p| lpower == p) { i.reset(&start); break 'parse; } From 161f9da1ea849c0786026cb3209c533b12f4a1e5 Mon Sep 17 00:00:00 2001 From: Igor Date: Wed, 20 Nov 2024 00:36:45 +0400 Subject: [PATCH 19/26] fix: remove 'static constraint from `Operand` --- src/combinator/precedence.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/combinator/precedence.rs b/src/combinator/precedence.rs index 4566694d4..a9758ac97 100644 --- a/src/combinator/precedence.rs +++ b/src/combinator/precedence.rs @@ -11,7 +11,7 @@ use crate::{ #[doc(alias = "shunting_yard")] #[doc(alias = "precedence_climbing")] #[inline(always)] -pub fn precedence( +pub fn precedence( start_power: i64, mut operand: ParseOperand, mut prefix: ParsePrefix, @@ -47,7 +47,7 @@ pub enum Assoc { } // recursive function -fn precedence_impl( +fn precedence_impl( i: &mut I, parse_operand: &mut ParseOperand, prefix: &mut ParsePrefix, From d53a32ef9e7e4774c282dc395c6f6608b934d7b6 Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 21 Nov 2024 00:36:16 +0400 Subject: [PATCH 20/26] refactor: rename to `expression.rs` --- src/combinator/{precedence.rs => expression.rs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/combinator/{precedence.rs => expression.rs} (100%) diff --git a/src/combinator/precedence.rs b/src/combinator/expression.rs similarity index 100% rename from src/combinator/precedence.rs rename to src/combinator/expression.rs From 4f690db449a420dbce4448423583602633c5e8a3 Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 21 Nov 2024 00:38:33 +0400 Subject: [PATCH 21/26] refactor: rename to `fn expression` --- src/combinator/expression.rs | 16 ++++++++-------- src/combinator/mod.rs | 3 +-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/combinator/expression.rs b/src/combinator/expression.rs index a9758ac97..03d98a4a2 100644 --- a/src/combinator/expression.rs +++ b/src/combinator/expression.rs @@ -11,7 +11,7 @@ use crate::{ #[doc(alias = "shunting_yard")] #[doc(alias = "precedence_climbing")] #[inline(always)] -pub fn precedence( +pub fn expression( start_power: i64, mut operand: ParseOperand, mut prefix: ParsePrefix, @@ -26,8 +26,8 @@ where ParsePostfix: Parser PResult), E>, E: ParserError, { - trace("precedence", move |i: &mut I| { - let result = precedence_impl( + trace("expression", move |i: &mut I| { + let result = expression_impl( i, &mut operand, &mut prefix, @@ -47,7 +47,7 @@ pub enum Assoc { } // recursive function -fn precedence_impl( +fn expression_impl( i: &mut I, parse_operand: &mut ParseOperand, prefix: &mut ParsePrefix, @@ -74,7 +74,7 @@ where if i.eof_offset() == len { return Err(ErrMode::assert(i, "`prefix` parsers must always consume")); } - let operand = precedence_impl(i, parse_operand, prefix, postfix, infix, power)?; + let operand = expression_impl(i, parse_operand, prefix, postfix, infix, power)?; fold_prefix(i, operand)? }; @@ -116,7 +116,7 @@ where break 'parse; } prev_op_is_neither = is_neither; - let rhs = precedence_impl(i, parse_operand, prefix, postfix, infix, rpower)?; + let rhs = expression_impl(i, parse_operand, prefix, postfix, infix, rpower)?; operand = fold_infix(i, operand, rhs)?; continue 'parse; @@ -147,7 +147,7 @@ mod tests { } fn parser<'i>() -> impl Parser<&'i str, i32, ContextError> { move |i: &mut &str| { - precedence( + expression( 0, trace( "operand", @@ -193,7 +193,7 @@ mod tests { } #[test] - fn test_precedence() { + fn test_expression() { // assert_eq!(parser().parse("-3!+-3 * 4"), Ok(-18)); // assert_eq!(parser().parse("+2 + 3 * 4"), Ok(14)); assert_eq!(parser().parse("2 * 3+4"), Ok(10)); diff --git a/src/combinator/mod.rs b/src/combinator/mod.rs index 574c906bf..3364942d5 100644 --- a/src/combinator/mod.rs +++ b/src/combinator/mod.rs @@ -166,7 +166,7 @@ mod multi; mod parser; mod sequence; -pub mod precedence; +pub mod expression; #[cfg(test)] mod tests; @@ -176,7 +176,6 @@ pub use self::core::*; pub use self::debug::*; pub use self::multi::*; pub use self::parser::*; -pub use self::precedence::*; pub use self::sequence::*; #[allow(unused_imports)] From 44546f21aa7e95c65196abe5244aa853952b3d8b Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 21 Nov 2024 13:29:29 +0400 Subject: [PATCH 22/26] feat: new api - builder pattern - `Prefix` `Postfix` `Infix` struct tuples --- src/combinator/expression.rs | 342 ++++++++++++++++++++++++----------- 1 file changed, 234 insertions(+), 108 deletions(-) diff --git a/src/combinator/expression.rs b/src/combinator/expression.rs index 03d98a4a2..2f5ebfebc 100644 --- a/src/combinator/expression.rs +++ b/src/combinator/expression.rs @@ -1,3 +1,5 @@ +use core::marker::PhantomData; + use crate::{ combinator::{opt, trace}, error::{ErrMode, ParserError}, @@ -5,71 +7,174 @@ use crate::{ PResult, Parser, }; +use super::{empty, fail}; + /// Parses an expression based on operator precedence. #[doc(alias = "pratt")] #[doc(alias = "separated")] #[doc(alias = "shunting_yard")] #[doc(alias = "precedence_climbing")] #[inline(always)] -pub fn expression( +pub fn expression( start_power: i64, - mut operand: ParseOperand, - mut prefix: ParsePrefix, - mut postfix: ParsePostfix, - mut infix: ParseInfix, -) -> impl Parser + parse_operand: ParseOperand, +) -> Expression< + I, + O, + ParseOperand, + impl Parser, E>, + impl Parser, E>, + impl Parser, E>, + E, +> +where + I: Stream + StreamIsPartial, + ParseOperand: Parser, + E: ParserError, +{ + Expression { + start_power, + parse_operand, + parse_prefix: fail, + parse_postfix: fail, + parse_infix: fail, + i: Default::default(), + o: Default::default(), + e: Default::default(), + } +} + +pub struct Expression +where + I: Stream + StreamIsPartial, + ParseOperand: Parser, + E: ParserError, +{ + start_power: i64, + parse_operand: ParseOperand, + parse_prefix: Pre, + parse_postfix: Post, + parse_infix: Pix, + i: PhantomData, + o: PhantomData, + e: PhantomData, +} + +impl Expression where + ParseOperand: Parser, I: Stream + StreamIsPartial, - ParseOperand: Parser, - ParseInfix: Parser PResult), E>, - ParsePrefix: Parser PResult), E>, - ParsePostfix: Parser PResult), E>, E: ParserError, { - trace("expression", move |i: &mut I| { - let result = expression_impl( - i, - &mut operand, - &mut prefix, - &mut postfix, - &mut infix, - start_power, - )?; - Ok(result) - }) + #[inline(always)] + pub fn prefix( + self, + parser: NewParsePrefix, + ) -> Expression + where + NewParsePrefix: Parser, E>, + { + Expression { + start_power: self.start_power, + parse_operand: self.parse_operand, + parse_prefix: parser, + parse_postfix: self.parse_postfix, + parse_infix: self.parse_infix, + i: Default::default(), + o: Default::default(), + e: Default::default(), + } + } + + #[inline(always)] + pub fn postfix( + self, + parser: NewParsePostfix, + ) -> Expression + where + NewParsePostfix: Parser, E>, + { + Expression { + start_power: self.start_power, + parse_operand: self.parse_operand, + parse_prefix: self.parse_prefix, + parse_postfix: parser, + parse_infix: self.parse_infix, + i: Default::default(), + o: Default::default(), + e: Default::default(), + } + } + + #[inline(always)] + pub fn infix( + self, + parser: NewParseInfix, + ) -> Expression + where + NewParseInfix: Parser, E>, + { + Expression { + start_power: self.start_power, + parse_operand: self.parse_operand, + parse_prefix: self.parse_prefix, + parse_postfix: self.parse_postfix, + parse_infix: parser, + i: Default::default(), + o: Default::default(), + e: Default::default(), + } + } } -#[derive(Debug, Clone, Copy)] -pub enum Assoc { - Left(i64), - Right(i64), - Neither(i64), +impl Parser for Expression +where + I: Stream + StreamIsPartial, + Pop: Parser, + Pix: Parser, E>, + Pre: Parser, E>, + Post: Parser, E>, + E: ParserError, +{ + #[inline(always)] + fn parse_next(&mut self, input: &mut I) -> PResult { + trace("expression", move |i: &mut I| { + expression_impl( + i, + &mut self.parse_operand, + &mut self.parse_prefix, + &mut self.parse_postfix, + &mut self.parse_infix, + self.start_power, + ) + }) + .parse_next(input) + } } -// recursive function -fn expression_impl( +fn expression_impl( i: &mut I, - parse_operand: &mut ParseOperand, - prefix: &mut ParsePrefix, - postfix: &mut ParsePostfix, - infix: &mut ParseInfix, + parse_operand: &mut Pop, + prefix: &mut Pre, + postfix: &mut Post, + infix: &mut Pix, min_power: i64, -) -> PResult +) -> PResult where I: Stream + StreamIsPartial, - ParseOperand: Parser, - ParseInfix: Parser PResult), E>, - ParsePrefix: Parser PResult), E>, - ParsePostfix: Parser PResult), E>, + Pop: Parser, + Pix: Parser, E>, + Pre: Parser, E>, + Post: Parser, E>, E: ParserError, { - let operand = opt(parse_operand.by_ref()).parse_next(i)?; + let operand = opt(trace("operand", parse_operand.by_ref())).parse_next(i)?; let mut operand = if let Some(operand) = operand { operand } else { // Prefix unary operators let len = i.eof_offset(); - let (power, fold_prefix) = prefix.parse_next(i)?; + let Prefix(power, fold_prefix) = trace("prefix", prefix.by_ref()).parse_next(i)?; // infinite loop check: the parser must always consume if i.eof_offset() == len { return Err(ErrMode::assert(i, "`prefix` parsers must always consume")); @@ -86,7 +191,9 @@ where 'parse: while i.eof_offset() > 0 { // Postfix unary operators let start = i.checkpoint(); - if let Some((power, fold_postfix)) = opt(postfix.by_ref()).parse_next(i)? { + if let Some(Postfix(power, fold_postfix)) = + opt(trace("postfix", postfix.by_ref())).parse_next(i)? + { // control precedence over the prefix e.g.: // `--(i++)` or `(--i)++` if power < min_power { @@ -100,15 +207,15 @@ where // Infix binary operators let start = i.checkpoint(); - let parse_result = opt(infix.by_ref()).parse_next(i)?; - if let Some((assoc, fold_infix)) = parse_result { + let parse_result = opt(trace("infix", infix.by_ref())).parse_next(i)?; + if let Some(infix_op) = parse_result { let mut is_neither = None; - let (lpower, rpower) = match assoc { - Assoc::Right(p) => (p, p - 1), - Assoc::Left(p) => (p, p + 1), - Assoc::Neither(p) => { + let (lpower, rpower, fold_infix) = match infix_op { + Infix::Right(p, f) => (p, p - 1, f), + Infix::Left(p, f) => (p, p + 1, f), + Infix::Neither(p, f) => { is_neither = Some(p); - (p, p + 1) + (p, p + 1, f) } }; if lpower < min_power || prev_op_is_neither.is_some_and(|p| lpower == p) { @@ -128,81 +235,100 @@ where Ok(operand) } +pub struct Prefix(i64, fn(&mut I, O) -> PResult); + +impl Clone for Prefix { + #[inline(always)] + fn clone(&self) -> Self { + Prefix(self.0, self.1.clone()) + } +} + +impl> Parser, E> for Prefix { + #[inline(always)] + fn parse_next(&mut self, input: &mut I) -> PResult, E> { + empty.value(self.clone()).parse_next(input) + } +} + +pub struct Postfix(i64, fn(&mut I, O) -> PResult); + +impl Clone for Postfix { + #[inline(always)] + fn clone(&self) -> Self { + Postfix(self.0, self.1) + } +} + +impl> Parser, E> + for (i64, fn(&mut I, O) -> PResult) +{ + #[inline(always)] + fn parse_next(&mut self, input: &mut I) -> PResult, E> { + empty.value(Postfix(self.0, self.1)).parse_next(input) + } +} + +pub enum Infix { + Left(i64, fn(&mut I, O, O) -> PResult), + Right(i64, fn(&mut I, O, O) -> PResult), + Neither(i64, fn(&mut I, O, O) -> PResult), +} + +impl Clone for Infix { + #[inline(always)] + fn clone(&self) -> Self { + match self { + Infix::Left(p, f) => Infix::Left(*p, *f), + Infix::Right(p, f) => Infix::Right(*p, *f), + Infix::Neither(p, f) => Infix::Neither(*p, *f), + } + } +} + +impl> Parser, E> for Infix { + #[inline(always)] + fn parse_next(&mut self, input: &mut I) -> PResult, E> { + empty.value(self.clone()).parse_next(input) + } +} + #[cfg(test)] mod tests { - use crate::ascii::{digit1, space0}; - use crate::combinator::{delimited, empty, fail, peek}; + use crate::ascii::digit1; + use crate::combinator::fail; use crate::dispatch; use crate::error::ContextError; use crate::token::any; use super::*; - fn factorial(x: i32) -> i32 { - if x == 0 { - 1 - } else { - x * factorial(x - 1) - } - } fn parser<'i>() -> impl Parser<&'i str, i32, ContextError> { move |i: &mut &str| { - expression( - 0, - trace( - "operand", - delimited( - space0, - dispatch! {peek(any); - '(' => delimited('(', parser(), ')'), - _ => digit1.parse_to::() - }, - space0, - ), - ), - trace( - "prefix", - dispatch! {any; - '+' => empty.value((9, (|_: &mut _, a| Ok(a)) as _)), - '-' => empty.value((9, (|_: &mut _, a: i32| Ok(-a)) as _)), - _ => fail - }, - ), - trace( - "postfix", - dispatch! {any; - '!' => empty.value((9, (|_: &mut _, a| {Ok(factorial(a))}) as _)), - _ => fail - }, - ), - trace( - "infix", - dispatch! {any; - '+' => empty.value((Assoc::Left(5), (|_: &mut _, a, b| Ok(a + b)) as _ )), - '-' => empty.value((Assoc::Left(5), (|_: &mut _, a, b| Ok(a - b)) as _)), - '*' => empty.value((Assoc::Left(7), (|_: &mut _, a, b| Ok(a * b)) as _)), - '/' => empty.value((Assoc::Left(7), (|_: &mut _, a, b| Ok(a / b)) as _)), - '%' => empty.value((Assoc::Left(7), (|_: &mut _, a, b| Ok(a % b)) as _)), - '^' => empty.value((Assoc::Right(9), (|_: &mut _, a, b| Ok(a ^ b)) as _)), - _ => fail - }, - ), - ) - .parse_next(i) + use Infix::*; + expression(0, digit1.parse_to::()) + .prefix(dispatch! {any; + '+' => Prefix(12, |_: &mut _, a: i32| Ok(a)), + '-' => Prefix(12, |_: &mut _, a: i32| Ok(-a)), + _ => fail + }) + .infix(dispatch! {any; + '+' => Left(5, |_: &mut _, a, b| Ok(a + b)), + '-' => Left(5, |_: &mut _, a, b| Ok(a - b)), + '*' => Left(7, |_: &mut _, a, b| Ok(a * b)), + '/' => Left(7, |_: &mut _, a, b| Ok(a / b)), + '%' => Left(7, |_: &mut _, a, b| Ok(a % b)), + '^' => Left(9, |_: &mut _, a, b| Ok(a ^ b)), + _ => fail + }) + .parse_next(i) } } #[test] fn test_expression() { - // assert_eq!(parser().parse("-3!+-3 * 4"), Ok(-18)); - // assert_eq!(parser().parse("+2 + 3 * 4"), Ok(14)); - assert_eq!(parser().parse("2 * 3+4"), Ok(10)); - } - #[test] - fn test_unary() { - assert_eq!(parser().parse("-2"), Ok(-2)); - assert_eq!(parser().parse("4!"), Ok(24)); - assert_eq!(parser().parse("2 + 4!"), Ok(26)); - assert_eq!(parser().parse("-2 + 2"), Ok(0)); + assert_eq!(parser().parse("-3+-3*4"), Ok(-15)); + assert_eq!(parser().parse("+2+3*4"), Ok(14)); + assert_eq!(parser().parse("2*3+4"), Ok(10)); } } From a583d24b599f459426111d0db8460d67bbdd2a0e Mon Sep 17 00:00:00 2001 From: Igor Date: Fri, 22 Nov 2024 23:31:56 +0400 Subject: [PATCH 23/26] fix: MSRV --- src/combinator/expression.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/combinator/expression.rs b/src/combinator/expression.rs index 2f5ebfebc..50a951efe 100644 --- a/src/combinator/expression.rs +++ b/src/combinator/expression.rs @@ -218,7 +218,13 @@ where (p, p + 1, f) } }; - if lpower < min_power || prev_op_is_neither.is_some_and(|p| lpower == p) { + if lpower < min_power + // MSRV: `is_some_and` + || match prev_op_is_neither { + None => false, + Some(p) => lpower == p, + } + { i.reset(&start); break 'parse; } From 431b6f61781d52537871e5090697ee4b0cd8084b Mon Sep 17 00:00:00 2001 From: Igor Date: Fri, 22 Nov 2024 23:40:18 +0400 Subject: [PATCH 24/26] fix: clippy --- src/combinator/expression.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/combinator/expression.rs b/src/combinator/expression.rs index 50a951efe..9e7d6c13b 100644 --- a/src/combinator/expression.rs +++ b/src/combinator/expression.rs @@ -246,7 +246,7 @@ pub struct Prefix(i64, fn(&mut I, O) -> PResult); impl Clone for Prefix { #[inline(always)] fn clone(&self) -> Self { - Prefix(self.0, self.1.clone()) + Prefix(self.0, self.1) } } From 482a162ef5d2e699fa6f24c152627d245315e57c Mon Sep 17 00:00:00 2001 From: Igor Date: Fri, 22 Nov 2024 23:47:06 +0400 Subject: [PATCH 25/26] style: no need to specify the input type in the `fold` closure --- src/combinator/expression.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/combinator/expression.rs b/src/combinator/expression.rs index 9e7d6c13b..04d8c0b27 100644 --- a/src/combinator/expression.rs +++ b/src/combinator/expression.rs @@ -314,17 +314,17 @@ mod tests { use Infix::*; expression(0, digit1.parse_to::()) .prefix(dispatch! {any; - '+' => Prefix(12, |_: &mut _, a: i32| Ok(a)), - '-' => Prefix(12, |_: &mut _, a: i32| Ok(-a)), + '+' => Prefix(12, |_, a| Ok(a)), + '-' => Prefix(12, |_, a: i32| Ok(-a)), _ => fail }) .infix(dispatch! {any; - '+' => Left(5, |_: &mut _, a, b| Ok(a + b)), - '-' => Left(5, |_: &mut _, a, b| Ok(a - b)), - '*' => Left(7, |_: &mut _, a, b| Ok(a * b)), - '/' => Left(7, |_: &mut _, a, b| Ok(a / b)), - '%' => Left(7, |_: &mut _, a, b| Ok(a % b)), - '^' => Left(9, |_: &mut _, a, b| Ok(a ^ b)), + '+' => Left(5, |_, a, b| Ok(a + b)), + '-' => Left(5, |_, a, b| Ok(a - b)), + '*' => Left(7, |_, a, b| Ok(a * b)), + '/' => Left(7, |_, a, b| Ok(a / b)), + '%' => Left(7, |_, a, b| Ok(a % b)), + '^' => Left(9, |_, a, b| Ok(a ^ b)), _ => fail }) .parse_next(i) From 81ba18512c2e8dbc98af236170fa76478bfd219b Mon Sep 17 00:00:00 2001 From: Igor Date: Sat, 23 Nov 2024 01:03:25 +0400 Subject: [PATCH 26/26] feat: fn current_precedence_level(self, level: i64) https://github.com/winnow-rs/winnow/pull/614#discussion_r1854580890 --- src/combinator/expression.rs | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/combinator/expression.rs b/src/combinator/expression.rs index 04d8c0b27..c2b3c1fe3 100644 --- a/src/combinator/expression.rs +++ b/src/combinator/expression.rs @@ -16,7 +16,6 @@ use super::{empty, fail}; #[doc(alias = "precedence_climbing")] #[inline(always)] pub fn expression( - start_power: i64, parse_operand: ParseOperand, ) -> Expression< I, @@ -33,7 +32,7 @@ where E: ParserError, { Expression { - start_power, + precedence_level: 0, parse_operand, parse_prefix: fail, parse_postfix: fail, @@ -50,7 +49,7 @@ where ParseOperand: Parser, E: ParserError, { - start_power: i64, + precedence_level: i64, parse_operand: ParseOperand, parse_prefix: Pre, parse_postfix: Post, @@ -75,7 +74,7 @@ where NewParsePrefix: Parser, E>, { Expression { - start_power: self.start_power, + precedence_level: self.precedence_level, parse_operand: self.parse_operand, parse_prefix: parser, parse_postfix: self.parse_postfix, @@ -95,7 +94,7 @@ where NewParsePostfix: Parser, E>, { Expression { - start_power: self.start_power, + precedence_level: self.precedence_level, parse_operand: self.parse_operand, parse_prefix: self.parse_prefix, parse_postfix: parser, @@ -115,7 +114,7 @@ where NewParseInfix: Parser, E>, { Expression { - start_power: self.start_power, + precedence_level: self.precedence_level, parse_operand: self.parse_operand, parse_prefix: self.parse_prefix, parse_postfix: self.parse_postfix, @@ -125,6 +124,15 @@ where e: Default::default(), } } + + #[inline(always)] + pub fn current_precedence_level( + mut self, + level: i64, + ) -> Expression { + self.precedence_level = level; + self + } } impl Parser for Expression @@ -145,7 +153,7 @@ where &mut self.parse_prefix, &mut self.parse_postfix, &mut self.parse_infix, - self.start_power, + self.precedence_level, ) }) .parse_next(input) @@ -312,7 +320,8 @@ mod tests { fn parser<'i>() -> impl Parser<&'i str, i32, ContextError> { move |i: &mut &str| { use Infix::*; - expression(0, digit1.parse_to::()) + expression(digit1.parse_to::()) + .current_precedence_level(0) .prefix(dispatch! {any; '+' => Prefix(12, |_, a| Ok(a)), '-' => Prefix(12, |_, a: i32| Ok(-a)),