From e9e2964f7a7a4fa1667bf57344e1bfa204029435 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Thu, 17 Mar 2016 16:51:48 +0100 Subject: [PATCH 01/63] WIP --- src/input.rs | 380 ++++++++++++++++++++++++++++++-------------- src/lib.rs | 24 +-- src/parse_result.rs | 112 +++++++------ src/parsers.rs | 193 +++++++++++----------- 4 files changed, 436 insertions(+), 273 deletions(-) diff --git a/src/input.rs b/src/input.rs index 3c2443de..c13a293e 100644 --- a/src/input.rs +++ b/src/input.rs @@ -1,66 +1,98 @@ use parse_result::{ParseResult, State}; use parse_result; -bitflags!{ - pub flags InputMode: u32 { - /// Default (empty) input state. - const DEFAULT = 0, - /// If set the current slice of input is the last one. - const END_OF_INPUT = 1, - } -} - -/// **Primitive:** Trait limiting the use of `Clone` for `Input`. -/// -/// # Primitive -/// -/// Only used by fundamental parsers and combinators. -/// -pub trait InputClone { - /// Creates a clone of the instance. - /// - /// # Primitive - /// - /// Only used by fundamental parsers and combinators. - #[inline(always)] - fn clone(&self) -> Self; -} +use primitives::Guard; -/// **Primitive:** Trait exposing the buffer of `Input`. -/// -/// # Primitive -/// -/// Only used by fundamental parsers and combinators. -/// -pub trait InputBuffer<'a> { - /// The type of each element of the buffer. - type Item: 'a; +pub mod primitives { + use {Input, ParseResult}; + use primitives::State; + use primitives::parse_result::new as new_result; - /// Reveals the internal buffer containig the remainder of the input. - /// - /// # Primitive + /// This is a zero-sized type used by the `Primitives` trait implementation to guarantee that + /// access to primitive methods on `Input` only happens when the `Primitives` trait has been + /// imported. /// - /// Only used by fundamental parsers and combinators. - #[inline(always)] - fn buffer(&self) -> &'a [Self::Item]; + /// It cannot be instantiated outside of the `Primitives` trait blanket implementation. + pub struct Guard(()); - /// Modifies the inner data without leaving the `Input` context. + /// **Primitive:** Trait enabling primitive actions on an `Input` type. /// /// # Primitive /// /// Only used by fundamental parsers and combinators. - #[inline(always)] - fn replace(self, &'a [Self::Item]) -> Self; - - /// Returns true if this is the last available slice of the input. - /// - /// # Primitive /// - /// Only used by fundamental parsers and combinators. - #[inline(always)] - fn is_last_slice(&self) -> bool; + // FIXME: Rename + pub trait Primitives: Input { + #[inline(always)] + /// **Primitive:** Notifies that a parser has reached the end of the currently supplied slice but + /// requires more data. + /// + /// # Primitive + /// + /// Only used by fundamental parsers and combinators. + #[inline] + fn incomplete(self, n: usize) -> ParseResult { + new_result(State::Incomplete(n)) + } + + #[inline(always)] + fn consume(self, n: usize) -> Self { + self._consume(Guard(()), n) + } + #[inline(always)] + fn buffer(&self) -> &[Self::Token] { + self._buffer(Guard(())) + } + + /// Returns true if this is the last available slice of the input. + /// + /// # Primitive + /// + /// Only used by fundamental parsers and combinators. + #[inline(always)] + fn is_end(&self) -> bool { + self._is_end(Guard(())) + } + /// + /// # Example + /// + /// ``` + /// use chomp::{take, Input}; + /// use chomp::primitives::Primitives; + /// use chomp::primitives::{InputBuffer, IntoInner, State}; + /// + /// let i = &b"Testing"; + /// + /// assert_eq!(i.buffer(), b"Testing"); + /// assert_eq!(i.is_end(), true); + /// + /// // mark and eat one token + /// let m = i.mark(); + /// let i = i.consume(1); + /// + /// assert_eq!(i.buffer(), b"esting"); + /// + /// // restore and continue parsing + /// let j = i.restore(m); + /// + /// let r = take(j, 4); + /// + /// assert_eq!(r.into_inner(), State::Data(input::new(input::END_OF_INPUT, b""), &b"Test"[..])); + /// ``` + #[inline(always)] + fn mark(&self) -> Self::Marker { + self._mark(Guard(())) + } + #[inline(always)] + fn restore(self, m: Self::Marker) -> Self { + self._restore(Guard(()), m) + } + } + + impl Primitives for I {} } +// FIXME: Update, do not refer to type or linear type /// Linear type containing the parser state, this type is threaded though `bind` and is also the /// initial type passed to a parser. /// @@ -73,20 +105,12 @@ pub trait InputBuffer<'a> { /// where ``Fn*`` is the appropriate closure/function trait, `I` the input token type (usually /// something like `u8`), `...` additional parameters to the parser, `T` the carried type and `E` /// the potential error type. -#[must_use] -#[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub struct Input<'a, I: 'a>(InputMode, &'a [I]); - -/// **Primitive:** Creates a new input from the given state and buffer. -/// -/// # Primitive -/// -/// Only used by fundamental parsers and combinators. -pub fn new(state: InputMode, buffer: &[I]) -> Input { - Input(state, buffer) -} +pub trait Input: Sized { + /// The token type of the input + type Token; + /// A marker type which is used to backtrack using `_mark` and `_restore`. + type Marker; -impl<'a, I> Input<'a, I> { /// Returns `t` as a success value in the parsing context. /// /// Equivalent to Haskell's `return` function in the `Monad` typeclass. @@ -104,7 +128,7 @@ impl<'a, I> Input<'a, I> { /// assert_eq!(r, Ok("Wohoo, success!")); /// ``` #[inline] - pub fn ret(self, t: T) -> ParseResult<'a, I, T, E> { + fn ret(self, t: T) -> ParseResult { parse_result::new(State::Data(self, t)) } @@ -125,19 +149,8 @@ impl<'a, I> Input<'a, I> { /// assert_eq!(r, Err(ParseError::Error(b"some input", "Something went wrong"))); /// ``` #[inline] - pub fn err(self, e: E) -> ParseResult<'a, I, T, E> { - parse_result::new(State::Error(self.1, e)) - } - - /// Notifies that a parser has reached the end of the currently supplied slice but requires - /// more data. - /// - /// # Primitive - /// - /// Only used by fundamental parsers and combinators. - #[inline] - pub fn incomplete(self, n: usize) -> ParseResult<'a, I, T, E> { - parse_result::new(State::Incomplete(n)) + fn err(self, e: E) -> ParseResult { + parse_result::new(State::Error(self, e)) } /// Converts a `Result` into a `ParseResult`, preserving parser state. @@ -161,12 +174,160 @@ impl<'a, I> Input<'a, I> { /// assert_eq!(r, Err(ParseError::Error(&b"test"[..], "error message"))); /// ``` #[inline] - pub fn from_result(self, r: Result) -> ParseResult<'a, I, T, E> { + fn from_result(self, r: Result) -> ParseResult { match r { Ok(t) => parse_result::new(State::Data(self, t)), - Err(e) => parse_result::new(State::Error(self.1, e)), + Err(e) => parse_result::new(State::Error(self, e)), } } + + // Primitive methods + + /// **Primitive:** See `Primitives::consume` for documentation. + #[inline] + fn _consume(self, Guard, usize) -> Self; + /// **Primitive:** See `Primitives::buffer` for documentation. + // TODO: Might be a better solution here to allow for not only slice-returns? + #[inline] + fn _buffer(&self, Guard) -> &[Self::Token]; + /// **Primitive:** See `Primitives::is_end` for documentation. + #[inline] + fn _is_end(&self, Guard) -> bool; + /// **Primitive:** See `Primitives::mark` for documentation. + #[inline] + fn _mark(&self, Guard) -> Self::Marker; + /// **Primitive:** See `Primitives::restore` for documentation. + #[inline] + fn _restore(self, Guard, Self::Marker) -> Self; +} + +impl<'a, I> Input for &'a [I] { + type Token = I; + type Marker = &'a [I]; + + #[inline(always)] + fn _consume(self, _g: Guard, n: usize) -> Self { + &self[n..] + } + #[inline(always)] + fn _buffer(&self, _g: Guard) -> &[Self::Token] { + &self + } + #[inline(always)] + fn _is_end(&self, _g: Guard) -> bool { + true + } + fn _mark(&self, _g: Guard) -> Self::Marker { + &self + } + fn _restore(self, _g: Guard, m: Self::Marker) -> Self { + m + } +} + +bitflags!{ + pub flags InputMode: u32 { + /// Default (empty) input state. + const DEFAULT = 0, + /// If set the current slice of input is the last one. + const END_OF_INPUT = 1, + } +} + + +/// Input buffer type which contains a flag which tells if we might have more data to read. +#[must_use] +#[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct InputBuf<'a, I: 'a>(InputMode, &'a [I]); + +/// **Primitive:** Creates a new input from the given state and buffer. +/// +/// # Primitive +/// +/// Only used by fundamental parsers and combinators. +pub fn new_buf(state: InputMode, buffer: &[I]) -> InputBuf { + InputBuf(state, buffer) +} + +impl<'a, I> Input for InputBuf<'a, I> { + type Token = I; + type Marker = &'a [I]; + + #[inline(always)] + fn _consume(mut self, _g: Guard, n: usize) -> Self { + self.1 = &self.1[n..]; + + self + } + #[inline(always)] + fn _buffer(&self, _g: Guard) -> &[Self::Token] { + &self.1 + } + #[inline(always)] + fn _is_end(&self, _g: Guard) -> bool { + self.0.contains(END_OF_INPUT) + } + fn _mark(&self, _g: Guard) -> Self::Marker { + &self.1 + } + fn _restore(mut self, _g: Guard, m: Self::Marker) -> Self { + self.1 = m; + + self + } +} + +// FIXME: Delete +/* +/// **Primitive:** Trait limiting the use of `Clone` for `Input`. +/// +/// # Primitive +/// +/// Only used by fundamental parsers and combinators. +/// +pub trait InputClone { + /// Creates a clone of the instance. + /// + /// # Primitive + /// + /// Only used by fundamental parsers and combinators. + #[inline(always)] + fn clone(&self) -> Self; +} + +/// **Primitive:** Trait exposing the buffer of `Input`. +/// +/// # Primitive +/// +/// Only used by fundamental parsers and combinators. +/// +pub trait InputBuffer<'a> { + /// The type of each element of the buffer. + type Item: 'a; + + /// Reveals the internal buffer containig the remainder of the input. + /// + /// # Primitive + /// + /// Only used by fundamental parsers and combinators. + #[inline(always)] + fn buffer(&self) -> &'a [Self::Item]; + + /// Modifies the inner data without leaving the `Input` context. + /// + /// # Primitive + /// + /// Only used by fundamental parsers and combinators. + #[inline(always)] + fn replace(self, &'a [Self::Item]) -> Self; + + /// Returns true if this is the last available slice of the input. + /// + /// # Primitive + /// + /// Only used by fundamental parsers and combinators. + #[inline(always)] + fn is_last_slice(&self) -> bool; } /// Trait limiting the use of `Clone` for `Input`. @@ -207,26 +368,6 @@ impl<'a, I: 'a> InputClone for Input<'a, I> { /// /// But to be able to parse data the contents of the `Input` type must be exposed in at least one /// point, so that data can be examined, and this trait that makes it possible. -/// -/// # Example -/// -/// ``` -/// use chomp::take; -/// use chomp::primitives::input; -/// use chomp::primitives::{InputBuffer, IntoInner, State}; -/// -/// let i = input::new(input::END_OF_INPUT, b"Testing"); -/// -/// assert_eq!(i.buffer(), b"Testing"); -/// assert_eq!(i.is_last_slice(), true); -/// -/// let b = i.buffer(); -/// let j = i.replace(&b[..4]); -/// -/// let r = take(j, 4); -/// -/// assert_eq!(r.into_inner(), State::Data(input::new(input::END_OF_INPUT, b""), &b"Test"[..])); -/// ``` impl<'a, I: 'a> InputBuffer<'a> for Input<'a, I> { type Item = I; @@ -245,44 +386,47 @@ impl<'a, I: 'a> InputBuffer<'a> for Input<'a, I> { self.0.contains(END_OF_INPUT) } } +*/ #[cfg(test)] mod test { - use super::{new, Input, InputBuffer, DEFAULT, END_OF_INPUT}; + use super::{new_buf, Input, InputBuf, DEFAULT, END_OF_INPUT}; use parse_result::ParseResult; use primitives::{IntoInner, State}; #[test] fn make_ret() { - let i1: Input = new(END_OF_INPUT, b"in1"); - let i2: Input = new(DEFAULT, b"in2"); + let i1: InputBuf = new_buf(END_OF_INPUT, b"in1"); + let i2: InputBuf = new_buf(DEFAULT, b"in2"); - let r1: ParseResult = i1.ret::<_, ()>(23u32); - let r2: ParseResult = i2.ret::<_, ()>(23i32); + let r1: ParseResult<_, u32, ()> = i1.ret::<_, ()>(23u32); + let r2: ParseResult<_, i32, ()> = i2.ret::<_, ()>(23i32); - assert_eq!(r1.into_inner(), State::Data(Input(END_OF_INPUT, b"in1"), 23u32)); - assert_eq!(r2.into_inner(), State::Data(Input(DEFAULT, b"in2"), 23i32)); + assert_eq!(r1.into_inner(), State::Data(InputBuf(END_OF_INPUT, b"in1"), 23u32)); + assert_eq!(r2.into_inner(), State::Data(InputBuf(DEFAULT, b"in2"), 23i32)); } #[test] fn make_err() { - let i1: Input = new(END_OF_INPUT, b"in1"); - let i2: Input = new(DEFAULT, b"in2"); + let i1: InputBuf = new_buf(END_OF_INPUT, b"in1"); + let i2: InputBuf = new_buf(DEFAULT, b"in2"); - let r1: ParseResult = i1.err::<(), _>(23u32); - let r2: ParseResult = i2.err::<(), _>(23i32); + let r1: ParseResult<_, (), u32> = i1.err::<(), _>(23u32); + let r2: ParseResult<_, (), i32> = i2.err::<(), _>(23i32); - assert_eq!(r1.into_inner(), State::Error(b"in1", 23u32)); - assert_eq!(r2.into_inner(), State::Error(b"in2", 23i32)); + assert_eq!(r1.into_inner(), State::Error(new_buf(END_OF_INPUT, b"in1"), 23u32)); + assert_eq!(r2.into_inner(), State::Error(new_buf(DEFAULT, b"in2"), 23i32)); } #[test] fn make_incomplete() { - let i1: Input = new(END_OF_INPUT, b"in1"); - let i2: Input = new(DEFAULT, b"in2"); + use primitives::Primitives; - let r1: ParseResult = i1.incomplete::<(), _>(23); - let r2: ParseResult = i2.incomplete::<(), _>(23); + let i1: InputBuf = new_buf(END_OF_INPUT, b"in1"); + let i2: InputBuf = new_buf(DEFAULT, b"in2"); + + let r1: ParseResult<_, (), u32> = i1.incomplete::<(), _>(23); + let r2: ParseResult<_, (), i32> = i2.incomplete::<(), _>(23); assert_eq!(r1.into_inner(), State::Incomplete(23)); assert_eq!(r2.into_inner(), State::Incomplete(23)); @@ -290,8 +434,10 @@ mod test { #[test] fn last_slice() { - let i = new(END_OF_INPUT, &b"foo"[..]); + use primitives::Primitives; + + let i = new_buf(END_OF_INPUT, &b"foo"[..]); - assert_eq!(i.is_last_slice(), true); + assert_eq!(i.is_end(), true); } } diff --git a/src/lib.rs b/src/lib.rs index 747334fe..819fab43 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -209,12 +209,17 @@ extern crate debugtrace; #[macro_use] mod macros; mod input; -mod parse; +// FIXME: Uncomment +//mod parse; mod parse_result; +// FIXME: Uncomment +pub mod parsers; + +pub use parsers::Error; +/* pub mod ascii; pub mod buffer; -pub mod parsers; pub mod combinators; pub use combinators::{ @@ -248,16 +253,17 @@ pub use parsers::{ take_while1, token, }; -pub use parsers::Error; -pub use input::Input; pub use parse::{ ParseError, parse_only, }; +*/ +pub use input::Input; pub use parse_result::{ ParseResult, + // FIXME: Uncomment SimpleResult, - U8Result, + //U8Result, }; /// Module used to construct fundamental parsers and combinators. @@ -266,9 +272,9 @@ pub use parse_result::{ /// /// Only used by fundamental parsers and combinators. pub mod primitives { - pub use input::{ - InputBuffer, - InputClone, + pub use input::primitives::{ + Primitives, + Guard, }; pub use parse_result::{ IntoInner, @@ -281,7 +287,7 @@ pub mod primitives { /// /// Only used by fundamental parsers and combinators. pub mod input { - pub use input::{DEFAULT, END_OF_INPUT, new}; + pub use input::{DEFAULT, END_OF_INPUT, new_buf}; } /// ParseResult utilities. diff --git a/src/parse_result.rs b/src/parse_result.rs index 84393ff2..88b4df41 100644 --- a/src/parse_result.rs +++ b/src/parse_result.rs @@ -1,10 +1,12 @@ -use parsers::Error; use input::Input; +use parsers::Error; /// Result for dealing with the basic parsers when parsing a stream of `u8`. -pub type U8Result<'a, T> = ParseResult<'a, u8, T, Error>; +// FIXME: Uncomment +//pub type U8Result = ParseResult>; /// Result returned from the basic parsers. -pub type SimpleResult<'a, I, T> = ParseResult<'a, I, T, Error>; +// FIXME: Uncomment +pub type SimpleResult = ParseResult::Token>>; /// **Primitive:** Primitive inner type containing the parse-state. /// @@ -13,16 +15,13 @@ pub type SimpleResult<'a, I, T> = ParseResult<'a, I, T, Error>; /// Only used by fundamental parsers and combinators. #[must_use] #[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub enum State<'a, I: 'a, T, E> - where I: 'a, - T: 'a, - E: 'a { +pub enum State { /// Successful parser state, first item is the input state and the second item is the contained /// value. - Data(Input<'a, I>, T), + Data(I, T), /// Parse error state, first item is a slice from where the error occurred in the input buffer /// to the end of the input buffer and the second item is the error value. - Error(&'a [I], E), + Error(I, E), /// Incomplete state, a parser attempted to request more data than available in the slice, the /// provided number is a guess at how many items are needed. Incomplete(usize), @@ -78,7 +77,7 @@ pub trait IntoInner { /// ``` #[must_use] #[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub struct ParseResult<'a, I: 'a, T: 'a, E: 'a>(State<'a, I, T, E>); +pub struct ParseResult(State); /// **Primitive:** Creates a new `ParseResult`. /// @@ -89,11 +88,11 @@ pub struct ParseResult<'a, I: 'a, T: 'a, E: 'a>(State<'a, I, T, E>); /// # Note /// /// Prefer to use ``Input::ret``, ``Input::err`` or ``Input::incomplete`` instead of using -pub fn new(s: State) -> ParseResult { +pub fn new(s: State) -> ParseResult { ParseResult(s) } -impl<'a, I, T, E> ParseResult<'a, I, T, E> { +impl ParseResult { /// Sequentially composes the result with a parse action ``f``, passing any produced value as /// the second parameter. /// @@ -137,8 +136,8 @@ impl<'a, I, T, E> ParseResult<'a, I, T, E> { /// assert_eq!(r, Ok(33)); /// ``` #[inline] - pub fn bind(self, f: F) -> ParseResult<'a, I, U, V> - where F: FnOnce(Input<'a, I>, T) -> ParseResult<'a, I, U, V>, + pub fn bind(self, f: F) -> ParseResult + where F: FnOnce(I, T) -> ParseResult, V: From { match self.0 { State::Data(i, t) => f(i, t).map_err(From::from), @@ -174,8 +173,8 @@ impl<'a, I, T, E> ParseResult<'a, I, T, E> { /// assert_eq!(r2, Ok("testing!")); /// ``` #[inline] - pub fn then(self, f: F) -> ParseResult<'a, I, U, V> - where F: FnOnce(Input<'a, I>) -> ParseResult<'a, I, U, V>, + pub fn then(self, f: F) -> ParseResult + where F: FnOnce(I) -> ParseResult, V: From { self.bind(|i, _| f(i)) } @@ -192,7 +191,7 @@ impl<'a, I, T, E> ParseResult<'a, I, T, E> { /// assert_eq!(r, Ok(b'm')); /// ``` #[inline] - pub fn map(self, f: F) -> ParseResult<'a, I, U, E> + pub fn map(self, f: F) -> ParseResult where F: FnOnce(T) -> U { match self.0 { State::Data(i, t) => ParseResult(State::Data(i, f(t))), @@ -215,7 +214,7 @@ impl<'a, I, T, E> ParseResult<'a, I, T, E> { /// assert_eq!(r, Err(ParseError::Error(b"foo", "this is an error".to_owned()))); /// ``` #[inline] - pub fn map_err(self, f: F) -> ParseResult<'a, I, T, V> + pub fn map_err(self, f: F) -> ParseResult where F: FnOnce(E) -> V { match self.0 { State::Data(i, t) => ParseResult(State::Data(i, t)), @@ -239,7 +238,7 @@ impl<'a, I, T, E> ParseResult<'a, I, T, E> { /// assert_eq!(r, Ok(&b"test"[..])); /// ``` #[inline] - pub fn inspect(self, f: F) -> ParseResult<'a, I, T, E> + pub fn inspect(self, f: F) -> ParseResult where F: FnOnce(&T) { if let State::Data(_, ref t) = self.0 { f(t) @@ -284,8 +283,8 @@ impl<'a, I, T, E> ParseResult<'a, I, T, E> { /// /// assert_eq!(r, Ok(&b"test"[..])); /// ``` -impl<'a, I, T, E> IntoInner for ParseResult<'a, I, T, E> { - type Inner = State<'a, I, T, E>; +impl IntoInner for ParseResult { + type Inner = State; #[inline(always)] fn into_inner(self) -> Self::Inner { @@ -303,12 +302,12 @@ mod test { #[test] fn monad_left_identity() { - fn f(i: Input, n: u32) -> ParseResult { + fn f(i: I, n: u32) -> ParseResult { i.ret(n + 1) } - let m1 = input::new(END_OF_INPUT, b"test"); - let m2 = input::new(END_OF_INPUT, b"test"); + let m1 = input::new_buf(END_OF_INPUT, b"test"); + let m2 = input::new_buf(END_OF_INPUT, b"test"); let a = 123; // return a >>= f @@ -316,43 +315,43 @@ mod test { // f a let rhs = f(m2, a); - assert_eq!(lhs.0, State::Data(input::new(END_OF_INPUT, b"test"), 124)); - assert_eq!(rhs.0, State::Data(input::new(END_OF_INPUT, b"test"), 124)); + assert_eq!(lhs.0, State::Data(input::new_buf(END_OF_INPUT, b"test"), 124)); + assert_eq!(rhs.0, State::Data(input::new_buf(END_OF_INPUT, b"test"), 124)); } #[test] fn monad_right_identity() { - let m1 = input::new(END_OF_INPUT, b"test").ret::<_, ()>(1); - let m2 = input::new(END_OF_INPUT, b"test").ret::<_, ()>(1); + let m1 = input::new_buf(END_OF_INPUT, b"test").ret::<_, ()>(1); + let m2 = input::new_buf(END_OF_INPUT, b"test").ret::<_, ()>(1); // m1 >>= ret === m2 let lhs = m1.bind::<_, _, ()>(Input::ret); let rhs = m2; - assert_eq!(lhs.0, State::Data(input::new(END_OF_INPUT, b"test"), 1)); - assert_eq!(rhs.0, State::Data(input::new(END_OF_INPUT, b"test"), 1)); + assert_eq!(lhs.0, State::Data(input::new_buf(END_OF_INPUT, b"test"), 1)); + assert_eq!(rhs.0, State::Data(input::new_buf(END_OF_INPUT, b"test"), 1)); } #[test] fn monad_associativity() { - fn f(i: Input, num: u32) -> ParseResult { + fn f(i: I, num: u32) -> ParseResult { i.ret((num + 1) as u64) } - fn g(i: Input, num: u64) -> ParseResult { + fn g(i: I, num: u64) -> ParseResult { i.ret(num * 2) } - let lhs_m = input::new(END_OF_INPUT, b"test").ret::<_, ()>(2); - let rhs_m = input::new(END_OF_INPUT, b"test").ret::<_, ()>(2); + let lhs_m = input::new_buf(END_OF_INPUT, b"test").ret::<_, ()>(2); + let rhs_m = input::new_buf(END_OF_INPUT, b"test").ret::<_, ()>(2); // (m >>= f) >>= g let lhs = lhs_m.bind(f).bind(g); // m >>= (\x -> f x >>= g) let rhs = rhs_m.bind(|i, x| f(i, x).bind(g)); - assert_eq!(lhs.0, State::Data(input::new(END_OF_INPUT, b"test"), 6)); - assert_eq!(rhs.0, State::Data(input::new(END_OF_INPUT, b"test"), 6)); + assert_eq!(lhs.0, State::Data(input::new_buf(END_OF_INPUT, b"test"), 6)); + assert_eq!(rhs.0, State::Data(input::new_buf(END_OF_INPUT, b"test"), 6)); } #[test] @@ -361,8 +360,8 @@ mod test { let mut n1 = 0; let mut n2 = 0; - let i1 = input::new(DEFAULT, b"test ").ret::(23); - let i2 = input::new(END_OF_INPUT, b"test ").ret::(23); + let i1 = input::new_buf(DEFAULT, b"test ").ret::(23); + let i2 = input::new_buf(END_OF_INPUT, b"test ").ret::(23); let r1 = i1.inspect(|d: &u32| { assert_eq!(d, &23); @@ -375,9 +374,9 @@ mod test { n2 += 1; }); - assert_eq!(r1.into_inner(), State::Data(input::new(DEFAULT, b"test "), 23)); + assert_eq!(r1.into_inner(), State::Data(input::new_buf(DEFAULT, b"test "), 23)); assert_eq!(n1, 1); - assert_eq!(r2.into_inner(), State::Data(input::new(END_OF_INPUT, b"test "), 23)); + assert_eq!(r2.into_inner(), State::Data(input::new_buf(END_OF_INPUT, b"test "), 23)); assert_eq!(n2, 1); } @@ -386,14 +385,14 @@ mod test { let mut n1_calls = 0; let mut n2_calls = 0; - let i1 = input::new(DEFAULT, b"test1").ret::<_, ()>(23); - let i2 = input::new(END_OF_INPUT, b"test2").ret::<_, ()>(24); + let i1 = input::new_buf(DEFAULT, b"test1").ret::<_, ()>(23); + let i2 = input::new_buf(END_OF_INPUT, b"test2").ret::<_, ()>(24); let r1: ParseResult<_, _, ()> = i1.bind(|i, t| { n1_calls += 1; i.ret(t) }); let r2: ParseResult<_, _, ()> = i2.bind(|i, t| { n2_calls += 1; i.ret(t) }); - assert_eq!(r1.0, State::Data(input::new(DEFAULT, b"test1"), 23)); - assert_eq!(r2.0, State::Data(input::new(END_OF_INPUT, b"test2"), 24)); + assert_eq!(r1.0, State::Data(input::new_buf(DEFAULT, b"test1"), 23)); + assert_eq!(r2.0, State::Data(input::new_buf(END_OF_INPUT, b"test2"), 24)); assert_eq!(n1_calls, 1); assert_eq!(n2_calls, 1); } @@ -403,25 +402,27 @@ mod test { let mut n1_calls = 0; let mut n2_calls = 0; - let i1 = input::new(DEFAULT, b"test1").err::<(), _>(23); - let i2 = input::new(END_OF_INPUT, b"test2").err::<(), _>(24); + let i1 = input::new_buf(DEFAULT, b"test1").err::<(), _>(23); + let i2 = input::new_buf(END_OF_INPUT, b"test2").err::<(), _>(24); let r1 = i1.bind(|i, t| { n1_calls += 1; i.ret(t) }); let r2 = i2.bind(|i, t| { n2_calls += 1; i.ret(t) }); - assert_eq!(r1.0, State::Error(b"test1", 23)); - assert_eq!(r2.0, State::Error(b"test2", 24)); + assert_eq!(r1.0, State::Error(input::new_buf(DEFAULT, b"test1"), 23)); + assert_eq!(r2.0, State::Error(input::new_buf(END_OF_INPUT, b"test2"), 24)); assert_eq!(n1_calls, 0); assert_eq!(n2_calls, 0); } #[test] fn incomplete_propagation() { + use primitives::Primitives; + let mut n1_calls = 0; let mut n2_calls = 0; - let i1 = input::new(DEFAULT, b"test1").incomplete::<(), ()>(23); - let i2 = input::new(END_OF_INPUT, b"test2").incomplete::<(), ()>(24); + let i1 = input::new_buf(DEFAULT, b"test1").incomplete::<(), ()>(23); + let i2 = input::new_buf(END_OF_INPUT, b"test2").incomplete::<(), ()>(24); let r1: ParseResult<_, _, ()> = i1.bind(|i, t| { n1_calls += 1; i.ret(t) }); let r2: ParseResult<_, _, ()> = i2.bind(|i, t| { n2_calls += 1; i.ret(t) }); @@ -431,4 +432,15 @@ mod test { assert_eq!(n1_calls, 0); assert_eq!(n2_calls, 0); } + + #[test] + fn slice() { + fn f(i: I, n: u32) -> ParseResult { i.ret(n + 1) } + + let lhs = (&b"test"[..]).ret(123).bind(f); + let rhs = f(&b"test"[..], 123); + + assert_eq!(lhs.0, State::Data(&b"test"[..], 124)); + assert_eq!(rhs.0, State::Data(&b"test"[..], 124)); + } } diff --git a/src/parsers.rs b/src/parsers.rs index 84cc1393..2571b9ea 100644 --- a/src/parsers.rs +++ b/src/parsers.rs @@ -4,7 +4,7 @@ use std::mem; use input::Input; use parse_result::SimpleResult; -use primitives::InputBuffer; +use primitives::Primitives; pub use self::error::Error; @@ -23,12 +23,12 @@ pub use debugtrace::StackFrame; /// assert_eq!(parse_only(any, b"abc"), Ok(b'a')); /// ``` #[inline] -pub fn any(i: Input) -> SimpleResult { +pub fn any(i: I) -> SimpleResult { let b = i.buffer(); match b.first() { None => i.incomplete(1), - Some(&c) => i.replace(&b[1..]).ret(c), + Some(&c) => i.consume(1).ret(c), } } @@ -43,13 +43,14 @@ pub fn any(i: Input) -> SimpleResult { /// assert_eq!(parse_only(|i| satisfy(i, |c| c == b'a'), b"abc"), Ok(b'a')); /// ``` #[inline] -pub fn satisfy(i: Input, f: F) -> SimpleResult - where F: FnOnce(I) -> bool { +pub fn satisfy(i: I, f: F) -> SimpleResult + where F: FnOnce(I::Token) -> bool, + I::Token: Copy { let b = i.buffer(); match b.first() { None => i.incomplete(1), - Some(&c) if f(c) => i.replace(&b[1..]).ret(c), + Some(&c) if f(c) => i.consume(1).ret(c), Some(_) => i.err(Error::unexpected()), } } @@ -69,17 +70,17 @@ pub fn satisfy(i: Input, f: F) -> SimpleResult /// assert_eq!(r, Ok(b'T')); /// ``` #[inline] -pub fn satisfy_with(i: Input, f: F, p: P) -> SimpleResult - where F: FnOnce(I) -> T, +pub fn satisfy_with(i: I, f: F, p: P) -> SimpleResult + where F: FnOnce(&I::Token) -> T, P: FnOnce(T) -> bool { let b = i.buffer(); - match b.first().cloned() { + match b.first() { Some(n) => { let t = f(n); if p(t.clone()) { - i.replace(&b[1..]).ret(t) + i.consume(1).ret(t) } else { i.err(Error::unexpected()) } @@ -98,12 +99,13 @@ pub fn satisfy_with(i: Input, f: F, p: P) -> SimpleR /// assert_eq!(parse_only(|i| token(i, b'a'), b"abc"), Ok(b'a')); /// ``` #[inline] -pub fn token(i: Input, t: I) -> SimpleResult { +pub fn token(i: I, t: I::Token) -> SimpleResult + where I::Token: PartialEq + Copy { let b = i.buffer(); match b.first() { None => i.incomplete(1), - Some(&c) if t == c => i.replace(&b[1..]).ret(c), + Some(&c) if t == c => i.consume(1).ret(c), Some(_) => i.err(Error::expected(t)), } } @@ -118,12 +120,13 @@ pub fn token(i: Input, t: I) -> SimpleResult { /// assert_eq!(parse_only(|i| not_token(i, b'b'), b"abc"), Ok(b'a')); /// ``` #[inline] -pub fn not_token(i: Input, t: I) -> SimpleResult { +pub fn not_token(i: I, t: I::Token) -> SimpleResult + where I::Token: PartialEq + Copy { let b = i.buffer(); match b.first() { None => i.incomplete(1), - Some(&c) if t != c => i.replace(&b[1..]).ret(c), + Some(&c) if t != c => i.consume(1).ret(c), Some(_) => i.err(Error::unexpected()), } } @@ -141,7 +144,8 @@ pub fn not_token(i: Input, t: I) -> SimpleResult { /// assert_eq!(parse_only(peek, b""), Ok(None)); /// ``` #[inline] -pub fn peek(i: Input) -> SimpleResult> { +pub fn peek(i: I) -> SimpleResult> + where I::Token: Clone { let d = i.buffer().first().cloned(); i.ret(d) @@ -157,7 +161,8 @@ pub fn peek(i: Input) -> SimpleResult> { /// assert_eq!(parse_only(peek_next, b"abc"), Ok(b'a')); /// ``` #[inline] -pub fn peek_next(i: Input) -> SimpleResult { +pub fn peek_next(i: I) -> SimpleResult + where I::Token: Clone { match i.buffer().first().cloned() { None => i.incomplete(1), Some(c) => i.ret(c), @@ -174,11 +179,11 @@ pub fn peek_next(i: Input) -> SimpleResult { /// assert_eq!(parse_only(|i| take(i, 3), b"abcd"), Ok(&b"abc"[..])); /// ``` #[inline] -pub fn take(i: Input, num: usize) -> SimpleResult { +pub fn take<'a, I: Input + 'a>(i: I, num: usize) -> SimpleResult { let b = i.buffer(); if num <= b.len() { - i.replace(&b[num..]).ret(&b[..num]) + i.consume(num).ret(&b[..num]) } else { i.incomplete(num - b.len()) } @@ -207,19 +212,17 @@ pub fn take(i: Input, num: usize) -> SimpleResult { /// assert_eq!(r, Ok(&b""[..])); /// ``` #[inline] -pub fn take_while(i: Input, f: F) -> SimpleResult - where F: Fn(I) -> bool { +pub fn take_while<'a, I: Input + 'a, F>(i: I, f: F) -> SimpleResult + where F: Fn(I::Token) -> bool { let b = i.buffer(); match b.iter().position(|c| f(*c) == false) { - Some(n) => i.replace(&b[n..]).ret(&b[..n]), + Some(n) => i.consume(n).ret(&b[..n]), // TODO: Should this following 1 be something else, seeing as take_while1 is potentially // infinite? - None => if i.is_last_slice() { + None => if i.is_end() { // Last slice and we have just read everything of it, replace with zero-sized slice: - // Hack to avoid branch and overflow, does not matter where this zero-sized slice is - // allocated - i.replace(&b[..0]).ret(b) + i.consume(b.len()).ret(b) } else { i.incomplete(1) }, @@ -240,20 +243,18 @@ pub fn take_while(i: Input, f: F) -> SimpleResult /// assert_eq!(r, Ok(&b"ab"[..])); /// ``` #[inline] -pub fn take_while1(i: Input, f: F) -> SimpleResult - where F: Fn(I) -> bool { +pub fn take_while1<'a, I: Input + 'a, F>(i: I, f: F) -> SimpleResult + where F: Fn(I::Token) -> bool { let b = i.buffer(); match b.iter().position(|c| f(*c) == false) { Some(0) => i.err(Error::unexpected()), - Some(n) => i.replace(&b[n..]).ret(&b[..n]), + Some(n) => i.consume(n).ret(&b[..n]), // TODO: Should this following 1 be something else, seeing as take_while1 is potentially // infinite? - None => if b.len() > 0 && i.is_last_slice() { + None => if b.len() > 0 && i.is_end() { // Last slice and we have just read everything of it, replace with zero-sized slice: - // Hack to avoid branch and overflow, does not matter where this zero-sized slice is - // allocated - i.replace(&b[..0]).ret(b) + i.consume(b.len()).ret(b) } else { i.incomplete(1) }, @@ -274,12 +275,13 @@ pub fn take_while1(i: Input, f: F) -> SimpleResult /// assert_eq!(r, Ok(&b"abc"[..])); /// ``` #[inline] -pub fn take_till(i: Input, f: F) -> SimpleResult - where F: Fn(I) -> bool { +pub fn take_till<'a, I: Input + 'a, F>(i: I, f: F) -> SimpleResult + where F: Fn(I::Token) -> bool, + I::Token: Clone { let b = i.buffer(); match b.iter().cloned().position(f) { - Some(n) => i.replace(&b[n..]).ret(&b[0..n]), + Some(n) => i.consume(n).ret(&b[0..n]), // TODO: Should this following 1 be something else, seeing as take_while1 is potentially // infinite? None => i.incomplete(1), @@ -301,13 +303,13 @@ pub fn take_till(i: Input, f: F) -> SimpleResult /// assert_eq!(parse_only(p, b"/*test*of*scan*/ foo"), Ok(&b"/*test*of*scan*"[..])); /// ``` #[inline] -pub fn scan(i: Input, s: S, mut f: F) -> SimpleResult - where F: FnMut(S, I) -> Option { +pub fn scan<'a, I: Input + 'a, S, F>(i: I, s: S, mut f: F) -> SimpleResult + where F: FnMut(S, I::Token) -> Option { let b = i.buffer(); let mut state = Some(s); match b.iter().position(|&c| { state = f(mem::replace(&mut state, None).unwrap(), c); state.is_none()}) { - Some(n) => i.replace(&b[n..]).ret(&b[0..n]), + Some(n) => i.consume(n).ret(&b[0..n]), // TODO: Should this following 1 be something else, seeing as take_while1 is potentially // infinite? None => i.incomplete(1), @@ -328,13 +330,13 @@ pub fn scan(i: Input, s: S, mut f: F) -> SimpleResult /// ``` #[inline] // TODO: Remove Copy bound on S -pub fn run_scanner(i: Input, s: S, mut f: F) -> SimpleResult - where F: FnMut(S, I) -> Option { +pub fn run_scanner<'a, I: Input + 'a, S: Copy, F>(i: I, s: S, mut f: F) -> SimpleResult + where F: FnMut(S, I::Token) -> Option { let b = i.buffer(); let mut state = s; match b.iter().position(|&c| { let t = f(state, c); match t { None => true, Some(v) => { state = v; false } } }) { - Some(n) => i.replace(&b[n..]).ret((&b[0..n], state)), + Some(n) => i.consume(n).ret((&b[0..n], state)), // TODO: Should this following 1 be something else, seeing as take_while1 is potentially // infinite? None => i.incomplete(1), @@ -349,13 +351,10 @@ pub fn run_scanner(i: Input, s: S, mut f: F) -> SimpleRe /// assert_eq!(parse_only(take_remainder, b"abcd"), Ok(&b"abcd"[..])); /// ``` #[inline] -pub fn take_remainder(i: Input) -> SimpleResult { +pub fn take_remainder<'a, I: Input + 'a>(i: I) -> SimpleResult { let b = i.buffer(); // Last slice and we have just read everything of it, replace with zero-sized slice: - // - // Hack to avoid branch and overflow, does not matter where this zero-sized slice is - // allocated as long as it is the same origin slice - i.replace(&b[..0]).ret(b) + i.consume(b.len()).ret(b) } /// Matches the given slice against the parser, returning the matched slice upon success. @@ -369,8 +368,8 @@ pub fn take_remainder(i: Input) -> SimpleResult { /// assert_eq!(parse_only(|i| string(i, b"abc"), b"abcdef"), Ok(&b"abc"[..])); /// ``` #[inline] -pub fn string<'a, 'b, I: Copy + PartialEq>(i: Input<'a, I>, s: &'b [I]) - -> SimpleResult<'a, I, &'a [I]> { +pub fn string<'a, 'b, T: Copy + PartialEq, I: 'a + Input>(i: I, s: &'b [T]) + -> SimpleResult { let b = i.buffer(); if s.len() > b.len() { @@ -381,11 +380,11 @@ pub fn string<'a, 'b, I: Copy + PartialEq>(i: Input<'a, I>, s: &'b [I]) for j in 0..s.len() { if s[j] != d[j] { - return i.replace(&b[j..]).err(Error::expected(d[j])) + return i.consume(j).err(Error::expected(d[j])) } } - i.replace(&b[s.len()..]).ret(d) + i.consume(s.len()).ret(d) } /// Matches the end of the input. @@ -398,8 +397,8 @@ pub fn string<'a, 'b, I: Copy + PartialEq>(i: Input<'a, I>, s: &'b [I]) /// assert_eq!(r, Ok(())); /// ``` #[inline] -pub fn eof(i: Input) -> SimpleResult { - if i.buffer().len() == 0 && i.is_last_slice() { +pub fn eof(i: I) -> SimpleResult { + if i.buffer().len() == 0 && i.is_end() { i.ret(()) } else { i.err(Error::unexpected()) @@ -563,7 +562,7 @@ mod error { #[cfg(test)] mod test { - use primitives::input::{new, DEFAULT, END_OF_INPUT}; + use primitives::input::{new_buf, DEFAULT, END_OF_INPUT}; use primitives::IntoInner; use primitives::State; use super::*; @@ -580,7 +579,7 @@ mod test { i.ret(bytes.iter().fold(0, |a, b| a * 10 + (b - b'0') as usize))) } - let i = new(END_OF_INPUT, b"123.4567 "); + let i = &b"123.4567 "[..]; let p = decimal(i).bind(|i, real| token(i, b'.').bind(|i, _| @@ -590,21 +589,21 @@ mod test { let d: SimpleResult<_, _> = p.bind(|i, num| take_remainder(i) .bind(|i, r| i.ret((r, num)))); - assert_eq!(d.into_inner(), State::Data(new(END_OF_INPUT, b""), (&b" "[..], (123, 4567)))); + assert_eq!(d.into_inner(), State::Data(&b""[..], (&b" "[..], (123, 4567)))); } #[test] fn parse_remainder_empty() { - let i = new(END_OF_INPUT, b""); + let i = &b""[..]; let r = take_remainder(i); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), &b""[..])); + assert_eq!(r.into_inner(), State::Data(&b""[..], &b""[..])); } #[test] fn take_while1_empty() { - let n = new(END_OF_INPUT, b""); + let n = &b""[..]; let r = take_while1(n, |_| true); @@ -613,80 +612,80 @@ mod test { #[test] fn token_test() { - assert_eq!(token(new(DEFAULT, b""), b'a').into_inner(), State::Incomplete(1)); - assert_eq!(token(new(DEFAULT, b"ab"), b'a').into_inner(), State::Data(new(DEFAULT, b"b"), b'a')); - assert_eq!(token(new(DEFAULT, b"bb"), b'a').into_inner(), State::Error(b"bb", Error::expected(b'a'))); + assert_eq!(token(new_buf(DEFAULT, b""), b'a').into_inner(), State::Incomplete(1)); + assert_eq!(token(new_buf(DEFAULT, b"ab"), b'a').into_inner(), State::Data(new_buf(DEFAULT, b"b"), b'a')); + assert_eq!(token(new_buf(DEFAULT, b"bb"), b'a').into_inner(), State::Error(b"bb", Error::expected(b'a'))); } #[test] fn take_test() { - assert_eq!(take(new(DEFAULT, b"a"), 1).into_inner(), State::Data(new(DEFAULT, b""), &b"a"[..])); - assert_eq!(take(new(DEFAULT, b"a"), 2).into_inner(), State::Incomplete(1)); - assert_eq!(take(new(DEFAULT, b"a"), 3).into_inner(), State::Incomplete(2)); - assert_eq!(take(new(DEFAULT, b"ab"), 1).into_inner(), State::Data(new(DEFAULT, b"b"), &b"a"[..])); - assert_eq!(take(new(DEFAULT, b"ab"), 2).into_inner(), State::Data(new(DEFAULT, b""), &b"ab"[..])); + assert_eq!(take(new_buf(DEFAULT, b"a"), 1).into_inner(), State::Data(new_buf(DEFAULT, b""), &b"a"[..])); + assert_eq!(take(new_buf(DEFAULT, b"a"), 2).into_inner(), State::Incomplete(1)); + assert_eq!(take(new_buf(DEFAULT, b"a"), 3).into_inner(), State::Incomplete(2)); + assert_eq!(take(new_buf(DEFAULT, b"ab"), 1).into_inner(), State::Data(new_buf(DEFAULT, b"b"), &b"a"[..])); + assert_eq!(take(new_buf(DEFAULT, b"ab"), 2).into_inner(), State::Data(new_buf(DEFAULT, b""), &b"ab"[..])); } #[test] fn take_while_test() { - assert_eq!(take_while(new(DEFAULT, b"abc"), |c| c != b'b').into_inner(), State::Data(new(DEFAULT, b"bc"), &b"a"[..])); - assert_eq!(take_while(new(DEFAULT, b"bbc"), |c| c != b'b').into_inner(), State::Data(new(DEFAULT, b"bbc"), &b""[..])); - assert_eq!(take_while(new(END_OF_INPUT, b"bbc"), |c| c != b'b').into_inner(), State::Data(new(END_OF_INPUT, b"bbc"), &b""[..])); - assert_eq!(take_while(new(END_OF_INPUT, b"abc"), |c| c != b'b').into_inner(), State::Data(new(END_OF_INPUT, b"bc"), &b"a"[..])); + assert_eq!(take_while(new_buf(DEFAULT, b"abc"), |c| c != b'b').into_inner(), State::Data(new_buf(DEFAULT, b"bc"), &b"a"[..])); + assert_eq!(take_while(new_buf(DEFAULT, b"bbc"), |c| c != b'b').into_inner(), State::Data(new_buf(DEFAULT, b"bbc"), &b""[..])); + assert_eq!(take_while(&b"bbc"[..], |c| c != b'b').into_inner(), State::Data(&b"bbc"[..], &b""[..])); + assert_eq!(take_while(&b"abc"[..], |c| c != b'b').into_inner(), State::Data(&b"bc"[..], &b"a"[..])); // TODO: Update when the incomplete type has been updated - assert_eq!(take_while(new(DEFAULT, b"acc"), |c| c != b'b').into_inner(), State::Incomplete(1)); - assert_eq!(take_while(new(END_OF_INPUT, b"acc"), |c| c != b'b').into_inner(), State::Data(new(END_OF_INPUT, b""), &b"acc"[..])); + assert_eq!(take_while(new_buf(DEFAULT, b"acc"), |c| c != b'b').into_inner(), State::Incomplete(1)); + assert_eq!(take_while(&b"acc"[..], |c| c != b'b').into_inner(), State::Data(&b""[..], &b"acc"[..])); } #[test] fn take_while1_test() { - assert_eq!(take_while1(new(DEFAULT, b"abc"), |c| c != b'b').into_inner(), State::Data(new(DEFAULT, b"bc"), &b"a"[..])); - assert_eq!(take_while1(new(DEFAULT, b"bbc"), |c| c != b'b').into_inner(), State::Error(&b"bbc"[..], Error::unexpected())); - assert_eq!(take_while1(new(END_OF_INPUT, b"bbc"), |c| c != b'b').into_inner(), State::Error(&b"bbc"[..], Error::unexpected())); - assert_eq!(take_while1(new(END_OF_INPUT, b"abc"), |c| c != b'b').into_inner(), State::Data(new(END_OF_INPUT, b"bc"), &b"a"[..])); + assert_eq!(take_while1(new_buf(DEFAULT, b"abc"), |c| c != b'b').into_inner(), State::Data(new_buf(DEFAULT, b"bc"), &b"a"[..])); + assert_eq!(take_while1(new_buf(DEFAULT, b"bbc"), |c| c != b'b').into_inner(), State::Error(&b"bbc"[..], Error::unexpected())); + assert_eq!(take_while1(&b"bbc"[..], |c| c != b'b').into_inner(), State::Error(&b"bbc"[..], Error::unexpected())); + assert_eq!(take_while1(&b"abc"[..], |c| c != b'b').into_inner(), State::Data(&b"bc"[..], &b"a"[..])); // TODO: Update when the incomplete type has been updated - assert_eq!(take_while1(new(DEFAULT, b"acc"), |c| c != b'b').into_inner(), State::Incomplete(1)); - assert_eq!(take_while1(new(END_OF_INPUT, b"acc"), |c| c != b'b').into_inner(), State::Data(new(END_OF_INPUT, b""), &b"acc"[..])); + assert_eq!(take_while1(new_buf(DEFAULT, b"acc"), |c| c != b'b').into_inner(), State::Incomplete(1)); + assert_eq!(take_while1(&b"acc"[..], |c| c != b'b').into_inner(), State::Data(&b""[..], &b"acc"[..])); } #[test] fn peek_next_test() { - assert_eq!(peek_next(new(DEFAULT, b"abc")).into_inner(), State::Data(new(DEFAULT, b"abc"), b'a')); - assert_eq!(peek_next(new(END_OF_INPUT, b"abc")).into_inner(), State::Data(new(END_OF_INPUT, b"abc"), b'a')); - assert_eq!(peek_next(new(DEFAULT, b"")).into_inner(), State::Incomplete(1)); - assert_eq!(peek_next(new(END_OF_INPUT, b"")).into_inner(), State::Incomplete(1)); + assert_eq!(peek_next(new_buf(DEFAULT, b"abc")).into_inner(), State::Data(new_buf(DEFAULT, b"abc"), b'a')); + assert_eq!(peek_next(&b"abc"[..]).into_inner(), State::Data(&b"abc"[..], b'a')); + assert_eq!(peek_next(new_buf(DEFAULT, b"")).into_inner(), State::Incomplete(1)); + assert_eq!(peek_next(&b""[..]).into_inner(), State::Incomplete(1)); } #[test] fn satisfy_with_test() { let mut m1 = 0; let mut n1 = 0; - assert_eq!(satisfy_with(new(DEFAULT, b"abc"), |m| { m1 += 1; m % 8 }, |n| { n1 += 1; n == 1 }).into_inner(), State::Data(new(DEFAULT, b"bc"), 1)); + assert_eq!(satisfy_with(new_buf(DEFAULT, b"abc"), |m| { m1 += 1; m % 8 }, |n| { n1 += 1; n == 1 }).into_inner(), State::Data(new_buf(DEFAULT, b"bc"), 1)); assert_eq!(m1, 1); assert_eq!(n1, 1); let mut m2 = 0; let mut n2 = 0; - assert_eq!(satisfy_with(new(DEFAULT, b""), |m| { m2 += 1; m % 8 }, |n| { n2 += 1; n == 1 }).into_inner(), State::Incomplete(1)); + assert_eq!(satisfy_with(new_buf(DEFAULT, b""), |m| { m2 += 1; m % 8 }, |n| { n2 += 1; n == 1 }).into_inner(), State::Incomplete(1)); assert_eq!(m2, 0); assert_eq!(n2, 0); } #[test] fn string_test() { - assert_eq!(string(new(DEFAULT, b"abc"), b"a").into_inner(), State::Data(new(DEFAULT, b"bc"), &b"a"[..])); - assert_eq!(string(new(DEFAULT, b"abc"), b"ab").into_inner(), State::Data(new(DEFAULT, b"c"), &b"ab"[..])); - assert_eq!(string(new(DEFAULT, b"abc"), b"abc").into_inner(), State::Data(new(DEFAULT, b""), &b"abc"[..])); - assert_eq!(string(new(DEFAULT, b"abc"), b"abcd").into_inner(), State::Incomplete(1)); - assert_eq!(string(new(DEFAULT, b"abc"), b"abcde").into_inner(), State::Incomplete(2)); - assert_eq!(string(new(DEFAULT, b"abc"), b"ac").into_inner(), State::Error(b"bc", Error::expected(b'b'))); - - assert_eq!(string(new(END_OF_INPUT, b"abc"), b"a").into_inner(), State::Data(new(END_OF_INPUT, b"bc"), &b"a"[..])); - assert_eq!(string(new(END_OF_INPUT, b"abc"), b"ab").into_inner(), State::Data(new(END_OF_INPUT, b"c"), &b"ab"[..])); - assert_eq!(string(new(END_OF_INPUT, b"abc"), b"abc").into_inner(), State::Data(new(END_OF_INPUT, b""), &b"abc"[..])); - assert_eq!(string(new(END_OF_INPUT, b"abc"), b"abcd").into_inner(), State::Incomplete(1)); - assert_eq!(string(new(END_OF_INPUT, b"abc"), b"abcde").into_inner(), State::Incomplete(2)); - assert_eq!(string(new(END_OF_INPUT, b"abc"), b"ac").into_inner(), State::Error(b"bc", Error::expected(b'b'))); + assert_eq!(string(new_buf(DEFAULT, b"abc"), b"a").into_inner(), State::Data(new_buf(DEFAULT, b"bc"), &b"a"[..])); + assert_eq!(string(new_buf(DEFAULT, b"abc"), b"ab").into_inner(), State::Data(new_buf(DEFAULT, b"c"), &b"ab"[..])); + assert_eq!(string(new_buf(DEFAULT, b"abc"), b"abc").into_inner(), State::Data(new_buf(DEFAULT, b""), &b"abc"[..])); + assert_eq!(string(new_buf(DEFAULT, b"abc"), b"abcd").into_inner(), State::Incomplete(1)); + assert_eq!(string(new_buf(DEFAULT, b"abc"), b"abcde").into_inner(), State::Incomplete(2)); + assert_eq!(string(new_buf(DEFAULT, b"abc"), b"ac").into_inner(), State::Error(b"bc", Error::expected(b'b'))); + + assert_eq!(string(&b"abc"[..], b"a").into_inner(), State::Data(&b"bc"[..], &b"a"[..])); + assert_eq!(string(&b"abc"[..], b"ab").into_inner(), State::Data(&b"c"[..], &b"ab"[..])); + assert_eq!(string(&b"abc"[..], b"abc").into_inner(), State::Data(&b""[..], &b"abc"[..])); + assert_eq!(string(&b"abc"[..], b"abcd").into_inner(), State::Incomplete(1)); + assert_eq!(string(&b"abc"[..], b"abcde").into_inner(), State::Incomplete(2)); + assert_eq!(string(&b"abc"[..], b"ac").into_inner(), State::Error(b"bc", Error::expected(b'b'))); } #[test] From 05fc3c26dac6d1594970a5af0fcc1471b6efaeea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Thu, 17 Mar 2016 22:24:56 +0100 Subject: [PATCH 02/63] Input trait: Made it possible to be even more generic over the input, the buffer-return does not need to be tied directly to the input itself --- src/input.rs | 109 +++++++++++++++++++++----- src/parsers.rs | 207 ++++++++++++++++++++++++++++--------------------- 2 files changed, 208 insertions(+), 108 deletions(-) diff --git a/src/input.rs b/src/input.rs index c13a293e..55331679 100644 --- a/src/input.rs +++ b/src/input.rs @@ -3,6 +3,9 @@ use parse_result; use primitives::Guard; +/// Primitive operations on `Input` types. +/// +/// All parsers only require a few primitive operations to parse data: pub mod primitives { use {Input, ParseResult}; use primitives::State; @@ -36,14 +39,26 @@ pub mod primitives { } #[inline(always)] - fn consume(self, n: usize) -> Self { + fn first(&self) -> Option { + self._first(Guard(())) + } + #[inline(always)] + fn iter(&self) -> Self::Iter { + self._iter(Guard(())) + } + #[inline(always)] + fn consume(&mut self, n: usize) -> Self::Buffer { self._consume(Guard(()), n) } #[inline(always)] - fn buffer(&self) -> &[Self::Token] { - self._buffer(Guard(())) + fn discard(&mut self, n: usize) { + self._discard(Guard(()), n) } + #[inline] + fn min_remaining(&self) -> usize { + self._min_remaining(Guard(())) + } /// Returns true if this is the last available slice of the input. /// /// # Primitive @@ -107,10 +122,14 @@ pub mod primitives { /// the potential error type. pub trait Input: Sized { /// The token type of the input - type Token; + // TODO: Maybe remove the copy bound at some point? + type Token: Copy; /// A marker type which is used to backtrack using `_mark` and `_restore`. type Marker; + type Iter: Iterator; + type Buffer; + /// Returns `t` as a success value in the parsing context. /// /// Equivalent to Haskell's `return` function in the `Monad` typeclass. @@ -182,36 +201,76 @@ pub trait Input: Sized { } // Primitive methods + /// **Primitive:** See `Primitives::first` for documentation. + /// + /// Returns the first remaining item if there are still data to read. + #[inline(always)] + fn _first(&self, g: Guard) -> Option { + self._iter(g).next() + } + /// **Primitive:** See `Primitives::iter` for documentation. + /// + /// Iterator over tokens in the input, does not consume any data. + #[inline] + fn _iter(&self, Guard) -> Self::Iter; /// **Primitive:** See `Primitives::consume` for documentation. + /// + /// Consumes a set amount of input tokens, returning a buffer containing them + // TODO: Should probably be combined with a ret + #[inline] + fn _consume(&mut self, Guard, usize) -> Self::Buffer; + /// **Primitive:** See `Primitives::discard` for documentation. + /// + /// Consumes a set amount of input tokens, discarding them. #[inline] - fn _consume(self, Guard, usize) -> Self; - /// **Primitive:** See `Primitives::buffer` for documentation. - // TODO: Might be a better solution here to allow for not only slice-returns? + fn _discard(&mut self, Guard, usize); + /// **Primitive:** See `Primitives::remaining` for documentation. + /// + /// Returns the number of tokens remaining in this input (only this part of it, more might + /// follow if `_is_end` is false). #[inline] - fn _buffer(&self, Guard) -> &[Self::Token]; + fn _min_remaining(&self, Guard) -> usize; /// **Primitive:** See `Primitives::is_end` for documentation. + /// + /// Returns true if there are no more input in `Self` besides what has already been populated. #[inline] - fn _is_end(&self, Guard) -> bool; + fn _is_end(&self, Guard) -> bool; /// **Primitive:** See `Primitives::mark` for documentation. + /// + /// Marks a position for backtracking along with relevant state. #[inline] fn _mark(&self, Guard) -> Self::Marker; /// **Primitive:** See `Primitives::restore` for documentation. + /// + /// Resumes from a previously marked state. #[inline] fn _restore(self, Guard, Self::Marker) -> Self; } -impl<'a, I> Input for &'a [I] { +impl<'a, I: Copy> Input for &'a [I] { type Token = I; type Marker = &'a [I]; + type Iter = ::std::iter::Cloned<::std::slice::Iter<'a, I>>; + type Buffer = &'a [I]; #[inline(always)] - fn _consume(self, _g: Guard, n: usize) -> Self { - &self[n..] + fn _iter(&self, _g: Guard) -> Self::Iter { + self.iter().cloned() } #[inline(always)] - fn _buffer(&self, _g: Guard) -> &[Self::Token] { - &self + fn _consume(&mut self, _g: Guard, n: usize) -> Self::Buffer { + let b = &self[..n]; + *self = &self[n..]; + b + } + #[inline(always)] + fn _discard(&mut self, _g: Guard, n: usize) { + *self = &self[n..]; + } + #[inline(always)] + fn _min_remaining(&self, _g: Guard) -> usize { + self.len() } #[inline(always)] fn _is_end(&self, _g: Guard) -> bool { @@ -249,19 +308,29 @@ pub fn new_buf(state: InputMode, buffer: &[I]) -> InputBuf { InputBuf(state, buffer) } -impl<'a, I> Input for InputBuf<'a, I> { +impl<'a, I: Copy> Input for InputBuf<'a, I> { type Token = I; type Marker = &'a [I]; + type Iter = ::std::iter::Cloned<::std::slice::Iter<'a, I>>; + type Buffer = &'a [I]; #[inline(always)] - fn _consume(mut self, _g: Guard, n: usize) -> Self { + fn _iter(&self, _g: Guard) -> Self::Iter { + self.1.iter().cloned() + } + #[inline(always)] + fn _consume(&mut self, _g: Guard, n: usize) -> Self::Buffer { + let b = &self.1[..n]; self.1 = &self.1[n..]; - - self + b } #[inline(always)] - fn _buffer(&self, _g: Guard) -> &[Self::Token] { - &self.1 + fn _discard(&mut self, _g: Guard, n: usize) { + self.1 = &self.1[n..]; + } + #[inline(always)] + fn _min_remaining(&self, _g: Guard) -> usize { + self.1.len() } #[inline(always)] fn _is_end(&self, _g: Guard) -> bool { diff --git a/src/parsers.rs b/src/parsers.rs index 2571b9ea..a5b1f7ec 100644 --- a/src/parsers.rs +++ b/src/parsers.rs @@ -23,12 +23,14 @@ pub use debugtrace::StackFrame; /// assert_eq!(parse_only(any, b"abc"), Ok(b'a')); /// ``` #[inline] -pub fn any(i: I) -> SimpleResult { - let b = i.buffer(); +pub fn any(mut i: I) -> SimpleResult { + match i.first() { + None => i.incomplete(1), + Some(c) => { + i.discard(1); - match b.first() { - None => i.incomplete(1), - Some(&c) => i.consume(1).ret(c), + i.ret(c) + }, } } @@ -43,15 +45,16 @@ pub fn any(i: I) -> SimpleResult { /// assert_eq!(parse_only(|i| satisfy(i, |c| c == b'a'), b"abc"), Ok(b'a')); /// ``` #[inline] -pub fn satisfy(i: I, f: F) -> SimpleResult - where F: FnOnce(I::Token) -> bool, - I::Token: Copy { - let b = i.buffer(); - - match b.first() { - None => i.incomplete(1), - Some(&c) if f(c) => i.consume(1).ret(c), - Some(_) => i.err(Error::unexpected()), +pub fn satisfy(mut i: I, f: F) -> SimpleResult + where F: FnOnce(I::Token) -> bool { + match i.first() { + None => i.incomplete(1), + Some(c) if f(c) => { + i.discard(1); + + i.ret(c) + }, + Some(_) => i.err(Error::unexpected()), } } @@ -70,17 +73,17 @@ pub fn satisfy(i: I, f: F) -> SimpleResult /// assert_eq!(r, Ok(b'T')); /// ``` #[inline] -pub fn satisfy_with(i: I, f: F, p: P) -> SimpleResult - where F: FnOnce(&I::Token) -> T, +pub fn satisfy_with(mut i: I, f: F, p: P) -> SimpleResult + where F: FnOnce(I::Token) -> T, P: FnOnce(T) -> bool { - let b = i.buffer(); - - match b.first() { + match i.first() { Some(n) => { let t = f(n); if p(t.clone()) { - i.consume(1).ret(t) + i.discard(1); + + i.ret(t) } else { i.err(Error::unexpected()) } @@ -99,14 +102,16 @@ pub fn satisfy_with(i: I, f: F, p: P) -> SimpleResult< /// assert_eq!(parse_only(|i| token(i, b'a'), b"abc"), Ok(b'a')); /// ``` #[inline] -pub fn token(i: I, t: I::Token) -> SimpleResult - where I::Token: PartialEq + Copy { - let b = i.buffer(); - - match b.first() { - None => i.incomplete(1), - Some(&c) if t == c => i.consume(1).ret(c), - Some(_) => i.err(Error::expected(t)), +pub fn token(mut i: I, t: I::Token) -> SimpleResult + where I::Token: PartialEq { + match i.first() { + None => i.incomplete(1), + Some(c) if t == c => { + i.discard(1); + + i.ret(c) + }, + Some(_) => i.err(Error::expected(t)), } } @@ -120,14 +125,15 @@ pub fn token(i: I, t: I::Token) -> SimpleResult /// assert_eq!(parse_only(|i| not_token(i, b'b'), b"abc"), Ok(b'a')); /// ``` #[inline] -pub fn not_token(i: I, t: I::Token) -> SimpleResult - where I::Token: PartialEq + Copy { - let b = i.buffer(); - - match b.first() { - None => i.incomplete(1), - Some(&c) if t != c => i.consume(1).ret(c), - Some(_) => i.err(Error::unexpected()), +pub fn not_token(mut i: I, t: I::Token) -> SimpleResult + where I::Token: PartialEq { + match i.first() { + None => i.incomplete(1), + Some(c) if t != c => { + i.discard(1); + i.ret(c) + }, + Some(_) => i.err(Error::unexpected()), } } @@ -146,7 +152,7 @@ pub fn not_token(i: I, t: I::Token) -> SimpleResult #[inline] pub fn peek(i: I) -> SimpleResult> where I::Token: Clone { - let d = i.buffer().first().cloned(); + let d = i.first(); i.ret(d) } @@ -163,7 +169,7 @@ pub fn peek(i: I) -> SimpleResult> #[inline] pub fn peek_next(i: I) -> SimpleResult where I::Token: Clone { - match i.buffer().first().cloned() { + match i.first() { None => i.incomplete(1), Some(c) => i.ret(c), } @@ -179,13 +185,15 @@ pub fn peek_next(i: I) -> SimpleResult /// assert_eq!(parse_only(|i| take(i, 3), b"abcd"), Ok(&b"abc"[..])); /// ``` #[inline] -pub fn take<'a, I: Input + 'a>(i: I, num: usize) -> SimpleResult { - let b = i.buffer(); +pub fn take(mut i: I, num: usize) -> SimpleResult { + let r = i.min_remaining(); - if num <= b.len() { - i.consume(num).ret(&b[..num]) + if num <= r { + let b = i.consume(num); + + i.ret(b) } else { - i.incomplete(num - b.len()) + i.incomplete(num - r) } } @@ -212,17 +220,22 @@ pub fn take<'a, I: Input + 'a>(i: I, num: usize) -> SimpleResult(i: I, f: F) -> SimpleResult +pub fn take_while(mut i: I, f: F) -> SimpleResult where F: Fn(I::Token) -> bool { - let b = i.buffer(); + match i.iter().position(|c| f(c) == false) { + Some(n) => { + let b = i.consume(n); - match b.iter().position(|c| f(*c) == false) { - Some(n) => i.consume(n).ret(&b[..n]), + i.ret(b) + }, // TODO: Should this following 1 be something else, seeing as take_while1 is potentially // infinite? None => if i.is_end() { // Last slice and we have just read everything of it, replace with zero-sized slice: - i.consume(b.len()).ret(b) + let r = i.min_remaining(); + let b = i.consume(r); + + i.ret(b) } else { i.incomplete(1) }, @@ -243,18 +256,23 @@ pub fn take_while<'a, I: Input + 'a, F>(i: I, f: F) -> SimpleResult(i: I, f: F) -> SimpleResult +pub fn take_while1(mut i: I, f: F) -> SimpleResult where F: Fn(I::Token) -> bool { - let b = i.buffer(); - - match b.iter().position(|c| f(*c) == false) { + match i.iter().position(|c| f(c) == false) { Some(0) => i.err(Error::unexpected()), - Some(n) => i.consume(n).ret(&b[..n]), + Some(n) => { + let b = i.consume(n); + + i.ret(b) + }, // TODO: Should this following 1 be something else, seeing as take_while1 is potentially // infinite? - None => if b.len() > 0 && i.is_end() { + None => if i.min_remaining() > 0 && i.is_end() { // Last slice and we have just read everything of it, replace with zero-sized slice: - i.consume(b.len()).ret(b) + let r = i.min_remaining(); + let b = i.consume(r); + + i.ret(b) } else { i.incomplete(1) }, @@ -275,13 +293,15 @@ pub fn take_while1<'a, I: Input + 'a, F>(i: I, f: F) -> SimpleResult(i: I, f: F) -> SimpleResult +pub fn take_till(mut i: I, f: F) -> SimpleResult where F: Fn(I::Token) -> bool, I::Token: Clone { - let b = i.buffer(); + match i.iter().position(f) { + Some(n) => { + let b = i.consume(n); - match b.iter().cloned().position(f) { - Some(n) => i.consume(n).ret(&b[0..n]), + i.ret(b) + }, // TODO: Should this following 1 be something else, seeing as take_while1 is potentially // infinite? None => i.incomplete(1), @@ -303,13 +323,16 @@ pub fn take_till<'a, I: Input + 'a, F>(i: I, f: F) -> SimpleResult(i: I, s: S, mut f: F) -> SimpleResult +pub fn scan(mut i: I, s: S, mut f: F) -> SimpleResult where F: FnMut(S, I::Token) -> Option { - let b = i.buffer(); let mut state = Some(s); - match b.iter().position(|&c| { state = f(mem::replace(&mut state, None).unwrap(), c); state.is_none()}) { - Some(n) => i.consume(n).ret(&b[0..n]), + match i.iter().position(|c| { state = f(mem::replace(&mut state, None).unwrap(), c); state.is_none()}) { + Some(n) => { + let b = i.consume(n); + + i.ret(b) + }, // TODO: Should this following 1 be something else, seeing as take_while1 is potentially // infinite? None => i.incomplete(1), @@ -330,13 +353,16 @@ pub fn scan<'a, I: Input + 'a, S, F>(i: I, s: S, mut f: F) -> SimpleResult(i: I, s: S, mut f: F) -> SimpleResult +pub fn run_scanner(mut i: I, s: S, mut f: F) -> SimpleResult where F: FnMut(S, I::Token) -> Option { - let b = i.buffer(); let mut state = s; - match b.iter().position(|&c| { let t = f(state, c); match t { None => true, Some(v) => { state = v; false } } }) { - Some(n) => i.consume(n).ret((&b[0..n], state)), + match i.iter().position(|c| { let t = f(state, c); match t { None => true, Some(v) => { state = v; false } } }) { + Some(n) => { + let b = i.consume(n); + + i.ret((b, state)) + }, // TODO: Should this following 1 be something else, seeing as take_while1 is potentially // infinite? None => i.incomplete(1), @@ -351,10 +377,12 @@ pub fn run_scanner<'a, I: Input + 'a, S: Copy, F>(i: I, s: S, mut f: F) -> Simpl /// assert_eq!(parse_only(take_remainder, b"abcd"), Ok(&b"abcd"[..])); /// ``` #[inline] -pub fn take_remainder<'a, I: Input + 'a>(i: I) -> SimpleResult { - let b = i.buffer(); +pub fn take_remainder(mut i: I) -> SimpleResult { // Last slice and we have just read everything of it, replace with zero-sized slice: - i.consume(b.len()).ret(b) + let r = i.min_remaining(); + let b = i.consume(r); + + i.ret(b) } /// Matches the given slice against the parser, returning the matched slice upon success. @@ -368,23 +396,26 @@ pub fn take_remainder<'a, I: Input + 'a>(i: I) -> SimpleResult>(i: I, s: &'b [T]) - -> SimpleResult { - let b = i.buffer(); +pub fn string<'b, T: Copy + PartialEq, I: Input>(mut i: I, s: &'b [T]) + -> SimpleResult { + let r = i.min_remaining(); - if s.len() > b.len() { - return i.incomplete(s.len() - b.len()); + if s.len() > r { + return i.incomplete(s.len() - r); } - let d = &b[..s.len()]; + // TODO: Check if this is efficient: + for (n, (a, b)) in i.iter().zip(s.iter()).enumerate() { + if a != *b { + i.discard(n); - for j in 0..s.len() { - if s[j] != d[j] { - return i.consume(j).err(Error::expected(d[j])) + return i.err(Error::expected(*b)); } } - i.consume(s.len()).ret(d) + let b = i.consume(s.len()); + + i.ret(b) } /// Matches the end of the input. @@ -398,7 +429,7 @@ pub fn string<'a, 'b, T: Copy + PartialEq, I: 'a + Input>(i: I, s: &'b /// ``` #[inline] pub fn eof(i: I) -> SimpleResult { - if i.buffer().len() == 0 && i.is_end() { + if i.min_remaining() == 0 && i.is_end() { i.ret(()) } else { i.err(Error::unexpected()) @@ -566,7 +597,7 @@ mod test { use primitives::IntoInner; use primitives::State; use super::*; - use {Input, SimpleResult}; + use {Input, SimpleResult, ParseResult}; #[test] fn parse_decimal() { @@ -574,7 +605,7 @@ mod test { c >= b'0' && c <= b'9' } - fn decimal(i: Input) -> SimpleResult { + fn decimal<'i, I: Input>(i: I) -> SimpleResult { take_while1(i, is_digit).bind(|i, bytes| i.ret(bytes.iter().fold(0, |a, b| a * 10 + (b - b'0') as usize))) } @@ -586,7 +617,7 @@ mod test { decimal(i).bind(|i, frac| i.ret((real, frac))))); - let d: SimpleResult<_, _> = p.bind(|i, num| take_remainder(i) + let d: ParseResult<_, _, Error> = p.bind(|i, num| take_remainder(i) .bind(|i, r| i.ret((r, num)))); assert_eq!(d.into_inner(), State::Data(&b""[..], (&b" "[..], (123, 4567)))); @@ -614,7 +645,7 @@ mod test { fn token_test() { assert_eq!(token(new_buf(DEFAULT, b""), b'a').into_inner(), State::Incomplete(1)); assert_eq!(token(new_buf(DEFAULT, b"ab"), b'a').into_inner(), State::Data(new_buf(DEFAULT, b"b"), b'a')); - assert_eq!(token(new_buf(DEFAULT, b"bb"), b'a').into_inner(), State::Error(b"bb", Error::expected(b'a'))); + assert_eq!(token(new_buf(DEFAULT, b"bb"), b'a').into_inner(), State::Error(new_buf(DEFAULT, b"bb"), Error::expected(b'a'))); } #[test] @@ -640,7 +671,7 @@ mod test { #[test] fn take_while1_test() { assert_eq!(take_while1(new_buf(DEFAULT, b"abc"), |c| c != b'b').into_inner(), State::Data(new_buf(DEFAULT, b"bc"), &b"a"[..])); - assert_eq!(take_while1(new_buf(DEFAULT, b"bbc"), |c| c != b'b').into_inner(), State::Error(&b"bbc"[..], Error::unexpected())); + assert_eq!(take_while1(new_buf(DEFAULT, b"bbc"), |c| c != b'b').into_inner(), State::Error(new_buf(DEFAULT, b"bbc"), Error::unexpected())); assert_eq!(take_while1(&b"bbc"[..], |c| c != b'b').into_inner(), State::Error(&b"bbc"[..], Error::unexpected())); assert_eq!(take_while1(&b"abc"[..], |c| c != b'b').into_inner(), State::Data(&b"bc"[..], &b"a"[..])); // TODO: Update when the incomplete type has been updated @@ -678,14 +709,14 @@ mod test { assert_eq!(string(new_buf(DEFAULT, b"abc"), b"abc").into_inner(), State::Data(new_buf(DEFAULT, b""), &b"abc"[..])); assert_eq!(string(new_buf(DEFAULT, b"abc"), b"abcd").into_inner(), State::Incomplete(1)); assert_eq!(string(new_buf(DEFAULT, b"abc"), b"abcde").into_inner(), State::Incomplete(2)); - assert_eq!(string(new_buf(DEFAULT, b"abc"), b"ac").into_inner(), State::Error(b"bc", Error::expected(b'b'))); + assert_eq!(string(new_buf(DEFAULT, b"abc"), b"ac").into_inner(), State::Error(new_buf(DEFAULT, b"bc"), Error::expected(b'c'))); assert_eq!(string(&b"abc"[..], b"a").into_inner(), State::Data(&b"bc"[..], &b"a"[..])); assert_eq!(string(&b"abc"[..], b"ab").into_inner(), State::Data(&b"c"[..], &b"ab"[..])); assert_eq!(string(&b"abc"[..], b"abc").into_inner(), State::Data(&b""[..], &b"abc"[..])); assert_eq!(string(&b"abc"[..], b"abcd").into_inner(), State::Incomplete(1)); assert_eq!(string(&b"abc"[..], b"abcde").into_inner(), State::Incomplete(2)); - assert_eq!(string(&b"abc"[..], b"ac").into_inner(), State::Error(b"bc", Error::expected(b'b'))); + assert_eq!(string(&b"abc"[..], b"ac").into_inner(), State::Error(&b"bc"[..], Error::expected(b'c'))); } #[test] From 64bec6acc0de815b99b4f5d8b8a0e566f8d24083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Fri, 18 Mar 2016 02:06:53 +0100 Subject: [PATCH 03/63] Updated combinators * State::Incomplete now also carries the input state, needed to be able to backtrack on incomplete --- src/combinators/mod.rs | 510 +++++++++++++++++++++-------------------- src/input.rs | 6 +- src/lib.rs | 38 +-- src/parse_result.rs | 26 +-- src/parsers.rs | 28 +-- 5 files changed, 313 insertions(+), 295 deletions(-) diff --git a/src/combinators/mod.rs b/src/combinators/mod.rs index c716c9e5..1901723d 100644 --- a/src/combinators/mod.rs +++ b/src/combinators/mod.rs @@ -3,14 +3,14 @@ #[macro_use] mod macros; -pub mod bounded; +//pub mod bounded; use std::iter::FromIterator; use {ParseResult, Input}; use primitives::State; -use primitives::{IntoInner, InputBuffer, InputClone}; +use primitives::{IntoInner, Primitives}; /// Applies the parser ``p`` exactly ``num`` times collecting all items into `T: FromIterator`. /// @@ -29,12 +29,12 @@ use primitives::{IntoInner, InputBuffer, InputClone}; /// assert_eq!(parse_only(with_remainder, b"aaa"), Ok((&b"a"[..], vec![b'a', b'a']))); /// ``` #[inline] -pub fn count<'a, I, T, E, F, U>(i: Input<'a, I>, num: usize, p: F) -> ParseResult<'a, I, T, E> - where I: Copy, - U: 'a, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, +pub fn count(i: I, num: usize, p: F) -> ParseResult + where F: FnMut(I) -> ParseResult, T: FromIterator { - bounded::many(i, num, p) + // FIXME: implement + //bounded::many(i, num, p) + unimplemented!() } /// Tries the parser ``f``, on success it yields the parsed value, on failure ``default`` will be @@ -53,16 +53,24 @@ pub fn count<'a, I, T, E, F, U>(i: Input<'a, I>, num: usize, p: F) -> ParseResul /// assert_eq!(parse_only(f, b"bbc"), Ok(b'd')); /// ``` #[inline] -pub fn option<'a, I, T, E, F>(i: Input<'a, I>, f: F, default: T) -> ParseResult<'a, I, T, E> - where I: 'a + Copy, - F: FnOnce(Input<'a, I>) -> ParseResult<'a, I, T, E> { - match f(i.clone()).into_inner() { +pub fn option(i: I, f: F, default: T) -> ParseResult + where F: FnOnce(I) -> ParseResult { + let m = i.mark(); + + // TODO: Make macro? Shared with or + match f(i).into_inner() { State::Data(b, d) => b.ret(d), - State::Error(_, _) => i.ret(default), - State::Incomplete(n) => if i.is_last_slice() { - i.ret(default) - } else { - i.incomplete(n) + State::Error(b, _) => { + b.restore(m).ret(default) + }, + State::Incomplete(b, n) => { + let b = b.restore(m); + + if b.is_end() { + b.ret(default) + } else { + b.incomplete(n) + } }, } } @@ -87,16 +95,22 @@ pub fn option<'a, I, T, E, F>(i: Input<'a, I>, f: F, default: T) -> ParseResult< /// assert_eq!(parse_only(&p, b"cbc"), Err(ParseError::Error(b"cbc", Error::expected(b'b')))); /// ``` #[inline] -pub fn or<'a, I, T, E, F, G>(i: Input<'a, I>, f: F, g: G) -> ParseResult<'a, I, T, E> - where F: FnOnce(Input<'a, I>) -> ParseResult<'a, I, T, E>, - G: FnOnce(Input<'a, I>) -> ParseResult<'a, I, T, E> { - match f(i.clone()).into_inner() { +pub fn or(i: I, f: F, g: G) -> ParseResult + where F: FnOnce(I) -> ParseResult, + G: FnOnce(I) -> ParseResult { + let m = i.mark(); + + match f(i).into_inner() { State::Data(b, d) => b.ret(d), - State::Error(_, _) => g(i), - State::Incomplete(n) => if i.is_last_slice() { - g(i) - } else { - i.incomplete(n) + State::Error(b, _) => g(b.restore(m)), + State::Incomplete(b, n) => { + let b = b.restore(m); + + if b.is_end() { + g(b) + } else { + b.incomplete(n) + } }, } } @@ -121,12 +135,12 @@ pub fn or<'a, I, T, E, F, G>(i: Input<'a, I>, f: F, g: G) -> ParseResult<'a, I, /// assert_eq!(r, Ok(vec![&b"a"[..], &b"bc"[..]])); /// ``` #[inline] -pub fn many<'a, I, T, E, F, U>(i: Input<'a, I>, f: F) -> ParseResult<'a, I, T, E> - where I: Copy, - U: 'a, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, +pub fn many(i: I, f: F) -> ParseResult + where F: FnMut(I) -> ParseResult, T: FromIterator { - bounded::many(i, .., f) + // FIXME: Implement + //bounded::many(i, .., f) + unimplemented!() } /// Parses at least one instance of ``f`` and continues until it does no longer match, collecting @@ -148,12 +162,12 @@ pub fn many<'a, I, T, E, F, U>(i: Input<'a, I>, f: F) -> ParseResult<'a, I, T, E /// assert_eq!(parse_only(&p, b"a, "), Ok(vec![&b"a"[..]])); /// ``` #[inline] -pub fn many1<'a, I, T, E, F, U>(i: Input<'a, I>, f: F) -> ParseResult<'a, I, T, E> - where I: Copy, - U: 'a, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, +pub fn many1(i: I, f: F) -> ParseResult + where F: FnMut(I) -> ParseResult, T: FromIterator { - bounded::many(i, 1.., f) + // FIXME: implement + //bounded::many(i, 1.., f) + unimplemented!() } /// Applies the parser `R` zero or more times, separated by the parser `F`. All matches from `R` @@ -173,16 +187,14 @@ pub fn many1<'a, I, T, E, F, U>(i: Input<'a, I>, f: F) -> ParseResult<'a, I, T, /// assert_eq!(r, Ok(vec![91, 03, 20])); /// ``` #[inline] -pub fn sep_by<'a, I, T, E, R, F, U, N, V>(i: Input<'a, I>, p: R, sep: F) -> ParseResult<'a, I, T, E> - where I: Copy, - U: 'a, - V: 'a, - N: 'a, - T: FromIterator, +pub fn sep_by(i: I, p: R, sep: F) -> ParseResult + where T: FromIterator, E: From, - R: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, V, N> { - bounded::sep_by(i, .., p, sep) + R: FnMut(I) -> ParseResult, + F: FnMut(I) -> ParseResult { + // FIXME: implement + //bounded::sep_by(i, .., p, sep) + unimplemented!() } @@ -203,16 +215,14 @@ pub fn sep_by<'a, I, T, E, R, F, U, N, V>(i: Input<'a, I>, p: R, sep: F) -> Pars /// assert_eq!(r, Ok(vec![91, 03, 20])); /// ``` #[inline] -pub fn sep_by1<'a, I, T, E, R, F, U, N, V>(i: Input<'a, I>, p: R, sep: F) -> ParseResult<'a, I, T, E> - where I: Copy, - U: 'a, - V: 'a, - N: 'a, - T: FromIterator, +pub fn sep_by1(i: I, p: R, sep: F) -> ParseResult + where T: FromIterator, E: From, - R: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, V, N> { - bounded::sep_by(i, 1.., p, sep) + R: FnMut(I) -> ParseResult, + F: FnMut(I) -> ParseResult { + // FIXME: Implement + //bounded::sep_by(i, 1.., p, sep) + unimplemented!() } /// Applies the parser `R` multiple times until the parser `F` succeeds and returns a @@ -230,15 +240,13 @@ pub fn sep_by1<'a, I, T, E, R, F, U, N, V>(i: Input<'a, I>, p: R, sep: F) -> Par /// assert_eq!(r, Ok(vec![b'a', b'b', b'c'])); /// ``` #[inline] -pub fn many_till<'a, I, T, E, R, F, U, N, V>(i: Input<'a, I>, p: R, end: F) -> ParseResult<'a, I, T, E> - where I: Copy, - U: 'a, - V: 'a, - N: 'a, - T: FromIterator, - R: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, V, N> { - bounded::many_till(i, .., p, end) +pub fn many_till(i: I, p: R, end: F) -> ParseResult + where T: FromIterator, + R: FnMut(I) -> ParseResult, + F: FnMut(I) -> ParseResult { + //bounded::many_till(i, .., p, end) + // FIXME: Implement + unimplemented!() } /// Runs the given parser until it fails, discarding matched input. @@ -256,10 +264,11 @@ pub fn many_till<'a, I, T, E, R, F, U, N, V>(i: Input<'a, I>, p: R, end: F) -> P /// assert_eq!(r, Ok(b'b')); /// ``` #[inline] -pub fn skip_many<'a, I, T, E, F>(i: Input<'a, I>, f: F) -> ParseResult<'a, I, (), E> - where T: 'a, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, T, E> { - bounded::skip_many(i, .., f) +pub fn skip_many(i: I, f: F) -> ParseResult + where F: FnMut(I) -> ParseResult { + //bounded::skip_many(i, .., f) + //FIXME: Implement + unimplemented!() } /// Runs the given parser until it fails, discarding matched input, expects at least one match. @@ -281,9 +290,11 @@ pub fn skip_many<'a, I, T, E, F>(i: Input<'a, I>, f: F) -> ParseResult<'a, I, () /// assert_eq!(parse_only(&p, b"bc"), Err(ParseError::Error(b"bc", Error::expected(b'a')))); /// ``` #[inline] -pub fn skip_many1<'a, I, T, E, F>(i: Input<'a, I>, f: F) -> ParseResult<'a, I, (), E> - where T: 'a, F: FnMut(Input<'a, I>) -> ParseResult<'a, I, T, E> { - bounded::skip_many(i, 1.., f) +pub fn skip_many1(i: I, f: F) -> ParseResult + where F: FnMut(I) -> ParseResult { + // FIXME: Implement + //bounded::skip_many(i, 1.., f) + unimplemented!() } /// Returns the result of the given parser as well as the slice which matched it. @@ -295,9 +306,11 @@ pub fn skip_many1<'a, I, T, E, F>(i: Input<'a, I>, f: F) -> ParseResult<'a, I, ( /// assert_eq!(parse_only(|i| matched_by(i, decimal), b"123"), Ok((&b"123"[..], 123u32))); /// ``` #[inline] -pub fn matched_by<'a, I, T, E, F>(i: Input<'a, I>, f: F) -> ParseResult<'a, I, (&'a [I], T), E> - where T: 'a, - F: FnOnce(Input<'a, I>) -> ParseResult<'a, I, T, E> { +pub fn matched_by(i: I, f: F) -> ParseResult + where F: FnOnce(I) -> ParseResult { + // TODO: Implement + unimplemented!() + /* let buf = i.buffer(); match f(i.clone()).into_inner() { @@ -311,6 +324,7 @@ pub fn matched_by<'a, I, T, E, F>(i: Input<'a, I>, f: F) -> ParseResult<'a, I, ( State::Error(b, e) => i.replace(b).err(e), State::Incomplete(n) => i.incomplete(n), } + */ } /// Applies the parser `F` without consuming any input. @@ -324,20 +338,22 @@ pub fn matched_by<'a, I, T, E, F>(i: Input<'a, I>, f: F) -> ParseResult<'a, I, ( /// assert_eq!(parse_only(p, b"testing"), Ok((&b"test"[..], &b"testing"[..]))); /// ``` #[inline] -pub fn look_ahead<'a, I, T, E, F>(i: Input<'a, I>, f: F) -> ParseResult<'a, I, T, E> - where F: FnOnce(Input<'a, I>) -> ParseResult<'a, I, T, E> { - match f(i.clone()).into_inner() { - State::Data(_, t) => i.ret(t), - State::Error(b, t) => i.replace(b).err(t), - State::Incomplete(n) => i.incomplete(n), +pub fn look_ahead(i: I, f: F) -> ParseResult + where F: FnOnce(I) -> ParseResult { + let m = i.mark(); + + match f(i).into_inner() { + State::Data(b, t) => b.restore(m).ret(t), + State::Error(b, t) => b.restore(m).err(t), + State::Incomplete(b, n) => b.restore(m).incomplete(n), } } #[cfg(test)] mod test { - use ParseResult; + use {Input, ParseResult}; use primitives::State; - use primitives::input::{new, DEFAULT, END_OF_INPUT}; + use primitives::input::{new_buf, DEFAULT, END_OF_INPUT}; use primitives::IntoInner; use super::*; @@ -345,239 +361,239 @@ mod test { #[test] fn option_test() { - assert_eq!(option(new(DEFAULT, b""), any, b'-').into_inner(), State::Incomplete(1)); - assert_eq!(option(new(DEFAULT, b"a"), any, b'-').into_inner(), State::Data(new(DEFAULT, b""), b'a')); - assert_eq!(option(new(DEFAULT, b""), |i| take(i, 2).map(ToOwned::to_owned), vec![]).into_inner(), State::Incomplete(2)); - assert_eq!(option(new(DEFAULT, b"a"), |i| take(i, 2).map(ToOwned::to_owned), vec![]).into_inner(), State::Incomplete(1)); - assert_eq!(option(new(DEFAULT, b"ab"), |i| take(i, 2).map(ToOwned::to_owned), vec![]).into_inner(), State::Data(new(DEFAULT, b""), vec![b'a', b'b'])); + assert_eq!(option(new_buf(DEFAULT, b""), any, b'-').into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + assert_eq!(option(new_buf(DEFAULT, b"a"), any, b'-').into_inner(), State::Data(new_buf(DEFAULT, b""), b'a')); + assert_eq!(option(new_buf(DEFAULT, b""), |i| take(i, 2).map(ToOwned::to_owned), vec![]).into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); + assert_eq!(option(new_buf(DEFAULT, b"a"), |i| take(i, 2).map(ToOwned::to_owned), vec![]).into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); + assert_eq!(option(new_buf(DEFAULT, b"ab"), |i| take(i, 2).map(ToOwned::to_owned), vec![]).into_inner(), State::Data(new_buf(DEFAULT, b""), vec![b'a', b'b'])); - assert_eq!(option(new(DEFAULT, b"a"), |i| token(i, b' ').map_err(|_| "token_err"), b'-').into_inner(), State::Data(new(DEFAULT, b"a"), b'-')); + assert_eq!(option(new_buf(DEFAULT, b"a"), |i| token(i, b' ').map_err(|_| "token_err"), b'-').into_inner(), State::Data(new_buf(DEFAULT, b"a"), b'-')); - assert_eq!(option(new(END_OF_INPUT, b""), any, b'-').into_inner(), State::Data(new(END_OF_INPUT, b""), b'-')); - assert_eq!(option(new(END_OF_INPUT, b"a"), any, b'-').into_inner(), State::Data(new(END_OF_INPUT, b""), b'a')); - assert_eq!(option(new(END_OF_INPUT, b""), |i| take(i, 2).map(ToOwned::to_owned), vec![]).into_inner(), State::Data(new(END_OF_INPUT, b""), vec![])); - assert_eq!(option(new(END_OF_INPUT, b"a"), |i| take(i, 2).map(ToOwned::to_owned), vec![]).into_inner(), State::Data(new(END_OF_INPUT, b"a"), vec![])); - assert_eq!(option(new(END_OF_INPUT, b"ab"), |i| take(i, 2).map(ToOwned::to_owned), vec![]).into_inner(), State::Data(new(END_OF_INPUT, b""), vec![b'a', b'b'])); + assert_eq!(option(new_buf(END_OF_INPUT, b""), any, b'-').into_inner(), State::Data(new_buf(END_OF_INPUT, b""), b'-')); + assert_eq!(option(new_buf(END_OF_INPUT, b"a"), any, b'-').into_inner(), State::Data(new_buf(END_OF_INPUT, b""), b'a')); + assert_eq!(option(new_buf(END_OF_INPUT, b""), |i| take(i, 2).map(ToOwned::to_owned), vec![]).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); + assert_eq!(option(new_buf(END_OF_INPUT, b"a"), |i| take(i, 2).map(ToOwned::to_owned), vec![]).into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![])); + assert_eq!(option(new_buf(END_OF_INPUT, b"ab"), |i| take(i, 2).map(ToOwned::to_owned), vec![]).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'b'])); - assert_eq!(option(new(END_OF_INPUT, b"a"), |i| token(i, b' ').map_err(|_| "token_err"), b'-').into_inner(), State::Data(new(END_OF_INPUT, b"a"), b'-')); + assert_eq!(option(new_buf(END_OF_INPUT, b"a"), |i| token(i, b' ').map_err(|_| "token_err"), b'-').into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), b'-')); } #[test] fn or_test() { - assert_eq!(or(new(DEFAULT, b""), any, any).into_inner(), State::Incomplete(1)); - assert_eq!(or(new(DEFAULT, b"a"), any, any).into_inner(), State::Data(new(DEFAULT, b""), b'a')); - assert_eq!(or(new(DEFAULT, b"a"), |i| take(i, 2), |i| take(i, 1)).into_inner(), State::Incomplete(1)); - assert_eq!(or(new(DEFAULT, b"ab"), |i| take(i, 2), |i| take(i, 1)).into_inner(), State::Data(new(DEFAULT, b""), &b"ab"[..])); - assert_eq!(or(new(DEFAULT, b"a"), |i| token(i, b'a'), |i| token(i, b'b')).into_inner(), State::Data(new(DEFAULT, b""), b'a')); - assert_eq!(or(new(DEFAULT, b"b"), |i| token(i, b'a'), |i| token(i, b'b')).into_inner(), State::Data(new(DEFAULT, b""), b'b')); - assert_eq!(or(new(DEFAULT, b"c"), |i| token(i, b'a').map_err(|_| "a err"), |i| token(i, b'b').map_err(|_| "b err")).into_inner(), State::Error(b"c", "b err")); - - assert_eq!(or(new(END_OF_INPUT, b""), any, any).into_inner(), State::Incomplete(1)); - assert_eq!(or(new(END_OF_INPUT, b"a"), any, any).into_inner(), State::Data(new(END_OF_INPUT, b""), b'a')); - assert_eq!(or(new(END_OF_INPUT, b"a"), |i| take(i, 2), |i| take(i, 1)).into_inner(), State::Data(new(END_OF_INPUT, b""), &b"a"[..])); - assert_eq!(or(new(END_OF_INPUT, b"ab"), |i| take(i, 2), |i| take(i, 1)).into_inner(), State::Data(new(END_OF_INPUT, b""), &b"ab"[..])); - assert_eq!(or(new(END_OF_INPUT, b"a"), |i| token(i, b'a'), |i| token(i, b'b')).into_inner(), State::Data(new(END_OF_INPUT, b""), b'a')); - assert_eq!(or(new(END_OF_INPUT, b"b"), |i| token(i, b'a'), |i| token(i, b'b')).into_inner(), State::Data(new(END_OF_INPUT, b""), b'b')); - assert_eq!(or(new(END_OF_INPUT, b"c"), |i| token(i, b'a').map_err(|_| "a err"), |i| token(i, b'b').map_err(|_| "b err")).into_inner(), State::Error(b"c", "b err")); + assert_eq!(or(new_buf(DEFAULT, b""), any, any).into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + assert_eq!(or(new_buf(DEFAULT, b"a"), any, any).into_inner(), State::Data(new_buf(DEFAULT, b""), b'a')); + assert_eq!(or(new_buf(DEFAULT, b"a"), |i| take(i, 2), |i| take(i, 1)).into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); + assert_eq!(or(new_buf(DEFAULT, b"ab"), |i| take(i, 2), |i| take(i, 1)).into_inner(), State::Data(new_buf(DEFAULT, b""), &b"ab"[..])); + assert_eq!(or(new_buf(DEFAULT, b"a"), |i| token(i, b'a'), |i| token(i, b'b')).into_inner(), State::Data(new_buf(DEFAULT, b""), b'a')); + assert_eq!(or(new_buf(DEFAULT, b"b"), |i| token(i, b'a'), |i| token(i, b'b')).into_inner(), State::Data(new_buf(DEFAULT, b""), b'b')); + assert_eq!(or(new_buf(DEFAULT, b"c"), |i| token(i, b'a').map_err(|_| "a err"), |i| token(i, b'b').map_err(|_| "b err")).into_inner(), State::Error(new_buf(DEFAULT, b"c"), "b err")); + + assert_eq!(or(new_buf(END_OF_INPUT, b""), any, any).into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); + assert_eq!(or(new_buf(END_OF_INPUT, b"a"), any, any).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), b'a')); + assert_eq!(or(new_buf(END_OF_INPUT, b"a"), |i| take(i, 2), |i| take(i, 1)).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), &b"a"[..])); + assert_eq!(or(new_buf(END_OF_INPUT, b"ab"), |i| take(i, 2), |i| take(i, 1)).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), &b"ab"[..])); + assert_eq!(or(new_buf(END_OF_INPUT, b"a"), |i| token(i, b'a'), |i| token(i, b'b')).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), b'a')); + assert_eq!(or(new_buf(END_OF_INPUT, b"b"), |i| token(i, b'a'), |i| token(i, b'b')).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), b'b')); + assert_eq!(or(new_buf(END_OF_INPUT, b"c"), |i| token(i, b'a').map_err(|_| "a err"), |i| token(i, b'b').map_err(|_| "b err")).into_inner(), State::Error(new_buf(END_OF_INPUT, b"c"), "b err")); } #[test] fn many_test() { - let r: State<_, Vec<_>, _> = many(new(DEFAULT, b""), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(1)); - let r: State<_, Vec<_>, _> = many(new(DEFAULT, b"a"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(1)); - let r: State<_, Vec<_>, _> = many(new(DEFAULT, b"aa"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(1)); - - let r: State<_, Vec<_>, _> = many(new(DEFAULT, b"bbb"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new(DEFAULT, b"bbb"), vec![])); - let r: State<_, Vec<_>, _> = many(new(DEFAULT, b"abb"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new(DEFAULT, b"bb"), vec![b'a'])); - let r: State<_, Vec<_>, _> = many(new(DEFAULT, b"aab"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new(DEFAULT, b"b"), vec![b'a', b'a'])); - - let r: State<_, Vec<_>, _> = many(new(END_OF_INPUT, b""), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new(END_OF_INPUT, b""), vec![])); - let r: State<_, Vec<_>, _> = many(new(END_OF_INPUT, b"a"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new(END_OF_INPUT, b""), vec![b'a'])); - let r: State<_, Vec<_>, _> = many(new(END_OF_INPUT, b"aa"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new(END_OF_INPUT, b""), vec![b'a', b'a'])); - - let r: State<_, Vec<_>, _> = many(new(END_OF_INPUT, b"aab"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new(END_OF_INPUT, b"b"), vec![b'a', b'a'])); + let r: State<_, Vec<_>, _> = many(new_buf(DEFAULT, b""), |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: State<_, Vec<_>, _> = many(new_buf(DEFAULT, b"a"), |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b"a"), 1)); + let r: State<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aa"), |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b"a"), 1)); + + let r: State<_, Vec<_>, _> = many(new_buf(DEFAULT, b"bbb"), |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Data(new_buf(DEFAULT, b"bbb"), vec![])); + let r: State<_, Vec<_>, _> = many(new_buf(DEFAULT, b"abb"), |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Data(new_buf(DEFAULT, b"bb"), vec![b'a'])); + let r: State<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aab"), |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Data(new_buf(DEFAULT, b"b"), vec![b'a', b'a'])); + + let r: State<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b""), |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Data(new_buf(END_OF_INPUT, b""), vec![])); + let r: State<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"a"), |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Data(new_buf(END_OF_INPUT, b""), vec![b'a'])); + let r: State<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aa"), |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'a'])); + + let r: State<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aab"), |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a', b'a'])); } #[test] fn many1_test() { - let r: State<_, Vec<_>, _> = many1(new(DEFAULT, b""), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(1)); - let r: State<_, Vec<_>, _> = many1(new(DEFAULT, b"a"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(1)); - let r: State<_, Vec<_>, _> = many1(new(DEFAULT, b"aa"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(1)); - - let r: State<_, Vec<_>, _> = many1(new(DEFAULT, b"bbb"), |i| token(i, b'a').map_err(|_| "token_error")).into_inner(); - assert_eq!(r, State::Error(b"bbb", "token_error")); - let r: State<_, Vec<_>, _> = many1(new(DEFAULT, b"abb"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new(DEFAULT, b"bb"), vec![b'a'])); - let r: State<_, Vec<_>, _> = many1(new(DEFAULT, b"aab"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new(DEFAULT, b"b"), vec![b'a', b'a'])); - - let r: State<_, Vec<_>, _> = many1(new(END_OF_INPUT, b""), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(1)); - let r: State<_, Vec<_>, _> = many1(new(END_OF_INPUT, b"a"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new(END_OF_INPUT, b""), vec![b'a'])); - let r: State<_, Vec<_>, _> = many1(new(END_OF_INPUT, b"aa"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new(END_OF_INPUT, b""), vec![b'a', b'a'])); - - let r: State<_, Vec<_>, _> = many1(new(END_OF_INPUT, b"aab"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new(END_OF_INPUT, b"b"), vec![b'a', b'a'])); + let r: State<_, Vec<_>, _> = many1(new_buf(DEFAULT, b""), |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: State<_, Vec<_>, _> = many1(new_buf(DEFAULT, b"a"), |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b"a"), 1)); + let r: State<_, Vec<_>, _> = many1(new_buf(DEFAULT, b"aa"), |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b"aa"), 1)); + + let r: State<_, Vec<_>, _> = many1(new_buf(DEFAULT, b"bbb"), |i| token(i, b'a').map_err(|_| "token_error")).into_inner(); + assert_eq!(r, State::Error(new_buf(DEFAULT, b"bbb"), "token_error")); + let r: State<_, Vec<_>, _> = many1(new_buf(DEFAULT, b"abb"), |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Data(new_buf(DEFAULT, b"bb"), vec![b'a'])); + let r: State<_, Vec<_>, _> = many1(new_buf(DEFAULT, b"aab"), |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Data(new_buf(DEFAULT, b"b"), vec![b'a', b'a'])); + + let r: State<_, Vec<_>, _> = many1(new_buf(END_OF_INPUT, b""), |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); + let r: State<_, Vec<_>, _> = many1(new_buf(END_OF_INPUT, b"a"), |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Data(new_buf(END_OF_INPUT, b""), vec![b'a'])); + let r: State<_, Vec<_>, _> = many1(new_buf(END_OF_INPUT, b"aa"), |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'a'])); + + let r: State<_, Vec<_>, _> = many1(new_buf(END_OF_INPUT, b"aab"), |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a', b'a'])); } #[test] fn count_test() { - let r: State<_, Vec<_>, _> = count(new(DEFAULT, b""), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(1)); - let r: State<_, Vec<_>, _> = count(new(DEFAULT, b"a"), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(1)); - let r: State<_, Vec<_>, _> = count(new(DEFAULT, b"aa"), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(1)); - let r: State<_, Vec<_>, _> = count(new(DEFAULT, b"aaa"), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new(DEFAULT, b""), vec![b'a', b'a', b'a'])); - let r: State<_, Vec<_>, _> = count(new(DEFAULT, b"aaaa"), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new(DEFAULT, b"a"), vec![b'a', b'a', b'a'])); - - let r: State<_, Vec<_>, _> = count(new(END_OF_INPUT, b""), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(1)); - let r: State<_, Vec<_>, _> = count(new(END_OF_INPUT, b"a"), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(1)); - let r: State<_, Vec<_>, _> = count(new(END_OF_INPUT, b"aa"), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(1)); - let r: State<_, Vec<_>, _> = count(new(END_OF_INPUT, b"aaa"), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new(END_OF_INPUT, b""), vec![b'a', b'a', b'a'])); - let r: State<_, Vec<_>, _> = count(new(END_OF_INPUT, b"aaaa"), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new(END_OF_INPUT, b"a"), vec![b'a', b'a', b'a'])); + let r: State<_, Vec<_>, _> = count(new_buf(DEFAULT, b""), 3, |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: State<_, Vec<_>, _> = count(new_buf(DEFAULT, b"a"), 3, |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b"a"), 1)); + let r: State<_, Vec<_>, _> = count(new_buf(DEFAULT, b"aa"), 3, |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b"aa"), 1)); + let r: State<_, Vec<_>, _> = count(new_buf(DEFAULT, b"aaa"), 3, |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Data(new_buf(DEFAULT, b""), vec![b'a', b'a', b'a'])); + let r: State<_, Vec<_>, _> = count(new_buf(DEFAULT, b"aaaa"), 3, |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Data(new_buf(DEFAULT, b"a"), vec![b'a', b'a', b'a'])); + + let r: State<_, Vec<_>, _> = count(new_buf(END_OF_INPUT, b""), 3, |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); + let r: State<_, Vec<_>, _> = count(new_buf(END_OF_INPUT, b"a"), 3, |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Incomplete(new_buf(END_OF_INPUT, b"a"), 1)); + let r: State<_, Vec<_>, _> = count(new_buf(END_OF_INPUT, b"aa"), 3, |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Incomplete(new_buf(END_OF_INPUT, b"aa"), 1)); + let r: State<_, Vec<_>, _> = count(new_buf(END_OF_INPUT, b"aaa"), 3, |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'a', b'a'])); + let r: State<_, Vec<_>, _> = count(new_buf(END_OF_INPUT, b"aaaa"), 3, |i| token(i, b'a')).into_inner(); + assert_eq!(r, State::Data(new_buf(END_OF_INPUT, b"a"), vec![b'a', b'a', b'a'])); } #[test] fn skip_many1_test() { - assert_eq!(skip_many1(new(DEFAULT, b"aabc"), |i| token(i, b'a')).into_inner(), State::Data(new(DEFAULT, b"bc"), ())); - assert_eq!(skip_many1(new(DEFAULT, b"abc"), |i| token(i, b'a')).into_inner(), State::Data(new(DEFAULT, b"bc"), ())); - assert_eq!(skip_many1(new(DEFAULT, b"bc"), |i| i.err::<(), _>("error")).into_inner(), State::Error(b"bc", "error")); - assert_eq!(skip_many1(new(DEFAULT, b"aaa"), |i| token(i, b'a')).into_inner(), State::Incomplete(1)); - assert_eq!(skip_many1(new(END_OF_INPUT, b"aabc"), |i| token(i, b'a')).into_inner(), State::Data(new(END_OF_INPUT, b"bc"), ())); - assert_eq!(skip_many1(new(END_OF_INPUT, b"abc"), |i| token(i, b'a')).into_inner(), State::Data(new(END_OF_INPUT, b"bc"), ())); - assert_eq!(skip_many1(new(END_OF_INPUT, b"bc"), |i| i.err::<(), _>("error")).into_inner(), State::Error(b"bc", "error")); - assert_eq!(skip_many1(new(END_OF_INPUT, b"aaa"), |i| token(i, b'a')).into_inner(), State::Data(new(END_OF_INPUT, b""), ())); + assert_eq!(skip_many1(new_buf(DEFAULT, b"aabc"), |i| token(i, b'a')).into_inner(), State::Data(new_buf(DEFAULT, b"bc"), ())); + assert_eq!(skip_many1(new_buf(DEFAULT, b"abc"), |i| token(i, b'a')).into_inner(), State::Data(new_buf(DEFAULT, b"bc"), ())); + assert_eq!(skip_many1(new_buf(DEFAULT, b"bc"), |i| i.err::<(), _>("error")).into_inner(), State::Error(new_buf(DEFAULT, b"bc"), "error")); + assert_eq!(skip_many1(new_buf(DEFAULT, b"aaa"), |i| token(i, b'a')).into_inner(), State::Incomplete(new_buf(DEFAULT, b"aaa"), 1)); + assert_eq!(skip_many1(new_buf(END_OF_INPUT, b"aabc"), |i| token(i, b'a')).into_inner(), State::Data(new_buf(END_OF_INPUT, b"bc"), ())); + assert_eq!(skip_many1(new_buf(END_OF_INPUT, b"abc"), |i| token(i, b'a')).into_inner(), State::Data(new_buf(END_OF_INPUT, b"bc"), ())); + assert_eq!(skip_many1(new_buf(END_OF_INPUT, b"bc"), |i| i.err::<(), _>("error")).into_inner(), State::Error(new_buf(END_OF_INPUT, b"bc"), "error")); + assert_eq!(skip_many1(new_buf(END_OF_INPUT, b"aaa"), |i| token(i, b'a')).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); } #[test] fn many_till_test() { - assert_eq!(many_till(new(DEFAULT, b"abcd"), any, |i| token(i, b'c')).into_inner(), State::Data(new(DEFAULT, b"d"), vec![b'a', b'b'])); - let r: ParseResult<_, Vec<_>, _> = many_till(new(DEFAULT, b"abd"), any, |i| token(i, b'c')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(many_till(new_buf(DEFAULT, b"abcd"), any, |i| token(i, b'c')).into_inner(), State::Data(new_buf(DEFAULT, b"d"), vec![b'a', b'b'])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abd"), any, |i| token(i, b'c')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"abd"), 1)); - let r: ParseResult<_, Vec, _> = many_till(new(DEFAULT, b"abcd"), |i| i.err(()), |i| token(i, b'c')); - assert_eq!(r.into_inner(), State::Error(b"abcd", ())); + let r: ParseResult<_, Vec, _> = many_till(new_buf(DEFAULT, b"abcd"), |i| i.err(()), |i| token(i, b'c')); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"abcd"), ())); // Variant to make sure error slice is propagated let mut n = 0; - let r: ParseResult<_, Vec<_>, _> = many_till(new(DEFAULT, b"abcd"), |i| if n == 0 { n += 1; any(i).map_err(|_| "any err") } else { i.err("the error") }, |i| token(i, b'c')); - assert_eq!(r.into_inner(), State::Error(b"bcd", "the error")); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abcd"), |i| if n == 0 { n += 1; any(i).map_err(|_| "any err") } else { i.err("the error") }, |i| token(i, b'c')); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"bcd"), "the error")); } #[test] fn matched_by_test() { - assert_eq!(matched_by(new(DEFAULT, b"abc"), any).into_inner(), State::Data(new(DEFAULT, b"bc"), (&b"a"[..], b'a'))); - assert_eq!(matched_by(new(DEFAULT, b"abc"), |i| i.err::<(), _>("my error")).into_inner(), State::Error(&b"abc"[..], "my error")); - assert_eq!(matched_by(new(DEFAULT, b"abc"), |i| any(i).map_err(|_| "any error").then(|i| i.err::<(), _>("my error"))).into_inner(), State::Error(&b"bc"[..], "my error")); - assert_eq!(matched_by(new(DEFAULT, b""), any).into_inner(), State::Incomplete(1)); + assert_eq!(matched_by(new_buf(DEFAULT, b"abc"), any).into_inner(), State::Data(new_buf(DEFAULT, b"bc"), (&b"a"[..], b'a'))); + assert_eq!(matched_by(new_buf(DEFAULT, b"abc"), |i| i.err::<(), _>("my error")).into_inner(), State::Error(new_buf(DEFAULT, b"abc"), "my error")); + assert_eq!(matched_by(new_buf(DEFAULT, b"abc"), |i| any(i).map_err(|_| "any error").then(|i| i.err::<(), _>("my error"))).into_inner(), State::Error(new_buf(DEFAULT, b"bc"), "my error")); + assert_eq!(matched_by(new_buf(DEFAULT, b""), any).into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); } #[test] fn sep_by_test() { - assert_eq!(sep_by(new(END_OF_INPUT, b""), any, |i| token(i, b';')).into_inner(), State::Data(new(END_OF_INPUT, b""), vec![])); - assert_eq!(sep_by(new(END_OF_INPUT, b"b"), |i| token(i, b'a'), |i| token(i, b';')).into_inner(), State::Data(new(END_OF_INPUT, b"b"), vec![])); - assert_eq!(sep_by(new(END_OF_INPUT, b"a"), any, |i| token(i, b';')).into_inner(), State::Data(new(END_OF_INPUT, b""), vec![b'a'])); - assert_eq!(sep_by(new(END_OF_INPUT, b"a;c"), any, |i| token(i, b';')).into_inner(), State::Data(new(END_OF_INPUT, b""), vec![b'a', b'c'])); - assert_eq!(sep_by(new(END_OF_INPUT, b"a;c;"), any, |i| token(i, b';')).into_inner(), State::Data(new(END_OF_INPUT, b";"), vec![b'a', b'c'])); - assert_eq!(sep_by(new(END_OF_INPUT, b"a--c-"), any, |i| string(i, b"--")).into_inner(), State::Data(new(END_OF_INPUT, b"-"), vec![b'a', b'c'])); - assert_eq!(sep_by(new(END_OF_INPUT, b"abc"), any, |i| token(i, b';')).into_inner(), State::Data(new(END_OF_INPUT, b"bc"), vec![b'a'])); - assert_eq!(sep_by(new(END_OF_INPUT, b"a;bc"), any, |i| token(i, b';')).into_inner(), State::Data(new(END_OF_INPUT, b"c"), vec![b'a', b'b'])); - - assert_eq!(sep_by(new(DEFAULT, b"abc"), any, |i| token(i, b';')).into_inner(), State::Data(new(DEFAULT, b"bc"), vec![b'a'])); - assert_eq!(sep_by(new(DEFAULT, b"a;bc"), any, |i| token(i, b';')).into_inner(), State::Data(new(DEFAULT, b"c"), vec![b'a', b'b'])); + assert_eq!(sep_by(new_buf(END_OF_INPUT, b""), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); + assert_eq!(sep_by(new_buf(END_OF_INPUT, b"b"), |i| token(i, b'a'), |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![])); + assert_eq!(sep_by(new_buf(END_OF_INPUT, b"a"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a'])); + assert_eq!(sep_by(new_buf(END_OF_INPUT, b"a;c"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'c'])); + assert_eq!(sep_by(new_buf(END_OF_INPUT, b"a;c;"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b";"), vec![b'a', b'c'])); + assert_eq!(sep_by(new_buf(END_OF_INPUT, b"a--c-"), any, |i| string(i, b"--")).into_inner(), State::Data(new_buf(END_OF_INPUT, b"-"), vec![b'a', b'c'])); + assert_eq!(sep_by(new_buf(END_OF_INPUT, b"abc"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b"bc"), vec![b'a'])); + assert_eq!(sep_by(new_buf(END_OF_INPUT, b"a;bc"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b"c"), vec![b'a', b'b'])); + + assert_eq!(sep_by(new_buf(DEFAULT, b"abc"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(DEFAULT, b"bc"), vec![b'a'])); + assert_eq!(sep_by(new_buf(DEFAULT, b"a;bc"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(DEFAULT, b"c"), vec![b'a', b'b'])); // Incomplete becasue there might be another separator or item to be read - let r: ParseResult<_, Vec<_>, _> = sep_by(new(DEFAULT, b""), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + let r: ParseResult<_, Vec<_>, _> = sep_by(new_buf(DEFAULT, b""), any, |i| token(i, b';')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: ParseResult<_, Vec<_>, _> = sep_by(new(DEFAULT, b"a"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + let r: ParseResult<_, Vec<_>, _> = sep_by(new_buf(DEFAULT, b"a"), any, |i| token(i, b';')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); - let r: ParseResult<_, Vec<_>, _> = sep_by(new(DEFAULT, b"a;"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + let r: ParseResult<_, Vec<_>, _> = sep_by(new_buf(DEFAULT, b"a;"), any, |i| token(i, b';')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a;"), 1)); - let r: ParseResult<_, Vec<_>, _> = sep_by(new(DEFAULT, b"a;c"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + let r: ParseResult<_, Vec<_>, _> = sep_by(new_buf(DEFAULT, b"a;c"), any, |i| token(i, b';')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a;c"), 1)); - let r: ParseResult<_, Vec<_>, _> = sep_by(new(DEFAULT, b"a;c;"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + let r: ParseResult<_, Vec<_>, _> = sep_by(new_buf(DEFAULT, b"a;c;"), any, |i| token(i, b';')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a;c;"), 1)); - let r: ParseResult<_, Vec<_>, _> = sep_by(new(DEFAULT, b"a--c-"), any, |i| string(i, b"--")); - assert_eq!(r.into_inner(), State::Incomplete(1)); + let r: ParseResult<_, Vec<_>, _> = sep_by(new_buf(DEFAULT, b"a--c-"), any, |i| string(i, b"--")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a--c-"), 1)); - let r: ParseResult<_, Vec<_>, _> = sep_by(new(DEFAULT, b"aaa--a"), |i| string(i, b"aaa"), |i| string(i, b"--")); - assert_eq!(r.into_inner(), State::Incomplete(2)); + let r: ParseResult<_, Vec<_>, _> = sep_by(new_buf(DEFAULT, b"aaa--a"), |i| string(i, b"aaa"), |i| string(i, b"--")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"aaa--a"), 2)); } #[test] fn sep_by1_test() { - let r: ParseResult<_, Vec<_>, _> = sep_by1(new(END_OF_INPUT, b""), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(END_OF_INPUT, b""), any, |i| token(i, b';')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); - let r: ParseResult<_, Vec<()>, _> = sep_by1(new(END_OF_INPUT, b"b"), |i| i.err("my err"), |i| token(i, b';').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "my err")); + let r: ParseResult<_, Vec<()>, _> = sep_by1(new_buf(END_OF_INPUT, b"b"), |i| i.err("my err"), |i| token(i, b';').map_err(|_| "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "my err")); - assert_eq!(sep_by1(new(END_OF_INPUT, b"a"), any, |i| token(i, b';')).into_inner(), State::Data(new(END_OF_INPUT, b""), vec![b'a'])); - assert_eq!(sep_by1(new(END_OF_INPUT, b"a;c"), any, |i| token(i, b';')).into_inner(), State::Data(new(END_OF_INPUT, b""), vec![b'a', b'c'])); - assert_eq!(sep_by1(new(END_OF_INPUT, b"a;c;"), any, |i| token(i, b';')).into_inner(), State::Data(new(END_OF_INPUT, b";"), vec![b'a', b'c'])); - assert_eq!(sep_by1(new(END_OF_INPUT, b"a--c-"), any, |i| string(i, b"--")).into_inner(), State::Data(new(END_OF_INPUT, b"-"), vec![b'a', b'c'])); - assert_eq!(sep_by1(new(END_OF_INPUT, b"abc"), any, |i| token(i, b';')).into_inner(), State::Data(new(END_OF_INPUT, b"bc"), vec![b'a'])); - assert_eq!(sep_by1(new(END_OF_INPUT, b"a;bc"), any, |i| token(i, b';')).into_inner(), State::Data(new(END_OF_INPUT, b"c"), vec![b'a', b'b'])); + assert_eq!(sep_by1(new_buf(END_OF_INPUT, b"a"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a'])); + assert_eq!(sep_by1(new_buf(END_OF_INPUT, b"a;c"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'c'])); + assert_eq!(sep_by1(new_buf(END_OF_INPUT, b"a;c;"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b";"), vec![b'a', b'c'])); + assert_eq!(sep_by1(new_buf(END_OF_INPUT, b"a--c-"), any, |i| string(i, b"--")).into_inner(), State::Data(new_buf(END_OF_INPUT, b"-"), vec![b'a', b'c'])); + assert_eq!(sep_by1(new_buf(END_OF_INPUT, b"abc"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b"bc"), vec![b'a'])); + assert_eq!(sep_by1(new_buf(END_OF_INPUT, b"a;bc"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b"c"), vec![b'a', b'b'])); - assert_eq!(sep_by1(new(DEFAULT, b"abc"), any, |i| token(i, b';')).into_inner(), State::Data(new(DEFAULT, b"bc"), vec![b'a'])); - assert_eq!(sep_by1(new(DEFAULT, b"a;bc"), any, |i| token(i, b';')).into_inner(), State::Data(new(DEFAULT, b"c"), vec![b'a', b'b'])); + assert_eq!(sep_by1(new_buf(DEFAULT, b"abc"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(DEFAULT, b"bc"), vec![b'a'])); + assert_eq!(sep_by1(new_buf(DEFAULT, b"a;bc"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(DEFAULT, b"c"), vec![b'a', b'b'])); // Incomplete becasue there might be another separator or item to be read - let r: ParseResult<_, Vec<_>, _> = sep_by1(new(DEFAULT, b""), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(DEFAULT, b""), any, |i| token(i, b';')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: ParseResult<_, Vec<_>, _> = sep_by1(new(DEFAULT, b"a"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(DEFAULT, b"a"), any, |i| token(i, b';')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); - let r: ParseResult<_, Vec<_>, _> = sep_by1(new(DEFAULT, b"a;"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(DEFAULT, b"a;"), any, |i| token(i, b';')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a;"), 1)); - let r: ParseResult<_, Vec<_>, _> = sep_by1(new(DEFAULT, b"a;c"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(DEFAULT, b"a;c"), any, |i| token(i, b';')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a;c"), 1)); - let r: ParseResult<_, Vec<_>, _> = sep_by1(new(DEFAULT, b"a;c;"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(DEFAULT, b"a;c;"), any, |i| token(i, b';')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a;c;"), 1)); - let r: ParseResult<_, Vec<_>, _> = sep_by1(new(DEFAULT, b"a--c-"), any, |i| string(i, b"--")); - assert_eq!(r.into_inner(), State::Incomplete(1)); + let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(DEFAULT, b"a--c-"), any, |i| string(i, b"--")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a--c-"), 1)); - let r: ParseResult<_, Vec<_>, _> = sep_by1(new(DEFAULT, b"aaa--a"), |i| string(i, b"aaa"), |i| string(i, b"--")); - assert_eq!(r.into_inner(), State::Incomplete(2)); + let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(DEFAULT, b"aaa--a"), |i| string(i, b"aaa"), |i| string(i, b"--")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"aaa--a"), 2)); } #[test] fn look_ahead_test() { - assert_eq!(look_ahead(new(DEFAULT, b"abc"), any).into_inner(), State::Data(new(DEFAULT, b"abc"), b'a')); - assert_eq!(look_ahead(new(DEFAULT, b"a"), |i| string(i, b"abc")).into_inner(), State::Incomplete(2)); - assert_eq!(look_ahead(new(DEFAULT, b"aa"), |i| token(i, b'a').then(|i| token(i, b'b')).map_err(|_| "err")).into_inner(), State::Error(b"a", "err")); + assert_eq!(look_ahead(new_buf(DEFAULT, b"abc"), any).into_inner(), State::Data(new_buf(DEFAULT, b"abc"), b'a')); + assert_eq!(look_ahead(new_buf(DEFAULT, b"a"), |i| string(i, b"abc")).into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 2)); + assert_eq!(look_ahead(new_buf(DEFAULT, b"aa"), |i| token(i, b'a').then(|i| token(i, b'b')).map_err(|_| "err")).into_inner(), State::Error(new_buf(DEFAULT, b"aa"), "err")); } } diff --git a/src/input.rs b/src/input.rs index 55331679..ba8dfe4a 100644 --- a/src/input.rs +++ b/src/input.rs @@ -35,7 +35,7 @@ pub mod primitives { /// Only used by fundamental parsers and combinators. #[inline] fn incomplete(self, n: usize) -> ParseResult { - new_result(State::Incomplete(n)) + new_result(State::Incomplete(self, n)) } #[inline(always)] @@ -497,8 +497,8 @@ mod test { let r1: ParseResult<_, (), u32> = i1.incomplete::<(), _>(23); let r2: ParseResult<_, (), i32> = i2.incomplete::<(), _>(23); - assert_eq!(r1.into_inner(), State::Incomplete(23)); - assert_eq!(r2.into_inner(), State::Incomplete(23)); + assert_eq!(r1.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b"in1"), 23)); + assert_eq!(r2.into_inner(), State::Incomplete(new_buf(DEFAULT, b"in2"), 23)); } #[test] diff --git a/src/lib.rs b/src/lib.rs index 819fab43..77953b00 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -214,27 +214,9 @@ mod input; mod parse_result; // FIXME: Uncomment -pub mod parsers; - -pub use parsers::Error; -/* -pub mod ascii; -pub mod buffer; pub mod combinators; +pub mod parsers; -pub use combinators::{ - count, - option, - or, - many, - many1, - sep_by, - sep_by1, - many_till, - skip_many, - skip_many1, - matched_by, -}; pub use parsers::{ any, eof, @@ -253,6 +235,24 @@ pub use parsers::{ take_while1, token, }; +pub use parsers::Error; +/* +pub mod ascii; +pub mod buffer; + +pub use combinators::{ + count, + option, + or, + many, + many1, + sep_by, + sep_by1, + many_till, + skip_many, + skip_many1, + matched_by, +}; pub use parse::{ ParseError, parse_only, diff --git a/src/parse_result.rs b/src/parse_result.rs index 88b4df41..f93873fc 100644 --- a/src/parse_result.rs +++ b/src/parse_result.rs @@ -3,7 +3,7 @@ use parsers::Error; /// Result for dealing with the basic parsers when parsing a stream of `u8`. // FIXME: Uncomment -//pub type U8Result = ParseResult>; +//pub type U8Result = ParseResult<, T, Error>; /// Result returned from the basic parsers. // FIXME: Uncomment pub type SimpleResult = ParseResult::Token>>; @@ -24,7 +24,7 @@ pub enum State { Error(I, E), /// Incomplete state, a parser attempted to request more data than available in the slice, the /// provided number is a guess at how many items are needed. - Incomplete(usize), + Incomplete(I, usize), } /// **Primitive:** Consumes self and reveals the inner state. @@ -140,9 +140,9 @@ impl ParseResult { where F: FnOnce(I, T) -> ParseResult, V: From { match self.0 { - State::Data(i, t) => f(i, t).map_err(From::from), - State::Error(i, e) => ParseResult(State::Error(i, From::from(e))), - State::Incomplete(n) => ParseResult(State::Incomplete(n)), + State::Data(i, t) => f(i, t).map_err(From::from), + State::Error(i, e) => ParseResult(State::Error(i, From::from(e))), + State::Incomplete(i, n) => ParseResult(State::Incomplete(i, n)), } } @@ -194,9 +194,9 @@ impl ParseResult { pub fn map(self, f: F) -> ParseResult where F: FnOnce(T) -> U { match self.0 { - State::Data(i, t) => ParseResult(State::Data(i, f(t))), - State::Error(i, e) => ParseResult(State::Error(i, e)), - State::Incomplete(n) => ParseResult(State::Incomplete(n)), + State::Data(i, t) => ParseResult(State::Data(i, f(t))), + State::Error(i, e) => ParseResult(State::Error(i, e)), + State::Incomplete(i, n) => ParseResult(State::Incomplete(i, n)), } } @@ -217,9 +217,9 @@ impl ParseResult { pub fn map_err(self, f: F) -> ParseResult where F: FnOnce(E) -> V { match self.0 { - State::Data(i, t) => ParseResult(State::Data(i, t)), - State::Error(i, e) => ParseResult(State::Error(i, f(e))), - State::Incomplete(n) => ParseResult(State::Incomplete(n)), + State::Data(i, t) => ParseResult(State::Data(i, t)), + State::Error(i, e) => ParseResult(State::Error(i, f(e))), + State::Incomplete(i, n) => ParseResult(State::Incomplete(i, n)), } } @@ -427,8 +427,8 @@ mod test { let r1: ParseResult<_, _, ()> = i1.bind(|i, t| { n1_calls += 1; i.ret(t) }); let r2: ParseResult<_, _, ()> = i2.bind(|i, t| { n2_calls += 1; i.ret(t) }); - assert_eq!(r1.0, State::Incomplete(23)); - assert_eq!(r2.0, State::Incomplete(24)); + assert_eq!(r1.0, State::Incomplete(input::new_buf(DEFAULT, b"test1"), 23)); + assert_eq!(r2.0, State::Incomplete(input::new_buf(END_OF_INPUT, b"test2"), 24)); assert_eq!(n1_calls, 0); assert_eq!(n2_calls, 0); } diff --git a/src/parsers.rs b/src/parsers.rs index a5b1f7ec..0ba2751b 100644 --- a/src/parsers.rs +++ b/src/parsers.rs @@ -617,6 +617,8 @@ mod test { decimal(i).bind(|i, frac| i.ret((real, frac))))); + // ParseResult necessary here due to inference, for some reason is + // `Error<::Token>` not specific enough to actually help inference. let d: ParseResult<_, _, Error> = p.bind(|i, num| take_remainder(i) .bind(|i, r| i.ret((r, num)))); @@ -638,12 +640,12 @@ mod test { let r = take_while1(n, |_| true); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(r.into_inner(), State::Incomplete(&b""[..], 1)); } #[test] fn token_test() { - assert_eq!(token(new_buf(DEFAULT, b""), b'a').into_inner(), State::Incomplete(1)); + assert_eq!(token(new_buf(DEFAULT, b""), b'a').into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); assert_eq!(token(new_buf(DEFAULT, b"ab"), b'a').into_inner(), State::Data(new_buf(DEFAULT, b"b"), b'a')); assert_eq!(token(new_buf(DEFAULT, b"bb"), b'a').into_inner(), State::Error(new_buf(DEFAULT, b"bb"), Error::expected(b'a'))); } @@ -651,8 +653,8 @@ mod test { #[test] fn take_test() { assert_eq!(take(new_buf(DEFAULT, b"a"), 1).into_inner(), State::Data(new_buf(DEFAULT, b""), &b"a"[..])); - assert_eq!(take(new_buf(DEFAULT, b"a"), 2).into_inner(), State::Incomplete(1)); - assert_eq!(take(new_buf(DEFAULT, b"a"), 3).into_inner(), State::Incomplete(2)); + assert_eq!(take(new_buf(DEFAULT, b"a"), 2).into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); + assert_eq!(take(new_buf(DEFAULT, b"a"), 3).into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 2)); assert_eq!(take(new_buf(DEFAULT, b"ab"), 1).into_inner(), State::Data(new_buf(DEFAULT, b"b"), &b"a"[..])); assert_eq!(take(new_buf(DEFAULT, b"ab"), 2).into_inner(), State::Data(new_buf(DEFAULT, b""), &b"ab"[..])); } @@ -664,7 +666,7 @@ mod test { assert_eq!(take_while(&b"bbc"[..], |c| c != b'b').into_inner(), State::Data(&b"bbc"[..], &b""[..])); assert_eq!(take_while(&b"abc"[..], |c| c != b'b').into_inner(), State::Data(&b"bc"[..], &b"a"[..])); // TODO: Update when the incomplete type has been updated - assert_eq!(take_while(new_buf(DEFAULT, b"acc"), |c| c != b'b').into_inner(), State::Incomplete(1)); + assert_eq!(take_while(new_buf(DEFAULT, b"acc"), |c| c != b'b').into_inner(), State::Incomplete(new_buf(DEFAULT, b"acc"), 1)); assert_eq!(take_while(&b"acc"[..], |c| c != b'b').into_inner(), State::Data(&b""[..], &b"acc"[..])); } @@ -675,7 +677,7 @@ mod test { assert_eq!(take_while1(&b"bbc"[..], |c| c != b'b').into_inner(), State::Error(&b"bbc"[..], Error::unexpected())); assert_eq!(take_while1(&b"abc"[..], |c| c != b'b').into_inner(), State::Data(&b"bc"[..], &b"a"[..])); // TODO: Update when the incomplete type has been updated - assert_eq!(take_while1(new_buf(DEFAULT, b"acc"), |c| c != b'b').into_inner(), State::Incomplete(1)); + assert_eq!(take_while1(new_buf(DEFAULT, b"acc"), |c| c != b'b').into_inner(), State::Incomplete(new_buf(DEFAULT, b"acc"), 1)); assert_eq!(take_while1(&b"acc"[..], |c| c != b'b').into_inner(), State::Data(&b""[..], &b"acc"[..])); } @@ -683,8 +685,8 @@ mod test { fn peek_next_test() { assert_eq!(peek_next(new_buf(DEFAULT, b"abc")).into_inner(), State::Data(new_buf(DEFAULT, b"abc"), b'a')); assert_eq!(peek_next(&b"abc"[..]).into_inner(), State::Data(&b"abc"[..], b'a')); - assert_eq!(peek_next(new_buf(DEFAULT, b"")).into_inner(), State::Incomplete(1)); - assert_eq!(peek_next(&b""[..]).into_inner(), State::Incomplete(1)); + assert_eq!(peek_next(new_buf(DEFAULT, b"")).into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + assert_eq!(peek_next(&b""[..]).into_inner(), State::Incomplete(&b""[..], 1)); } #[test] @@ -697,7 +699,7 @@ mod test { let mut m2 = 0; let mut n2 = 0; - assert_eq!(satisfy_with(new_buf(DEFAULT, b""), |m| { m2 += 1; m % 8 }, |n| { n2 += 1; n == 1 }).into_inner(), State::Incomplete(1)); + assert_eq!(satisfy_with(new_buf(DEFAULT, b""), |m| { m2 += 1; m % 8 }, |n| { n2 += 1; n == 1 }).into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); assert_eq!(m2, 0); assert_eq!(n2, 0); } @@ -707,15 +709,15 @@ mod test { assert_eq!(string(new_buf(DEFAULT, b"abc"), b"a").into_inner(), State::Data(new_buf(DEFAULT, b"bc"), &b"a"[..])); assert_eq!(string(new_buf(DEFAULT, b"abc"), b"ab").into_inner(), State::Data(new_buf(DEFAULT, b"c"), &b"ab"[..])); assert_eq!(string(new_buf(DEFAULT, b"abc"), b"abc").into_inner(), State::Data(new_buf(DEFAULT, b""), &b"abc"[..])); - assert_eq!(string(new_buf(DEFAULT, b"abc"), b"abcd").into_inner(), State::Incomplete(1)); - assert_eq!(string(new_buf(DEFAULT, b"abc"), b"abcde").into_inner(), State::Incomplete(2)); + assert_eq!(string(new_buf(DEFAULT, b"abc"), b"abcd").into_inner(), State::Incomplete(new_buf(DEFAULT, b"abc"), 1)); + assert_eq!(string(new_buf(DEFAULT, b"abc"), b"abcde").into_inner(), State::Incomplete(new_buf(DEFAULT, b"abc"), 2)); assert_eq!(string(new_buf(DEFAULT, b"abc"), b"ac").into_inner(), State::Error(new_buf(DEFAULT, b"bc"), Error::expected(b'c'))); assert_eq!(string(&b"abc"[..], b"a").into_inner(), State::Data(&b"bc"[..], &b"a"[..])); assert_eq!(string(&b"abc"[..], b"ab").into_inner(), State::Data(&b"c"[..], &b"ab"[..])); assert_eq!(string(&b"abc"[..], b"abc").into_inner(), State::Data(&b""[..], &b"abc"[..])); - assert_eq!(string(&b"abc"[..], b"abcd").into_inner(), State::Incomplete(1)); - assert_eq!(string(&b"abc"[..], b"abcde").into_inner(), State::Incomplete(2)); + assert_eq!(string(&b"abc"[..], b"abcd").into_inner(), State::Incomplete(&b"abc"[..], 1)); + assert_eq!(string(&b"abc"[..], b"abcde").into_inner(), State::Incomplete(&b"abc"[..], 2)); assert_eq!(string(&b"abc"[..], b"ac").into_inner(), State::Error(&b"bc"[..], Error::expected(b'c'))); } From 2d68c4f9edf6c1fa3a1ee9631626e85193a70c95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Fri, 18 Mar 2016 02:49:42 +0100 Subject: [PATCH 04/63] Updated ascii and parse_only --- src/ascii.rs | 13 +++++++------ src/lib.rs | 24 +++++++++++------------- src/parse.rs | 39 +++++++++++++++++++++------------------ 3 files changed, 39 insertions(+), 37 deletions(-) diff --git a/src/ascii.rs b/src/ascii.rs index e9f1c333..868c9eeb 100644 --- a/src/ascii.rs +++ b/src/ascii.rs @@ -5,7 +5,7 @@ use conv::errors::UnwrapOk; use std::ops::{Add, Mul}; -use {Input, U8Result}; +use {Input, SimpleResult}; use combinators::option; use parsers::{take_while, take_while1, satisfy}; @@ -79,7 +79,7 @@ pub fn is_alphanumeric(c: u8) -> bool { /// assert_eq!(parse_only(skip_whitespace, b" \t "), Ok(())); /// ``` #[inline] -pub fn skip_whitespace(i: Input) -> U8Result<()> { +pub fn skip_whitespace>(i: I) -> SimpleResult { take_while(i, is_whitespace).map(|_| ()) } @@ -98,7 +98,7 @@ pub fn skip_whitespace(i: Input) -> U8Result<()> { /// assert_eq!(parse_only(digit, b"1"), Ok(b'1')); /// ``` #[inline] -pub fn digit(i: Input) -> U8Result { +pub fn digit>(i: I) -> SimpleResult { satisfy(i, is_digit) } @@ -120,9 +120,9 @@ pub fn digit(i: Input) -> U8Result { /// assert_eq!(r, Ok(-123i16)); /// ``` #[inline] -pub fn signed(i: Input, f: F) -> U8Result +pub fn signed, T, F>(i: I, f: F) -> SimpleResult where T: Copy + ValueFrom + Add + Mul, - F: FnOnce(Input) -> U8Result { + F: FnOnce(I) -> SimpleResult { option(i, |i| satisfy(i, |c| c == b'-' || c == b'+') .map(|s| T::value_from(if s == b'+' { 1 } else { -1 }).unwrap_ok()), @@ -146,8 +146,9 @@ pub fn signed(i: Input, f: F) -> U8Result /// /// assert_eq!(r, Ok(123u8)); /// ``` +// TODO: Use methods on `Buffer` to implement `to_decimal` #[inline] -pub fn decimal + Add + Mul>(i: Input) -> U8Result { +pub fn decimal<'a, I: Input, T: Copy + ValueFrom + Add + Mul>(i: I) -> SimpleResult { take_while1(i, is_digit).map(to_decimal) } diff --git a/src/lib.rs b/src/lib.rs index 77953b00..84483072 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -210,10 +210,11 @@ extern crate debugtrace; mod macros; mod input; // FIXME: Uncomment -//mod parse; +mod parse; mod parse_result; // FIXME: Uncomment +pub mod ascii; pub mod combinators; pub mod parsers; @@ -236,28 +237,25 @@ pub use parsers::{ token, }; pub use parsers::Error; -/* -pub mod ascii; -pub mod buffer; +//pub mod buffer; pub use combinators::{ - count, + //count, option, or, - many, - many1, - sep_by, - sep_by1, - many_till, - skip_many, - skip_many1, + //many, + //many1, + //sep_by, + //sep_by1, + //many_till, + //skip_many, + //skip_many1, matched_by, }; pub use parse::{ ParseError, parse_only, }; -*/ pub use input::Input; pub use parse_result::{ ParseResult, diff --git a/src/parse.rs b/src/parse.rs index b911cafe..7ed332db 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -1,13 +1,12 @@ -use {Input, ParseResult}; +use ParseResult; use primitives::{IntoInner, State}; -use primitives::input; +use primitives::Primitives; /// Simple error type returned from `parse_only`. #[derive(Debug, Eq, PartialEq)] -pub enum ParseError<'a, I, E> - where I: 'a { +pub enum ParseError { /// A parse error occurred. - Error(&'a [I], E), + Error(I, E), /// The parser attempted to read more data than available. Incomplete(usize), } @@ -49,21 +48,25 @@ pub enum ParseError<'a, I, E> /// Err(ParseError::Error(&b" and more"[..], Error::new()))); /// # } /// ``` -pub fn parse_only<'a, I, T, E, F>(parser: F, input: &'a [I]) -> Result> +pub fn parse_only<'a, I: Copy, T, E, F>(parser: F, input: &'a [I]) -> Result> where T: 'a, E: 'a, - F: FnOnce(Input<'a, I>) -> ParseResult<'a, I, T, E> { - match parser(input::new(input::END_OF_INPUT, input)).into_inner() { + F: FnOnce(&'a [I]) -> ParseResult<&'a [I], T, E> { + match parser(input).into_inner() { State::Data(_, t) => Ok(t), - State::Error(b, e) => Err(ParseError::Error(b, e)), - State::Incomplete(n) => Err(ParseError::Incomplete(n)), + State::Error(mut b, e) => Err(ParseError::Error({ + let r = b.min_remaining(); + + b.consume(r) + }, e)), + State::Incomplete(_, n) => Err(ParseError::Incomplete(n)), } } #[cfg(test)] mod test { - use primitives::InputBuffer; - + use Input; + use primitives::Primitives; use super::{ ParseError, parse_only, @@ -75,22 +78,22 @@ mod test { let mut input = None; assert_eq!(parse_only(|i| { - state = Some(i.is_last_slice()); - input = Some(i.buffer()); + state = Some(i.is_end()); + input = Some(i.iter().cloned().collect()); i.ret::<_, ()>("the result") }, b"the input"), Ok("the result")); - assert_eq!(input, Some(&b"the input"[..])); + assert_eq!(input, Some(b"the input".to_vec())); assert_eq!(state, Some(true)); } #[test] fn err() { - assert_eq!(parse_only(|i| { - let buf = i.buffer(); + assert_eq!(parse_only(|mut i| { + i.consume(4); - i.replace(&buf[4..]).err::<(), _>("my error") + i.err::<(), _>("my error") }, b"the input"), Err(ParseError::Error(&b"input"[..], "my error"))); } From 3c041ae7c64dca3c14653fcb45e5dd45366b56da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Fri, 18 Mar 2016 16:20:10 +0100 Subject: [PATCH 05/63] Updated combinators::bounded::many --- src/combinators/bounded.rs | 926 ++++++++++++++++++++----------------- src/combinators/macros.rs | 53 ++- src/combinators/mod.rs | 29 +- src/lib.rs | 1 - src/parsers.rs | 2 +- 5 files changed, 533 insertions(+), 478 deletions(-) diff --git a/src/combinators/bounded.rs b/src/combinators/bounded.rs index 1416f409..d4d4e69c 100644 --- a/src/combinators/bounded.rs +++ b/src/combinators/bounded.rs @@ -26,10 +26,12 @@ use std::ops::{ use std::cmp::max; use {Input, ParseResult}; -use primitives::{InputClone, InputBuffer, IntoInner, State}; +use primitives::{Primitives, IntoInner, State}; /// Trait for applying a parser multiple times based on a range. pub trait BoundedRange { + // TODO: Update documentation regarding input state. Incomplete will point to the last + // successful parsed data. mark a backtrack point to be able to restart parsing. /// Applies the parser `F` multiple times until it fails or the maximum value of the range has /// been reached, collecting the successful values into a `T: FromIterator`. /// @@ -47,12 +49,12 @@ pub trait BoundedRange { /// * If the last parser succeeds on the last input item then this parser is still considered /// incomplete if the input flag END_OF_INPUT is not set as there might be more data to fill. #[inline] - fn parse_many<'a, I, T, E, F, U>(self, Input<'a, I>, F) -> ParseResult<'a, I, T, E> - where I: Copy, - U: 'a, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, + fn parse_many(self, I, F) -> ParseResult + where F: FnMut(I) -> ParseResult, T: FromIterator; + // FIXME: Uncomment + /* /// Applies the parser `F` multiple times until it fails or the maximum value of the range has /// been reached, throwing away any produced value. /// @@ -97,14 +99,13 @@ pub trait BoundedRange { T: FromIterator, R: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, F: FnMut(Input<'a, I>) -> ParseResult<'a, I, V, N>; + */ } impl BoundedRange for Range { #[inline] - fn parse_many<'a, I, T, E, F, U>(self, i: Input<'a, I>, f: F) -> ParseResult<'a, I, T, E> - where I: Copy, - U: 'a, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, + fn parse_many(self, i: I, f: F) -> ParseResult + where F: FnMut(I) -> ParseResult, T: FromIterator { // Range does not perform this assertion assert!(self.start <= self.end); @@ -133,23 +134,28 @@ impl BoundedRange for Range { => result : T { // Got all occurrences of the parser + // TODO: Backtrack to last good (s, (0, 0), _) => s.ret(result), // Ok, last parser failed and we have reached minimum, we have iterated all. // Return remainder of buffer and the collected result - (s, (0, _), EndState::Error(_, _)) => s.ret(result), + // TODO: Backtrack to last good + (s, (0, _), EndState::Error(m, _)) => s.restore(m).ret(result), // Nested parser incomplete but reached at least minimum, propagate if not at end - (s, (0, _), EndState::Incomplete(n)) => if s.is_last_slice() { + (s, (0, _), EndState::Incomplete(n)) => if s.is_end() { + // TODO: Backtrack to last good s.ret(result) } else { s.incomplete(n) }, // Did not reach minimum, propagate - (s, (_, _), EndState::Error(b, e)) => s.replace(b).err(e), + (s, (_, _), EndState::Error(_, e)) => s.err(e), (s, (_, _), EndState::Incomplete(n)) => s.incomplete(n) } } } + // FIXME: Uncomment + /* #[inline] fn skip_many<'a, I, T, E, F>(self, mut i: Input<'a, I>, mut f: F) -> ParseResult<'a, I, (), E> where T: 'a, @@ -178,7 +184,7 @@ impl BoundedRange for Range { // Not enough iterations return i.replace(b).err(e); }, - State::Incomplete(n) => if min == 0 && i.is_last_slice() { + State::Incomplete(n) => if min == 0 && i.is_end() { break; } else { // We have not done the minimum amount of iterations @@ -246,14 +252,13 @@ impl BoundedRange for Range { } } } + */ } impl BoundedRange for RangeFrom { #[inline] - fn parse_many<'a, I, T, E, F, U>(self, i: Input<'a, I>, f: F) -> ParseResult<'a, I, T, E> - where I: Copy, - U: 'a, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, + fn parse_many(self, i: I, f: F) -> ParseResult + where F: FnMut(I) -> ParseResult, T: FromIterator { run_iter!{ input: i, @@ -274,20 +279,23 @@ impl BoundedRange for RangeFrom { => result : T { // We got at least n items - (s, 0, EndState::Error(_, _)) => s.ret(result), + (s, 0, EndState::Error(m, _)) => s.restore(m).ret(result), // Nested parser incomplete, propagate if not at end - (s, 0, EndState::Incomplete(n)) => if s.is_last_slice() { + (s, 0, EndState::Incomplete(n)) => if s.is_end() { + // TODO: Backtrack to last good s.ret(result) } else { s.incomplete(n) }, // Items still remaining, propagate - (s, _, EndState::Error(b, e)) => s.replace(b).err(e), + (s, _, EndState::Error(_, e)) => s.err(e), (s, _, EndState::Incomplete(n)) => s.incomplete(n) } } } + // FIXME: Uncomment + /* #[inline] fn skip_many<'a, I, T, E, F>(self, mut i: Input<'a, I>, mut f: F) -> ParseResult<'a, I, (), E> where T: 'a, @@ -308,7 +316,7 @@ impl BoundedRange for RangeFrom { // Not enough iterations return i.replace(b).err(e); }, - State::Incomplete(n) => if min == 0 && i.is_last_slice() { + State::Incomplete(n) => if min == 0 && i.is_end() { break; } else { // We have not done the minimum amount of iterations @@ -364,14 +372,13 @@ impl BoundedRange for RangeFrom { } } } + */ } impl BoundedRange for RangeFull { #[inline] - fn parse_many<'a, I, T, E, F, U>(self, i: Input<'a, I>, f: F) -> ParseResult<'a, I, T, E> - where I: Copy, - U: 'a, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, + fn parse_many(self, i: I, f: F) -> ParseResult + where F: FnMut(I) -> ParseResult, T: FromIterator { run_iter!{ input: i, @@ -388,9 +395,9 @@ impl BoundedRange for RangeFull { } => result : T { - (s, (), EndState::Error(_, _)) => s.ret(result), + (s, (), EndState::Error(m, _)) => s.restore(m).ret(result), // Nested parser incomplete, propagate if not at end - (s, (), EndState::Incomplete(n)) => if s.is_last_slice() { + (s, (), EndState::Incomplete(n)) => if s.is_end() { s.ret(result) } else { s.incomplete(n) @@ -399,6 +406,7 @@ impl BoundedRange for RangeFull { } } + /* #[inline] fn skip_many<'a, I, T, E, F>(self, mut i: Input<'a, I>, mut f: F) -> ParseResult<'a, I, (), E> where T: 'a, @@ -407,7 +415,7 @@ impl BoundedRange for RangeFull { match f(i.clone()).into_inner() { State::Data(b, _) => i = b, State::Error(_, _) => break, - State::Incomplete(n) => if i.is_last_slice() { + State::Incomplete(n) => if i.is_end() { break; } else { return i.incomplete(n); @@ -453,14 +461,13 @@ impl BoundedRange for RangeFull { } } } + */ } impl BoundedRange for RangeTo { #[inline] - fn parse_many<'a, I, T, E, F, U>(self, i: Input<'a, I>, f: F) -> ParseResult<'a, I, T, E> - where I: Copy, - U: 'a, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, + fn parse_many(self, i: I, f: F) -> ParseResult + where F: FnMut(I) -> ParseResult, T: FromIterator { run_iter!{ input: i, @@ -485,11 +492,13 @@ impl BoundedRange for RangeTo { => result : T { // Either error or incomplete after the end + // TODO: Backtrack to last good (s, 0, _) => s.ret(result), // Inside of range, never outside - (s, _, EndState::Error(_, _)) => s.ret(result), + (s, _, EndState::Error(m, _)) => s.restore(m).ret(result), // Nested parser incomplete, propagate if not at end - (s, _, EndState::Incomplete(n)) => if s.is_last_slice() { + (s, _, EndState::Incomplete(n)) => if s.is_end() { + // TODO: Backtrack to last good s.ret(result) } else { s.incomplete(n) @@ -498,6 +507,8 @@ impl BoundedRange for RangeTo { } } + // FIXME: Uncomment + /* #[inline] fn skip_many<'a, I, T, E, F>(self, mut i: Input<'a, I>, mut f: F) -> ParseResult<'a, I, (), E> where T: 'a, @@ -518,7 +529,7 @@ impl BoundedRange for RangeTo { }, // Always ok to end iteration State::Error(_, _) => break, - State::Incomplete(n) => if i.is_last_slice() { + State::Incomplete(n) => if i.is_end() { break; } else { return i.incomplete(n); @@ -576,14 +587,13 @@ impl BoundedRange for RangeTo { } } } + */ } impl BoundedRange for usize { #[inline] - fn parse_many<'a, I, T, E, F, U>(self, i: Input<'a, I>, f: F) -> ParseResult<'a, I, T, E> - where I: Copy, - U: 'a, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, + fn parse_many(self, i: I, f: F) -> ParseResult + where F: FnMut(I) -> ParseResult, T: FromIterator { run_iter!{ input: i, @@ -608,15 +618,18 @@ impl BoundedRange for usize { => result : T { // Got exact + // TODO: Backtrack to last good (s, 0, _) => s.ret(result), // We have got too few items, propagate error - (s, _, EndState::Error(b, e)) => s.replace(b).err(e), + (s, _, EndState::Error(_, e)) => s.err(e), // Nested parser incomplete, propagate (s, _, EndState::Incomplete(n)) => s.incomplete(n) } } } + // FIXME: Uncomment + /* #[inline] fn skip_many<'a, I, T, E, F>(self, mut i: Input<'a, I>, mut f: F) -> ParseResult<'a, I, (), E> where T: 'a, @@ -696,6 +709,7 @@ impl BoundedRange for usize { } } } + */ } /// Applies the parser `F` multiple times until it fails or the maximum value of the range has @@ -714,15 +728,14 @@ impl BoundedRange for usize { /// * If the last parser succeeds on the last input item then this parser is still considered /// incomplete if the input flag END_OF_INPUT is not set as there might be more data to fill. #[inline] -pub fn many<'a, I, T, E, F, U, R>(i: Input<'a, I>, r: R, f: F) -> ParseResult<'a, I, T, E> - where I: Copy, - R: BoundedRange, - U: 'a, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, +pub fn many(i: I, r: R, f: F) -> ParseResult + where R: BoundedRange, + F: FnMut(I) -> ParseResult, T: FromIterator { BoundedRange::parse_many(r, i, f) } +/* /// Applies the parser `F` multiple times until it fails or the maximum value of the range has /// been reached, throwing away any produced value. /// @@ -771,6 +784,7 @@ pub fn many_till<'a, I, T, E, R, F, U, N, P, V>(i: Input<'a, I>, r: R, p: P, end F: FnMut(Input<'a, I>) -> ParseResult<'a, I, V, N> { BoundedRange::many_till(r, i, p, end) } +*/ /// Applies the parser `p` multiple times, separated by the parser `sep` and returns a value /// populated with the values yielded by `p`. If the number of items yielded by `p` does not fall @@ -788,16 +802,12 @@ pub fn many_till<'a, I, T, E, R, F, U, N, P, V>(i: Input<'a, I>, r: R, p: P, end /// * If the last parser succeeds on the last input item then this combinator is still considered /// incomplete unless the parser `F` matches or the lower bound has not been met. #[inline] -pub fn sep_by<'a, I, T, E, R, F, U, N, P, V>(i: Input<'a, I>, r: R, mut p: P, mut sep: F) -> ParseResult<'a, I, T, E> - where I: Copy, - U: 'a, - V: 'a, - N: 'a, - T: FromIterator, +pub fn sep_by(i: I, r: R, mut p: P, mut sep: F) -> ParseResult + where T: FromIterator, E: From, R: BoundedRange, - P: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, V, N> { + P: FnMut(I) -> ParseResult, + F: FnMut(I) -> ParseResult { // If we have parsed at least one item let mut item = false; // Add sep in front of p if we have read at least one item @@ -814,484 +824,536 @@ pub fn sep_by<'a, I, T, E, R, F, U, N, P, V>(i: Input<'a, I>, r: R, mut p: P, mu #[cfg(test)] mod test { - use ParseResult; - use parsers::{any, token}; + use {Error, ParseResult}; + use parsers::{any, token, string}; use primitives::input::*; use primitives::{IntoInner, State}; use super::{ many, - many_till, - skip_many, + //many_till, + //skip_many, }; #[test] fn many_range_full() { - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b""), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"a"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"aa"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); - - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"b"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"b"), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"ab"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"b"), vec![b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"aab"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"b"), vec![b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b""), .., any); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"a"), .., any); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), vec![b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"aa"), .., any); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), vec![b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"b"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"ab"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), vec![b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"aab"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), vec![b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b""), .., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"a"), .., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aa"), .., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"b"), .., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), vec![])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"ab"), .., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), vec![b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aab"), .., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), vec![b'a', b'a'])); + + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b""), .., any); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"a"), .., any); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aa"), .., any); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'a'])); + + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"b"), .., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"ab"), .., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aab"), .., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a', b'a'])); + + // Test where we error inside of the inner parser + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"abac"), .., |i| string(i, b"ab")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ac"), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"abac"), .., |i| string(i, b"ab")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ac"), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aba"), .., |i| string(i, b"ab")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![&b"ab"[..]])); } #[test] fn many_range_to() { - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b""), ..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"a"), ..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"a"), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b""), ..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"a"), ..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"a"), vec![])); - - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b""), ..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"a"), ..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"a"), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b""), ..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"a"), ..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"a"), vec![])); - - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b""), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"a"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"aa"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b""), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"aaa"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"a"), vec![b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"b"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"b"), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"ab"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"b"), vec![b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"aab"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"b"), vec![b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b""), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"a"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), vec![b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"aa"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), vec![b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"b"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"ab"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), vec![b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"aab"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"aaab"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"ab"), vec![b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b""), ..0, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"a"), ..0, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), vec![])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b""), ..0, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"a"), ..0, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![])); + + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b""), ..1, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"a"), ..1, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), vec![])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b""), ..1, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"a"), ..1, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![])); + + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b""), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"a"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aa"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aaa"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), vec![b'a', b'a'])); + + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"b"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), vec![])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"ab"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), vec![b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aab"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), vec![b'a', b'a'])); + + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b""), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"a"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aa"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'a'])); + + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"b"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"ab"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aab"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aaab"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ab"), vec![b'a', b'a'])); + + // Test where we error inside of the inner parser + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"abac"), ..3, |i| string(i, b"ab")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ac"), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"abac"), ..3, |i| string(i, b"ab")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ac"), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aba"), ..3, |i| string(i, b"ab")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![&b"ab"[..]])); } #[test] fn many_range_from() { - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b""), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"a"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"aa"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"aaa"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); - - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"b"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"ab"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"aab"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"b"), vec![b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b""), 2.., any); - assert_eq!(r.into_inner(), State::Incomplete(1)); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"a"), 2.., any); - assert_eq!(r.into_inner(), State::Incomplete(1)); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"aa"), 2.., any); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"aaa"), 2.., any); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), vec![b'a', b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"b"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"ab"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"aab"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"aaab"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), vec![b'a', b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b""), 2.., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"a"), 2.., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aa"), 2.., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aaa"), 2.., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"b"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"ab"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aab"), 2.., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), vec![b'a', b'a'])); + + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b""), 2.., any); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"a"), 2.., any); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aa"), 2.., any); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aaa"), 2.., any); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'a', b'a'])); + + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"b"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"ab"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aab"), 2.., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aaab"), 2.., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a', b'a', b'a'])); + + // Test where we error inside of the inner parser + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"ababac"), 2.., |i| string(i, b"ab")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ac"), vec![&b"ab"[..], &b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"ababac"), 2.., |i| string(i, b"ab")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ac"), vec![&b"ab"[..], &b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"ababa"), 2.., |i| string(i, b"ab")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![&b"ab"[..], &b"ab"[..]])); } #[test] fn many_range() { - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b""), 0..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"a"), 0..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"a"), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b""), 0..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"a"), 0..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"a"), vec![])); - - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b""), 0..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"a"), 0..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"a"), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b""), 0..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"a"), 0..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"a"), vec![])); - - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b""), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"a"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"aa"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"aaa"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b""), vec![b'a', b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"aaaa"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"a"), vec![b'a', b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"b"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"ab"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"aab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"b"), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"aaab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"b"), vec![b'a', b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"aaaab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"ab"), vec![b'a', b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b""), 2..4, any); - assert_eq!(r.into_inner(), State::Incomplete(1)); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"a"), 2..4, any); - assert_eq!(r.into_inner(), State::Incomplete(1)); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"aa"), 2..4, any); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"aaa"), 2..4, any); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), vec![b'a', b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"aaaa"), 2..4, any); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"a"), vec![b'a', b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"b"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"ab"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"aab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"aaab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), vec![b'a', b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"aaaab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"ab"), vec![b'a', b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b""), 0..0, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"a"), 0..0, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), vec![])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b""), 0..0, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"a"), 0..0, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![])); + + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b""), 0..1, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"a"), 0..1, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), vec![])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b""), 0..1, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"a"), 0..1, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![])); + + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b""), 2..4, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"a"), 2..4, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aa"), 2..4, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aaa"), 2..4, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![b'a', b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aaaa"), 2..4, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), vec![b'a', b'a', b'a'])); + + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"b"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"ab"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aab"), 2..4, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), vec![b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aaab"), 2..4, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), vec![b'a', b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aaaab"), 2..4, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ab"), vec![b'a', b'a', b'a'])); + + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b""), 2..4, any); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"a"), 2..4, any); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aa"), 2..4, any); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aaa"), 2..4, any); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aaaa"), 2..4, any); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![b'a', b'a', b'a'])); + + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"b"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"ab"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aab"), 2..4, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aaab"), 2..4, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a', b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aaaab"), 2..4, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ab"), vec![b'a', b'a', b'a'])); + + // Test where we error inside of the inner parser + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"abac"), 1..3, |i| string(i, b"ab")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ac"), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"ababac"), 1..3, |i| string(i, b"ab")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ac"), vec![&b"ab"[..], &b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"abac"), 1..3, |i| string(i, b"ab")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ac"), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"ababac"), 1..3, |i| string(i, b"ab")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ac"), vec![&b"ab"[..], &b"ab"[..]])); } #[test] fn many_exact() { - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b""), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"a"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"aa"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b""), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"aaa"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"a"), vec![b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"b"), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"ab"), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"aab"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"b"), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"aaab"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"ab"), vec![b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b""), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Incomplete(1)); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"a"), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Incomplete(1)); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"aa"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"aaa"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"a"), vec![b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"b"), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"ab"), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"aab"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new(END_OF_INPUT, b"aaab"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"ab"), vec![b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b""), 2, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"a"), 2, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aa"), 2, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aaa"), 2, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), vec![b'a', b'a'])); + + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"b"), 2, |i| token(i, b'a').map_err(|_| "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"ab"), 2, |i| token(i, b'a').map_err(|_| "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aab"), 2, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), vec![b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aaab"), 2, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ab"), vec![b'a', b'a'])); + + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b""), 2, |i| token(i, b'a').map_err(|_| "token_err")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"a"), 2, |i| token(i, b'a').map_err(|_| "token_err")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aa"), 2, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aaa"), 2, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![b'a', b'a'])); + + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"b"), 2, |i| token(i, b'a').map_err(|_| "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"ab"), 2, |i| token(i, b'a').map_err(|_| "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aab"), 2, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aaab"), 2, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ab"), vec![b'a', b'a'])); + + // Test where we error inside of the inner parser + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"abac"), 2, |i| string(i, b"ab")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"c"), Error::expected(b'b'))); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"ababa"), 2, |i| string(i, b"ab")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), vec![&b"ab"[..], &b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"abac"), 2, |i| string(i, b"ab")); + assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"c"), Error::expected(b'b'))); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"ababac"), 2, |i| string(i, b"ab")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ac"), vec![&b"ab"[..], &b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"ababa"), 2, |i| string(i, b"ab")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![&b"ab"[..], &b"ab"[..]])); } + // FIXME: Uncomment + /* #[test] fn skip_range_full() { - let r = skip_many(new(DEFAULT, b""), .., |i| token(i, b'a')); + let r = skip_many(new_buf(DEFAULT, b""), .., |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Incomplete(1)); - let r = skip_many(new(DEFAULT, b"a"), .., |i| token(i, b'a')); + let r = skip_many(new_buf(DEFAULT, b"a"), .., |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Incomplete(1)); - let r = skip_many(new(DEFAULT, b"aa"), .., |i| token(i, b'a')); + let r = skip_many(new_buf(DEFAULT, b"aa"), .., |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Incomplete(1)); - let r = skip_many(new(DEFAULT, b"b"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"b"), ())); - let r = skip_many(new(DEFAULT, b"ab"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"b"), ())); - let r = skip_many(new(DEFAULT, b"aab"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"b"), ())); - - let r = skip_many(new(END_OF_INPUT, b""), .., any); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), ())); - let r = skip_many(new(END_OF_INPUT, b"a"), .., any); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), ())); - let r = skip_many(new(END_OF_INPUT, b"aa"), .., any); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), ())); - - let r = skip_many(new(END_OF_INPUT, b"b"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), ())); - let r = skip_many(new(END_OF_INPUT, b"ab"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), ())); - let r = skip_many(new(END_OF_INPUT, b"aab"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), ())); + let r = skip_many(new_buf(DEFAULT, b"b"), .., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); + let r = skip_many(new_buf(DEFAULT, b"ab"), .., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); + let r = skip_many(new_buf(DEFAULT, b"aab"), .., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); + + let r = skip_many(new_buf(END_OF_INPUT, b""), .., any); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"a"), .., any); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"aa"), .., any); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); + + let r = skip_many(new_buf(END_OF_INPUT, b"b"), .., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"ab"), .., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"aab"), .., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); } #[test] fn skip_range_to() { - let r = skip_many(new(DEFAULT, b""), ..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b""), ())); - let r = skip_many(new(DEFAULT, b"a"), ..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"a"), ())); - let r = skip_many(new(END_OF_INPUT, b""), ..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), ())); - let r = skip_many(new(END_OF_INPUT, b"a"), ..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"a"), ())); - - let r = skip_many(new(DEFAULT, b""), ..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b""), ())); - let r = skip_many(new(DEFAULT, b"a"), ..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"a"), ())); - let r = skip_many(new(END_OF_INPUT, b""), ..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), ())); - let r = skip_many(new(END_OF_INPUT, b"a"), ..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"a"), ())); - - let r = skip_many(new(DEFAULT, b""), ..3, |i| token(i, b'a')); + let r = skip_many(new_buf(DEFAULT, b""), ..0, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), ())); + let r = skip_many(new_buf(DEFAULT, b"a"), ..0, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), ())); + let r = skip_many(new_buf(END_OF_INPUT, b""), ..0, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"a"), ..0, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), ())); + + let r = skip_many(new_buf(DEFAULT, b""), ..1, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), ())); + let r = skip_many(new_buf(DEFAULT, b"a"), ..1, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), ())); + let r = skip_many(new_buf(END_OF_INPUT, b""), ..1, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"a"), ..1, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), ())); + + let r = skip_many(new_buf(DEFAULT, b""), ..3, |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Incomplete(1)); - let r = skip_many(new(DEFAULT, b"a"), ..3, |i| token(i, b'a')); + let r = skip_many(new_buf(DEFAULT, b"a"), ..3, |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Incomplete(1)); - let r = skip_many(new(DEFAULT, b"aa"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b""), ())); - let r = skip_many(new(DEFAULT, b"aaa"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"a"), ())); - - let r = skip_many(new(DEFAULT, b"b"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"b"), ())); - let r = skip_many(new(DEFAULT, b"ab"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"b"), ())); - let r = skip_many(new(DEFAULT, b"aab"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"b"), ())); - - let r = skip_many(new(END_OF_INPUT, b""), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), ())); - let r = skip_many(new(END_OF_INPUT, b"a"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), ())); - let r = skip_many(new(END_OF_INPUT, b"aa"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), ())); - - let r = skip_many(new(END_OF_INPUT, b"b"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), ())); - let r = skip_many(new(END_OF_INPUT, b"ab"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), ())); - let r = skip_many(new(END_OF_INPUT, b"aab"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), ())); - let r = skip_many(new(END_OF_INPUT, b"aaab"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"ab"), ())); + let r = skip_many(new_buf(DEFAULT, b"aa"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), ())); + let r = skip_many(new_buf(DEFAULT, b"aaa"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), ())); + + let r = skip_many(new_buf(DEFAULT, b"b"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); + let r = skip_many(new_buf(DEFAULT, b"ab"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); + let r = skip_many(new_buf(DEFAULT, b"aab"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); + + let r = skip_many(new_buf(END_OF_INPUT, b""), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"a"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"aa"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); + + let r = skip_many(new_buf(END_OF_INPUT, b"b"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"ab"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"aab"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"aaab"), ..3, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ab"), ())); } #[test] fn skip_range_from() { - let r = skip_many(new(DEFAULT, b""), 2.., |i| token(i, b'a')); + let r = skip_many(new_buf(DEFAULT, b""), 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Incomplete(1)); - let r = skip_many(new(DEFAULT, b"a"), 2.., |i| token(i, b'a')); + let r = skip_many(new_buf(DEFAULT, b"a"), 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Incomplete(1)); - let r = skip_many(new(DEFAULT, b"aa"), 2.., |i| token(i, b'a')); + let r = skip_many(new_buf(DEFAULT, b"aa"), 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Incomplete(1)); - let r = skip_many(new(DEFAULT, b"aaa"), 2.., |i| token(i, b'a')); + let r = skip_many(new_buf(DEFAULT, b"aaa"), 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Incomplete(1)); - let r = skip_many(new(DEFAULT, b"b"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); + let r = skip_many(new_buf(DEFAULT, b"b"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r = skip_many(new(DEFAULT, b"ab"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); + let r = skip_many(new_buf(DEFAULT, b"ab"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r = skip_many(new(DEFAULT, b"aab"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"b"), ())); + let r = skip_many(new_buf(DEFAULT, b"aab"), 2.., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); - let r = skip_many(new(END_OF_INPUT, b""), 2.., any); + let r = skip_many(new_buf(END_OF_INPUT, b""), 2.., any); assert_eq!(r.into_inner(), State::Incomplete(1)); - let r = skip_many(new(END_OF_INPUT, b"a"), 2.., any); + let r = skip_many(new_buf(END_OF_INPUT, b"a"), 2.., any); assert_eq!(r.into_inner(), State::Incomplete(1)); - let r = skip_many(new(END_OF_INPUT, b"aa"), 2.., any); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), ())); - let r = skip_many(new(END_OF_INPUT, b"aaa"), 2.., any); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"aa"), 2.., any); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"aaa"), 2.., any); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); - let r = skip_many(new(END_OF_INPUT, b"b"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); + let r = skip_many(new_buf(END_OF_INPUT, b"b"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r = skip_many(new(END_OF_INPUT, b"ab"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); + let r = skip_many(new_buf(END_OF_INPUT, b"ab"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r = skip_many(new(END_OF_INPUT, b"aab"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), ())); - let r = skip_many(new(END_OF_INPUT, b"aaab"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"aab"), 2.., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"aaab"), 2.., |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); } #[test] fn skip_range() { - let r = skip_many(new(DEFAULT, b""), 0..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b""), ())); - let r = skip_many(new(DEFAULT, b"a"), 0..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"a"), ())); - let r = skip_many(new(END_OF_INPUT, b""), 0..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), ())); - let r = skip_many(new(END_OF_INPUT, b"a"), 0..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"a"), ())); - - let r = skip_many(new(DEFAULT, b""), 0..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b""), ())); - let r = skip_many(new(DEFAULT, b"a"), 0..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"a"), ())); - let r = skip_many(new(END_OF_INPUT, b""), 0..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), ())); - let r = skip_many(new(END_OF_INPUT, b"a"), 0..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"a"), ())); - - let r = skip_many(new(DEFAULT, b""), 2..4, |i| token(i, b'a')); + let r = skip_many(new_buf(DEFAULT, b""), 0..0, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), ())); + let r = skip_many(new_buf(DEFAULT, b"a"), 0..0, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), ())); + let r = skip_many(new_buf(END_OF_INPUT, b""), 0..0, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"a"), 0..0, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), ())); + + let r = skip_many(new_buf(DEFAULT, b""), 0..1, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), ())); + let r = skip_many(new_buf(DEFAULT, b"a"), 0..1, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), ())); + let r = skip_many(new_buf(END_OF_INPUT, b""), 0..1, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"a"), 0..1, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), ())); + + let r = skip_many(new_buf(DEFAULT, b""), 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Incomplete(1)); - let r = skip_many(new(DEFAULT, b"a"), 2..4, |i| token(i, b'a')); + let r = skip_many(new_buf(DEFAULT, b"a"), 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Incomplete(1)); - let r = skip_many(new(DEFAULT, b"aa"), 2..4, |i| token(i, b'a')); + let r = skip_many(new_buf(DEFAULT, b"aa"), 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Incomplete(1)); - let r = skip_many(new(DEFAULT, b"aaa"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b""), ())); - let r = skip_many(new(DEFAULT, b"aaaa"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"a"), ())); + let r = skip_many(new_buf(DEFAULT, b"aaa"), 2..4, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), ())); + let r = skip_many(new_buf(DEFAULT, b"aaaa"), 2..4, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), ())); - let r = skip_many(new(DEFAULT, b"b"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); + let r = skip_many(new_buf(DEFAULT, b"b"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r = skip_many(new(DEFAULT, b"ab"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); + let r = skip_many(new_buf(DEFAULT, b"ab"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r = skip_many(new(DEFAULT, b"aab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"b"), ())); - let r = skip_many(new(DEFAULT, b"aaab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"b"), ())); - let r = skip_many(new(DEFAULT, b"aaaab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"ab"), ())); - - let r = skip_many(new(END_OF_INPUT, b""), 2..4, any); + let r = skip_many(new_buf(DEFAULT, b"aab"), 2..4, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); + let r = skip_many(new_buf(DEFAULT, b"aaab"), 2..4, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); + let r = skip_many(new_buf(DEFAULT, b"aaaab"), 2..4, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ab"), ())); + + let r = skip_many(new_buf(END_OF_INPUT, b""), 2..4, any); assert_eq!(r.into_inner(), State::Incomplete(1)); - let r = skip_many(new(END_OF_INPUT, b"a"), 2..4, any); + let r = skip_many(new_buf(END_OF_INPUT, b"a"), 2..4, any); assert_eq!(r.into_inner(), State::Incomplete(1)); - let r = skip_many(new(END_OF_INPUT, b"aa"), 2..4, any); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), ())); - let r = skip_many(new(END_OF_INPUT, b"aaa"), 2..4, any); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), ())); - let r = skip_many(new(END_OF_INPUT, b"aaaa"), 2..4, any); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"a"), ())); - - let r = skip_many(new(END_OF_INPUT, b"b"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); + let r = skip_many(new_buf(END_OF_INPUT, b"aa"), 2..4, any); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"aaa"), 2..4, any); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"aaaa"), 2..4, any); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), ())); + + let r = skip_many(new_buf(END_OF_INPUT, b"b"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r = skip_many(new(END_OF_INPUT, b"ab"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); + let r = skip_many(new_buf(END_OF_INPUT, b"ab"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r = skip_many(new(END_OF_INPUT, b"aab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), ())); - let r = skip_many(new(END_OF_INPUT, b"aaab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), ())); - let r = skip_many(new(END_OF_INPUT, b"aaaab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"ab"), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"aab"), 2..4, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"aaab"), 2..4, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"aaaab"), 2..4, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ab"), ())); } #[test] fn skip_exact() { - let r = skip_many(new(DEFAULT, b""), 2, |i| token(i, b'a')); + let r = skip_many(new_buf(DEFAULT, b""), 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Incomplete(1)); - let r = skip_many(new(DEFAULT, b"a"), 2, |i| token(i, b'a')); + let r = skip_many(new_buf(DEFAULT, b"a"), 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Incomplete(1)); - let r = skip_many(new(DEFAULT, b"aa"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b""), ())); - let r = skip_many(new(DEFAULT, b"aaa"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"a"), ())); + let r = skip_many(new_buf(DEFAULT, b"aa"), 2, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), ())); + let r = skip_many(new_buf(DEFAULT, b"aaa"), 2, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), ())); - let r = skip_many(new(DEFAULT, b"b"), 2, |i| token(i, b'a').map_err(|_| "token_err")); + let r = skip_many(new_buf(DEFAULT, b"b"), 2, |i| token(i, b'a').map_err(|_| "token_err")); assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r = skip_many(new(DEFAULT, b"ab"), 2, |i| token(i, b'a').map_err(|_| "token_err")); + let r = skip_many(new_buf(DEFAULT, b"ab"), 2, |i| token(i, b'a').map_err(|_| "token_err")); assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r = skip_many(new(DEFAULT, b"aab"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"b"), ())); - let r = skip_many(new(DEFAULT, b"aaab"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"ab"), ())); + let r = skip_many(new_buf(DEFAULT, b"aab"), 2, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); + let r = skip_many(new_buf(DEFAULT, b"aaab"), 2, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ab"), ())); - let r = skip_many(new(END_OF_INPUT, b""), 2, |i| token(i, b'a').map_err(|_| "token_err")); + let r = skip_many(new_buf(END_OF_INPUT, b""), 2, |i| token(i, b'a').map_err(|_| "token_err")); assert_eq!(r.into_inner(), State::Incomplete(1)); - let r = skip_many(new(END_OF_INPUT, b"a"), 2, |i| token(i, b'a').map_err(|_| "token_err")); + let r = skip_many(new_buf(END_OF_INPUT, b"a"), 2, |i| token(i, b'a').map_err(|_| "token_err")); assert_eq!(r.into_inner(), State::Incomplete(1)); - let r = skip_many(new(END_OF_INPUT, b"aa"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b""), ())); - let r = skip_many(new(END_OF_INPUT, b"aaa"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"a"), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"aa"), 2, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"aaa"), 2, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), ())); - let r = skip_many(new(END_OF_INPUT, b"b"), 2, |i| token(i, b'a').map_err(|_| "token_err")); + let r = skip_many(new_buf(END_OF_INPUT, b"b"), 2, |i| token(i, b'a').map_err(|_| "token_err")); assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r = skip_many(new(END_OF_INPUT, b"ab"), 2, |i| token(i, b'a').map_err(|_| "token_err")); + let r = skip_many(new_buf(END_OF_INPUT, b"ab"), 2, |i| token(i, b'a').map_err(|_| "token_err")); assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); - let r = skip_many(new(END_OF_INPUT, b"aab"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"b"), ())); - let r = skip_many(new(END_OF_INPUT, b"aaab"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(END_OF_INPUT, b"ab"), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"aab"), 2, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); + let r = skip_many(new_buf(END_OF_INPUT, b"aaab"), 2, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ab"), ())); } + */ #[test] #[should_panic] fn panic_many_range_lt() { - let r: ParseResult<_, Vec<_>, _> = many(new(DEFAULT, b"aaaab"), 2..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"ab"), vec![b'a', b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aaaab"), 2..1, |i| token(i, b'a')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ab"), vec![b'a', b'a', b'a'])); } + // FIXME: Uncomment + /* #[test] #[should_panic] fn panic_skip_many_range_lt() { - assert_eq!(skip_many(new(DEFAULT, b"aaaab"), 2..1, |i| token(i, b'a')).into_inner(), State::Data(new(DEFAULT, b"ab"), ())); + assert_eq!(skip_many(new_buf(DEFAULT, b"aaaab"), 2..1, |i| token(i, b'a')).into_inner(), State::Data(new_buf(DEFAULT, b"ab"), ())); } #[test] #[should_panic] fn panic_many_till_range_lt() { - let r: ParseResult<_, Vec<_>, _> = many_till(new(DEFAULT, b"aaaab"), 2..1, |i| token(i, b'a'), |i| token(i, b'b')); - assert_eq!(r.into_inner(), State::Data(new(DEFAULT, b"ab"), vec![b'a', b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"aaaab"), 2..1, |i| token(i, b'a'), |i| token(i, b'b')); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ab"), vec![b'a', b'a', b'a'])); } + */ } diff --git a/src/combinators/macros.rs b/src/combinators/macros.rs index b7462ed0..dc822f43 100644 --- a/src/combinators/macros.rs +++ b/src/combinators/macros.rs @@ -17,44 +17,38 @@ macro_rules! run_iter { $($pat:pat => $arm:expr),* } ) => { { - enum EndState<'a, I, E> - where I: 'a { - Error(&'a [I], E), + enum EndState { + // We have an interest to maybe backtrack to last good state in the case of an error. + Error(M, E), Incomplete(usize), } - struct Iter<'a, I, T, E, F> - where I: 'a, - T: 'a, - E: 'a, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, T, E> { + struct Iter + where F: FnMut(I) -> ParseResult { /// Last state of the parser - state: EndState<'a, I, E>, + state: EndState, /// Parser to execute once for each iteration parser: F, /// Remaining buffer - buf: Input<'a, I>, + // TODO: Use UnsafeCell or similar to omit branches, only way to avoid this being set + // on next read would be panic I think? (since self is mutably borrowed already by + // next) + buf: Option, /// Nested state data: $data_ty, _t: PhantomData, } - impl<'a, I, T, E, F> Iter<'a, I, T, E, F> - where I: 'a, - T: 'a, - E: 'a, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, T, E> { + impl Iter + where F: FnMut(I) -> ParseResult { #[inline] - fn end_state(self) -> (Input<'a, I>, $data_ty, EndState<'a, I, E>) { - (self.buf, self.data, self.state) + fn end_state(self) -> (I, $data_ty, EndState) { + (self.buf.unwrap(), self.data, self.state) } } - impl<'a, I, T, E, F> Iterator for Iter<'a, I, T, E, F> - where I: 'a, - T: 'a, - E: 'a, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, T, E> { + impl Iterator for Iter + where F: FnMut(I) -> ParseResult { type Item = T; #[inline] @@ -66,20 +60,25 @@ macro_rules! run_iter { fn next(&mut $next_self) -> Option { $pre_next - match ($next_self.parser)($next_self.buf.clone()).into_inner() { + // TODO: Remove the branches here (ie. take + unwrap) + let i = $next_self.buf.take().unwrap(); + let m = i.mark(); + match ($next_self.parser)(i).into_inner() { State::Data(b, v) => { - $next_self.buf = b; + $next_self.buf = Some(b); $on_next Some(v) }, State::Error(b, e) => { - $next_self.state = EndState::Error(b, e); + $next_self.buf = Some(b); + $next_self.state = EndState::Error(m, e); None }, - State::Incomplete(n) => { + State::Incomplete(b, n) => { + $next_self.buf = Some(b); $next_self.state = EndState::Incomplete(n); None @@ -91,7 +90,7 @@ macro_rules! run_iter { let mut iter = Iter { state: EndState::Incomplete(1), parser: $parser, - buf: $input, + buf: Some($input), data: $data, _t: PhantomData, }; diff --git a/src/combinators/mod.rs b/src/combinators/mod.rs index 1901723d..9bc8ad0f 100644 --- a/src/combinators/mod.rs +++ b/src/combinators/mod.rs @@ -3,7 +3,7 @@ #[macro_use] mod macros; -//pub mod bounded; +pub mod bounded; use std::iter::FromIterator; @@ -32,9 +32,7 @@ use primitives::{IntoInner, Primitives}; pub fn count(i: I, num: usize, p: F) -> ParseResult where F: FnMut(I) -> ParseResult, T: FromIterator { - // FIXME: implement - //bounded::many(i, num, p) - unimplemented!() + bounded::many(i, num, p) } /// Tries the parser ``f``, on success it yields the parsed value, on failure ``default`` will be @@ -138,9 +136,7 @@ pub fn or(i: I, f: F, g: G) -> ParseResult pub fn many(i: I, f: F) -> ParseResult where F: FnMut(I) -> ParseResult, T: FromIterator { - // FIXME: Implement - //bounded::many(i, .., f) - unimplemented!() + bounded::many(i, .., f) } /// Parses at least one instance of ``f`` and continues until it does no longer match, collecting @@ -166,8 +162,7 @@ pub fn many1(i: I, f: F) -> ParseResult where F: FnMut(I) -> ParseResult, T: FromIterator { // FIXME: implement - //bounded::many(i, 1.., f) - unimplemented!() + bounded::many(i, 1.., f) } /// Applies the parser `R` zero or more times, separated by the parser `F`. All matches from `R` @@ -402,9 +397,9 @@ mod test { let r: State<_, Vec<_>, _> = many(new_buf(DEFAULT, b""), |i| token(i, b'a')).into_inner(); assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b""), 1)); let r: State<_, Vec<_>, _> = many(new_buf(DEFAULT, b"a"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b"a"), 1)); + assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b""), 1)); let r: State<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aa"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b"a"), 1)); + assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b""), 1)); let r: State<_, Vec<_>, _> = many(new_buf(DEFAULT, b"bbb"), |i| token(i, b'a')).into_inner(); assert_eq!(r, State::Data(new_buf(DEFAULT, b"bbb"), vec![])); @@ -429,9 +424,9 @@ mod test { let r: State<_, Vec<_>, _> = many1(new_buf(DEFAULT, b""), |i| token(i, b'a')).into_inner(); assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b""), 1)); let r: State<_, Vec<_>, _> = many1(new_buf(DEFAULT, b"a"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b"a"), 1)); + assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b""), 1)); let r: State<_, Vec<_>, _> = many1(new_buf(DEFAULT, b"aa"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b"aa"), 1)); + assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b""), 1)); let r: State<_, Vec<_>, _> = many1(new_buf(DEFAULT, b"bbb"), |i| token(i, b'a').map_err(|_| "token_error")).into_inner(); assert_eq!(r, State::Error(new_buf(DEFAULT, b"bbb"), "token_error")); @@ -456,9 +451,9 @@ mod test { let r: State<_, Vec<_>, _> = count(new_buf(DEFAULT, b""), 3, |i| token(i, b'a')).into_inner(); assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b""), 1)); let r: State<_, Vec<_>, _> = count(new_buf(DEFAULT, b"a"), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b"a"), 1)); + assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b""), 1)); let r: State<_, Vec<_>, _> = count(new_buf(DEFAULT, b"aa"), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b"aa"), 1)); + assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b""), 1)); let r: State<_, Vec<_>, _> = count(new_buf(DEFAULT, b"aaa"), 3, |i| token(i, b'a')).into_inner(); assert_eq!(r, State::Data(new_buf(DEFAULT, b""), vec![b'a', b'a', b'a'])); let r: State<_, Vec<_>, _> = count(new_buf(DEFAULT, b"aaaa"), 3, |i| token(i, b'a')).into_inner(); @@ -467,9 +462,9 @@ mod test { let r: State<_, Vec<_>, _> = count(new_buf(END_OF_INPUT, b""), 3, |i| token(i, b'a')).into_inner(); assert_eq!(r, State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); let r: State<_, Vec<_>, _> = count(new_buf(END_OF_INPUT, b"a"), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(new_buf(END_OF_INPUT, b"a"), 1)); + assert_eq!(r, State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); let r: State<_, Vec<_>, _> = count(new_buf(END_OF_INPUT, b"aa"), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(new_buf(END_OF_INPUT, b"aa"), 1)); + assert_eq!(r, State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); let r: State<_, Vec<_>, _> = count(new_buf(END_OF_INPUT, b"aaa"), 3, |i| token(i, b'a')).into_inner(); assert_eq!(r, State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'a', b'a'])); let r: State<_, Vec<_>, _> = count(new_buf(END_OF_INPUT, b"aaaa"), 3, |i| token(i, b'a')).into_inner(); diff --git a/src/lib.rs b/src/lib.rs index 84483072..b174a003 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -209,7 +209,6 @@ extern crate debugtrace; #[macro_use] mod macros; mod input; -// FIXME: Uncomment mod parse; mod parse_result; diff --git a/src/parsers.rs b/src/parsers.rs index 0ba2751b..6aa30e26 100644 --- a/src/parsers.rs +++ b/src/parsers.rs @@ -593,7 +593,7 @@ mod error { #[cfg(test)] mod test { - use primitives::input::{new_buf, DEFAULT, END_OF_INPUT}; + use primitives::input::{new_buf, DEFAULT}; use primitives::IntoInner; use primitives::State; use super::*; From a829059453d38d638f483491400da5e0a0e04b5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Fri, 18 Mar 2016 18:07:02 +0100 Subject: [PATCH 06/63] test: Tests for primitives on slice and partial slice --- src/input.rs | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/src/input.rs b/src/input.rs index ba8dfe4a..b06c9944 100644 --- a/src/input.rs +++ b/src/input.rs @@ -459,6 +459,8 @@ impl<'a, I: 'a> InputBuffer<'a> for Input<'a, I> { #[cfg(test)] mod test { + use std::fmt::Debug; + use super::{new_buf, Input, InputBuf, DEFAULT, END_OF_INPUT}; use parse_result::ParseResult; use primitives::{IntoInner, State}; @@ -502,11 +504,44 @@ mod test { } #[test] - fn last_slice() { - use primitives::Primitives; + fn primitives_slice() { + run_primitives_test(&b"abc"[..], true); + } + + #[test] + fn primitives_input_buf_default() { + run_primitives_test(new_buf(DEFAULT, b"abc"), false); + } - let i = new_buf(END_OF_INPUT, &b"foo"[..]); + #[test] + fn primitives_input_buf_end() { + run_primitives_test(new_buf(END_OF_INPUT, b"abc"), true); + } + + fn run_primitives_test PartialEq<&'a [u8]>, I: Input>(mut s: I, last: bool) { + use primitives::Primitives; - assert_eq!(i.is_end(), true); + assert_eq!(s.is_end(), last); + assert_eq!(s.min_remaining(), 3); + let m = s.mark(); + assert_eq!(s.min_remaining(), 3); + assert_eq!(s.first(), Some(b'a')); + assert_eq!(s.min_remaining(), 3); + assert_eq!(s.iter().collect::>(), vec![b'a', b'b', b'c']); + assert_eq!(s.min_remaining(), 3); + assert_eq!(s.consume(2), &b"ab"[..]); + assert_eq!(s.min_remaining(), 1); + assert_eq!(s.iter().collect::>(), vec![b'c']); + assert_eq!(s.consume(1), &b"c"[..]); + assert_eq!(s.min_remaining(), 0); + assert_eq!(s.iter().collect::>(), vec![]); + assert_eq!(s.is_end(), last); + let mut s = s.restore(m); + assert_eq!(s.min_remaining(), 3); + assert_eq!(s.iter().collect::>(), vec![b'a', b'b', b'c']); + s.discard(1); + assert_eq!(s.first(), Some(b'b')); + assert_eq!(s.min_remaining(), 2); + assert_eq!(s.iter().collect::>(), vec![b'b', b'c']); } } From 37479cce8fa3493eec88cf7dff562e79b94d1891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Fri, 18 Mar 2016 18:35:08 +0100 Subject: [PATCH 07/63] updated combinators::bounded::many_till --- src/combinators/bounded.rs | 171 ++++++++++++++++--------------------- src/combinators/macros.rs | 111 ++++++++++++------------ src/combinators/mod.rs | 15 +--- 3 files changed, 136 insertions(+), 161 deletions(-) diff --git a/src/combinators/bounded.rs b/src/combinators/bounded.rs index d4d4e69c..ed1d7e92 100644 --- a/src/combinators/bounded.rs +++ b/src/combinators/bounded.rs @@ -74,7 +74,9 @@ pub trait BoundedRange { fn skip_many<'a, I, T, E, F>(self, Input<'a, I>, F) -> ParseResult<'a, I, (), E> where T: 'a, F: FnMut(Input<'a, I>) -> ParseResult<'a, I, T, E>; + */ + // TODO: Fix documentation regarding incomplete /// Applies the parser `P` multiple times until the parser `F` succeeds and returns a value /// populated by the values yielded by `P`. Consumes the matched part of `F`. If `F` does not /// succeed within the given range `R` this combinator will propagate any failure from `P`. @@ -91,15 +93,10 @@ pub trait BoundedRange { /// * If the last parser succeeds on the last input item then this combinator is still considered /// incomplete unless the parser `F` matches or the lower bound has not been met. #[inline] - fn many_till<'a, I, T, E, R, F, U, N, V>(self, i: Input<'a, I>, p: R, end: F) -> ParseResult<'a, I, T, E> - where I: Copy, - U: 'a, - V: 'a, - N: 'a, - T: FromIterator, - R: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, V, N>; - */ + fn many_till(self, I, R, F) -> ParseResult + where T: FromIterator, + R: FnMut(I) -> ParseResult, + F: FnMut(I) -> ParseResult; } impl BoundedRange for Range { @@ -134,22 +131,20 @@ impl BoundedRange for Range { => result : T { // Got all occurrences of the parser - // TODO: Backtrack to last good - (s, (0, 0), _) => s.ret(result), + // TODO: Can this even happen? + (s, (0, 0), _, _) => unreachable!(), // s.ret(result), // Ok, last parser failed and we have reached minimum, we have iterated all. // Return remainder of buffer and the collected result - // TODO: Backtrack to last good - (s, (0, _), EndState::Error(m, _)) => s.restore(m).ret(result), + (s, (0, _), m, EndState::Error(_)) => s.restore(m).ret(result), // Nested parser incomplete but reached at least minimum, propagate if not at end - (s, (0, _), EndState::Incomplete(n)) => if s.is_end() { - // TODO: Backtrack to last good - s.ret(result) + (s, (0, _), m, EndState::Incomplete(n)) => if s.is_end() { + s.restore(m).ret(result) } else { s.incomplete(n) }, // Did not reach minimum, propagate - (s, (_, _), EndState::Error(_, e)) => s.err(e), - (s, (_, _), EndState::Incomplete(n)) => s.incomplete(n) + (s, (_, _), _, EndState::Error(e)) => s.err(e), + (s, (_, _), _, EndState::Incomplete(n)) => s.incomplete(n) } } } @@ -195,16 +190,13 @@ impl BoundedRange for Range { i.ret(()) } + */ #[inline] - fn many_till<'a, I, T, E, R, F, U, N, V>(self, i: Input<'a, I>, p: R, end: F) -> ParseResult<'a, I, T, E> - where I: Copy, - U: 'a, - V: 'a, - N: 'a, - T: FromIterator, - R: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, V, N> { + fn many_till(self, i: I, p: R, end: F) -> ParseResult + where T: FromIterator, + R: FnMut(I) -> ParseResult, + F: FnMut(I) -> ParseResult { // Range does not perform this assertion assert!(self.start <= self.end); @@ -244,7 +236,7 @@ impl BoundedRange for Range { // Got all occurrences of the parser (s, (0, _), EndStateTill::EndSuccess) => s.ret(result), // Did not reach minimum or a failure, propagate - (s, (_, _), EndStateTill::Error(b, e)) => s.replace(b).err(e), + (s, (_, _), EndStateTill::Error(e)) => s.err(e), (s, (_, _), EndStateTill::Incomplete(n)) => s.incomplete(n), // We cannot reach this since we only run the end test once we have reached the // minimum number of matches @@ -252,7 +244,6 @@ impl BoundedRange for Range { } } } - */ } impl BoundedRange for RangeFrom { @@ -279,17 +270,16 @@ impl BoundedRange for RangeFrom { => result : T { // We got at least n items - (s, 0, EndState::Error(m, _)) => s.restore(m).ret(result), + (s, 0, m, EndState::Error(_)) => s.restore(m).ret(result), // Nested parser incomplete, propagate if not at end - (s, 0, EndState::Incomplete(n)) => if s.is_end() { - // TODO: Backtrack to last good - s.ret(result) + (s, 0, m, EndState::Incomplete(n)) => if s.is_end() { + s.restore(m).ret(result) } else { s.incomplete(n) }, // Items still remaining, propagate - (s, _, EndState::Error(_, e)) => s.err(e), - (s, _, EndState::Incomplete(n)) => s.incomplete(n) + (s, _, _, EndState::Error(e)) => s.err(e), + (s, _, _, EndState::Incomplete(n)) => s.incomplete(n) } } } @@ -327,16 +317,13 @@ impl BoundedRange for RangeFrom { i.ret(()) } + */ #[inline] - fn many_till<'a, I, T, E, R, F, U, N, V>(self, i: Input<'a, I>, p: R, end: F) -> ParseResult<'a, I, T, E> - where I: Copy, - U: 'a, - V: 'a, - N: 'a, - T: FromIterator, - R: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, V, N> { + fn many_till(self, i: I, p: R, end: F) -> ParseResult + where T: FromIterator, + R: FnMut(I) -> ParseResult, + F: FnMut(I) -> ParseResult { run_iter_till!{ input: i, parser: p, @@ -364,7 +351,7 @@ impl BoundedRange for RangeFrom { // Got all occurrences of the parser (s, 0, EndStateTill::EndSuccess) => s.ret(result), // Did not reach minimum or a failure, propagate - (s, _, EndStateTill::Error(b, e)) => s.replace(b).err(e), + (s, _, EndStateTill::Error(e)) => s.err(e), (s, _, EndStateTill::Incomplete(n)) => s.incomplete(n), // We cannot reach this since we only run the end test once we have reached the // minimum number of matches @@ -372,7 +359,6 @@ impl BoundedRange for RangeFrom { } } } - */ } impl BoundedRange for RangeFull { @@ -395,10 +381,10 @@ impl BoundedRange for RangeFull { } => result : T { - (s, (), EndState::Error(m, _)) => s.restore(m).ret(result), + (s, (), m, EndState::Error(_)) => s.restore(m).ret(result), // Nested parser incomplete, propagate if not at end - (s, (), EndState::Incomplete(n)) => if s.is_end() { - s.ret(result) + (s, (), m, EndState::Incomplete(n)) => if s.is_end() { + s.restore(m).ret(result) } else { s.incomplete(n) } @@ -425,16 +411,13 @@ impl BoundedRange for RangeFull { i.ret(()) } + */ #[inline] - fn many_till<'a, I, T, E, R, F, U, N, V>(self, i: Input<'a, I>, p: R, end: F) -> ParseResult<'a, I, T, E> - where I: Copy, - U: 'a, - V: 'a, - N: 'a, - T: FromIterator, - R: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, V, N> { + fn many_till(self, i: I, p: R, end: F) -> ParseResult + where T: FromIterator, + R: FnMut(I) -> ParseResult, + F: FnMut(I) -> ParseResult { run_iter_till!{ input: i, parser: p, @@ -455,13 +438,12 @@ impl BoundedRange for RangeFull { => result : T { (s, (), EndStateTill::EndSuccess) => s.ret(result), - (s, (), EndStateTill::Error(b, e)) => s.replace(b).err(e), + (s, (), EndStateTill::Error(e)) => s.err(e), // Nested parser incomplete, propagate if not at end (s, (), EndStateTill::Incomplete(n)) => s.incomplete(n) } } } - */ } impl BoundedRange for RangeTo { @@ -492,14 +474,13 @@ impl BoundedRange for RangeTo { => result : T { // Either error or incomplete after the end - // TODO: Backtrack to last good - (s, 0, _) => s.ret(result), + // TODO: Can this even happen? + (s, 0, _, _) => unreachable!(), // s.ret(result), // Inside of range, never outside - (s, _, EndState::Error(m, _)) => s.restore(m).ret(result), + (s, _, m, EndState::Error(_)) => s.restore(m).ret(result), // Nested parser incomplete, propagate if not at end - (s, _, EndState::Incomplete(n)) => if s.is_end() { - // TODO: Backtrack to last good - s.ret(result) + (s, _, m, EndState::Incomplete(n)) => if s.is_end() { + s.restore(m).ret(result) } else { s.incomplete(n) } @@ -539,16 +520,13 @@ impl BoundedRange for RangeTo { i.ret(()) } + */ #[inline] - fn many_till<'a, I, T, E, R, F, U, N, V>(self, i: Input<'a, I>, p: R, end: F) -> ParseResult<'a, I, T, E> - where I: Copy, - U: 'a, - V: 'a, - N: 'a, - T: FromIterator, - R: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, V, N> { + fn many_till(self, i: I, p: R, end: F) -> ParseResult + where T: FromIterator, + R: FnMut(I) -> ParseResult, + F: FnMut(I) -> ParseResult { run_iter_till!{ input: i, parser: p, @@ -579,7 +557,7 @@ impl BoundedRange for RangeTo { // Got all occurrences of the parser (s, 0, EndStateTill::EndSuccess) => s.ret(result), // Did not reach minimum or a failure, propagate - (s, _, EndStateTill::Error(b, e)) => s.replace(b).err(e), + (s, _, EndStateTill::Error(e)) => s.err(e), (s, _, EndStateTill::Incomplete(n)) => s.incomplete(n), // We cannot reach this since we only run the end test once we have reached the // minimum number of matches @@ -587,10 +565,10 @@ impl BoundedRange for RangeTo { } } } - */ } impl BoundedRange for usize { + // TODO: Any way to avoid marking for backtracking here? #[inline] fn parse_many(self, i: I, f: F) -> ParseResult where F: FnMut(I) -> ParseResult, @@ -618,12 +596,11 @@ impl BoundedRange for usize { => result : T { // Got exact - // TODO: Backtrack to last good - (s, 0, _) => s.ret(result), + (s, 0, _, _) => s.ret(result), // We have got too few items, propagate error - (s, _, EndState::Error(_, e)) => s.err(e), + (s, _, _, EndState::Error(e)) => s.err(e), // Nested parser incomplete, propagate - (s, _, EndState::Incomplete(n)) => s.incomplete(n) + (s, _, _, EndState::Incomplete(n)) => s.incomplete(n) } } } @@ -663,16 +640,13 @@ impl BoundedRange for usize { i.ret(()) } + */ #[inline] - fn many_till<'a, I, T, E, R, F, U, N, V>(self, i: Input<'a, I>, p: R, end: F) -> ParseResult<'a, I, T, E> - where I: Copy, - U: 'a, - V: 'a, - N: 'a, - T: FromIterator, - R: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, V, N> { + fn many_till(self, i: I, p: R, end: F) -> ParseResult + where T: FromIterator, + R: FnMut(I) -> ParseResult, + F: FnMut(I) -> ParseResult { run_iter_till!{ input: i, parser: p, @@ -701,7 +675,7 @@ impl BoundedRange for usize { // Got all occurrences of the parser (s, 0, EndStateTill::EndSuccess) => s.ret(result), // Did not reach minimum or a failure, propagate - (s, _, EndStateTill::Error(b, e)) => s.replace(b).err(e), + (s, _, EndStateTill::Error(e)) => s.err(e), (s, _, EndStateTill::Incomplete(n)) => s.incomplete(n), // We cannot reach this since we only run the end test once we have reached the // minimum number of matches @@ -709,7 +683,6 @@ impl BoundedRange for usize { } } } - */ } /// Applies the parser `F` multiple times until it fails or the maximum value of the range has @@ -757,7 +730,9 @@ pub fn skip_many<'a, I, T, E, F, R>(i: Input<'a, I>, r: R, f: F) -> ParseResult< F: FnMut(Input<'a, I>) -> ParseResult<'a, I, T, E> { BoundedRange::skip_many(r, i, f) } +*/ +// TODO: Update documentation regarding incomplete behaviour /// Applies the parser `P` multiple times until the parser `F` succeeds and returns a value /// populated by the values yielded by `P`. Consumes the matched part of `F`. If `F` does not /// succeed within the given range `R` this combinator will propagate any failure from `P`. @@ -773,18 +748,13 @@ pub fn skip_many<'a, I, T, E, F, R>(i: Input<'a, I>, r: R, f: F) -> ParseResult< /// * If the last parser succeeds on the last input item then this combinator is still considered /// incomplete unless the parser `F` matches or the lower bound has not been met. #[inline] -pub fn many_till<'a, I, T, E, R, F, U, N, P, V>(i: Input<'a, I>, r: R, p: P, end: F) -> ParseResult<'a, I, T, E> - where I: Copy, - U: 'a, - V: 'a, - N: 'a, - R: BoundedRange, +pub fn many_till(i: I, r: R, p: P, end: F) -> ParseResult + where R: BoundedRange, T: FromIterator, - P: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, V, N> { + P: FnMut(I) -> ParseResult, + F: FnMut(I) -> ParseResult { BoundedRange::many_till(r, i, p, end) } -*/ /// Applies the parser `p` multiple times, separated by the parser `sep` and returns a value /// populated with the values yielded by `p`. If the number of items yielded by `p` does not fall @@ -831,7 +801,7 @@ mod test { use super::{ many, - //many_till, + many_till, //skip_many, }; @@ -1106,6 +1076,13 @@ mod test { assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![&b"ab"[..], &b"ab"[..]])); } + // TODO: Write tests for many_till + #[test] + fn many_till_range_full() { + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), .., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); + } + // FIXME: Uncomment /* #[test] diff --git a/src/combinators/macros.rs b/src/combinators/macros.rs index dc822f43..798a339e 100644 --- a/src/combinators/macros.rs +++ b/src/combinators/macros.rs @@ -17,23 +17,25 @@ macro_rules! run_iter { $($pat:pat => $arm:expr),* } ) => { { - enum EndState { - // We have an interest to maybe backtrack to last good state in the case of an error. - Error(M, E), + enum EndState { + Error(E), Incomplete(usize), } struct Iter where F: FnMut(I) -> ParseResult { /// Last state of the parser - state: EndState, + state: EndState, /// Parser to execute once for each iteration parser: F, /// Remaining buffer - // TODO: Use UnsafeCell or similar to omit branches, only way to avoid this being set - // on next read would be panic I think? (since self is mutably borrowed already by - // next) + /// + /// Wrapped in option to prevent two calls to destructors. buf: Option, + /// Last good state. + /// + /// Wrapped in option to prevent two calls to destructors. + mark: Option, /// Nested state data: $data_ty, _t: PhantomData, @@ -42,8 +44,9 @@ macro_rules! run_iter { impl Iter where F: FnMut(I) -> ParseResult { #[inline] - fn end_state(self) -> (I, $data_ty, EndState) { - (self.buf.unwrap(), self.data, self.state) + fn end_state(self) -> (I, $data_ty, I::Marker, EndState) { + // TODO: Avoid branch, check if this can be guaranteed to always be Some(T) + (self.buf.expect("Iter.buf was None"), self.data, self.mark.expect("Iter.mark was None"), self.state) } } @@ -61,8 +64,10 @@ macro_rules! run_iter { $pre_next // TODO: Remove the branches here (ie. take + unwrap) - let i = $next_self.buf.take().unwrap(); - let m = i.mark(); + let i = $next_self.buf.take().expect("Iter.buf was None"); + + $next_self.mark = Some(i.mark()); + match ($next_self.parser)(i).into_inner() { State::Data(b, v) => { $next_self.buf = Some(b); @@ -73,7 +78,7 @@ macro_rules! run_iter { }, State::Error(b, e) => { $next_self.buf = Some(b); - $next_self.state = EndState::Error(m, e); + $next_self.state = EndState::Error(e); None }, @@ -91,6 +96,7 @@ macro_rules! run_iter { state: EndState::Incomplete(1), parser: $parser, buf: Some($input), + mark: None, data: $data, _t: PhantomData, }; @@ -122,54 +128,39 @@ macro_rules! run_iter_till { $($pat:pat => $arm:expr),* } ) => { { - enum EndStateTill<'a, I, E> - where I: 'a { - Error(&'a [I], E), + enum EndStateTill { + Error(E), Incomplete(usize), EndSuccess, } /// Iterator used by ``many_till`` and ``many1``. - struct IterTill<'a, I, T, U, E, F, P, N> - where I: 'a, - T: 'a, - E: 'a, - U: 'a, - N: 'a, - P: FnMut(Input<'a, I>) -> ParseResult<'a, I, T, E>, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, N> { - state: EndStateTill<'a, I, E>, + struct IterTill + where P: FnMut(I) -> ParseResult, + F: FnMut(I) -> ParseResult { + state: EndStateTill, parser: P, end: F, - buf: Input<'a, I>, + buf: Option, data: $data_ty, _t: PhantomData<(T, U, N)>, } - impl<'a, I, T, U, E, F, P, N> IterTill<'a, I, T, U, E, F, P, N> - where I: 'a, - T: 'a, - E: 'a, - U: 'a, - N: 'a, - P: FnMut(Input<'a, I>) -> ParseResult<'a, I, T, E>, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, N> { + impl IterTill + where P: FnMut(I) -> ParseResult, + F: FnMut(I) -> ParseResult { /// Destructures the iterator returning the position just after the last successful parse as /// well as the state of the last attempt to parse data. #[inline] - fn end_state(self) -> (Input<'a, I>, $data_ty, EndStateTill<'a, I, E>) { - (self.buf, self.data, self.state) + fn end_state(self) -> (I, $data_ty, EndStateTill) { + // TODO: Avoid branch, check if this can be guaranteed to always be Some(T) + (self.buf.expect("Iter.buf was None"), self.data, self.state) } } - impl<'a, I, T, U, E, F, P, N> Iterator for IterTill<'a, I, T, U, E, F, P, N> - where I: 'a, - T: 'a, - E: 'a, - U: 'a, - N: 'a, - P: FnMut(Input<'a, I>) -> ParseResult<'a, I, T, E>, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, N> { + impl Iterator for IterTill + where P: FnMut(I) -> ParseResult, + F: FnMut(I) -> ParseResult { type Item = T; #[inline] @@ -181,20 +172,25 @@ macro_rules! run_iter_till { fn next(&mut $next_self) -> Option { $pre_next - match ($next_self.parser)($next_self.buf.clone()).into_inner() { + // TODO: Remove the branches here (ie. take + unwrap) + let i = $next_self.buf.take().expect("Iter.buf was None"); + + match ($next_self.parser)(i).into_inner() { State::Data(b, v) => { - $next_self.buf = b; + $next_self.buf = Some(b); $on_next Some(v) }, State::Error(b, e) => { - $next_self.state = EndStateTill::Error(b, e); + $next_self.buf = Some(b); + $next_self.state = EndStateTill::Error(e); None }, - State::Incomplete(n) => { + State::Incomplete(b, n) => { + $next_self.buf = Some(b); $next_self.state = EndStateTill::Incomplete(n); None @@ -207,7 +203,7 @@ macro_rules! run_iter_till { state: EndStateTill::Incomplete(1), parser: $parser, end: $end, - buf: $input, + buf: Some($input), data: $data, _t: PhantomData, }; @@ -225,11 +221,20 @@ macro_rules! run_iter_till { /// will be returned, stopping the iteration. If the test fails execution continues. macro_rules! iter_till_end_test { ( $the_self:ident ) => { { - if let State::Data(b, _) = ($the_self.end)($the_self.buf.clone()).into_inner() { - $the_self.buf = b; - $the_self.state = EndStateTill::EndSuccess; - - return None + // TODO: Remove the branches here (ie. take + unwrap) + let i = $the_self.buf.take().expect("Iter.buf was None"); + let m = i.mark(); + + match ($the_self.end)(i).into_inner() { + State::Data(b, _) => { + $the_self.buf = Some(b); + $the_self.state = EndStateTill::EndSuccess; + + return None + }, + // Failed to end, restore and continue + State::Error(b, _) => $the_self.buf = Some(b.restore(m)), + State::Incomplete(b, _) => $the_self.buf = Some(b.restore(m)), } } } } diff --git a/src/combinators/mod.rs b/src/combinators/mod.rs index 9bc8ad0f..0e95f50d 100644 --- a/src/combinators/mod.rs +++ b/src/combinators/mod.rs @@ -161,7 +161,6 @@ pub fn many(i: I, f: F) -> ParseResult pub fn many1(i: I, f: F) -> ParseResult where F: FnMut(I) -> ParseResult, T: FromIterator { - // FIXME: implement bounded::many(i, 1.., f) } @@ -187,9 +186,7 @@ pub fn sep_by(i: I, p: R, sep: F) -> ParseResult< E: From, R: FnMut(I) -> ParseResult, F: FnMut(I) -> ParseResult { - // FIXME: implement - //bounded::sep_by(i, .., p, sep) - unimplemented!() + bounded::sep_by(i, .., p, sep) } @@ -215,9 +212,7 @@ pub fn sep_by1(i: I, p: R, sep: F) -> ParseResult E: From, R: FnMut(I) -> ParseResult, F: FnMut(I) -> ParseResult { - // FIXME: Implement - //bounded::sep_by(i, 1.., p, sep) - unimplemented!() + bounded::sep_by(i, 1.., p, sep) } /// Applies the parser `R` multiple times until the parser `F` succeeds and returns a @@ -239,9 +234,7 @@ pub fn many_till(i: I, p: R, end: F) -> ParseResu where T: FromIterator, R: FnMut(I) -> ParseResult, F: FnMut(I) -> ParseResult { - //bounded::many_till(i, .., p, end) - // FIXME: Implement - unimplemented!() + bounded::many_till(i, .., p, end) } /// Runs the given parser until it fails, discarding matched input. @@ -487,7 +480,7 @@ mod test { fn many_till_test() { assert_eq!(many_till(new_buf(DEFAULT, b"abcd"), any, |i| token(i, b'c')).into_inner(), State::Data(new_buf(DEFAULT, b"d"), vec![b'a', b'b'])); let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abd"), any, |i| token(i, b'c')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"abd"), 1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r: ParseResult<_, Vec, _> = many_till(new_buf(DEFAULT, b"abcd"), |i| i.err(()), |i| token(i, b'c')); assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"abcd"), ())); From 82eeed43315da3ed3fd2fe3d2bc29cb818c38aa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Fri, 18 Mar 2016 19:44:27 +0100 Subject: [PATCH 08/63] combinators::bounded::many: Fixed backtracking and sep_by tests --- src/combinators/bounded.rs | 16 +++++++++------- src/combinators/macros.rs | 12 ++++++++---- src/combinators/mod.rs | 24 ++++++++++++------------ 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/combinators/bounded.rs b/src/combinators/bounded.rs index ed1d7e92..6ff66f67 100644 --- a/src/combinators/bounded.rs +++ b/src/combinators/bounded.rs @@ -124,6 +124,7 @@ impl BoundedRange for Range { } } on { + // TODO: Saturating sub? self.data.0 = if self.data.0 == 0 { 0 } else { self.data.0 - 1 }; self.data.1 -= 1; } @@ -131,8 +132,9 @@ impl BoundedRange for Range { => result : T { // Got all occurrences of the parser - // TODO: Can this even happen? - (s, (0, 0), _, _) => unreachable!(), // s.ret(result), + // First state or reached max => do not restore to mark since it is from last + // iteration + (s, (0, 0), _, _) => s.ret(result), // Ok, last parser failed and we have reached minimum, we have iterated all. // Return remainder of buffer and the collected result (s, (0, _), m, EndState::Error(_)) => s.restore(m).ret(result), @@ -381,7 +383,7 @@ impl BoundedRange for RangeFull { } => result : T { - (s, (), m, EndState::Error(_)) => s.restore(m).ret(result), + (s, (), m, EndState::Error(_)) => s.restore(m).ret(result), // Nested parser incomplete, propagate if not at end (s, (), m, EndState::Incomplete(n)) => if s.is_end() { s.restore(m).ret(result) @@ -438,7 +440,7 @@ impl BoundedRange for RangeFull { => result : T { (s, (), EndStateTill::EndSuccess) => s.ret(result), - (s, (), EndStateTill::Error(e)) => s.err(e), + (s, (), EndStateTill::Error(e)) => s.err(e), // Nested parser incomplete, propagate if not at end (s, (), EndStateTill::Incomplete(n)) => s.incomplete(n) } @@ -473,9 +475,9 @@ impl BoundedRange for RangeTo { } => result : T { - // Either error or incomplete after the end - // TODO: Can this even happen? - (s, 0, _, _) => unreachable!(), // s.ret(result), + // First state or reached max => do not restore to mark since it is from last + // iteration + (s, 0, _, _) => s.ret(result), // Inside of range, never outside (s, _, m, EndState::Error(_)) => s.restore(m).ret(result), // Nested parser incomplete, propagate if not at end diff --git a/src/combinators/macros.rs b/src/combinators/macros.rs index 798a339e..c4f6cf48 100644 --- a/src/combinators/macros.rs +++ b/src/combinators/macros.rs @@ -35,7 +35,7 @@ macro_rules! run_iter { /// Last good state. /// /// Wrapped in option to prevent two calls to destructors. - mark: Option, + mark: I::Marker, /// Nested state data: $data_ty, _t: PhantomData, @@ -46,7 +46,7 @@ macro_rules! run_iter { #[inline] fn end_state(self) -> (I, $data_ty, I::Marker, EndState) { // TODO: Avoid branch, check if this can be guaranteed to always be Some(T) - (self.buf.expect("Iter.buf was None"), self.data, self.mark.expect("Iter.mark was None"), self.state) + (self.buf.expect("Iter.buf was None"), self.data, self.mark, self.state) } } @@ -66,7 +66,8 @@ macro_rules! run_iter { // TODO: Remove the branches here (ie. take + unwrap) let i = $next_self.buf.take().expect("Iter.buf was None"); - $next_self.mark = Some(i.mark()); + // TODO: Any way to prevent marking here since it is not used at all times? + $next_self.mark = i.mark(); match ($next_self.parser)(i).into_inner() { State::Data(b, v) => { @@ -92,11 +93,14 @@ macro_rules! run_iter { } } + // TODO: Not always used + let m = $input.mark(); + let mut iter = Iter { state: EndState::Incomplete(1), parser: $parser, buf: Some($input), - mark: None, + mark: m, data: $data, _t: PhantomData, }; diff --git a/src/combinators/mod.rs b/src/combinators/mod.rs index 0e95f50d..773fa99a 100644 --- a/src/combinators/mod.rs +++ b/src/combinators/mod.rs @@ -518,22 +518,22 @@ mod test { assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r: ParseResult<_, Vec<_>, _> = sep_by(new_buf(DEFAULT, b"a"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r: ParseResult<_, Vec<_>, _> = sep_by(new_buf(DEFAULT, b"a;"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a;"), 1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r: ParseResult<_, Vec<_>, _> = sep_by(new_buf(DEFAULT, b"a;c"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a;c"), 1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r: ParseResult<_, Vec<_>, _> = sep_by(new_buf(DEFAULT, b"a;c;"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a;c;"), 1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r: ParseResult<_, Vec<_>, _> = sep_by(new_buf(DEFAULT, b"a--c-"), any, |i| string(i, b"--")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a--c-"), 1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"-"), 1)); let r: ParseResult<_, Vec<_>, _> = sep_by(new_buf(DEFAULT, b"aaa--a"), |i| string(i, b"aaa"), |i| string(i, b"--")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"aaa--a"), 2)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 2)); } @@ -560,22 +560,22 @@ mod test { assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(DEFAULT, b"a"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(DEFAULT, b"a;"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a;"), 1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(DEFAULT, b"a;c"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a;c"), 1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(DEFAULT, b"a;c;"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a;c;"), 1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(DEFAULT, b"a--c-"), any, |i| string(i, b"--")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a--c-"), 1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"-"), 1)); let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(DEFAULT, b"aaa--a"), |i| string(i, b"aaa"), |i| string(i, b"--")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"aaa--a"), 2)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 2)); } #[test] From 98443f3644e6edb7c6b5f0522a5a0df651f4310e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Fri, 18 Mar 2016 20:52:56 +0100 Subject: [PATCH 09/63] combinators::bounded::many_till: Added tests and fixed bugs --- src/combinators/bounded.rs | 282 ++++++++++++++++++++++++++++++++++--- src/combinators/macros.rs | 9 +- src/combinators/mod.rs | 11 +- 3 files changed, 272 insertions(+), 30 deletions(-) diff --git a/src/combinators/bounded.rs b/src/combinators/bounded.rs index 6ff66f67..b30f34ba 100644 --- a/src/combinators/bounded.rs +++ b/src/combinators/bounded.rs @@ -95,6 +95,7 @@ pub trait BoundedRange { #[inline] fn many_till(self, I, R, F) -> ParseResult where T: FromIterator, + E: From, R: FnMut(I) -> ParseResult, F: FnMut(I) -> ParseResult; } @@ -197,6 +198,7 @@ impl BoundedRange for Range { #[inline] fn many_till(self, i: I, p: R, end: F) -> ParseResult where T: FromIterator, + E: From, R: FnMut(I) -> ParseResult, F: FnMut(I) -> ParseResult { // Range does not perform this assertion @@ -217,15 +219,36 @@ impl BoundedRange for Range { pre { if self.data.0 == 0 { // We have reached minimum, we can attempt to end now - iter_till_end_test!(self); - } - // Maximum reached, stop iteration and check error state - if self.data.1 == 0 { - // Attempt to make a successful end - iter_till_end_test!(self); - - return None; + // TODO: Remove the branches here (ie. take + unwrap) + let i = self.buf.take().expect("Iter.buf was None"); + let m = i.mark(); + + match (self.data.1, (self.end)(i).into_inner()) { + // We can always end + (_, State::Data(b, _)) => { + self.buf = Some(b); + self.state = EndStateTill::EndSuccess; + + return None + }, + // We have reached end, end must match or it is an error + (0, State::Error(b, e)) => { + self.buf = Some(b); + self.state = EndStateTill::Error(From::from(e)); + + return None; + }, + (0, State::Incomplete(b, n)) => { + self.buf = Some(b); + self.state = EndStateTill::Incomplete(n); + + return None; + }, + // Failed to end, restore and continue since we can parse more + (_, State::Error(b, _)) => self.buf = Some(b.restore(m)), + (_, State::Incomplete(b, _)) => self.buf = Some(b.restore(m)), + } } } on { @@ -324,6 +347,7 @@ impl BoundedRange for RangeFrom { #[inline] fn many_till(self, i: I, p: R, end: F) -> ParseResult where T: FromIterator, + E: From, R: FnMut(I) -> ParseResult, F: FnMut(I) -> ParseResult { run_iter_till!{ @@ -418,6 +442,7 @@ impl BoundedRange for RangeFull { #[inline] fn many_till(self, i: I, p: R, end: F) -> ParseResult where T: FromIterator, + E: From, R: FnMut(I) -> ParseResult, F: FnMut(I) -> ParseResult { run_iter_till!{ @@ -527,6 +552,7 @@ impl BoundedRange for RangeTo { #[inline] fn many_till(self, i: I, p: R, end: F) -> ParseResult where T: FromIterator, + E: From, R: FnMut(I) -> ParseResult, F: FnMut(I) -> ParseResult { run_iter_till!{ @@ -542,12 +568,34 @@ impl BoundedRange for RangeTo { next(self) { pre { - // Can end at any time - iter_till_end_test!(self); - - // Maximum reached, stop iteration and check error state - if self.data == 0 { - return None; + // TODO: Remove the branches here (ie. take + unwrap) + let i = self.buf.take().expect("Iter.buf was None"); + let m = i.mark(); + + match (self.data, (self.end)(i).into_inner()) { + // We can always end + (_, State::Data(b, _)) => { + self.buf = Some(b); + self.state = EndStateTill::EndSuccess; + + return None + }, + // We have reached end, end must match or it is an error + (0, State::Error(b, e)) => { + self.buf = Some(b); + self.state = EndStateTill::Error(From::from(e)); + + return None; + }, + (0, State::Incomplete(b, n)) => { + self.buf = Some(b); + self.state = EndStateTill::Incomplete(n); + + return None; + }, + // Failed to end, restore and continue since we can parse more + (_, State::Error(b, _)) => self.buf = Some(b.restore(m)), + (_, State::Incomplete(b, _)) => self.buf = Some(b.restore(m)), } } on { @@ -556,14 +604,11 @@ impl BoundedRange for RangeTo { } => result : T { - // Got all occurrences of the parser - (s, 0, EndStateTill::EndSuccess) => s.ret(result), + // Got all occurrences of the parser since we have no minimum bound + (s, _, EndStateTill::EndSuccess) => s.ret(result), // Did not reach minimum or a failure, propagate (s, _, EndStateTill::Error(e)) => s.err(e), - (s, _, EndStateTill::Incomplete(n)) => s.incomplete(n), - // We cannot reach this since we only run the end test once we have reached the - // minimum number of matches - (_, _, EndStateTill::EndSuccess) => unreachable!() + (s, _, EndStateTill::Incomplete(n)) => s.incomplete(n) } } } @@ -647,6 +692,7 @@ impl BoundedRange for usize { #[inline] fn many_till(self, i: I, p: R, end: F) -> ParseResult where T: FromIterator, + E: From, R: FnMut(I) -> ParseResult, F: FnMut(I) -> ParseResult { run_iter_till!{ @@ -677,7 +723,7 @@ impl BoundedRange for usize { // Got all occurrences of the parser (s, 0, EndStateTill::EndSuccess) => s.ret(result), // Did not reach minimum or a failure, propagate - (s, _, EndStateTill::Error(e)) => s.err(e), + (s, _, EndStateTill::Error(e)) => s.err(e), (s, _, EndStateTill::Incomplete(n)) => s.incomplete(n), // We cannot reach this since we only run the end test once we have reached the // minimum number of matches @@ -753,6 +799,7 @@ pub fn skip_many<'a, I, T, E, F, R>(i: Input<'a, I>, r: R, f: F) -> ParseResult< pub fn many_till(i: I, r: R, p: P, end: F) -> ParseResult where R: BoundedRange, T: FromIterator, + E: From, P: FnMut(I) -> ParseResult, F: FnMut(I) -> ParseResult { BoundedRange::many_till(r, i, p, end) @@ -1078,11 +1125,202 @@ mod test { assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![&b"ab"[..], &b"ab"[..]])); } - // TODO: Write tests for many_till #[test] fn many_till_range_full() { + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b""), .., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), .., |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), .., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ababac"), .., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..], &b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ababab"), .., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abababa"), .., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); + + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b""), .., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 2)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), .., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abac"), .., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ababac"), .., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..], &b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ababab"), .., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 2)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abababa"), .., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b"a"), 1)); + } + + #[test] + fn many_till_range_from() { + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b""), 0.., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), 0.., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), 1.., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"c"), Error::expected(b'b'))); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), 0.., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), 1.., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), 2.., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"c"), Error::expected(b'b'))); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ababac"), 2.., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..], &b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ababab"), 2.., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abababa"), 2.., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); + + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b""), 0.., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 2)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), 0.., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), 1.., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"c"), Error::expected(b'b'))); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abac"), 0.., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abac"), 1.., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abac"), 2.., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"c"), Error::expected(b'b'))); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ababac"), 2.., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..], &b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ababab"), 2.., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 2)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abababa"), 2.., |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b"a"), 1)); + } + + #[test] + fn many_till_range_to() { + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b""), ..0, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"a"), ..0, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), ..0, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b""), ..1, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), ..1, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), ..2, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b""), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ababac"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..], &b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abababac"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"bac"), Error::expected(b'c'))); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ababab"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), Error::expected(b'c'))); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ababa"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); + + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b""), ..0, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 2)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"a"), ..0, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b"a"), 1)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), ..0, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b""), ..1, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 2)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), ..1, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abac"), ..2, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abac"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ababac"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..], &b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abababac"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"bac"), Error::expected(b'c'))); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ababa"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b"a"), 1)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abababa"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"ba"), Error::expected(b'c'))); + } + + #[test] + fn many_till_range() { + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b""), 0..0, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"a"), 0..0, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), 0..0, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b""), 0..1, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), 0..1, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), 0..2, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), 0..2, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b""), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ababac"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..], &b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abababac"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"bac"), Error::expected(b'c'))); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ababab"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), Error::expected(b'c'))); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ababa"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); + + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), 1..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"c"), Error::expected(b'b'))); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), 1..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), 2..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"c"), Error::expected(b'b'))); + + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b""), 0..0, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 2)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"a"), 0..0, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b"a"), 1)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), 0..0, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b""), 0..1, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 2)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), 0..1, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), 0..2, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abac"), 0..2, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abac"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ababac"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..], &b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abababac"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"bac"), Error::expected(b'c'))); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ababa"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b"a"), 1)); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abababa"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"ba"), Error::expected(b'c'))); + + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), 1..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"c"), Error::expected(b'b'))); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abac"), 1..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), 2..3, |i| string(i, b"ab"), |i| string(i, b"ac")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"c"), Error::expected(b'b'))); } // FIXME: Uncomment diff --git a/src/combinators/macros.rs b/src/combinators/macros.rs index c4f6cf48..b6f4bfce 100644 --- a/src/combinators/macros.rs +++ b/src/combinators/macros.rs @@ -140,7 +140,8 @@ macro_rules! run_iter_till { /// Iterator used by ``many_till`` and ``many1``. struct IterTill - where P: FnMut(I) -> ParseResult, + where E: From, + P: FnMut(I) -> ParseResult, F: FnMut(I) -> ParseResult { state: EndStateTill, parser: P, @@ -151,7 +152,8 @@ macro_rules! run_iter_till { } impl IterTill - where P: FnMut(I) -> ParseResult, + where E: From, + P: FnMut(I) -> ParseResult, F: FnMut(I) -> ParseResult { /// Destructures the iterator returning the position just after the last successful parse as /// well as the state of the last attempt to parse data. @@ -163,7 +165,8 @@ macro_rules! run_iter_till { } impl Iterator for IterTill - where P: FnMut(I) -> ParseResult, + where E: From, + P: FnMut(I) -> ParseResult, F: FnMut(I) -> ParseResult { type Item = T; diff --git a/src/combinators/mod.rs b/src/combinators/mod.rs index 773fa99a..fe6974af 100644 --- a/src/combinators/mod.rs +++ b/src/combinators/mod.rs @@ -232,6 +232,7 @@ pub fn sep_by1(i: I, p: R, sep: F) -> ParseResult #[inline] pub fn many_till(i: I, p: R, end: F) -> ParseResult where T: FromIterator, + E: From, R: FnMut(I) -> ParseResult, F: FnMut(I) -> ParseResult { bounded::many_till(i, .., p, end) @@ -339,7 +340,7 @@ pub fn look_ahead(i: I, f: F) -> ParseResult #[cfg(test)] mod test { - use {Input, ParseResult}; + use {Input, Error, ParseResult}; use primitives::State; use primitives::input::{new_buf, DEFAULT, END_OF_INPUT}; use primitives::IntoInner; @@ -482,13 +483,13 @@ mod test { let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abd"), any, |i| token(i, b'c')); assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: ParseResult<_, Vec, _> = many_till(new_buf(DEFAULT, b"abcd"), |i| i.err(()), |i| token(i, b'c')); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"abcd"), ())); + let r: ParseResult<_, Vec, _> = many_till(new_buf(DEFAULT, b"abcd"), |i| i.err(Error::expected(b'@')), |i| token(i, b'c')); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"abcd"), Error::expected(b'@'))); // Variant to make sure error slice is propagated let mut n = 0; - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abcd"), |i| if n == 0 { n += 1; any(i).map_err(|_| "any err") } else { i.err("the error") }, |i| token(i, b'c')); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"bcd"), "the error")); + let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abcd"), |i| if n == 0 { n += 1; any(i).map_err(|_| Error::expected(b'i')) } else { i.err(Error::expected(b'@')) }, |i| token(i, b'c')); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"bcd"), Error::expected(b'@'))); } #[test] From 001e14235d4fb7350141db7308494b986417ac5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Fri, 18 Mar 2016 21:19:02 +0100 Subject: [PATCH 10/63] Updated combinators::bounded::skip_many --- src/combinators/bounded.rs | 208 +++++++++++++++++++------------------ src/combinators/mod.rs | 10 +- 2 files changed, 108 insertions(+), 110 deletions(-) diff --git a/src/combinators/bounded.rs b/src/combinators/bounded.rs index b30f34ba..a25d1ad1 100644 --- a/src/combinators/bounded.rs +++ b/src/combinators/bounded.rs @@ -53,8 +53,6 @@ pub trait BoundedRange { where F: FnMut(I) -> ParseResult, T: FromIterator; - // FIXME: Uncomment - /* /// Applies the parser `F` multiple times until it fails or the maximum value of the range has /// been reached, throwing away any produced value. /// @@ -71,10 +69,8 @@ pub trait BoundedRange { /// * If the last parser succeeds on the last input item then this parser is still considered /// incomplete if the input flag END_OF_INPUT is not set as there might be more data to fill. #[inline] - fn skip_many<'a, I, T, E, F>(self, Input<'a, I>, F) -> ParseResult<'a, I, (), E> - where T: 'a, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, T, E>; - */ + fn skip_many(self, I, F) -> ParseResult + where F: FnMut(I) -> ParseResult; // TODO: Fix documentation regarding incomplete /// Applies the parser `P` multiple times until the parser `F` succeeds and returns a value @@ -152,12 +148,9 @@ impl BoundedRange for Range { } } - // FIXME: Uncomment - /* #[inline] - fn skip_many<'a, I, T, E, F>(self, mut i: Input<'a, I>, mut f: F) -> ParseResult<'a, I, (), E> - where T: 'a, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, T, E> { + fn skip_many(self, mut i: I, mut f: F) -> ParseResult + where F: FnMut(I) -> ParseResult { // Range does not perform this assertion assert!(self.start <= self.end); @@ -169,7 +162,9 @@ impl BoundedRange for Range { break; } - match f(i.clone()).into_inner() { + let m = i.mark(); + + match f(i).into_inner() { State::Data(b, _) => { min = if min == 0 { 0 } else { min - 1 }; max -= 1; @@ -177,23 +172,26 @@ impl BoundedRange for Range { i = b }, State::Error(b, e) => if min == 0 { + i = b.restore(m); + break; } else { - // Not enough iterations - return i.replace(b).err(e); + // Not enough iterations, propagate + return b.err(e); }, - State::Incomplete(n) => if min == 0 && i.is_end() { + State::Incomplete(b, n) => if min == 0 && b.is_end() { + i = b.restore(m); + break; } else { // We have not done the minimum amount of iterations - return i.incomplete(n); + return b.incomplete(n); } } } i.ret(()) } - */ #[inline] fn many_till(self, i: I, p: R, end: F) -> ParseResult @@ -309,40 +307,42 @@ impl BoundedRange for RangeFrom { } } - // FIXME: Uncomment - /* #[inline] - fn skip_many<'a, I, T, E, F>(self, mut i: Input<'a, I>, mut f: F) -> ParseResult<'a, I, (), E> - where T: 'a, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, T, E> { + fn skip_many(self, mut i: I, mut f: F) -> ParseResult + where F: FnMut(I) -> ParseResult { // Closed on left side, open on right let mut min = self.start; loop { - match f(i.clone()).into_inner() { + let m = i.mark(); + + match f(i).into_inner() { State::Data(b, _) => { min = if min == 0 { 0 } else { min - 1 }; i = b }, State::Error(b, e) => if min == 0 { + i = b.restore(m); + break; } else { - // Not enough iterations - return i.replace(b).err(e); + // Not enough iterations, propagate + return b.err(e); }, - State::Incomplete(n) => if min == 0 && i.is_end() { + State::Incomplete(b, n) => if min == 0 && b.is_end() { + i = b.restore(m); + break; } else { // We have not done the minimum amount of iterations - return i.incomplete(n); + return b.incomplete(n); } } } i.ret(()) } - */ #[inline] fn many_till(self, i: I, p: R, end: F) -> ParseResult @@ -418,26 +418,31 @@ impl BoundedRange for RangeFull { } } - /* #[inline] - fn skip_many<'a, I, T, E, F>(self, mut i: Input<'a, I>, mut f: F) -> ParseResult<'a, I, (), E> - where T: 'a, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, T, E> { + fn skip_many(self, mut i: I, mut f: F) -> ParseResult + where F: FnMut(I) -> ParseResult { loop { - match f(i.clone()).into_inner() { + let m = i.mark(); + + match f(i).into_inner() { State::Data(b, _) => i = b, - State::Error(_, _) => break, - State::Incomplete(n) => if i.is_end() { + State::Error(b, _) => { + i = b.restore(m); + + break; + }, + State::Incomplete(b, n) => if b.is_end() { + i = b.restore(m); + break; } else { - return i.incomplete(n); + return b.incomplete(n); } } } i.ret(()) } - */ #[inline] fn many_till(self, i: I, p: R, end: F) -> ParseResult @@ -515,12 +520,9 @@ impl BoundedRange for RangeTo { } } - // FIXME: Uncomment - /* #[inline] - fn skip_many<'a, I, T, E, F>(self, mut i: Input<'a, I>, mut f: F) -> ParseResult<'a, I, (), E> - where T: 'a, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, T, E> { + fn skip_many(self, mut i: I, mut f: F) -> ParseResult + where F: FnMut(I) -> ParseResult { // [0, n) let mut max = max(self.end, 1) - 1; @@ -529,25 +531,32 @@ impl BoundedRange for RangeTo { break; } - match f(i.clone()).into_inner() { + let m = i.mark(); + + match f(i).into_inner() { State::Data(b, _) => { max -= 1; i = b }, // Always ok to end iteration - State::Error(_, _) => break, - State::Incomplete(n) => if i.is_end() { + State::Error(b, _) => { + i = b.restore(m); + + break; + }, + State::Incomplete(b, n) => if b.is_end() { + i = b.restore(m); + break; } else { - return i.incomplete(n); + return b.incomplete(n); } } } i.ret(()) } - */ #[inline] fn many_till(self, i: I, p: R, end: F) -> ParseResult @@ -652,12 +661,9 @@ impl BoundedRange for usize { } } - // FIXME: Uncomment - /* #[inline] - fn skip_many<'a, I, T, E, F>(self, mut i: Input<'a, I>, mut f: F) -> ParseResult<'a, I, (), E> - where T: 'a, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, T, E> { + fn skip_many(self, mut i: I, mut f: F) -> ParseResult + where F: FnMut(I) -> ParseResult { let mut n = self; loop { @@ -665,29 +671,34 @@ impl BoundedRange for usize { break; } - match f(i.clone()).into_inner() { + let m = i.mark(); + + match f(i).into_inner() { State::Data(b, _) => { n -= 1; i = b }, State::Error(b, e) => if n == 0 { + i = b.restore(m); + break; } else { - // Not enough iterations - return i.replace(b).err(e); + // Not enough iterations, propagate + return b.err(e); }, - State::Incomplete(n) => if n == 0 { + State::Incomplete(b, n) => if n == 0 { + i = b.restore(m); + break; } else { - return i.incomplete(n); + return b.incomplete(n); } } } i.ret(()) } - */ #[inline] fn many_till(self, i: I, p: R, end: F) -> ParseResult @@ -756,7 +767,6 @@ pub fn many(i: I, r: R, f: F) -> ParseResult BoundedRange::parse_many(r, i, f) } -/* /// Applies the parser `F` multiple times until it fails or the maximum value of the range has /// been reached, throwing away any produced value. /// @@ -772,13 +782,11 @@ pub fn many(i: I, r: R, f: F) -> ParseResult /// * If the last parser succeeds on the last input item then this parser is still considered /// incomplete if the input flag END_OF_INPUT is not set as there might be more data to fill. #[inline] -pub fn skip_many<'a, I, T, E, F, R>(i: Input<'a, I>, r: R, f: F) -> ParseResult<'a, I, (), E> - where T: 'a, - R: BoundedRange, - F: FnMut(Input<'a, I>) -> ParseResult<'a, I, T, E> { +pub fn skip_many(i: I, r: R, f: F) -> ParseResult + where R: BoundedRange, + F: FnMut(I) -> ParseResult { BoundedRange::skip_many(r, i, f) } -*/ // TODO: Update documentation regarding incomplete behaviour /// Applies the parser `P` multiple times until the parser `F` succeeds and returns a value @@ -851,7 +859,7 @@ mod test { use super::{ many, many_till, - //skip_many, + skip_many, }; #[test] @@ -1323,16 +1331,14 @@ mod test { assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"c"), Error::expected(b'b'))); } - // FIXME: Uncomment - /* #[test] fn skip_range_full() { let r = skip_many(new_buf(DEFAULT, b""), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r = skip_many(new_buf(DEFAULT, b"a"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r = skip_many(new_buf(DEFAULT, b"aa"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r = skip_many(new_buf(DEFAULT, b"b"), .., |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); @@ -1377,9 +1383,9 @@ mod test { assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), ())); let r = skip_many(new_buf(DEFAULT, b""), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r = skip_many(new_buf(DEFAULT, b"a"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r = skip_many(new_buf(DEFAULT, b"aa"), ..3, |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), ())); let r = skip_many(new_buf(DEFAULT, b"aaa"), ..3, |i| token(i, b'a')); @@ -1412,34 +1418,34 @@ mod test { #[test] fn skip_range_from() { let r = skip_many(new_buf(DEFAULT, b""), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r = skip_many(new_buf(DEFAULT, b"a"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r = skip_many(new_buf(DEFAULT, b"aa"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r = skip_many(new_buf(DEFAULT, b"aaa"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r = skip_many(new_buf(DEFAULT, b"b"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); let r = skip_many(new_buf(DEFAULT, b"ab"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); let r = skip_many(new_buf(DEFAULT, b"aab"), 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); let r = skip_many(new_buf(END_OF_INPUT, b""), 2.., any); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); let r = skip_many(new_buf(END_OF_INPUT, b"a"), 2.., any); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); let r = skip_many(new_buf(END_OF_INPUT, b"aa"), 2.., any); assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); let r = skip_many(new_buf(END_OF_INPUT, b"aaa"), 2.., any); assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); let r = skip_many(new_buf(END_OF_INPUT, b"b"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); let r = skip_many(new_buf(END_OF_INPUT, b"ab"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); let r = skip_many(new_buf(END_OF_INPUT, b"aab"), 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); let r = skip_many(new_buf(END_OF_INPUT, b"aaab"), 2.., |i| token(i, b'a')); @@ -1467,20 +1473,20 @@ mod test { assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), ())); let r = skip_many(new_buf(DEFAULT, b""), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r = skip_many(new_buf(DEFAULT, b"a"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r = skip_many(new_buf(DEFAULT, b"aa"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r = skip_many(new_buf(DEFAULT, b"aaa"), 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), ())); let r = skip_many(new_buf(DEFAULT, b"aaaa"), 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), ())); let r = skip_many(new_buf(DEFAULT, b"b"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); let r = skip_many(new_buf(DEFAULT, b"ab"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); let r = skip_many(new_buf(DEFAULT, b"aab"), 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); let r = skip_many(new_buf(DEFAULT, b"aaab"), 2..4, |i| token(i, b'a')); @@ -1489,9 +1495,9 @@ mod test { assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ab"), ())); let r = skip_many(new_buf(END_OF_INPUT, b""), 2..4, any); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); let r = skip_many(new_buf(END_OF_INPUT, b"a"), 2..4, any); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); let r = skip_many(new_buf(END_OF_INPUT, b"aa"), 2..4, any); assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); let r = skip_many(new_buf(END_OF_INPUT, b"aaa"), 2..4, any); @@ -1500,9 +1506,9 @@ mod test { assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), ())); let r = skip_many(new_buf(END_OF_INPUT, b"b"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); let r = skip_many(new_buf(END_OF_INPUT, b"ab"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); let r = skip_many(new_buf(END_OF_INPUT, b"aab"), 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); let r = skip_many(new_buf(END_OF_INPUT, b"aaab"), 2..4, |i| token(i, b'a')); @@ -1514,42 +1520,41 @@ mod test { #[test] fn skip_exact() { let r = skip_many(new_buf(DEFAULT, b""), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r = skip_many(new_buf(DEFAULT, b"a"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); let r = skip_many(new_buf(DEFAULT, b"aa"), 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), ())); let r = skip_many(new_buf(DEFAULT, b"aaa"), 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), ())); let r = skip_many(new_buf(DEFAULT, b"b"), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); let r = skip_many(new_buf(DEFAULT, b"ab"), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); let r = skip_many(new_buf(DEFAULT, b"aab"), 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); let r = skip_many(new_buf(DEFAULT, b"aaab"), 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ab"), ())); let r = skip_many(new_buf(END_OF_INPUT, b""), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); let r = skip_many(new_buf(END_OF_INPUT, b"a"), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Incomplete(1)); + assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); let r = skip_many(new_buf(END_OF_INPUT, b"aa"), 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); let r = skip_many(new_buf(END_OF_INPUT, b"aaa"), 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), ())); let r = skip_many(new_buf(END_OF_INPUT, b"b"), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); let r = skip_many(new_buf(END_OF_INPUT, b"ab"), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(b"b", "token_err")); + assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); let r = skip_many(new_buf(END_OF_INPUT, b"aab"), 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); let r = skip_many(new_buf(END_OF_INPUT, b"aaab"), 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ab"), ())); } - */ #[test] #[should_panic] @@ -1558,8 +1563,6 @@ mod test { assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ab"), vec![b'a', b'a', b'a'])); } - // FIXME: Uncomment - /* #[test] #[should_panic] fn panic_skip_many_range_lt() { @@ -1572,5 +1575,4 @@ mod test { let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"aaaab"), 2..1, |i| token(i, b'a'), |i| token(i, b'b')); assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ab"), vec![b'a', b'a', b'a'])); } - */ } diff --git a/src/combinators/mod.rs b/src/combinators/mod.rs index fe6974af..f628b510 100644 --- a/src/combinators/mod.rs +++ b/src/combinators/mod.rs @@ -255,9 +255,7 @@ pub fn many_till(i: I, p: R, end: F) -> ParseResu #[inline] pub fn skip_many(i: I, f: F) -> ParseResult where F: FnMut(I) -> ParseResult { - //bounded::skip_many(i, .., f) - //FIXME: Implement - unimplemented!() + bounded::skip_many(i, .., f) } /// Runs the given parser until it fails, discarding matched input, expects at least one match. @@ -281,9 +279,7 @@ pub fn skip_many(i: I, f: F) -> ParseResult #[inline] pub fn skip_many1(i: I, f: F) -> ParseResult where F: FnMut(I) -> ParseResult { - // FIXME: Implement - //bounded::skip_many(i, 1.., f) - unimplemented!() + bounded::skip_many(i, 1.., f) } /// Returns the result of the given parser as well as the slice which matched it. @@ -470,7 +466,7 @@ mod test { assert_eq!(skip_many1(new_buf(DEFAULT, b"aabc"), |i| token(i, b'a')).into_inner(), State::Data(new_buf(DEFAULT, b"bc"), ())); assert_eq!(skip_many1(new_buf(DEFAULT, b"abc"), |i| token(i, b'a')).into_inner(), State::Data(new_buf(DEFAULT, b"bc"), ())); assert_eq!(skip_many1(new_buf(DEFAULT, b"bc"), |i| i.err::<(), _>("error")).into_inner(), State::Error(new_buf(DEFAULT, b"bc"), "error")); - assert_eq!(skip_many1(new_buf(DEFAULT, b"aaa"), |i| token(i, b'a')).into_inner(), State::Incomplete(new_buf(DEFAULT, b"aaa"), 1)); + assert_eq!(skip_many1(new_buf(DEFAULT, b"aaa"), |i| token(i, b'a')).into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); assert_eq!(skip_many1(new_buf(END_OF_INPUT, b"aabc"), |i| token(i, b'a')).into_inner(), State::Data(new_buf(END_OF_INPUT, b"bc"), ())); assert_eq!(skip_many1(new_buf(END_OF_INPUT, b"abc"), |i| token(i, b'a')).into_inner(), State::Data(new_buf(END_OF_INPUT, b"bc"), ())); assert_eq!(skip_many1(new_buf(END_OF_INPUT, b"bc"), |i| i.err::<(), _>("error")).into_inner(), State::Error(new_buf(END_OF_INPUT, b"bc"), "error")); From a8711c448b1a59787feb84ce89b872a323baefef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Fri, 18 Mar 2016 21:36:48 +0100 Subject: [PATCH 11/63] Updated combinators::matched_by --- src/combinators/mod.rs | 18 +++++--------- src/input.rs | 56 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 12 deletions(-) diff --git a/src/combinators/mod.rs b/src/combinators/mod.rs index f628b510..f60cfa65 100644 --- a/src/combinators/mod.rs +++ b/src/combinators/mod.rs @@ -293,23 +293,17 @@ pub fn skip_many1(i: I, f: F) -> ParseResult #[inline] pub fn matched_by(i: I, f: F) -> ParseResult where F: FnOnce(I) -> ParseResult { - // TODO: Implement - unimplemented!() - /* - let buf = i.buffer(); + let m = i.mark(); - match f(i.clone()).into_inner() { - State::Data(b, t) => { - // b is remainder, find out how much the parser used - let diff = buf.len() - b.buffer().len(); - let n = &buf[..diff]; + match f(i).into_inner() { + State::Data(mut b, t) => { + let n = b.consume_from(m); b.ret((n, t)) }, - State::Error(b, e) => i.replace(b).err(e), - State::Incomplete(n) => i.incomplete(n), + State::Error(b, e) => b.err(e), + State::Incomplete(b, n) => b.incomplete(n), } - */ } /// Applies the parser `F` without consuming any input. diff --git a/src/input.rs b/src/input.rs index b06c9944..8818d6d8 100644 --- a/src/input.rs +++ b/src/input.rs @@ -42,23 +42,34 @@ pub mod primitives { fn first(&self) -> Option { self._first(Guard(())) } + #[inline(always)] fn iter(&self) -> Self::Iter { self._iter(Guard(())) } + #[inline(always)] fn consume(&mut self, n: usize) -> Self::Buffer { self._consume(Guard(()), n) } + #[inline(always)] fn discard(&mut self, n: usize) { self._discard(Guard(()), n) } + /// Returns the buffer from the marker `m` to the current position, discarding the + /// backtracking position carried by `m`. + #[inline(always)] + fn consume_from(&mut self, m: Self::Marker) -> Self::Buffer { + self._consume_from(Guard(()), m) + } + #[inline] fn min_remaining(&self) -> usize { self._min_remaining(Guard(())) } + /// Returns true if this is the last available slice of the input. /// /// # Primitive @@ -68,6 +79,8 @@ pub mod primitives { fn is_end(&self) -> bool { self._is_end(Guard(())) } + + /// Marks the current position to be able to backtrack to it using `restore`. /// /// # Example /// @@ -98,6 +111,7 @@ pub mod primitives { fn mark(&self) -> Self::Marker { self._mark(Guard(())) } + #[inline(always)] fn restore(self, m: Self::Marker) -> Self { self._restore(Guard(()), m) @@ -214,33 +228,46 @@ pub trait Input: Sized { /// Iterator over tokens in the input, does not consume any data. #[inline] fn _iter(&self, Guard) -> Self::Iter; + /// **Primitive:** See `Primitives::consume` for documentation. /// /// Consumes a set amount of input tokens, returning a buffer containing them // TODO: Should probably be combined with a ret #[inline] fn _consume(&mut self, Guard, usize) -> Self::Buffer; + /// **Primitive:** See `Primitives::discard` for documentation. /// /// Consumes a set amount of input tokens, discarding them. #[inline] fn _discard(&mut self, Guard, usize); + + /// **Primitive:** See `Primitives::consume_from for documentation. + /// + /// Returns the buffer from the marker to the current position, discarding the + /// backtracking position carried by the marker. + #[inline] + fn _consume_from(&mut self, Guard, Self::Marker) -> Self::Buffer; + /// **Primitive:** See `Primitives::remaining` for documentation. /// /// Returns the number of tokens remaining in this input (only this part of it, more might /// follow if `_is_end` is false). #[inline] fn _min_remaining(&self, Guard) -> usize; + /// **Primitive:** See `Primitives::is_end` for documentation. /// /// Returns true if there are no more input in `Self` besides what has already been populated. #[inline] fn _is_end(&self, Guard) -> bool; + /// **Primitive:** See `Primitives::mark` for documentation. /// /// Marks a position for backtracking along with relevant state. #[inline] fn _mark(&self, Guard) -> Self::Marker; + /// **Primitive:** See `Primitives::restore` for documentation. /// /// Resumes from a previously marked state. @@ -268,6 +295,10 @@ impl<'a, I: Copy> Input for &'a [I] { fn _discard(&mut self, _g: Guard, n: usize) { *self = &self[n..]; } + #[inline] + fn _consume_from(&mut self, _g: Guard, m: Self::Marker) -> Self::Buffer { + &m[..m.len() - self.len()] + } #[inline(always)] fn _min_remaining(&self, _g: Guard) -> usize { self.len() @@ -328,6 +359,10 @@ impl<'a, I: Copy> Input for InputBuf<'a, I> { fn _discard(&mut self, _g: Guard, n: usize) { self.1 = &self.1[n..]; } + #[inline] + fn _consume_from(&mut self, _g: Guard, m: Self::Marker) -> Self::Buffer { + &m[..m.len() - self.1.len()] + } #[inline(always)] fn _min_remaining(&self, _g: Guard) -> usize { self.1.len() @@ -505,17 +540,38 @@ mod test { #[test] fn primitives_slice() { + use primitives::Primitives; run_primitives_test(&b"abc"[..], true); + + let mut s = &b"abc"[..]; + let m = s.mark(); + s.discard(2); + assert_eq!(s.consume_from(m), &b"ab"[..]); + assert_eq!(s, &b"c"[..]); } #[test] fn primitives_input_buf_default() { + use primitives::Primitives; run_primitives_test(new_buf(DEFAULT, b"abc"), false); + + let mut s = new_buf(DEFAULT, b"abc"); + let m = s.mark(); + s.discard(2); + assert_eq!(s.consume_from(m), &b"ab"[..]); + assert_eq!(s, new_buf(DEFAULT, b"c")); } #[test] fn primitives_input_buf_end() { + use primitives::Primitives; run_primitives_test(new_buf(END_OF_INPUT, b"abc"), true); + + let mut s = new_buf(END_OF_INPUT, b"abc"); + let m = s.mark(); + s.discard(2); + assert_eq!(s.consume_from(m), &b"ab"[..]); + assert_eq!(s, new_buf(END_OF_INPUT, b"c")); } fn run_primitives_test PartialEq<&'a [u8]>, I: Input>(mut s: I, last: bool) { From 2e05b8ff774258759b494431b63ef3b812982509 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Fri, 18 Mar 2016 21:58:54 +0100 Subject: [PATCH 12/63] Updated benchmarks --- benches/combinators.rs | 65 +++++++++++++++++++++++++++++++++--------- benches/http_bench.rs | 37 ++++++++++++------------ src/lib.rs | 16 +++++------ 3 files changed, 79 insertions(+), 39 deletions(-) diff --git a/benches/combinators.rs b/benches/combinators.rs index cbb4a832..2244cc91 100644 --- a/benches/combinators.rs +++ b/benches/combinators.rs @@ -8,18 +8,18 @@ use test::Bencher; use std::iter; use chomp::*; -use chomp::buffer::{Stream, IntoStream}; +use chomp::primitives::input::{new_buf, DEFAULT}; #[bench] fn count_vec_1k(b: &mut Bencher) { let data = iter::repeat(b'a').take(1024).collect::>(); - fn count_vec(i: Input) -> ParseResult, Error> { + fn count_vec(i: I) -> ParseResult, Error> { count(i, 1024, any) } b.iter(|| { - data.into_stream().parse(count_vec) + parse_only(count_vec, &data) }) } @@ -27,12 +27,12 @@ fn count_vec_1k(b: &mut Bencher) { fn count_vec_10k(b: &mut Bencher) { let data = iter::repeat(b'a').take(10024).collect::>(); - fn count_vec(i: Input) -> ParseResult, Error> { + fn count_vec(i: I) -> ParseResult, Error> { count(i, 10024, any) } b.iter(|| { - data.into_stream().parse(count_vec) + parse_only(count_vec, &data) }) } @@ -40,12 +40,12 @@ fn count_vec_10k(b: &mut Bencher) { fn many_vec_1k(b: &mut Bencher) { let data = iter::repeat(b'a').take(1024).collect::>(); - fn many_vec(i: Input) -> ParseResult, Error> { + fn many_vec(i: I) -> ParseResult, Error> { many(i, any) } b.iter(|| { - data.into_stream().parse(many_vec) + parse_only(many_vec, &data) }) } @@ -53,12 +53,12 @@ fn many_vec_1k(b: &mut Bencher) { fn many_vec_10k(b: &mut Bencher) { let data = iter::repeat(b'a').take(10024).collect::>(); - fn many_vec(i: Input) -> ParseResult, Error> { + fn many_vec(i: I) -> ParseResult, Error> { many(i, any) } b.iter(|| { - data.into_stream().parse(many_vec) + parse_only(many_vec, &data) }) } @@ -66,12 +66,12 @@ fn many_vec_10k(b: &mut Bencher) { fn many1_vec_1k(b: &mut Bencher) { let data = iter::repeat(b'a').take(1024).collect::>(); - fn many1_vec(i: Input) -> ParseResult, Error> { + fn many1_vec(i: I) -> ParseResult, Error> { many1(i, any) } b.iter(|| { - data.into_stream().parse(many1_vec) + parse_only(many1_vec, &data) }) } @@ -79,11 +79,50 @@ fn many1_vec_1k(b: &mut Bencher) { fn many1_vec_10k(b: &mut Bencher) { let data = iter::repeat(b'a').take(10024).collect::>(); - fn many1_vec(i: Input) -> ParseResult, Error> { + fn many1_vec(i: I) -> ParseResult, Error> { many1(i, any) } b.iter(|| { - data.into_stream().parse(many1_vec) + parse_only(many1_vec, &data) + }) +} + +#[bench] +fn count_vec_10k_maybe_incomplete(b: &mut Bencher) { + let data = iter::repeat(b'a').take(10024).collect::>(); + + fn count_vec(i: I) -> ParseResult, Error> { + count(i, 10024, any) + } + + b.iter(|| { + count_vec(new_buf(DEFAULT, &data)) + }) +} + +#[bench] +fn many_vec_10k_maybe_incomplete(b: &mut Bencher) { + let data = iter::repeat(b'a').take(10024).collect::>(); + + fn many_vec(i: I) -> ParseResult, Error> { + many(i, any) + } + + b.iter(|| { + many_vec(new_buf(DEFAULT, &data)) + }) +} + +#[bench] +fn many1_vec_10k_maybe_incomplete(b: &mut Bencher) { + let data = iter::repeat(b'a').take(10024).collect::>(); + + fn many1_vec(i: I) -> ParseResult, Error> { + many1(i, any) + } + + b.iter(|| { + many1_vec(new_buf(DEFAULT, &data)) }) } diff --git a/benches/http_bench.rs b/benches/http_bench.rs index 4c0c408d..d9db76e3 100644 --- a/benches/http_bench.rs +++ b/benches/http_bench.rs @@ -5,19 +5,18 @@ extern crate chomp; use test::Bencher; use chomp::*; -use chomp::buffer::{Stream, IntoStream}; #[derive(Debug)] -struct Request<'a> { - method: &'a [u8], - uri: &'a [u8], - version: &'a [u8], +struct Request { + method: B, + uri: B, + version: B, } #[derive(Debug)] -struct Header<'a> { - name: &'a [u8], - value: Vec<&'a [u8]>, +struct Header { + name: B, + value: Vec, } fn is_token(c: u8) -> bool { @@ -52,7 +51,7 @@ fn is_not_space(c: u8) -> bool { c != b' ' } fn is_end_of_line(c: u8) -> bool { c == b'\r' || c == b'\n' } fn is_http_version(c: u8) -> bool { c >= b'0' && c <= b'9' || c == b'.' } -fn end_of_line(i: Input) -> U8Result { +fn end_of_line>(i: I) -> SimpleResult { or(i, |i| parse!{i; token(b'\r'); token(b'\n'); @@ -60,14 +59,14 @@ fn end_of_line(i: Input) -> U8Result { |i| token(i, b'\n')) } -fn http_version(i: Input) -> U8Result<&[u8]> { +fn http_version>(i: I) -> SimpleResult { parse!{i; string(b"HTTP/"); take_while1(is_http_version) } } -fn request_line(i: Input) -> U8Result { +fn request_line>(i: I) -> SimpleResult> { parse!{i; let method = take_while1(is_token); take_while1(is_space); @@ -83,7 +82,7 @@ fn request_line(i: Input) -> U8Result { } } -fn message_header_line(i: Input) -> U8Result<&[u8]> { +fn message_header_line>(i: I) -> SimpleResult { parse!{i; take_while1(is_horizontal_space); let line = take_till(is_end_of_line); @@ -93,7 +92,7 @@ fn message_header_line(i: Input) -> U8Result<&[u8]> { } } -fn message_header(i: Input) -> U8Result
{ +fn message_header>(i: I) -> SimpleResult> { parse!{i; let name = take_while1(is_token); token(b':'); @@ -106,7 +105,7 @@ fn message_header(i: Input) -> U8Result
{ } } -fn request(i: Input) -> U8Result<(Request, Vec
)> { +fn request>(i: I) -> SimpleResult, Vec>)> { parse!{i; let r = request_line(); end_of_line(); @@ -130,7 +129,7 @@ Connection: keep-alive\r \r"; b.iter(|| { - data.into_stream().parse(request) + parse_only(request, data) }) } @@ -142,7 +141,7 @@ Host: localhost\r \r"; b.iter(|| { - data.into_stream().parse(request) + parse_only(request, data) }) } @@ -161,7 +160,7 @@ Cookie: azk=ue1-5eb08aeed9a7401c9195cb933eb7c966\r \r"; b.iter(|| { - data.into_stream().parse(request) + parse_only(request, data) }) } @@ -170,6 +169,8 @@ fn multiple_requests(b: &mut Bencher) { let data = include_bytes!("./data/http-requests.txt"); b.iter(|| { - data.into_stream().parse::<_, Vec<_>, _>(parser!{many(request)}) + let r: Result, _> = parse_only(parser!{many(request)}, data); + + r }) } diff --git a/src/lib.rs b/src/lib.rs index b174a003..ed1a5d41 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -239,16 +239,16 @@ pub use parsers::Error; //pub mod buffer; pub use combinators::{ - //count, + count, option, or, - //many, - //many1, - //sep_by, - //sep_by1, - //many_till, - //skip_many, - //skip_many1, + many, + many1, + sep_by, + sep_by1, + many_till, + skip_many, + skip_many1, matched_by, }; pub use parse::{ From 0ceda8d569d4b44e65a79160a8c51b0b89687dac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Fri, 18 Mar 2016 22:34:28 +0100 Subject: [PATCH 13/63] Updated buffers --- examples/http_parser.rs | 26 +++++++++++++------------- src/buffer/buffer.rs | 2 +- src/buffer/data_source.rs | 5 +++-- src/buffer/mod.rs | 20 +++++++++++++------- src/buffer/slice.rs | 26 +++++++++++++++----------- src/buffer/stateful.rs | 27 ++++++++++++++++----------- src/input.rs | 2 ++ src/lib.rs | 10 ++++++---- 8 files changed, 69 insertions(+), 49 deletions(-) diff --git a/examples/http_parser.rs b/examples/http_parser.rs index 78b71e96..254f14aa 100644 --- a/examples/http_parser.rs +++ b/examples/http_parser.rs @@ -20,16 +20,16 @@ use chomp::*; use chomp::buffer::{Source, Stream, StreamError}; #[derive(Debug)] -struct Request<'a> { - method: &'a [u8], - uri: &'a [u8], - version: &'a [u8], +struct Request { + method: B, + uri: B, + version: B, } #[derive(Debug)] -struct Header<'a> { - name: &'a [u8], - value: Vec<&'a [u8]>, +struct Header { + name: B, + value: Vec, } fn is_token(c: u8) -> bool { @@ -64,7 +64,7 @@ fn is_not_space(c: u8) -> bool { c != b' ' } fn is_end_of_line(c: u8) -> bool { c == b'\r' || c == b'\n' } fn is_http_version(c: u8) -> bool { c >= b'0' && c <= b'9' || c == b'.' } -fn end_of_line(i: Input) -> U8Result { +fn end_of_line>(i: I) -> SimpleResult { or(i, |i| parse!{i; token(b'\r'); token(b'\n'); @@ -72,14 +72,14 @@ fn end_of_line(i: Input) -> U8Result { |i| token(i, b'\n')) } -fn http_version(i: Input) -> U8Result<&[u8]> { +fn http_version>(i: I) -> SimpleResult { parse!{i; string(b"HTTP/"); take_while1(is_http_version) } } -fn request_line(i: Input) -> U8Result { +fn request_line>(i: I) -> SimpleResult> { parse!{i; let method = take_while1(is_token); take_while1(is_space); @@ -95,7 +95,7 @@ fn request_line(i: Input) -> U8Result { } } -fn message_header_line(i: Input) -> U8Result<&[u8]> { +fn message_header_line>(i: I) -> SimpleResult { parse!{i; take_while1(is_horizontal_space); let line = take_till(is_end_of_line); @@ -105,7 +105,7 @@ fn message_header_line(i: Input) -> U8Result<&[u8]> { } } -fn message_header(i: Input) -> U8Result
{ +fn message_header>(i: I) -> SimpleResult> { parse!{i; let name = take_while1(is_token); token(b':'); @@ -118,7 +118,7 @@ fn message_header(i: Input) -> U8Result
{ } } -fn request(i: Input) -> U8Result<(Request, Vec
)> { +fn request>(i: I) -> SimpleResult, Vec>)> { parse!{i; let r = request_line(); end_of_line(); diff --git a/src/buffer/buffer.rs b/src/buffer/buffer.rs index 05ce1129..43b0e19c 100644 --- a/src/buffer/buffer.rs +++ b/src/buffer/buffer.rs @@ -12,7 +12,7 @@ const DEFAULT_BUFFER_SIZE: usize = 6 * 1024; /// /// Enables the consumer to request specific amounts of data and only consume partial parts of the /// buffer. -pub trait Buffer: ops::Deref { +pub trait Buffer: ops::Deref { /// Attempt to fill the buffer using the closure `F`. /// /// The successful return from `F` should contain the number of items successfully written to diff --git a/src/buffer/data_source.rs b/src/buffer/data_source.rs index 49afcda5..9eeb5546 100644 --- a/src/buffer/data_source.rs +++ b/src/buffer/data_source.rs @@ -5,7 +5,7 @@ use std::io; /// Abstraction over `io::Read`, `Iterator` and others. pub trait DataSource { /// The type of items this data source produces. - type Item; + type Item: Copy; /// Populates the supplied buffer with data, returns the number of items written. /// @@ -68,7 +68,8 @@ impl IteratorDataSource { } } -impl DataSource for IteratorDataSource { +impl DataSource for IteratorDataSource + where I::Item: Copy { type Item = I::Item; #[inline] diff --git a/src/buffer/mod.rs b/src/buffer/mod.rs index df933bd6..7c579ba2 100644 --- a/src/buffer/mod.rs +++ b/src/buffer/mod.rs @@ -35,7 +35,7 @@ pub mod data_source; use std::io; -use {ParseResult, Input}; +use {ParseResult, Input, InputBuf}; use parse::ParseError; pub use self::slice::SliceStream; @@ -79,11 +79,17 @@ impl<'a, I, E> PartialEq for StreamError<'a, I, E> } } -impl<'a, I, E> From> for StreamError<'a, I, E> +impl<'a, T, I: Input, E> From> for StreamError<'a, T, E> where I: 'a { - fn from(e: ParseError<'a, I, E>) -> Self { + fn from(e: ParseError) -> Self { + use primitives::Primitives; + match e { - ParseError::Error(b, e) => StreamError::ParseError(b, e), + ParseError::Error(mut b, e) => { + let r = b.min_remaining(); + + StreamError::ParseError(b.consume(r), e) + }, ParseError::Incomplete(n) => StreamError::Incomplete(n), } } @@ -92,7 +98,7 @@ impl<'a, I, E> From> for StreamError<'a, I, E> /// Trait wrapping the state management in reading from a data source while parsing. pub trait Stream<'a, 'i> { /// The input item type, usually depending on which `DataSource` is used. - type Item: 'i; + type Item: 'i + Copy; /// Attempts to run the supplied parser `F` once on the currently populated data in this /// stream, providing a borrow of the inner data storage. @@ -101,7 +107,7 @@ pub trait Stream<'a, 'i> { /// (the implementation might require a separate call to refill the stream). #[inline] fn parse(&'a mut self, f: F) -> Result> - where F: FnOnce(Input<'i, Self::Item>) -> ParseResult<'i, Self::Item, T, E>, + where F: FnOnce(InputBuf<'i, Self::Item>) -> ParseResult, T, E>, T: 'i, E: 'i; } @@ -109,7 +115,7 @@ pub trait Stream<'a, 'i> { /// Trait for conversion into a `Stream`. pub trait IntoStream<'a, 'i> { /// The input item type provided by the stream. - type Item: 'i; + type Item: 'i + Copy; /// The `Stream` instance type. type Into: Stream<'a, 'i, Item=Self::Item>; diff --git a/src/buffer/slice.rs b/src/buffer/slice.rs index 97d61c68..08f39d6e 100644 --- a/src/buffer/slice.rs +++ b/src/buffer/slice.rs @@ -1,7 +1,7 @@ use primitives::input; -use primitives::{State, InputBuffer, IntoInner}; +use primitives::{State, IntoInner}; -use {Input, ParseResult}; +use {InputBuf, ParseResult}; use buffer::{IntoStream, StreamError, Stream}; /// Stream implementation for immutable slices. @@ -68,7 +68,7 @@ impl<'i, I: 'i> SliceStream<'i, I> { } } -impl<'a, 'i, I: 'i> IntoStream<'a, 'i> for &'i [I] { +impl<'a, 'i, I: 'i + Copy> IntoStream<'a, 'i> for &'i [I] { type Item = I; type Into = SliceStream<'i, I>; @@ -78,33 +78,37 @@ impl<'a, 'i, I: 'i> IntoStream<'a, 'i> for &'i [I] { } } -impl<'a, 'i, I: 'i> Stream<'a, 'i> for SliceStream<'i, I> { +impl<'a, 'i, I: 'i + Copy> Stream<'a, 'i> for SliceStream<'i, I> { type Item = I; #[inline] fn parse(&'a mut self, f: F) -> Result> - where F: FnOnce(Input<'i, Self::Item>) -> ParseResult<'i, Self::Item, T, E>, + where F: FnOnce(InputBuf<'i, Self::Item>) -> ParseResult, T, E>, T: 'i, E: 'i { + use primitives::Primitives; + if self.is_empty() { return Err(StreamError::EndOfInput); } - match f(input::new(input::END_OF_INPUT, &self.slice[self.pos..])).into_inner() { + match f(input::new_buf(input::END_OF_INPUT, &self.slice[self.pos..])).into_inner() { State::Data(remainder, data) => { // TODO: Do something neater with the remainder - self.pos += self.len() - remainder.buffer().len(); + self.pos += self.len() - remainder.min_remaining(); Ok(data) }, - State::Error(remainder, err) => { + State::Error(mut remainder, err) => { // TODO: Do something neater with the remainder // TODO: Detail this behaviour, maybe make it configurable - self.pos += self.len() - remainder.len(); + let r = remainder.min_remaining(); + + self.pos += self.len() - r; - Err(StreamError::ParseError(remainder, err)) + Err(StreamError::ParseError(remainder.consume(r), err)) }, - State::Incomplete(n) => Err(StreamError::Incomplete(n + self.len())), + State::Incomplete(_, n) => Err(StreamError::Incomplete(n + self.len())), } } } diff --git a/src/buffer/stateful.rs b/src/buffer/stateful.rs index 86ae7e1b..1e2d7e33 100644 --- a/src/buffer/stateful.rs +++ b/src/buffer/stateful.rs @@ -1,9 +1,9 @@ use std::io; use std::cmp; -use {Input, ParseResult}; +use {InputBuf, ParseResult}; use primitives::input; -use primitives::{InputBuffer, State, IntoInner}; +use primitives::{State, IntoInner}; use buffer::{ Buffer, @@ -56,7 +56,8 @@ impl> Source, B> { } } -impl> Source, B> { +impl> Source, B> + where I::Item: Copy { /// Creates a new `Source` from `Iterator` and `Buffer` instances. #[inline] pub fn from_iter(source: I, buffer: B) -> Self { @@ -204,9 +205,11 @@ impl<'a, S: DataSource, B: Buffer> Stream<'a, 'a> for Source #[inline] fn parse(&'a mut self, f: F) -> Result> - where F: FnOnce(Input<'a, Self::Item>) -> ParseResult<'a, Self::Item, T, E>, + where F: FnOnce(InputBuf<'a, Self::Item>) -> ParseResult, T, E>, T: 'a, E: 'a { + use primitives::Primitives; + if self.state.contains(INCOMPLETE | AUTOMATIC_FILL) { try!(self.fill().map_err(StreamError::IoError)); } @@ -217,21 +220,23 @@ impl<'a, S: DataSource, B: Buffer> Stream<'a, 'a> for Source let input_state = if self.state.contains(END_OF_INPUT) { input::END_OF_INPUT } else { input::DEFAULT }; - match f(input::new(input_state, &self.buffer)).into_inner() { + match f(input::new_buf(input_state, &self.buffer)).into_inner() { State::Data(remainder, data) => { // TODO: Do something neater with the remainder - self.buffer.consume(self.buffer.len() - remainder.buffer().len()); + self.buffer.consume(self.buffer.len() - remainder.min_remaining()); Ok(data) }, - State::Error(remainder, err) => { + State::Error(mut remainder, err) => { // TODO: Do something neater with the remainder // TODO: Detail this behaviour, maybe make it configurable - self.buffer.consume(self.buffer.len() - remainder.len()); + let r = remainder.min_remaining(); + + self.buffer.consume(self.buffer.len() - r); - Err(StreamError::ParseError(remainder, err)) + Err(StreamError::ParseError(remainder.consume(r), err)) }, - State::Incomplete(n) => { + State::Incomplete(_, n) => { self.request = self.buffer.len() + n; if self.state.contains(END_OF_INPUT) { @@ -249,7 +254,7 @@ impl<'a, S: DataSource, B: Buffer> Stream<'a, 'a> for Source #[cfg(test)] mod test { use std::io; - use {any, take}; + use {Input, any, take}; use Error; use buffer::{ FixedSizeBuffer, diff --git a/src/input.rs b/src/input.rs index 8818d6d8..98af944b 100644 --- a/src/input.rs +++ b/src/input.rs @@ -3,6 +3,8 @@ use parse_result; use primitives::Guard; +pub trait U8Input: Input {} + /// Primitive operations on `Input` types. /// /// All parsers only require a few primitive operations to parse data: diff --git a/src/lib.rs b/src/lib.rs index ed1a5d41..0663eba2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -236,7 +236,7 @@ pub use parsers::{ token, }; pub use parsers::Error; -//pub mod buffer; +pub mod buffer; pub use combinators::{ count, @@ -255,12 +255,14 @@ pub use parse::{ ParseError, parse_only, }; -pub use input::Input; +pub use input::{ + Input, + InputBuf, + U8Input, +}; pub use parse_result::{ ParseResult, - // FIXME: Uncomment SimpleResult, - //U8Result, }; /// Module used to construct fundamental parsers and combinators. From 4e6ba8ee24c4330325ddf1fb0005acd03455aea9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Sun, 20 Mar 2016 13:48:59 +0100 Subject: [PATCH 14/63] WIP on removing incomplete --- src/buffer/slice.rs | 16 ++- src/buffer/stateful.rs | 43 ++++--- src/combinators/bounded.rs | 110 +++--------------- src/combinators/macros.rs | 34 ++---- src/combinators/mod.rs | 20 ---- src/input.rs | 146 +++++++++++++++++------- src/lib.rs | 2 +- src/parse.rs | 1 - src/parse_result.rs | 25 ----- src/parsers.rs | 224 +++++++++++++++++++------------------ 10 files changed, 286 insertions(+), 335 deletions(-) diff --git a/src/buffer/slice.rs b/src/buffer/slice.rs index 08f39d6e..aaac9c7c 100644 --- a/src/buffer/slice.rs +++ b/src/buffer/slice.rs @@ -100,15 +100,19 @@ impl<'a, 'i, I: 'i + Copy> Stream<'a, 'i> for SliceStream<'i, I> { Ok(data) }, State::Error(mut remainder, err) => { - // TODO: Do something neater with the remainder - // TODO: Detail this behaviour, maybe make it configurable - let r = remainder.min_remaining(); + if remainder.is_incomplete() { + // TODO: 1 is not correct, n is expected len but we can't obtain that right now + Err(StreamError::Incomplete(self.len() + 1)) + } else { + // TODO: Do something neater with the remainder + // TODO: Detail this behaviour, maybe make it configurable + let r = remainder.min_remaining(); - self.pos += self.len() - r; + self.pos += self.len() - r; - Err(StreamError::ParseError(remainder.consume(r), err)) + Err(StreamError::ParseError(remainder.consume(r), err)) + } }, - State::Incomplete(_, n) => Err(StreamError::Incomplete(n + self.len())), } } } diff --git a/src/buffer/stateful.rs b/src/buffer/stateful.rs index 1e2d7e33..77a23d85 100644 --- a/src/buffer/stateful.rs +++ b/src/buffer/stateful.rs @@ -222,29 +222,40 @@ impl<'a, S: DataSource, B: Buffer> Stream<'a, 'a> for Source match f(input::new_buf(input_state, &self.buffer)).into_inner() { State::Data(remainder, data) => { - // TODO: Do something neater with the remainder - self.buffer.consume(self.buffer.len() - remainder.min_remaining()); + if remainder.is_incomplete() && self.state.contains(END_OF_INPUT) { + // We can't accept this since we might have hit a premature end + self.request = self.buffer.len() + 1; - Ok(data) - }, - State::Error(mut remainder, err) => { - // TODO: Do something neater with the remainder - // TODO: Detail this behaviour, maybe make it configurable - let r = remainder.min_remaining(); + self.state.insert(INCOMPLETE); - self.buffer.consume(self.buffer.len() - r); + Err(StreamError::Retry) + } else { + // TODO: Do something neater with the remainder + self.buffer.consume(self.buffer.len() - remainder.min_remaining()); - Err(StreamError::ParseError(remainder.consume(r), err)) + Ok(data) + } }, - State::Incomplete(_, n) => { - self.request = self.buffer.len() + n; + State::Error(mut remainder, err) => { + if remainder.is_incomplete() { + // TODO: How to deal with n, no longer present? + self.request = self.buffer.len() + 1; + + if self.state.contains(END_OF_INPUT) { + Err(StreamError::Incomplete(self.request)) + } else { + self.state.insert(INCOMPLETE); - if self.state.contains(END_OF_INPUT) { - Err(StreamError::Incomplete(self.request)) + Err(StreamError::Retry) + } } else { - self.state.insert(INCOMPLETE); + // TODO: Do something neater with the remainder + // TODO: Detail this behaviour, maybe make it configurable + let r = remainder.min_remaining(); - Err(StreamError::Retry) + self.buffer.consume(self.buffer.len() - r); + + Err(StreamError::ParseError(remainder.consume(r), err)) } }, } diff --git a/src/combinators/bounded.rs b/src/combinators/bounded.rs index a25d1ad1..6c347f8b 100644 --- a/src/combinators/bounded.rs +++ b/src/combinators/bounded.rs @@ -131,19 +131,13 @@ impl BoundedRange for Range { // Got all occurrences of the parser // First state or reached max => do not restore to mark since it is from last // iteration - (s, (0, 0), _, _) => s.ret(result), + (s, (0, 0), _, _) => s.ret(result), // Ok, last parser failed and we have reached minimum, we have iterated all. // Return remainder of buffer and the collected result - (s, (0, _), m, EndState::Error(_)) => s.restore(m).ret(result), - // Nested parser incomplete but reached at least minimum, propagate if not at end - (s, (0, _), m, EndState::Incomplete(n)) => if s.is_end() { - s.restore(m).ret(result) - } else { - s.incomplete(n) - }, + (s, (0, _), m, Some(_)) => s.restore(m).ret(result), // Did not reach minimum, propagate - (s, (_, _), _, EndState::Error(e)) => s.err(e), - (s, (_, _), _, EndState::Incomplete(n)) => s.incomplete(n) + (s, (_, _), _, Some(e)) => s.err(e), + (_, _, _, None) => unreachable!(), } } } @@ -179,14 +173,6 @@ impl BoundedRange for Range { // Not enough iterations, propagate return b.err(e); }, - State::Incomplete(b, n) => if min == 0 && b.is_end() { - i = b.restore(m); - - break; - } else { - // We have not done the minimum amount of iterations - return b.incomplete(n); - } } } @@ -237,15 +223,8 @@ impl BoundedRange for Range { return None; }, - (0, State::Incomplete(b, n)) => { - self.buf = Some(b); - self.state = EndStateTill::Incomplete(n); - - return None; - }, // Failed to end, restore and continue since we can parse more (_, State::Error(b, _)) => self.buf = Some(b.restore(m)), - (_, State::Incomplete(b, _)) => self.buf = Some(b.restore(m)), } } } @@ -260,7 +239,7 @@ impl BoundedRange for Range { (s, (0, _), EndStateTill::EndSuccess) => s.ret(result), // Did not reach minimum or a failure, propagate (s, (_, _), EndStateTill::Error(e)) => s.err(e), - (s, (_, _), EndStateTill::Incomplete(n)) => s.incomplete(n), + (s, (_, _), EndStateTill::Incomplete) => unreachable!(), // We cannot reach this since we only run the end test once we have reached the // minimum number of matches (_, (_, _), EndStateTill::EndSuccess) => unreachable!() @@ -293,16 +272,10 @@ impl BoundedRange for RangeFrom { => result : T { // We got at least n items - (s, 0, m, EndState::Error(_)) => s.restore(m).ret(result), - // Nested parser incomplete, propagate if not at end - (s, 0, m, EndState::Incomplete(n)) => if s.is_end() { - s.restore(m).ret(result) - } else { - s.incomplete(n) - }, + (s, 0, m, Some(_)) => s.restore(m).ret(result), // Items still remaining, propagate - (s, _, _, EndState::Error(e)) => s.err(e), - (s, _, _, EndState::Incomplete(n)) => s.incomplete(n) + (s, _, _, Some(e)) => s.err(e), + (_, _, _, None) => unreachable!(), } } } @@ -330,14 +303,6 @@ impl BoundedRange for RangeFrom { // Not enough iterations, propagate return b.err(e); }, - State::Incomplete(b, n) => if min == 0 && b.is_end() { - i = b.restore(m); - - break; - } else { - // We have not done the minimum amount of iterations - return b.incomplete(n); - } } } @@ -378,7 +343,7 @@ impl BoundedRange for RangeFrom { (s, 0, EndStateTill::EndSuccess) => s.ret(result), // Did not reach minimum or a failure, propagate (s, _, EndStateTill::Error(e)) => s.err(e), - (s, _, EndStateTill::Incomplete(n)) => s.incomplete(n), + (s, _, EndStateTill::Incomplete) => unreachable!(), // We cannot reach this since we only run the end test once we have reached the // minimum number of matches (_, _, EndStateTill::EndSuccess) => unreachable!() @@ -407,13 +372,8 @@ impl BoundedRange for RangeFull { } => result : T { - (s, (), m, EndState::Error(_)) => s.restore(m).ret(result), - // Nested parser incomplete, propagate if not at end - (s, (), m, EndState::Incomplete(n)) => if s.is_end() { - s.restore(m).ret(result) - } else { - s.incomplete(n) - } + (s, (), m, Some(_)) => s.restore(m).ret(result), + (_, _, _, None) => unreachable!(), } } } @@ -431,13 +391,6 @@ impl BoundedRange for RangeFull { break; }, - State::Incomplete(b, n) => if b.is_end() { - i = b.restore(m); - - break; - } else { - return b.incomplete(n); - } } } @@ -472,7 +425,7 @@ impl BoundedRange for RangeFull { (s, (), EndStateTill::EndSuccess) => s.ret(result), (s, (), EndStateTill::Error(e)) => s.err(e), // Nested parser incomplete, propagate if not at end - (s, (), EndStateTill::Incomplete(n)) => s.incomplete(n) + (s, (), EndStateTill::Incomplete) => unreachable!() } } } @@ -509,13 +462,8 @@ impl BoundedRange for RangeTo { // iteration (s, 0, _, _) => s.ret(result), // Inside of range, never outside - (s, _, m, EndState::Error(_)) => s.restore(m).ret(result), - // Nested parser incomplete, propagate if not at end - (s, _, m, EndState::Incomplete(n)) => if s.is_end() { - s.restore(m).ret(result) - } else { - s.incomplete(n) - } + (s, _, m, Some(_)) => s.restore(m).ret(result), + (_, _, _, None) => unreachable!(), } } } @@ -545,13 +493,6 @@ impl BoundedRange for RangeTo { break; }, - State::Incomplete(b, n) => if b.is_end() { - i = b.restore(m); - - break; - } else { - return b.incomplete(n); - } } } @@ -596,15 +537,8 @@ impl BoundedRange for RangeTo { return None; }, - (0, State::Incomplete(b, n)) => { - self.buf = Some(b); - self.state = EndStateTill::Incomplete(n); - - return None; - }, // Failed to end, restore and continue since we can parse more (_, State::Error(b, _)) => self.buf = Some(b.restore(m)), - (_, State::Incomplete(b, _)) => self.buf = Some(b.restore(m)), } } on { @@ -617,7 +551,7 @@ impl BoundedRange for RangeTo { (s, _, EndStateTill::EndSuccess) => s.ret(result), // Did not reach minimum or a failure, propagate (s, _, EndStateTill::Error(e)) => s.err(e), - (s, _, EndStateTill::Incomplete(n)) => s.incomplete(n) + (s, _, EndStateTill::Incomplete) => unreachable!(), } } } @@ -654,9 +588,8 @@ impl BoundedRange for usize { // Got exact (s, 0, _, _) => s.ret(result), // We have got too few items, propagate error - (s, _, _, EndState::Error(e)) => s.err(e), - // Nested parser incomplete, propagate - (s, _, _, EndState::Incomplete(n)) => s.incomplete(n) + (s, _, _, Some(e)) => s.err(e), + (_, _, _, None) => unreachable!(), } } } @@ -687,13 +620,6 @@ impl BoundedRange for usize { // Not enough iterations, propagate return b.err(e); }, - State::Incomplete(b, n) => if n == 0 { - i = b.restore(m); - - break; - } else { - return b.incomplete(n); - } } } @@ -735,7 +661,7 @@ impl BoundedRange for usize { (s, 0, EndStateTill::EndSuccess) => s.ret(result), // Did not reach minimum or a failure, propagate (s, _, EndStateTill::Error(e)) => s.err(e), - (s, _, EndStateTill::Incomplete(n)) => s.incomplete(n), + (s, _, EndStateTill::Incomplete) => unreachable!(), // We cannot reach this since we only run the end test once we have reached the // minimum number of matches (_, _, EndStateTill::EndSuccess) => unreachable!() diff --git a/src/combinators/macros.rs b/src/combinators/macros.rs index b6f4bfce..f85b0379 100644 --- a/src/combinators/macros.rs +++ b/src/combinators/macros.rs @@ -14,18 +14,13 @@ macro_rules! run_iter { } => $result:ident : $t:ty { - $($pat:pat => $arm:expr),* + $($pat:pat => $arm:expr),*$(,)* } ) => { { - enum EndState { - Error(E), - Incomplete(usize), - } - struct Iter where F: FnMut(I) -> ParseResult { /// Last state of the parser - state: EndState, + state: Option, /// Parser to execute once for each iteration parser: F, /// Remaining buffer @@ -44,7 +39,7 @@ macro_rules! run_iter { impl Iter where F: FnMut(I) -> ParseResult { #[inline] - fn end_state(self) -> (I, $data_ty, I::Marker, EndState) { + fn end_state(self) -> (I, $data_ty, I::Marker, Option) { // TODO: Avoid branch, check if this can be guaranteed to always be Some(T) (self.buf.expect("Iter.buf was None"), self.data, self.mark, self.state) } @@ -79,13 +74,7 @@ macro_rules! run_iter { }, State::Error(b, e) => { $next_self.buf = Some(b); - $next_self.state = EndState::Error(e); - - None - }, - State::Incomplete(b, n) => { - $next_self.buf = Some(b); - $next_self.state = EndState::Incomplete(n); + $next_self.state = Some(e); None }, @@ -97,7 +86,7 @@ macro_rules! run_iter { let m = $input.mark(); let mut iter = Iter { - state: EndState::Incomplete(1), + state: None, parser: $parser, buf: Some($input), mark: m, @@ -129,12 +118,12 @@ macro_rules! run_iter_till { } => $result:ident : $t:ty { - $($pat:pat => $arm:expr),* + $($pat:pat => $arm:expr),*$(,)* } ) => { { enum EndStateTill { Error(E), - Incomplete(usize), + Incomplete, EndSuccess, } @@ -194,12 +183,6 @@ macro_rules! run_iter_till { $next_self.buf = Some(b); $next_self.state = EndStateTill::Error(e); - None - }, - State::Incomplete(b, n) => { - $next_self.buf = Some(b); - $next_self.state = EndStateTill::Incomplete(n); - None }, } @@ -207,7 +190,7 @@ macro_rules! run_iter_till { } let mut iter = IterTill { - state: EndStateTill::Incomplete(1), + state: EndStateTill::Incomplete, parser: $parser, end: $end, buf: Some($input), @@ -241,7 +224,6 @@ macro_rules! iter_till_end_test { }, // Failed to end, restore and continue State::Error(b, _) => $the_self.buf = Some(b.restore(m)), - State::Incomplete(b, _) => $the_self.buf = Some(b.restore(m)), } } } } diff --git a/src/combinators/mod.rs b/src/combinators/mod.rs index f60cfa65..98708305 100644 --- a/src/combinators/mod.rs +++ b/src/combinators/mod.rs @@ -61,15 +61,6 @@ pub fn option(i: I, f: F, default: T) -> ParseResult State::Error(b, _) => { b.restore(m).ret(default) }, - State::Incomplete(b, n) => { - let b = b.restore(m); - - if b.is_end() { - b.ret(default) - } else { - b.incomplete(n) - } - }, } } @@ -101,15 +92,6 @@ pub fn or(i: I, f: F, g: G) -> ParseResult match f(i).into_inner() { State::Data(b, d) => b.ret(d), State::Error(b, _) => g(b.restore(m)), - State::Incomplete(b, n) => { - let b = b.restore(m); - - if b.is_end() { - g(b) - } else { - b.incomplete(n) - } - }, } } @@ -302,7 +284,6 @@ pub fn matched_by(i: I, f: F) -> ParseResult b.err(e), - State::Incomplete(b, n) => b.incomplete(n), } } @@ -324,7 +305,6 @@ pub fn look_ahead(i: I, f: F) -> ParseResult match f(i).into_inner() { State::Data(b, t) => b.restore(m).ret(t), State::Error(b, t) => b.restore(m).err(t), - State::Incomplete(b, n) => b.restore(m).incomplete(n), } } diff --git a/src/input.rs b/src/input.rs index 98af944b..339b8b16 100644 --- a/src/input.rs +++ b/src/input.rs @@ -5,6 +5,78 @@ use primitives::Guard; pub trait U8Input: Input {} +// TODO: More restrictions? Buffer=&[u8]? +impl U8Input for T + where T: Input {} + +/* +Behaviour on incomplete: + +any: + error, unexpected eof +satisfy: + error, unexpected eof +satisfy_with: + error, unexpected eof +token: + error, unexpected eof +not_token: + error, unexpected eof +peek: + return None +peek_next + error, unexpected eof +take: + error, unexpected eof +take_while: + try refill (success: resume scanning, fail: return remainder) +take_while1: + try refill (success: resume scanning, fail: return remainder) +take_till: + try refill (success: resume scanning, fail: error, unexpected eof) +scan: + try refill (success: resume scanning, fail: error, unexpected eof) +run_scanner: + try refill (success: resume scanning, fail: error, unexpected eof) +take_remainder: + try refill, return remainder +string: + try refill (success: resume scanning, fail: error, unexpected eof) +eof: + try refill (success: fail, unexpected, fail: success) + +option: + failed incomplete == error +or: + failed incomplete == error +matched_by: + failed incomplete == error +look_ahead: + failed incomplete == error + +Range + many: + failed incomplete == error + skip: + failed incomplete == error + many_till: + failed incomplete == error +RangeFrom + many: + failed incomplete == error + skip: + failed incomplete == error + many_till: + failed incomplete == error +RangeFull + many: + failed incomplete == error + skip: + failed incomplete == error + many_till: + failed incomplete == error +*/ + /// Primitive operations on `Input` types. /// /// All parsers only require a few primitive operations to parse data: @@ -28,18 +100,6 @@ pub mod primitives { /// // FIXME: Rename pub trait Primitives: Input { - #[inline(always)] - /// **Primitive:** Notifies that a parser has reached the end of the currently supplied slice but - /// requires more data. - /// - /// # Primitive - /// - /// Only used by fundamental parsers and combinators. - #[inline] - fn incomplete(self, n: usize) -> ParseResult { - new_result(State::Incomplete(self, n)) - } - #[inline(always)] fn first(&self) -> Option { self._first(Guard(())) @@ -55,8 +115,9 @@ pub mod primitives { self._consume(Guard(()), n) } + // TODO: Change to self -> self? #[inline(always)] - fn discard(&mut self, n: usize) { + fn discard(self, n: usize) -> Self { self._discard(Guard(()), n) } @@ -72,14 +133,15 @@ pub mod primitives { self._min_remaining(Guard(())) } - /// Returns true if this is the last available slice of the input. + /// Attempts to populate the input with more data, returning the number of additional + /// tokens which were added. /// /// # Primitive /// /// Only used by fundamental parsers and combinators. #[inline(always)] - fn is_end(&self) -> bool { - self._is_end(Guard(())) + fn fill(&mut self) -> usize { + self._fill(Guard(())) } /// Marks the current position to be able to backtrack to it using `restore`. @@ -242,7 +304,7 @@ pub trait Input: Sized { /// /// Consumes a set amount of input tokens, discarding them. #[inline] - fn _discard(&mut self, Guard, usize); + fn _discard(self, Guard, usize) -> Self; /// **Primitive:** See `Primitives::consume_from for documentation. /// @@ -258,11 +320,12 @@ pub trait Input: Sized { #[inline] fn _min_remaining(&self, Guard) -> usize; - /// **Primitive:** See `Primitives::is_end` for documentation. + /// **Primitive:** See `Primitives::fill` for documentation. /// - /// Returns true if there are no more input in `Self` besides what has already been populated. - #[inline] - fn _is_end(&self, Guard) -> bool; + /// Attempts to populate the input with more data, returning the number of additional + /// tokens which were added. + #[inline(always)] + fn _fill(&mut self, Guard) -> usize; /// **Primitive:** See `Primitives::mark` for documentation. /// @@ -294,8 +357,8 @@ impl<'a, I: Copy> Input for &'a [I] { b } #[inline(always)] - fn _discard(&mut self, _g: Guard, n: usize) { - *self = &self[n..]; + fn _discard(self, _g: Guard, n: usize) -> Self { + &self[n..] } #[inline] fn _consume_from(&mut self, _g: Guard, m: Self::Marker) -> Self::Buffer { @@ -306,8 +369,8 @@ impl<'a, I: Copy> Input for &'a [I] { self.len() } #[inline(always)] - fn _is_end(&self, _g: Guard) -> bool { - true + fn _fill(&mut self, _g: Guard) -> usize { + 0 } fn _mark(&self, _g: Guard) -> Self::Marker { &self @@ -323,6 +386,8 @@ bitflags!{ const DEFAULT = 0, /// If set the current slice of input is the last one. const END_OF_INPUT = 1, + /// If a parser has attempted to read incomplete + const INCOMPLETE = 2, } } @@ -341,8 +406,15 @@ pub fn new_buf(state: InputMode, buffer: &[I]) -> InputBuf { InputBuf(state, buffer) } +impl<'a, I: 'a> InputBuf<'a, I> { + pub fn is_incomplete(&self) -> bool { + self.0.contains(INCOMPLETE) + } +} + impl<'a, I: Copy> Input for InputBuf<'a, I> { type Token = I; + // TODO: InputMode? INCOMPLETE must be set no matter what type Marker = &'a [I]; type Iter = ::std::iter::Cloned<::std::slice::Iter<'a, I>>; type Buffer = &'a [I]; @@ -358,8 +430,10 @@ impl<'a, I: Copy> Input for InputBuf<'a, I> { b } #[inline(always)] - fn _discard(&mut self, _g: Guard, n: usize) { + fn _discard(mut self, _g: Guard, n: usize) -> Self { self.1 = &self.1[n..]; + + self } #[inline] fn _consume_from(&mut self, _g: Guard, m: Self::Marker) -> Self::Buffer { @@ -370,8 +444,10 @@ impl<'a, I: Copy> Input for InputBuf<'a, I> { self.1.len() } #[inline(always)] - fn _is_end(&self, _g: Guard) -> bool { - self.0.contains(END_OF_INPUT) + fn _fill(&mut self, _g: Guard) -> usize { + self.0.insert(INCOMPLETE); + + 0 } fn _mark(&self, _g: Guard) -> Self::Marker { &self.1 @@ -526,20 +602,6 @@ mod test { assert_eq!(r2.into_inner(), State::Error(new_buf(DEFAULT, b"in2"), 23i32)); } - #[test] - fn make_incomplete() { - use primitives::Primitives; - - let i1: InputBuf = new_buf(END_OF_INPUT, b"in1"); - let i2: InputBuf = new_buf(DEFAULT, b"in2"); - - let r1: ParseResult<_, (), u32> = i1.incomplete::<(), _>(23); - let r2: ParseResult<_, (), i32> = i2.incomplete::<(), _>(23); - - assert_eq!(r1.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b"in1"), 23)); - assert_eq!(r2.into_inner(), State::Incomplete(new_buf(DEFAULT, b"in2"), 23)); - } - #[test] fn primitives_slice() { use primitives::Primitives; diff --git a/src/lib.rs b/src/lib.rs index 0663eba2..9a137aae 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -286,7 +286,7 @@ pub mod primitives { /// /// Only used by fundamental parsers and combinators. pub mod input { - pub use input::{DEFAULT, END_OF_INPUT, new_buf}; + pub use input::{DEFAULT, END_OF_INPUT, INCOMPLETE, new_buf}; } /// ParseResult utilities. diff --git a/src/parse.rs b/src/parse.rs index 7ed332db..f439f0fa 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -59,7 +59,6 @@ pub fn parse_only<'a, I: Copy, T, E, F>(parser: F, input: &'a [I]) -> Result Err(ParseError::Incomplete(n)), } } diff --git a/src/parse_result.rs b/src/parse_result.rs index f93873fc..a5d75a06 100644 --- a/src/parse_result.rs +++ b/src/parse_result.rs @@ -22,9 +22,6 @@ pub enum State { /// Parse error state, first item is a slice from where the error occurred in the input buffer /// to the end of the input buffer and the second item is the error value. Error(I, E), - /// Incomplete state, a parser attempted to request more data than available in the slice, the - /// provided number is a guess at how many items are needed. - Incomplete(I, usize), } /// **Primitive:** Consumes self and reveals the inner state. @@ -142,7 +139,6 @@ impl ParseResult { match self.0 { State::Data(i, t) => f(i, t).map_err(From::from), State::Error(i, e) => ParseResult(State::Error(i, From::from(e))), - State::Incomplete(i, n) => ParseResult(State::Incomplete(i, n)), } } @@ -196,7 +192,6 @@ impl ParseResult { match self.0 { State::Data(i, t) => ParseResult(State::Data(i, f(t))), State::Error(i, e) => ParseResult(State::Error(i, e)), - State::Incomplete(i, n) => ParseResult(State::Incomplete(i, n)), } } @@ -219,7 +214,6 @@ impl ParseResult { match self.0 { State::Data(i, t) => ParseResult(State::Data(i, t)), State::Error(i, e) => ParseResult(State::Error(i, f(e))), - State::Incomplete(i, n) => ParseResult(State::Incomplete(i, n)), } } @@ -414,25 +408,6 @@ mod test { assert_eq!(n2_calls, 0); } - #[test] - fn incomplete_propagation() { - use primitives::Primitives; - - let mut n1_calls = 0; - let mut n2_calls = 0; - - let i1 = input::new_buf(DEFAULT, b"test1").incomplete::<(), ()>(23); - let i2 = input::new_buf(END_OF_INPUT, b"test2").incomplete::<(), ()>(24); - - let r1: ParseResult<_, _, ()> = i1.bind(|i, t| { n1_calls += 1; i.ret(t) }); - let r2: ParseResult<_, _, ()> = i2.bind(|i, t| { n2_calls += 1; i.ret(t) }); - - assert_eq!(r1.0, State::Incomplete(input::new_buf(DEFAULT, b"test1"), 23)); - assert_eq!(r2.0, State::Incomplete(input::new_buf(END_OF_INPUT, b"test2"), 24)); - assert_eq!(n1_calls, 0); - assert_eq!(n2_calls, 0); - } - #[test] fn slice() { fn f(i: I, n: u32) -> ParseResult { i.ret(n + 1) } diff --git a/src/parsers.rs b/src/parsers.rs index 6aa30e26..fd1ce181 100644 --- a/src/parsers.rs +++ b/src/parsers.rs @@ -24,14 +24,7 @@ pub use debugtrace::StackFrame; /// ``` #[inline] pub fn any(mut i: I) -> SimpleResult { - match i.first() { - None => i.incomplete(1), - Some(c) => { - i.discard(1); - - i.ret(c) - }, - } + peek_next(i).bind(|i, c| i.discard(1).ret(c)) } /// Matches an item using ``f``, the item is returned if ``f`` yields true, otherwise this parser @@ -47,15 +40,11 @@ pub fn any(mut i: I) -> SimpleResult { #[inline] pub fn satisfy(mut i: I, f: F) -> SimpleResult where F: FnOnce(I::Token) -> bool { - match i.first() { - None => i.incomplete(1), - Some(c) if f(c) => { - i.discard(1); - - i.ret(c) - }, - Some(_) => i.err(Error::unexpected()), - } + peek_next(i).bind(|i, c| if f(c) { + i.discard(1).ret(c) + } else { + i.err(Error::unexpected()) + }) } /// Reads a single token, applies the transformation `F` and then succeeds with the transformed @@ -76,20 +65,15 @@ pub fn satisfy(mut i: I, f: F) -> SimpleResult pub fn satisfy_with(mut i: I, f: F, p: P) -> SimpleResult where F: FnOnce(I::Token) -> T, P: FnOnce(T) -> bool { - match i.first() { - Some(n) => { - let t = f(n); + peek_next(i).bind(|i, c| { + let t = f(c); - if p(t.clone()) { - i.discard(1); - - i.ret(t) - } else { - i.err(Error::unexpected()) - } - }, - None => i.incomplete(1), - } + if p(t.clone()) { + i.discard(1).ret(t) + } else { + i.err(Error::unexpected()) + } + }) } /// Matches a single token, returning the match on success. @@ -104,15 +88,11 @@ pub fn satisfy_with(mut i: I, f: F, p: P) -> SimpleRes #[inline] pub fn token(mut i: I, t: I::Token) -> SimpleResult where I::Token: PartialEq { - match i.first() { - None => i.incomplete(1), - Some(c) if t == c => { - i.discard(1); - - i.ret(c) - }, - Some(_) => i.err(Error::expected(t)), - } + peek_next(i).bind(|i, c| if t == c { + i.discard(1).ret(c) + } else { + i.err(Error::expected(t)) + }) } /// Matches a single token as long as it is not equal to `t`, returning the match on success. @@ -127,14 +107,11 @@ pub fn token(mut i: I, t: I::Token) -> SimpleResult #[inline] pub fn not_token(mut i: I, t: I::Token) -> SimpleResult where I::Token: PartialEq { - match i.first() { - None => i.incomplete(1), - Some(c) if t != c => { - i.discard(1); - i.ret(c) - }, - Some(_) => i.err(Error::unexpected()), - } + peek_next(i).bind(|i, c| if t != c { + i.discard(1).ret(c) + } else { + i.err(Error::unexpected()) + }) } /// Matches any item but does not consume it, on success it gives ``Some`` but if no input remains @@ -149,12 +126,12 @@ pub fn not_token(mut i: I, t: I::Token) -> SimpleResult /// /// assert_eq!(parse_only(peek, b""), Ok(None)); /// ``` -#[inline] +#[inline(always)] pub fn peek(i: I) -> SimpleResult> where I::Token: Clone { - let d = i.first(); + let c = i.first(); - i.ret(d) + i.ret(c) } /// Matches any item but does not consume it. @@ -166,12 +143,19 @@ pub fn peek(i: I) -> SimpleResult> /// /// assert_eq!(parse_only(peek_next, b"abc"), Ok(b'a')); /// ``` -#[inline] -pub fn peek_next(i: I) -> SimpleResult +#[inline(always)] +pub fn peek_next(mut i: I) -> SimpleResult where I::Token: Clone { match i.first() { - None => i.incomplete(1), Some(c) => i.ret(c), + None => if i.fill() > 0 { + let c = i.first() + .expect("peek_next: Failed to obtain first token after fill() returned > 0"); + + i.ret(c) + } else { + i.err(Error::unexpected()) + }, } } @@ -186,15 +170,22 @@ pub fn peek_next(i: I) -> SimpleResult /// ``` #[inline] pub fn take(mut i: I, num: usize) -> SimpleResult { - let r = i.min_remaining(); + let mut r = i.min_remaining(); - if num <= r { - let b = i.consume(num); + while r < num { + let n = i.fill(); - i.ret(b) - } else { - i.incomplete(num - r) + if n == 0 { + // TODO: Proper incomplete error here? + return i.err(Error::unexpected()); + } + + r += n } + + let b = i.consume(num); + + i.ret(b) } /// Matches all items while ``f`` returns false, returns a slice of all the matched items. @@ -222,23 +213,23 @@ pub fn take(mut i: I, num: usize) -> SimpleResult { #[inline] pub fn take_while(mut i: I, f: F) -> SimpleResult where F: Fn(I::Token) -> bool { - match i.iter().position(|c| f(c) == false) { - Some(n) => { + loop { + if let Some(n) = i.iter().position(|c| f(c) == false) { let b = i.consume(n); - i.ret(b) - }, - // TODO: Should this following 1 be something else, seeing as take_while1 is potentially - // infinite? - None => if i.is_end() { - // Last slice and we have just read everything of it, replace with zero-sized slice: - let r = i.min_remaining(); - let b = i.consume(r); + return i.ret(b); + } - i.ret(b) - } else { - i.incomplete(1) - }, + // Attempt to fill + if i.fill() > 0 { + continue; + } + + // Last slice and we fail to get more, take remainder + let r = i.min_remaining(); + let b = i.consume(r); + + return i.ret(b); } } @@ -258,24 +249,27 @@ pub fn take_while(mut i: I, f: F) -> SimpleResult #[inline] pub fn take_while1(mut i: I, f: F) -> SimpleResult where F: Fn(I::Token) -> bool { - match i.iter().position(|c| f(c) == false) { - Some(0) => i.err(Error::unexpected()), - Some(n) => { - let b = i.consume(n); + loop { + match i.iter().position(|c| f(c) == false) { + Some(0) => return i.err(Error::unexpected()), + Some(n) => { + let b = i.consume(n); + + return i.ret(b); + }, + None => {}, + } - i.ret(b) - }, - // TODO: Should this following 1 be something else, seeing as take_while1 is potentially - // infinite? - None => if i.min_remaining() > 0 && i.is_end() { - // Last slice and we have just read everything of it, replace with zero-sized slice: - let r = i.min_remaining(); - let b = i.consume(r); + // Attempt to fill + if i.fill() > 0 { + continue; + } - i.ret(b) - } else { - i.incomplete(1) - }, + // Last slice and we fail to get more, take remainder + let r = i.min_remaining(); + let b = i.consume(r); + + return i.ret(b); } } @@ -293,18 +287,23 @@ pub fn take_while1(mut i: I, f: F) -> SimpleResult /// assert_eq!(r, Ok(&b"abc"[..])); /// ``` #[inline] -pub fn take_till(mut i: I, f: F) -> SimpleResult +pub fn take_till, F>(mut i: I, f: F) -> SimpleResult where F: Fn(I::Token) -> bool, I::Token: Clone { - match i.iter().position(f) { - Some(n) => { + loop { + if let Some(n) = i.iter().position(&f) { let b = i.consume(n); - i.ret(b) - }, - // TODO: Should this following 1 be something else, seeing as take_while1 is potentially - // infinite? - None => i.incomplete(1), + return i.ret(b); + } + + // Attempt to fill + if i.fill() > 0 { + continue; + } + + // TODO: Incomplete + return i.err(Error::unexpected()); } } @@ -325,6 +324,8 @@ pub fn take_till(mut i: I, f: F) -> SimpleResult #[inline] pub fn scan(mut i: I, s: S, mut f: F) -> SimpleResult where F: FnMut(S, I::Token) -> Option { + unimplemented!() + /* let mut state = Some(s); match i.iter().position(|c| { state = f(mem::replace(&mut state, None).unwrap(), c); state.is_none()}) { @@ -337,6 +338,7 @@ pub fn scan(mut i: I, s: S, mut f: F) -> SimpleResult i.incomplete(1), } + */ } /// Like `scan` but generalized to return the final state of the scanner. @@ -355,6 +357,8 @@ pub fn scan(mut i: I, s: S, mut f: F) -> SimpleResult(mut i: I, s: S, mut f: F) -> SimpleResult where F: FnMut(S, I::Token) -> Option { + unimplemented!() + /* let mut state = s; match i.iter().position(|c| { let t = f(state, c); match t { None => true, Some(v) => { state = v; false } } }) { @@ -367,6 +371,7 @@ pub fn run_scanner(mut i: I, s: S, mut f: F) -> SimpleResu // infinite? None => i.incomplete(1), } + */ } /// Matches the remainder of the buffer and returns it, always succeeds. @@ -378,7 +383,9 @@ pub fn run_scanner(mut i: I, s: S, mut f: F) -> SimpleResu /// ``` #[inline] pub fn take_remainder(mut i: I) -> SimpleResult { - // Last slice and we have just read everything of it, replace with zero-sized slice: + + while i.fill() > 0 {} + let r = i.min_remaining(); let b = i.consume(r); @@ -398,18 +405,23 @@ pub fn take_remainder(mut i: I) -> SimpleResult { #[inline] pub fn string<'b, T: Copy + PartialEq, I: Input>(mut i: I, s: &'b [T]) -> SimpleResult { - let r = i.min_remaining(); + let mut r = i.min_remaining(); + + while r < s.len() { + let n = i.fill(); - if s.len() > r { - return i.incomplete(s.len() - r); + if n == 0 { + // TODO: Proper incomplete error here? + return i.err(Error::unexpected()); + } + + r += n } // TODO: Check if this is efficient: for (n, (a, b)) in i.iter().zip(s.iter()).enumerate() { if a != *b { - i.discard(n); - - return i.err(Error::expected(*b)); + return i.discard(n).err(Error::expected(*b)); } } @@ -428,8 +440,8 @@ pub fn string<'b, T: Copy + PartialEq, I: Input>(mut i: I, s: &'b [T]) /// assert_eq!(r, Ok(())); /// ``` #[inline] -pub fn eof(i: I) -> SimpleResult { - if i.min_remaining() == 0 && i.is_end() { +pub fn eof(mut i: I) -> SimpleResult { + if i.min_remaining() == 0 && i.fill() == 0 { i.ret(()) } else { i.err(Error::unexpected()) From f5f5e97da0f1572e8328884ce6bf6c4a48478142 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Thu, 24 Mar 2016 01:57:57 +0100 Subject: [PATCH 15/63] WIP --- src/buffer/buffer.rs | 22 +- src/buffer/data_source.rs | 4 +- src/buffer/mod.rs | 12 +- src/buffer/slice.rs | 12 +- src/buffer/stateful.rs | 14 +- src/combinators/bounded.rs | 3 +- src/combinators/mod.rs | 2 + src/input.rs | 521 ++++++++++++++++++------------------- src/lib.rs | 3 +- src/parse.rs | 17 +- src/parse_result.rs | 2 + src/parsers.rs | 252 +++++++----------- 12 files changed, 397 insertions(+), 467 deletions(-) diff --git a/src/buffer/buffer.rs b/src/buffer/buffer.rs index 43b0e19c..60368e3d 100644 --- a/src/buffer/buffer.rs +++ b/src/buffer/buffer.rs @@ -12,7 +12,7 @@ const DEFAULT_BUFFER_SIZE: usize = 6 * 1024; /// /// Enables the consumer to request specific amounts of data and only consume partial parts of the /// buffer. -pub trait Buffer: ops::Deref { +pub trait Buffer: ops::Deref { /// Attempt to fill the buffer using the closure `F`. /// /// The successful return from `F` should contain the number of items successfully written to @@ -59,7 +59,7 @@ pub trait Buffer: ops::Deref { /// Only allocates when created. // TODO: Tests #[derive(Debug, Eq, PartialEq)] -pub struct FixedSizeBuffer { +pub struct FixedSizeBuffer { /// Backing memory. buffer: Vec, /// Number of items of `buffer` which contain actual data. @@ -70,7 +70,7 @@ pub struct FixedSizeBuffer { used: Cell, } -impl FixedSizeBuffer { +impl FixedSizeBuffer { /// Creates a fixed-size buffer with the default buffer size. #[inline] pub fn new() -> Self { @@ -101,7 +101,7 @@ impl FixedSizeBuffer { } } -impl ops::Deref for FixedSizeBuffer { +impl ops::Deref for FixedSizeBuffer { type Target = [I]; #[inline] @@ -110,14 +110,14 @@ impl ops::Deref for FixedSizeBuffer { } } -impl ops::DerefMut for FixedSizeBuffer { +impl ops::DerefMut for FixedSizeBuffer { #[inline] fn deref_mut(&mut self) -> &mut [I] { &mut self.buffer[self.used.get()..self.populated] } } -impl Buffer for FixedSizeBuffer { +impl Buffer for FixedSizeBuffer { #[inline] fn fill>(&mut self, s: &mut S) -> io::Result { s.read(&mut self.buffer[self.populated..]).map(|n| { @@ -171,7 +171,7 @@ impl Buffer for FixedSizeBuffer { /// Will not decrease in size. // TODO: Tests #[derive(Debug)] -pub struct GrowingBuffer { +pub struct GrowingBuffer { /// Backing memory. buffer: Vec, /// Number of items of `buffer` which contain actual data. @@ -184,7 +184,7 @@ pub struct GrowingBuffer { used: Cell, } -impl GrowingBuffer { +impl GrowingBuffer { /// Creates a new unlimited `GrowingBuffer`. #[inline] pub fn new() -> Self { @@ -208,7 +208,7 @@ impl GrowingBuffer { } } -impl ops::Deref for GrowingBuffer { +impl ops::Deref for GrowingBuffer { type Target = [I]; #[inline] @@ -217,14 +217,14 @@ impl ops::Deref for GrowingBuffer { } } -impl ops::DerefMut for GrowingBuffer { +impl ops::DerefMut for GrowingBuffer { #[inline] fn deref_mut(&mut self) -> &mut [I] { &mut self.buffer[self.used.get()..self.populated] } } -impl Buffer for GrowingBuffer { +impl Buffer for GrowingBuffer { #[inline] fn fill>(&mut self, s: &mut S) -> io::Result { s.read(&mut self.buffer[self.populated..]).map(|n| { diff --git a/src/buffer/data_source.rs b/src/buffer/data_source.rs index 9eeb5546..c574d96c 100644 --- a/src/buffer/data_source.rs +++ b/src/buffer/data_source.rs @@ -5,7 +5,7 @@ use std::io; /// Abstraction over `io::Read`, `Iterator` and others. pub trait DataSource { /// The type of items this data source produces. - type Item: Copy; + type Item: Copy + PartialEq; /// Populates the supplied buffer with data, returns the number of items written. /// @@ -69,7 +69,7 @@ impl IteratorDataSource { } impl DataSource for IteratorDataSource - where I::Item: Copy { + where I::Item: Copy + PartialEq { type Item = I::Item; #[inline] diff --git a/src/buffer/mod.rs b/src/buffer/mod.rs index 7c579ba2..aa27a996 100644 --- a/src/buffer/mod.rs +++ b/src/buffer/mod.rs @@ -85,12 +85,8 @@ impl<'a, T, I: Input, E> From> for StreamError< use primitives::Primitives; match e { - ParseError::Error(mut b, e) => { - let r = b.min_remaining(); - - StreamError::ParseError(b.consume(r), e) - }, - ParseError::Incomplete(n) => StreamError::Incomplete(n), + ParseError::Error(mut b, e) => StreamError::ParseError(b.consume_remaining(), e), + ParseError::Incomplete(n) => StreamError::Incomplete(n), } } } @@ -98,7 +94,7 @@ impl<'a, T, I: Input, E> From> for StreamError< /// Trait wrapping the state management in reading from a data source while parsing. pub trait Stream<'a, 'i> { /// The input item type, usually depending on which `DataSource` is used. - type Item: 'i + Copy; + type Item: 'i + Copy + PartialEq; /// Attempts to run the supplied parser `F` once on the currently populated data in this /// stream, providing a borrow of the inner data storage. @@ -115,7 +111,7 @@ pub trait Stream<'a, 'i> { /// Trait for conversion into a `Stream`. pub trait IntoStream<'a, 'i> { /// The input item type provided by the stream. - type Item: 'i + Copy; + type Item: 'i + Copy + PartialEq; /// The `Stream` instance type. type Into: Stream<'a, 'i, Item=Self::Item>; diff --git a/src/buffer/slice.rs b/src/buffer/slice.rs index aaac9c7c..22046fea 100644 --- a/src/buffer/slice.rs +++ b/src/buffer/slice.rs @@ -68,7 +68,7 @@ impl<'i, I: 'i> SliceStream<'i, I> { } } -impl<'a, 'i, I: 'i + Copy> IntoStream<'a, 'i> for &'i [I] { +impl<'a, 'i, I: 'i + Copy + PartialEq> IntoStream<'a, 'i> for &'i [I] { type Item = I; type Into = SliceStream<'i, I>; @@ -78,7 +78,7 @@ impl<'a, 'i, I: 'i + Copy> IntoStream<'a, 'i> for &'i [I] { } } -impl<'a, 'i, I: 'i + Copy> Stream<'a, 'i> for SliceStream<'i, I> { +impl<'a, 'i, I: 'i + Copy + PartialEq> Stream<'a, 'i> for SliceStream<'i, I> { type Item = I; #[inline] @@ -92,10 +92,10 @@ impl<'a, 'i, I: 'i + Copy> Stream<'a, 'i> for SliceStream<'i, I> { return Err(StreamError::EndOfInput); } - match f(input::new_buf(input::END_OF_INPUT, &self.slice[self.pos..])).into_inner() { + match f(input::new_buf(input::DEFAULT, &self.slice[self.pos..])).into_inner() { State::Data(remainder, data) => { // TODO: Do something neater with the remainder - self.pos += self.len() - remainder.min_remaining(); + self.pos += self.len() - remainder.len(); Ok(data) }, @@ -106,11 +106,11 @@ impl<'a, 'i, I: 'i + Copy> Stream<'a, 'i> for SliceStream<'i, I> { } else { // TODO: Do something neater with the remainder // TODO: Detail this behaviour, maybe make it configurable - let r = remainder.min_remaining(); + let r = remainder.len(); self.pos += self.len() - r; - Err(StreamError::ParseError(remainder.consume(r), err)) + Err(StreamError::ParseError(remainder.consume_remaining(), err)) } }, } diff --git a/src/buffer/stateful.rs b/src/buffer/stateful.rs index 77a23d85..56ef7393 100644 --- a/src/buffer/stateful.rs +++ b/src/buffer/stateful.rs @@ -57,7 +57,7 @@ impl> Source, B> { } impl> Source, B> - where I::Item: Copy { + where I::Item: Copy + PartialEq { /// Creates a new `Source` from `Iterator` and `Buffer` instances. #[inline] pub fn from_iter(source: I, buffer: B) -> Self { @@ -218,9 +218,7 @@ impl<'a, S: DataSource, B: Buffer> Stream<'a, 'a> for Source return Err(StreamError::EndOfInput); } - let input_state = if self.state.contains(END_OF_INPUT) { input::END_OF_INPUT } else { input::DEFAULT }; - - match f(input::new_buf(input_state, &self.buffer)).into_inner() { + match f(input::new_buf(input::DEFAULT, &self.buffer)).into_inner() { State::Data(remainder, data) => { if remainder.is_incomplete() && self.state.contains(END_OF_INPUT) { // We can't accept this since we might have hit a premature end @@ -231,7 +229,7 @@ impl<'a, S: DataSource, B: Buffer> Stream<'a, 'a> for Source Err(StreamError::Retry) } else { // TODO: Do something neater with the remainder - self.buffer.consume(self.buffer.len() - remainder.min_remaining()); + self.buffer.consume(self.buffer.len() - remainder.len()); Ok(data) } @@ -251,11 +249,9 @@ impl<'a, S: DataSource, B: Buffer> Stream<'a, 'a> for Source } else { // TODO: Do something neater with the remainder // TODO: Detail this behaviour, maybe make it configurable - let r = remainder.min_remaining(); - - self.buffer.consume(self.buffer.len() - r); + self.buffer.consume(self.buffer.len() - remainder.len()); - Err(StreamError::ParseError(remainder.consume(r), err)) + Err(StreamError::ParseError(remainder.consume_remaining(), err)) } }, } diff --git a/src/combinators/bounded.rs b/src/combinators/bounded.rs index 6c347f8b..65f8a6f3 100644 --- a/src/combinators/bounded.rs +++ b/src/combinators/bounded.rs @@ -775,6 +775,7 @@ pub fn sep_by(i: I, r: R, mut p: P, mut sep: F BoundedRange::parse_many(r, i, parser) } +/* #[cfg(test)] mod test { use {Error, ParseResult}; @@ -1501,4 +1502,4 @@ mod test { let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"aaaab"), 2..1, |i| token(i, b'a'), |i| token(i, b'b')); assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ab"), vec![b'a', b'a', b'a'])); } -} +}*/ diff --git a/src/combinators/mod.rs b/src/combinators/mod.rs index 98708305..44d440bf 100644 --- a/src/combinators/mod.rs +++ b/src/combinators/mod.rs @@ -308,6 +308,7 @@ pub fn look_ahead(i: I, f: F) -> ParseResult } } +/* #[cfg(test)] mod test { use {Input, Error, ParseResult}; @@ -556,3 +557,4 @@ mod test { assert_eq!(look_ahead(new_buf(DEFAULT, b"aa"), |i| token(i, b'a').then(|i| token(i, b'b')).map_err(|_| "err")).into_inner(), State::Error(new_buf(DEFAULT, b"aa"), "err")); } } +*/ diff --git a/src/input.rs b/src/input.rs index 339b8b16..2d45e9fb 100644 --- a/src/input.rs +++ b/src/input.rs @@ -9,81 +9,11 @@ pub trait U8Input: Input {} impl U8Input for T where T: Input {} -/* -Behaviour on incomplete: - -any: - error, unexpected eof -satisfy: - error, unexpected eof -satisfy_with: - error, unexpected eof -token: - error, unexpected eof -not_token: - error, unexpected eof -peek: - return None -peek_next - error, unexpected eof -take: - error, unexpected eof -take_while: - try refill (success: resume scanning, fail: return remainder) -take_while1: - try refill (success: resume scanning, fail: return remainder) -take_till: - try refill (success: resume scanning, fail: error, unexpected eof) -scan: - try refill (success: resume scanning, fail: error, unexpected eof) -run_scanner: - try refill (success: resume scanning, fail: error, unexpected eof) -take_remainder: - try refill, return remainder -string: - try refill (success: resume scanning, fail: error, unexpected eof) -eof: - try refill (success: fail, unexpected, fail: success) - -option: - failed incomplete == error -or: - failed incomplete == error -matched_by: - failed incomplete == error -look_ahead: - failed incomplete == error - -Range - many: - failed incomplete == error - skip: - failed incomplete == error - many_till: - failed incomplete == error -RangeFrom - many: - failed incomplete == error - skip: - failed incomplete == error - many_till: - failed incomplete == error -RangeFull - many: - failed incomplete == error - skip: - failed incomplete == error - many_till: - failed incomplete == error -*/ - /// Primitive operations on `Input` types. /// /// All parsers only require a few primitive operations to parse data: pub mod primitives { - use {Input, ParseResult}; - use primitives::State; - use primitives::parse_result::new as new_result; + use Input; /// This is a zero-sized type used by the `Primitives` trait implementation to guarantee that /// access to primitive methods on `Input` only happens when the `Primitives` trait has been @@ -98,27 +28,27 @@ pub mod primitives { /// /// Only used by fundamental parsers and combinators. /// - // FIXME: Rename + // FIXME: Rename and documentation pub trait Primitives: Input { #[inline(always)] - fn first(&self) -> Option { - self._first(Guard(())) + fn peek(&mut self) -> Option { + self._peek(Guard(())) } #[inline(always)] - fn iter(&self) -> Self::Iter { - self._iter(Guard(())) + fn pop(&mut self) -> Option { + self._pop(Guard(())) } - #[inline(always)] - fn consume(&mut self, n: usize) -> Self::Buffer { + #[inline] + fn consume(&mut self, n: usize) -> Option { self._consume(Guard(()), n) } - // TODO: Change to self -> self? - #[inline(always)] - fn discard(self, n: usize) -> Self { - self._discard(Guard(()), n) + #[inline] + fn consume_while(&mut self, f: F) -> Self::Buffer + where F: FnMut(Self::Token) -> bool { + self._consume_while(Guard(()), f) } /// Returns the buffer from the marker `m` to the current position, discarding the @@ -128,20 +58,9 @@ pub mod primitives { self._consume_from(Guard(()), m) } - #[inline] - fn min_remaining(&self) -> usize { - self._min_remaining(Guard(())) - } - - /// Attempts to populate the input with more data, returning the number of additional - /// tokens which were added. - /// - /// # Primitive - /// - /// Only used by fundamental parsers and combinators. #[inline(always)] - fn fill(&mut self) -> usize { - self._fill(Guard(())) + fn consume_remaining(&mut self) -> Self::Buffer { + self._consume_remaining(Guard(())) } /// Marks the current position to be able to backtrack to it using `restore`. @@ -185,6 +104,16 @@ pub mod primitives { impl Primitives for I {} } +// FIXME: Docs +// TODO: More methods? +pub trait Buffer { + fn len(&self) -> usize; + + fn is_empty(&self) -> bool { + self.len() == 0 + } +} + // FIXME: Update, do not refer to type or linear type /// Linear type containing the parser state, this type is threaded though `bind` and is also the /// initial type passed to a parser. @@ -201,12 +130,14 @@ pub mod primitives { pub trait Input: Sized { /// The token type of the input // TODO: Maybe remove the copy bound at some point? - type Token: Copy; + type Token: Copy + PartialEq; /// A marker type which is used to backtrack using `_mark` and `_restore`. type Marker; + /// An iterator over the loaded data in the input. type Iter: Iterator; - type Buffer; + // TODO: Bounds for common usage (eg. to iterator?) + type Buffer: Buffer; /// Returns `t` as a success value in the parsing context. /// @@ -279,32 +210,39 @@ pub trait Input: Sized { } // Primitive methods - /// **Primitive:** See `Primitives::first` for documentation. + + /// **Primitive:** See `Primitives::peek` for documentation. /// - /// Returns the first remaining item if there are still data to read. - #[inline(always)] - fn _first(&self, g: Guard) -> Option { - self._iter(g).next() - } + /// Peeks at the next token in the input without consuming it. `None` if no more input is + /// available. + /// + /// Note: Will refill automatically. + #[inline] + fn _peek(&mut self, Guard) -> Option; - /// **Primitive:** See `Primitives::iter` for documentation. + /// **Primitive:** See `Primitives::pop` for documentation. /// - /// Iterator over tokens in the input, does not consume any data. + /// Pops a token off the start of the input. `None` if no more input is available. + /// + /// Note: Will refill automatically. #[inline] - fn _iter(&self, Guard) -> Self::Iter; + fn _pop(&mut self, Guard) -> Option; /// **Primitive:** See `Primitives::consume` for documentation. /// - /// Consumes a set amount of input tokens, returning a buffer containing them - // TODO: Should probably be combined with a ret + /// Attempt to consume `n` tokens, if it fails do not advance the position but return `None`. #[inline] - fn _consume(&mut self, Guard, usize) -> Self::Buffer; + fn _consume(&mut self, Guard, usize) -> Option; - /// **Primitive:** See `Primitives::discard` for documentation. + /// **Primitive:** See `Primitives::consume_while` for documentation. + /// + /// Runs the closure `F` on the tokens in order until it returns false, all tokens up to that + /// token will be returned as a buffer and discarded from the current input. /// - /// Consumes a set amount of input tokens, discarding them. + /// Note: Will refill automatically. #[inline] - fn _discard(self, Guard, usize) -> Self; + fn _consume_while(&mut self, Guard, F) -> Self::Buffer + where F: FnMut(Self::Token) -> bool; /// **Primitive:** See `Primitives::consume_from for documentation. /// @@ -313,19 +251,13 @@ pub trait Input: Sized { #[inline] fn _consume_from(&mut self, Guard, Self::Marker) -> Self::Buffer; - /// **Primitive:** See `Primitives::remaining` for documentation. + /// **Primitive:** See `Primitives::consume_remaining` for documentation. /// - /// Returns the number of tokens remaining in this input (only this part of it, more might - /// follow if `_is_end` is false). - #[inline] - fn _min_remaining(&self, Guard) -> usize; - - /// **Primitive:** See `Primitives::fill` for documentation. + /// Returns the remainder of the input in a buffer. /// - /// Attempts to populate the input with more data, returning the number of additional - /// tokens which were added. - #[inline(always)] - fn _fill(&mut self, Guard) -> usize; + /// Note: Will refill the intenal buffer until no more data is available if the underlying + /// implementation supports it. + fn _consume_remaining(&mut self, Guard) -> Self::Buffer; /// **Primitive:** See `Primitives::mark` for documentation. /// @@ -340,41 +272,79 @@ pub trait Input: Sized { fn _restore(self, Guard, Self::Marker) -> Self; } -impl<'a, I: Copy> Input for &'a [I] { +impl<'a, I> Buffer for &'a [I] { + fn len(&self) -> usize { + (&self[..]).len() + } +} + +impl<'a, I: Copy + PartialEq> Input for &'a [I] { type Token = I; type Marker = &'a [I]; type Iter = ::std::iter::Cloned<::std::slice::Iter<'a, I>>; type Buffer = &'a [I]; - #[inline(always)] - fn _iter(&self, _g: Guard) -> Self::Iter { - self.iter().cloned() + #[inline] + fn _peek(&mut self, _g: Guard) -> Option { + self.first().cloned() } - #[inline(always)] - fn _consume(&mut self, _g: Guard, n: usize) -> Self::Buffer { - let b = &self[..n]; - *self = &self[n..]; - b + #[inline] + fn _pop(&mut self, _g: Guard) -> Option { + self.first().cloned().map(|c| { + *self = &self[1..]; + + c + }) } - #[inline(always)] - fn _discard(self, _g: Guard, n: usize) -> Self { - &self[n..] + #[inline] + fn _consume(&mut self, _g: Guard, n: usize) -> Option { + if n > self.len() { + None + } else { + let b = &self[..n]; + + *self = &self[n..]; + + Some(b) + } + } + #[inline] + fn _consume_while(&mut self, _g: Guard, mut f: F) -> Self::Buffer + where F: FnMut(Self::Token) -> bool { + match self.iter().position(|c| !f(*c)) { + Some(n) => { + let b = &self[..n]; + + *self = &self[n..]; + + b + }, + None => { + let b = &self[..]; + + *self = &self[..0]; + + b + } + } } #[inline] fn _consume_from(&mut self, _g: Guard, m: Self::Marker) -> Self::Buffer { &m[..m.len() - self.len()] } - #[inline(always)] - fn _min_remaining(&self, _g: Guard) -> usize { - self.len() - } - #[inline(always)] - fn _fill(&mut self, _g: Guard) -> usize { - 0 + #[inline] + fn _consume_remaining(&mut self, _g: Guard) -> Self::Buffer { + let b = &self[..]; + + *self = &self[..0]; + + b } + #[inline] fn _mark(&self, _g: Guard) -> Self::Marker { &self } + #[inline] fn _restore(self, _g: Guard, m: Self::Marker) -> Self { m } @@ -384,10 +354,8 @@ bitflags!{ pub flags InputMode: u32 { /// Default (empty) input state. const DEFAULT = 0, - /// If set the current slice of input is the last one. - const END_OF_INPUT = 1, /// If a parser has attempted to read incomplete - const INCOMPLETE = 2, + const INCOMPLETE = 1, } } @@ -410,48 +378,95 @@ impl<'a, I: 'a> InputBuf<'a, I> { pub fn is_incomplete(&self) -> bool { self.0.contains(INCOMPLETE) } + + pub fn len(&self) -> usize { + self.1.len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } } -impl<'a, I: Copy> Input for InputBuf<'a, I> { +impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { type Token = I; // TODO: InputMode? INCOMPLETE must be set no matter what type Marker = &'a [I]; type Iter = ::std::iter::Cloned<::std::slice::Iter<'a, I>>; type Buffer = &'a [I]; - #[inline(always)] - fn _iter(&self, _g: Guard) -> Self::Iter { - self.1.iter().cloned() + #[inline] + fn _peek(&mut self, _g: Guard) -> Option { + match self.1.first() { + Some(c) => Some(*c), + None => { + self.0.insert(INCOMPLETE); + + None + }, + } } - #[inline(always)] - fn _consume(&mut self, _g: Guard, n: usize) -> Self::Buffer { - let b = &self.1[..n]; - self.1 = &self.1[n..]; - b + #[inline] + fn _pop(&mut self, g: Guard) -> Option { + self._peek(g).map(|c| { + self.1 = &self.1[1..]; + + c + }) } - #[inline(always)] - fn _discard(mut self, _g: Guard, n: usize) -> Self { - self.1 = &self.1[n..]; + #[inline] + fn _consume(&mut self, _g: Guard, n: usize) -> Option { + if n > self.1.len() { + self.0.insert(INCOMPLETE); - self + None + } else { + let b = &self.1[..n]; + + self.1 = &self.1[n..]; + + Some(b) + } + } + #[inline] + fn _consume_while(&mut self, _g: Guard, mut f: F) -> Self::Buffer + where F: FnMut(Self::Token) -> bool { + match self.1.iter().position(|c| !f(*c)) { + Some(n) => { + let b = &self.1[..n]; + + self.1 = &self.1[n..]; + + b + }, + None => { + self.0.insert(INCOMPLETE); + + let b = self.1; + + self.1 = &self.1[..0]; + + b + } + } } #[inline] fn _consume_from(&mut self, _g: Guard, m: Self::Marker) -> Self::Buffer { &m[..m.len() - self.1.len()] } - #[inline(always)] - fn _min_remaining(&self, _g: Guard) -> usize { - self.1.len() - } - #[inline(always)] - fn _fill(&mut self, _g: Guard) -> usize { - self.0.insert(INCOMPLETE); + #[inline] + fn _consume_remaining(&mut self, _g: Guard) -> Self::Buffer { + let b = self.1; + + self.1 = &self.1[..0]; - 0 + b } + #[inline] fn _mark(&self, _g: Guard) -> Self::Marker { &self.1 } + #[inline] fn _restore(mut self, _g: Guard, m: Self::Marker) -> Self { self.1 = m; @@ -459,117 +474,93 @@ impl<'a, I: Copy> Input for InputBuf<'a, I> { } } -// FIXME: Delete -/* -/// **Primitive:** Trait limiting the use of `Clone` for `Input`. -/// -/// # Primitive -/// -/// Only used by fundamental parsers and combinators. -/// -pub trait InputClone { - /// Creates a clone of the instance. - /// - /// # Primitive - /// - /// Only used by fundamental parsers and combinators. - #[inline(always)] - fn clone(&self) -> Self; +impl<'a> Buffer for &'a str { + fn len(&self) -> usize { + (&self[..]).len() + } } -/// **Primitive:** Trait exposing the buffer of `Input`. -/// -/// # Primitive -/// -/// Only used by fundamental parsers and combinators. -/// -pub trait InputBuffer<'a> { - /// The type of each element of the buffer. - type Item: 'a; - - /// Reveals the internal buffer containig the remainder of the input. - /// - /// # Primitive - /// - /// Only used by fundamental parsers and combinators. - #[inline(always)] - fn buffer(&self) -> &'a [Self::Item]; +impl<'a> Input for &'a str { + type Token = char; + type Marker = &'a str; + type Iter = ::std::str::Chars<'a>; + type Buffer = &'a str; - /// Modifies the inner data without leaving the `Input` context. - /// - /// # Primitive - /// - /// Only used by fundamental parsers and combinators. - #[inline(always)] - fn replace(self, &'a [Self::Item]) -> Self; + #[inline] + fn _peek(&mut self, _g: Guard) -> Option { + self.chars().next() + } + #[inline] + fn _pop(&mut self, _g: Guard) -> Option { + let mut iter = self.char_indices(); - /// Returns true if this is the last available slice of the input. - /// - /// # Primitive - /// - /// Only used by fundamental parsers and combinators. - #[inline(always)] - fn is_last_slice(&self) -> bool; -} + iter.next().map(|(_, c)| { + match iter.next().map(|(p, _)| p) { + Some(n) => *self = &self[n..], + None => *self = &self[..0], + } -/// Trait limiting the use of `Clone` for `Input`. -/// -/// # Primitive -/// -/// Only used by fundamental parsers and combinators. -/// -/// # Motivation -/// -/// The `Input` type is supposed to be an approximation of a linear type when observed in the -/// monadic parser context. This means that it should not be possible to duplicate or accidentally -/// throw it away as well as restrict when and where an `Input` can be constructed. Not -/// implementing `Clone` or `Copy` solves the first issue. -/// -/// However, cloning an `Input` is necessary for backtracking and also allows for slightly more -/// efficient iteration in combinators. This trait allows us to enable cloning selectively. -impl<'a, I: 'a> InputClone for Input<'a, I> { - #[inline(always)] - fn clone(&self) -> Self { - Input(self.0, self.1) + c + }) } -} + #[inline] + fn _consume(&mut self, _g: Guard, n: usize) -> Option { + match self.char_indices().enumerate().take(n + 1).last() { + // n always less than num if self contains more than n characters + Some((num, (pos, _))) if n < num => { + let b = &self[..pos]; -/// Trait exposing the buffer of `Input`. -/// -/// # Primitive -/// -/// Only used by fundamental parsers and combinators. -/// -/// # Motivation -/// -/// The `Input` type is supposed to be an approximation of a linear type when observed in the -/// monadic parser context. This means that it should not be possible to duplicate or accidentally -/// throw it away as well as restrict when and where an `Input` can be constructed. Not exposing -/// the constructor (to allow destructuring) as well as using `#[must_use]` solves the second -/// issue. -/// -/// But to be able to parse data the contents of the `Input` type must be exposed in at least one -/// point, so that data can be examined, and this trait that makes it possible. -impl<'a, I: 'a> InputBuffer<'a> for Input<'a, I> { - type Item = I; + *self = &self[pos..]; - #[inline(always)] - fn buffer(&self) -> &'a [Self::Item] { - self.1 + Some(b) + }, + _ => None, + } } + #[inline] + fn _consume_while(&mut self, _g: Guard, mut f: F) -> Self::Buffer + where F: FnMut(Self::Token) -> bool { + // We need to find the character following the one which did not match + match self.char_indices().skip_while(|&(_, c)| f(c)).next() { + Some((pos, _)) => { + let b = &self[..pos]; + + *self = &self[pos..]; + + b + }, + None => { + let b = &self[..]; + + *self = &self[..0]; - #[inline(always)] - fn replace(self, b: &'a [Self::Item]) -> Self { - Input(self.0, b) + b + } + } + } + #[inline] + fn _consume_from(&mut self, _g: Guard, m: Self::Marker) -> Self::Buffer { + &m[..m.len() - self.len()] } + #[inline] + fn _consume_remaining(&mut self, _g: Guard) -> Self::Buffer { + let b = &self[..]; + + *self = &self[..0]; - #[inline(always)] - fn is_last_slice(&self) -> bool { - self.0.contains(END_OF_INPUT) + b + } + #[inline] + fn _mark(&self, _g: Guard) -> Self::Marker { + &self + } + #[inline] + fn _restore(self, _g: Guard, m: Self::Marker) -> Self { + m } } -*/ +/* #[cfg(test)] mod test { use std::fmt::Debug; @@ -602,6 +593,7 @@ mod test { assert_eq!(r2.into_inner(), State::Error(new_buf(DEFAULT, b"in2"), 23i32)); } + /* #[test] fn primitives_slice() { use primitives::Primitives; @@ -641,7 +633,6 @@ mod test { fn run_primitives_test PartialEq<&'a [u8]>, I: Input>(mut s: I, last: bool) { use primitives::Primitives; - assert_eq!(s.is_end(), last); assert_eq!(s.min_remaining(), 3); let m = s.mark(); assert_eq!(s.min_remaining(), 3); @@ -655,7 +646,6 @@ mod test { assert_eq!(s.consume(1), &b"c"[..]); assert_eq!(s.min_remaining(), 0); assert_eq!(s.iter().collect::>(), vec![]); - assert_eq!(s.is_end(), last); let mut s = s.restore(m); assert_eq!(s.min_remaining(), 3); assert_eq!(s.iter().collect::>(), vec![b'a', b'b', b'c']); @@ -664,4 +654,5 @@ mod test { assert_eq!(s.min_remaining(), 2); assert_eq!(s.iter().collect::>(), vec![b'b', b'c']); } -} + */ +}*/ diff --git a/src/lib.rs b/src/lib.rs index 9a137aae..fc00eb57 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -271,6 +271,7 @@ pub use parse_result::{ /// /// Only used by fundamental parsers and combinators. pub mod primitives { + pub use input::Buffer; pub use input::primitives::{ Primitives, Guard, @@ -286,7 +287,7 @@ pub mod primitives { /// /// Only used by fundamental parsers and combinators. pub mod input { - pub use input::{DEFAULT, END_OF_INPUT, INCOMPLETE, new_buf}; + pub use input::{DEFAULT, INCOMPLETE, new_buf}; } /// ParseResult utilities. diff --git a/src/parse.rs b/src/parse.rs index f439f0fa..730fbfb2 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -48,17 +48,12 @@ pub enum ParseError { /// Err(ParseError::Error(&b" and more"[..], Error::new()))); /// # } /// ``` -pub fn parse_only<'a, I: Copy, T, E, F>(parser: F, input: &'a [I]) -> Result> - where T: 'a, - E: 'a, +pub fn parse_only<'a, I, T, E, F>(parser: F, input: &'a [I]) -> Result> + where I: Copy + PartialEq, F: FnOnce(&'a [I]) -> ParseResult<&'a [I], T, E> { match parser(input).into_inner() { - State::Data(_, t) => Ok(t), - State::Error(mut b, e) => Err(ParseError::Error({ - let r = b.min_remaining(); - - b.consume(r) - }, e)), + State::Data(_, t) => Ok(t), + State::Error(mut b, e) => Err(ParseError::Error(b.consume_remaining(), e)), } } @@ -71,6 +66,7 @@ mod test { parse_only, }; + /* #[test] fn inspect_input() { let mut state = None; @@ -86,6 +82,7 @@ mod test { assert_eq!(input, Some(b"the input".to_vec())); assert_eq!(state, Some(true)); } + */ #[test] fn err() { @@ -96,8 +93,10 @@ mod test { }, b"the input"), Err(ParseError::Error(&b"input"[..], "my error"))); } + /* #[test] fn incomplete() { assert_eq!(parse_only(|i| i.incomplete::<(), ()>(23), b"the input"), Err(ParseError::Incomplete(23))); } + */ } diff --git a/src/parse_result.rs b/src/parse_result.rs index a5d75a06..e8448d2b 100644 --- a/src/parse_result.rs +++ b/src/parse_result.rs @@ -286,6 +286,7 @@ impl IntoInner for ParseResult { } } +/* #[cfg(test)] mod test { use input; @@ -419,3 +420,4 @@ mod test { assert_eq!(rhs.0, State::Data(&b"test"[..], 124)); } } +*/ diff --git a/src/parsers.rs b/src/parsers.rs index fd1ce181..3c40dbd2 100644 --- a/src/parsers.rs +++ b/src/parsers.rs @@ -4,7 +4,7 @@ use std::mem; use input::Input; use parse_result::SimpleResult; -use primitives::Primitives; +use primitives::{Primitives, Buffer}; pub use self::error::Error; @@ -24,7 +24,10 @@ pub use debugtrace::StackFrame; /// ``` #[inline] pub fn any(mut i: I) -> SimpleResult { - peek_next(i).bind(|i, c| i.discard(1).ret(c)) + match i.pop() { + Some(c) => i.ret(c), + None => i.err(Error::unexpected()), + } } /// Matches an item using ``f``, the item is returned if ``f`` yields true, otherwise this parser @@ -40,11 +43,10 @@ pub fn any(mut i: I) -> SimpleResult { #[inline] pub fn satisfy(mut i: I, f: F) -> SimpleResult where F: FnOnce(I::Token) -> bool { - peek_next(i).bind(|i, c| if f(c) { - i.discard(1).ret(c) - } else { - i.err(Error::unexpected()) - }) + match i.peek() { + Some(c) if f(c) => { i.pop(); i.ret(c) }, + _ => i.err(Error::unexpected()), + } } /// Reads a single token, applies the transformation `F` and then succeeds with the transformed @@ -65,15 +67,18 @@ pub fn satisfy(mut i: I, f: F) -> SimpleResult pub fn satisfy_with(mut i: I, f: F, p: P) -> SimpleResult where F: FnOnce(I::Token) -> T, P: FnOnce(T) -> bool { - peek_next(i).bind(|i, c| { - let t = f(c); - - if p(t.clone()) { - i.discard(1).ret(t) - } else { - i.err(Error::unexpected()) - } - }) + match i.peek().map(f) { + Some(c) => { + if p(c.clone()) { + i.pop(); + + i.ret(c) + } else { + i.err(Error::unexpected()) + } + }, + _ => i.err(Error::unexpected()), + } } /// Matches a single token, returning the match on success. @@ -86,13 +91,11 @@ pub fn satisfy_with(mut i: I, f: F, p: P) -> SimpleRes /// assert_eq!(parse_only(|i| token(i, b'a'), b"abc"), Ok(b'a')); /// ``` #[inline] -pub fn token(mut i: I, t: I::Token) -> SimpleResult - where I::Token: PartialEq { - peek_next(i).bind(|i, c| if t == c { - i.discard(1).ret(c) - } else { - i.err(Error::expected(t)) - }) +pub fn token(mut i: I, t: I::Token) -> SimpleResult { + match i.peek() { + Some(c) if c == t => { i.pop(); i.ret(c) }, + _ => i.err(Error::unexpected()), + } } /// Matches a single token as long as it is not equal to `t`, returning the match on success. @@ -105,13 +108,11 @@ pub fn token(mut i: I, t: I::Token) -> SimpleResult /// assert_eq!(parse_only(|i| not_token(i, b'b'), b"abc"), Ok(b'a')); /// ``` #[inline] -pub fn not_token(mut i: I, t: I::Token) -> SimpleResult - where I::Token: PartialEq { - peek_next(i).bind(|i, c| if t != c { - i.discard(1).ret(c) - } else { - i.err(Error::unexpected()) - }) +pub fn not_token(mut i: I, t: I::Token) -> SimpleResult { + match i.peek() { + Some(c) if c != t => { i.pop(); i.ret(c) }, + _ => i.err(Error::unexpected()), + } } /// Matches any item but does not consume it, on success it gives ``Some`` but if no input remains @@ -127,11 +128,10 @@ pub fn not_token(mut i: I, t: I::Token) -> SimpleResult /// assert_eq!(parse_only(peek, b""), Ok(None)); /// ``` #[inline(always)] -pub fn peek(i: I) -> SimpleResult> - where I::Token: Clone { - let c = i.first(); +pub fn peek(mut i: I) -> SimpleResult> { + let t = i.peek(); - i.ret(c) + i.ret(t) } /// Matches any item but does not consume it. @@ -144,18 +144,10 @@ pub fn peek(i: I) -> SimpleResult> /// assert_eq!(parse_only(peek_next, b"abc"), Ok(b'a')); /// ``` #[inline(always)] -pub fn peek_next(mut i: I) -> SimpleResult - where I::Token: Clone { - match i.first() { +pub fn peek_next(mut i: I) -> SimpleResult { + match i.peek() { Some(c) => i.ret(c), - None => if i.fill() > 0 { - let c = i.first() - .expect("peek_next: Failed to obtain first token after fill() returned > 0"); - - i.ret(c) - } else { - i.err(Error::unexpected()) - }, + None => i.err(Error::unexpected()), } } @@ -170,22 +162,11 @@ pub fn peek_next(mut i: I) -> SimpleResult /// ``` #[inline] pub fn take(mut i: I, num: usize) -> SimpleResult { - let mut r = i.min_remaining(); - - while r < num { - let n = i.fill(); - - if n == 0 { - // TODO: Proper incomplete error here? - return i.err(Error::unexpected()); - } - - r += n + match i.consume(num) { + Some(b) => i.ret(b), + // TODO: Proper incomplete error here? + None => i.err(Error::unexpected()), } - - let b = i.consume(num); - - i.ret(b) } /// Matches all items while ``f`` returns false, returns a slice of all the matched items. @@ -213,24 +194,9 @@ pub fn take(mut i: I, num: usize) -> SimpleResult { #[inline] pub fn take_while(mut i: I, f: F) -> SimpleResult where F: Fn(I::Token) -> bool { - loop { - if let Some(n) = i.iter().position(|c| f(c) == false) { - let b = i.consume(n); - - return i.ret(b); - } - - // Attempt to fill - if i.fill() > 0 { - continue; - } - - // Last slice and we fail to get more, take remainder - let r = i.min_remaining(); - let b = i.consume(r); + let b = i.consume_while(f); - return i.ret(b); - } + i.ret(b) } /// Matches all items while ``f`` returns true, if at least one item matched this parser succeeds @@ -249,27 +215,12 @@ pub fn take_while(mut i: I, f: F) -> SimpleResult #[inline] pub fn take_while1(mut i: I, f: F) -> SimpleResult where F: Fn(I::Token) -> bool { - loop { - match i.iter().position(|c| f(c) == false) { - Some(0) => return i.err(Error::unexpected()), - Some(n) => { - let b = i.consume(n); - - return i.ret(b); - }, - None => {}, - } - - // Attempt to fill - if i.fill() > 0 { - continue; - } - - // Last slice and we fail to get more, take remainder - let r = i.min_remaining(); - let b = i.consume(r); + let b = i.consume_while(f); - return i.ret(b); + if b.is_empty() { + i.err(Error::unexpected()) + } else { + i.ret(b) } } @@ -290,20 +241,24 @@ pub fn take_while1(mut i: I, f: F) -> SimpleResult pub fn take_till, F>(mut i: I, f: F) -> SimpleResult where F: Fn(I::Token) -> bool, I::Token: Clone { - loop { - if let Some(n) = i.iter().position(&f) { - let b = i.consume(n); + // TODO: How to check the last token? to make sure f succeeded on it? + let mut ok = false; - return i.ret(b); - } + let b = i.consume_while(|c| { + if !f(c) { + true + } else { + ok = true; - // Attempt to fill - if i.fill() > 0 { - continue; + false } + }); - // TODO: Incomplete - return i.err(Error::unexpected()); + if ok { + i.ret(b) + } else { + // TODO: Error could not find + i.err(Error::unexpected()) } } @@ -324,21 +279,11 @@ pub fn take_till, F>(mut i: I, f: F) -> SimpleResult(mut i: I, s: S, mut f: F) -> SimpleResult where F: FnMut(S, I::Token) -> Option { - unimplemented!() - /* let mut state = Some(s); - match i.iter().position(|c| { state = f(mem::replace(&mut state, None).unwrap(), c); state.is_none()}) { - Some(n) => { - let b = i.consume(n); + let b = i.consume_while(|c| { state = f(mem::replace(&mut state, None).unwrap(), c); state.is_none()}); - i.ret(b) - }, - // TODO: Should this following 1 be something else, seeing as take_while1 is potentially - // infinite? - None => i.incomplete(1), - } - */ + i.ret(b) } /// Like `scan` but generalized to return the final state of the scanner. @@ -357,21 +302,17 @@ pub fn scan(mut i: I, s: S, mut f: F) -> SimpleResult(mut i: I, s: S, mut f: F) -> SimpleResult where F: FnMut(S, I::Token) -> Option { - unimplemented!() - /* let mut state = s; - match i.iter().position(|c| { let t = f(state, c); match t { None => true, Some(v) => { state = v; false } } }) { - Some(n) => { - let b = i.consume(n); + let b = i.consume_while(|c| { + let t = f(state, c); + match t { + None => true, + Some(v) => { state = v; false } + } + }); - i.ret((b, state)) - }, - // TODO: Should this following 1 be something else, seeing as take_while1 is potentially - // infinite? - None => i.incomplete(1), - } - */ + i.ret((b, state)) } /// Matches the remainder of the buffer and returns it, always succeeds. @@ -383,11 +324,7 @@ pub fn run_scanner(mut i: I, s: S, mut f: F) -> SimpleResu /// ``` #[inline] pub fn take_remainder(mut i: I) -> SimpleResult { - - while i.fill() > 0 {} - - let r = i.min_remaining(); - let b = i.consume(r); + let b = i.consume_remaining(); i.ret(b) } @@ -405,29 +342,32 @@ pub fn take_remainder(mut i: I) -> SimpleResult { #[inline] pub fn string<'b, T: Copy + PartialEq, I: Input>(mut i: I, s: &'b [T]) -> SimpleResult { - let mut r = i.min_remaining(); - - while r < s.len() { - let n = i.fill(); - - if n == 0 { - // TODO: Proper incomplete error here? - return i.err(Error::unexpected()); + let mut n = 0; + let len = s.len(); + let mut ok = true; + + // TODO: There has to be some more efficient way here + // also doesn't seem to work nicely with string yet + let b = i.consume_while(|c| { + if n >= len { + false } + else if c == s[n] { + n += 1; - r += n - } + true + } else { + ok = false; - // TODO: Check if this is efficient: - for (n, (a, b)) in i.iter().zip(s.iter()).enumerate() { - if a != *b { - return i.discard(n).err(Error::expected(*b)); + false } - } - - let b = i.consume(s.len()); + }); - i.ret(b) + if ok { + i.ret(b) + } else { + i.err(Error::expected(s[b.len()])) + } } /// Matches the end of the input. @@ -441,7 +381,7 @@ pub fn string<'b, T: Copy + PartialEq, I: Input>(mut i: I, s: &'b [T]) /// ``` #[inline] pub fn eof(mut i: I) -> SimpleResult { - if i.min_remaining() == 0 && i.fill() == 0 { + if i.peek() == None { i.ret(()) } else { i.err(Error::unexpected()) @@ -603,6 +543,7 @@ mod error { } } +/* #[cfg(test)] mod test { use primitives::input::{new_buf, DEFAULT}; @@ -766,3 +707,4 @@ mod test { assert!(this.name.as_ref().map(|n| n.contains("parsers::test::backtrace_test")).unwrap_or(false), "Expected trace to contain \"parsers::test::backtrace_test\", got: {:?}", this.name.as_ref()); } } +*/ From 4c7c3a3b7238ff366a315fdc9ad55f1a02b3eb57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Thu, 24 Mar 2016 21:08:49 +0100 Subject: [PATCH 16/63] Indent, #inline and comments --- src/input.rs | 55 +++++++++++++++++++++++++++++++++++++++----------- src/parsers.rs | 7 ++++--- 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/src/input.rs b/src/input.rs index 2d45e9fb..b8ccf3b1 100644 --- a/src/input.rs +++ b/src/input.rs @@ -40,12 +40,12 @@ pub mod primitives { self._pop(Guard(())) } - #[inline] + #[inline(always)] fn consume(&mut self, n: usize) -> Option { self._consume(Guard(()), n) } - #[inline] + #[inline(always)] fn consume_while(&mut self, f: F) -> Self::Buffer where F: FnMut(Self::Token) -> bool { self._consume_while(Guard(()), f) @@ -131,11 +131,11 @@ pub trait Input: Sized { /// The token type of the input // TODO: Maybe remove the copy bound at some point? type Token: Copy + PartialEq; - /// A marker type which is used to backtrack using `_mark` and `_restore`. + /// A marker type which is used to backtrack using `_mark` and `_restore`. It should also be + /// possible to use this type to consume the data from the marked position to the current + /// position. type Marker; - /// An iterator over the loaded data in the input. - type Iter: Iterator; // TODO: Bounds for common usage (eg. to iterator?) type Buffer: Buffer; @@ -236,8 +236,9 @@ pub trait Input: Sized { /// **Primitive:** See `Primitives::consume_while` for documentation. /// - /// Runs the closure `F` on the tokens in order until it returns false, all tokens up to that - /// token will be returned as a buffer and discarded from the current input. + /// Runs the closure `F` on the tokens *in order* until it returns false, all tokens up to that + /// token will be returned as a buffer and discarded from the current input. MUST never run the + /// closure more than once on the exact same token. /// /// Note: Will refill automatically. #[inline] @@ -257,6 +258,7 @@ pub trait Input: Sized { /// /// Note: Will refill the intenal buffer until no more data is available if the underlying /// implementation supports it. + #[inline] fn _consume_remaining(&mut self, Guard) -> Self::Buffer; /// **Primitive:** See `Primitives::mark` for documentation. @@ -274,6 +276,7 @@ pub trait Input: Sized { impl<'a, I> Buffer for &'a [I] { fn len(&self) -> usize { + // Slice to reach inherent method to prevent infinite recursion (&self[..]).len() } } @@ -281,13 +284,13 @@ impl<'a, I> Buffer for &'a [I] { impl<'a, I: Copy + PartialEq> Input for &'a [I] { type Token = I; type Marker = &'a [I]; - type Iter = ::std::iter::Cloned<::std::slice::Iter<'a, I>>; type Buffer = &'a [I]; #[inline] fn _peek(&mut self, _g: Guard) -> Option { self.first().cloned() } + #[inline] fn _pop(&mut self, _g: Guard) -> Option { self.first().cloned().map(|c| { @@ -296,6 +299,7 @@ impl<'a, I: Copy + PartialEq> Input for &'a [I] { c }) } + #[inline] fn _consume(&mut self, _g: Guard, n: usize) -> Option { if n > self.len() { @@ -308,6 +312,7 @@ impl<'a, I: Copy + PartialEq> Input for &'a [I] { Some(b) } } + #[inline] fn _consume_while(&mut self, _g: Guard, mut f: F) -> Self::Buffer where F: FnMut(Self::Token) -> bool { @@ -328,10 +333,12 @@ impl<'a, I: Copy + PartialEq> Input for &'a [I] { } } } + #[inline] fn _consume_from(&mut self, _g: Guard, m: Self::Marker) -> Self::Buffer { &m[..m.len() - self.len()] } + #[inline] fn _consume_remaining(&mut self, _g: Guard) -> Self::Buffer { let b = &self[..]; @@ -340,16 +347,19 @@ impl<'a, I: Copy + PartialEq> Input for &'a [I] { b } + #[inline] fn _mark(&self, _g: Guard) -> Self::Marker { &self } + #[inline] fn _restore(self, _g: Guard, m: Self::Marker) -> Self { m } } +// TODO: Replace with boolean flag bitflags!{ pub flags InputMode: u32 { /// Default (empty) input state. @@ -359,8 +369,9 @@ bitflags!{ } } - +// FIXME: Docs /// Input buffer type which contains a flag which tells if we might have more data to read. +// TODO: Move to buffer module and make public? #[must_use] #[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] pub struct InputBuf<'a, I: 'a>(InputMode, &'a [I]); @@ -370,19 +381,25 @@ pub struct InputBuf<'a, I: 'a>(InputMode, &'a [I]); /// # Primitive /// /// Only used by fundamental parsers and combinators. +// TODO: Remove and replace with new? +#[inline] pub fn new_buf(state: InputMode, buffer: &[I]) -> InputBuf { InputBuf(state, buffer) } +// FIXME: Docs impl<'a, I: 'a> InputBuf<'a, I> { + #[inline] pub fn is_incomplete(&self) -> bool { self.0.contains(INCOMPLETE) } + #[inline] pub fn len(&self) -> usize { self.1.len() } + #[inline] pub fn is_empty(&self) -> bool { self.len() == 0 } @@ -390,9 +407,7 @@ impl<'a, I: 'a> InputBuf<'a, I> { impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { type Token = I; - // TODO: InputMode? INCOMPLETE must be set no matter what type Marker = &'a [I]; - type Iter = ::std::iter::Cloned<::std::slice::Iter<'a, I>>; type Buffer = &'a [I]; #[inline] @@ -406,6 +421,7 @@ impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { }, } } + #[inline] fn _pop(&mut self, g: Guard) -> Option { self._peek(g).map(|c| { @@ -414,6 +430,7 @@ impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { c }) } + #[inline] fn _consume(&mut self, _g: Guard, n: usize) -> Option { if n > self.1.len() { @@ -428,6 +445,7 @@ impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { Some(b) } } + #[inline] fn _consume_while(&mut self, _g: Guard, mut f: F) -> Self::Buffer where F: FnMut(Self::Token) -> bool { @@ -450,10 +468,12 @@ impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { } } } + #[inline] fn _consume_from(&mut self, _g: Guard, m: Self::Marker) -> Self::Buffer { &m[..m.len() - self.1.len()] } + #[inline] fn _consume_remaining(&mut self, _g: Guard) -> Self::Buffer { let b = self.1; @@ -462,10 +482,14 @@ impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { b } + #[inline] fn _mark(&self, _g: Guard) -> Self::Marker { + // Incomplete state is separate from the parsed state, no matter how we hit incomplete we + // want to keep it. &self.1 } + #[inline] fn _restore(mut self, _g: Guard, m: Self::Marker) -> Self { self.1 = m; @@ -476,6 +500,7 @@ impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { impl<'a> Buffer for &'a str { fn len(&self) -> usize { + // Slice to reach inherent method to prevent infinite recursion (&self[..]).len() } } @@ -483,13 +508,13 @@ impl<'a> Buffer for &'a str { impl<'a> Input for &'a str { type Token = char; type Marker = &'a str; - type Iter = ::std::str::Chars<'a>; type Buffer = &'a str; #[inline] fn _peek(&mut self, _g: Guard) -> Option { self.chars().next() } + #[inline] fn _pop(&mut self, _g: Guard) -> Option { let mut iter = self.char_indices(); @@ -503,6 +528,7 @@ impl<'a> Input for &'a str { c }) } + #[inline] fn _consume(&mut self, _g: Guard, n: usize) -> Option { match self.char_indices().enumerate().take(n + 1).last() { @@ -517,6 +543,7 @@ impl<'a> Input for &'a str { _ => None, } } + #[inline] fn _consume_while(&mut self, _g: Guard, mut f: F) -> Self::Buffer where F: FnMut(Self::Token) -> bool { @@ -538,10 +565,12 @@ impl<'a> Input for &'a str { } } } + #[inline] fn _consume_from(&mut self, _g: Guard, m: Self::Marker) -> Self::Buffer { &m[..m.len() - self.len()] } + #[inline] fn _consume_remaining(&mut self, _g: Guard) -> Self::Buffer { let b = &self[..]; @@ -550,10 +579,12 @@ impl<'a> Input for &'a str { b } + #[inline] fn _mark(&self, _g: Guard) -> Self::Marker { &self } + #[inline] fn _restore(self, _g: Guard, m: Self::Marker) -> Self { m diff --git a/src/parsers.rs b/src/parsers.rs index 3c40dbd2..ac870876 100644 --- a/src/parsers.rs +++ b/src/parsers.rs @@ -127,7 +127,7 @@ pub fn not_token(mut i: I, t: I::Token) -> SimpleResult { /// /// assert_eq!(parse_only(peek, b""), Ok(None)); /// ``` -#[inline(always)] +#[inline] pub fn peek(mut i: I) -> SimpleResult> { let t = i.peek(); @@ -143,7 +143,7 @@ pub fn peek(mut i: I) -> SimpleResult> { /// /// assert_eq!(parse_only(peek_next, b"abc"), Ok(b'a')); /// ``` -#[inline(always)] +#[inline] pub fn peek_next(mut i: I) -> SimpleResult { match i.peek() { Some(c) => i.ret(c), @@ -339,6 +339,7 @@ pub fn take_remainder(mut i: I) -> SimpleResult { /// /// assert_eq!(parse_only(|i| string(i, b"abc"), b"abcdef"), Ok(&b"abc"[..])); /// ``` +// TODO: Does not actually work with &str yet #[inline] pub fn string<'b, T: Copy + PartialEq, I: Input>(mut i: I, s: &'b [T]) -> SimpleResult { @@ -347,7 +348,6 @@ pub fn string<'b, T: Copy + PartialEq, I: Input>(mut i: I, s: &'b [T]) let mut ok = true; // TODO: There has to be some more efficient way here - // also doesn't seem to work nicely with string yet let b = i.consume_while(|c| { if n >= len { false @@ -543,6 +543,7 @@ mod error { } } +// FIXME: Uncomment and fix /* #[cfg(test)] mod test { From 46ae0aab7794070c115df91748a194b8fd647e67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Thu, 24 Mar 2016 21:44:18 +0100 Subject: [PATCH 17/63] Moved Input and ParseResult into types module, made primitives a real module --- src/ascii.rs | 18 ++-- src/buffer/mod.rs | 2 +- src/buffer/slice.rs | 5 +- src/buffer/stateful.rs | 5 +- src/combinators/bounded.rs | 2 +- src/combinators/mod.rs | 14 +++- src/lib.rs | 131 ++++++++++------------------- src/parse.rs | 9 +- src/parsers.rs | 12 ++- src/primitives.rs | 129 ++++++++++++++++++++++++++++ src/{ => types}/input.rs | 143 +++++--------------------------- src/types/mod.rs | 12 +++ src/{ => types}/parse_result.rs | 45 +--------- 13 files changed, 252 insertions(+), 275 deletions(-) create mode 100644 src/primitives.rs rename src/{ => types}/input.rs (80%) create mode 100644 src/types/mod.rs rename src/{ => types}/parse_result.rs (90%) diff --git a/src/ascii.rs b/src/ascii.rs index 868c9eeb..eeae2ed3 100644 --- a/src/ascii.rs +++ b/src/ascii.rs @@ -1,13 +1,21 @@ //! Utilities and parsers for dealing with ASCII data in `u8` format. -use conv::{NoError, ValueFrom}; -use conv::errors::UnwrapOk; - use std::ops::{Add, Mul}; -use {Input, SimpleResult}; +use conv::{ + NoError, + ValueFrom, +}; +use conv::errors::UnwrapOk; + +use types::Input; use combinators::option; -use parsers::{take_while, take_while1, satisfy}; +use parsers::{ + SimpleResult, + satisfy, + take_while, + take_while1, +}; /// Lowercase ASCII predicate. #[inline] diff --git a/src/buffer/mod.rs b/src/buffer/mod.rs index aa27a996..dbefac12 100644 --- a/src/buffer/mod.rs +++ b/src/buffer/mod.rs @@ -35,7 +35,7 @@ pub mod data_source; use std::io; -use {ParseResult, Input, InputBuf}; +use types::{ParseResult, Input, InputBuf}; use parse::ParseError; pub use self::slice::SliceStream; diff --git a/src/buffer/slice.rs b/src/buffer/slice.rs index 22046fea..378e4900 100644 --- a/src/buffer/slice.rs +++ b/src/buffer/slice.rs @@ -1,7 +1,6 @@ -use primitives::input; use primitives::{State, IntoInner}; -use {InputBuf, ParseResult}; +use types::{InputBuf, ParseResult}; use buffer::{IntoStream, StreamError, Stream}; /// Stream implementation for immutable slices. @@ -92,7 +91,7 @@ impl<'a, 'i, I: 'i + Copy + PartialEq> Stream<'a, 'i> for SliceStream<'i, I> { return Err(StreamError::EndOfInput); } - match f(input::new_buf(input::DEFAULT, &self.slice[self.pos..])).into_inner() { + match f(InputBuf::new(&self.slice[self.pos..])).into_inner() { State::Data(remainder, data) => { // TODO: Do something neater with the remainder self.pos += self.len() - remainder.len(); diff --git a/src/buffer/stateful.rs b/src/buffer/stateful.rs index 56ef7393..918a6410 100644 --- a/src/buffer/stateful.rs +++ b/src/buffer/stateful.rs @@ -1,8 +1,7 @@ use std::io; use std::cmp; -use {InputBuf, ParseResult}; -use primitives::input; +use types::{InputBuf, ParseResult}; use primitives::{State, IntoInner}; use buffer::{ @@ -218,7 +217,7 @@ impl<'a, S: DataSource, B: Buffer> Stream<'a, 'a> for Source return Err(StreamError::EndOfInput); } - match f(input::new_buf(input::DEFAULT, &self.buffer)).into_inner() { + match f(InputBuf::new(&self.buffer)).into_inner() { State::Data(remainder, data) => { if remainder.is_incomplete() && self.state.contains(END_OF_INPUT) { // We can't accept this since we might have hit a premature end diff --git a/src/combinators/bounded.rs b/src/combinators/bounded.rs index 65f8a6f3..6a0be65e 100644 --- a/src/combinators/bounded.rs +++ b/src/combinators/bounded.rs @@ -25,7 +25,7 @@ use std::ops::{ }; use std::cmp::max; -use {Input, ParseResult}; +use types::{Input, ParseResult}; use primitives::{Primitives, IntoInner, State}; /// Trait for applying a parser multiple times based on a range. diff --git a/src/combinators/mod.rs b/src/combinators/mod.rs index 44d440bf..2181e058 100644 --- a/src/combinators/mod.rs +++ b/src/combinators/mod.rs @@ -7,10 +7,16 @@ pub mod bounded; use std::iter::FromIterator; -use {ParseResult, Input}; - -use primitives::State; -use primitives::{IntoInner, Primitives}; +use types::{ + ParseResult, + Input, +}; + +use primitives::{ + IntoInner, + Primitives, + State, +}; /// Applies the parser ``p`` exactly ``num`` times collecting all items into `T: FromIterator`. /// diff --git a/src/lib.rs b/src/lib.rs index fc00eb57..530387e3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -208,99 +208,60 @@ extern crate debugtrace; #[macro_use] mod macros; -mod input; mod parse; -mod parse_result; -// FIXME: Uncomment pub mod ascii; +pub mod buffer; pub mod combinators; pub mod parsers; +pub mod primitives; +pub mod types; -pub use parsers::{ - any, - eof, - not_token, - peek, - peek_next, - satisfy, - satisfy_with, - scan, - string, - run_scanner, - take, - take_remainder, - take_till, - take_while, - take_while1, - token, -}; -pub use parsers::Error; -pub mod buffer; +pub mod prelude { + pub use parsers::{ + any, + eof, + not_token, + peek, + peek_next, + satisfy, + satisfy_with, + scan, + string, + run_scanner, + take, + take_remainder, + take_till, + take_while, + take_while1, + token, + }; + pub use parsers::{ + Error, + SimpleResult, + }; + + pub use combinators::{ + count, + option, + or, + many, + many1, + sep_by, + sep_by1, + many_till, + skip_many, + skip_many1, + matched_by, + }; + pub use types::{ + Input, + U8Input, + ParseResult, + }; +} -pub use combinators::{ - count, - option, - or, - many, - many1, - sep_by, - sep_by1, - many_till, - skip_many, - skip_many1, - matched_by, -}; pub use parse::{ ParseError, parse_only, }; -pub use input::{ - Input, - InputBuf, - U8Input, -}; -pub use parse_result::{ - ParseResult, - SimpleResult, -}; - -/// Module used to construct fundamental parsers and combinators. -/// -/// # Primitive -/// -/// Only used by fundamental parsers and combinators. -pub mod primitives { - pub use input::Buffer; - pub use input::primitives::{ - Primitives, - Guard, - }; - pub use parse_result::{ - IntoInner, - State, - }; - - /// Input utilities. - /// - /// # Primitive - /// - /// Only used by fundamental parsers and combinators. - pub mod input { - pub use input::{DEFAULT, INCOMPLETE, new_buf}; - } - - /// ParseResult utilities. - /// - /// # Primitive - /// - /// Only used by fundamental parsers and combinators. - /// - /// # Note - /// - /// Prefer to use ``Input::ret``, ``Input::err`` or ``Input::incomplete`` instead of using - /// ``parse_result::new``. - pub mod parse_result { - pub use parse_result::new; - } -} diff --git a/src/parse.rs b/src/parse.rs index 730fbfb2..542125ed 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -1,6 +1,9 @@ -use ParseResult; -use primitives::{IntoInner, State}; -use primitives::Primitives; +use types::ParseResult; +use primitives::{ + IntoInner, + Primitives, + State, +}; /// Simple error type returned from `parse_only`. #[derive(Debug, Eq, PartialEq)] diff --git a/src/parsers.rs b/src/parsers.rs index ac870876..e334e108 100644 --- a/src/parsers.rs +++ b/src/parsers.rs @@ -2,12 +2,18 @@ use std::mem; -use input::Input; -use parse_result::SimpleResult; -use primitives::{Primitives, Buffer}; +use types::{ + Buffer, + Input, + ParseResult, +}; +use primitives::Primitives; pub use self::error::Error; +/// Result returned from the basic parsers. +pub type SimpleResult = ParseResult::Token>>; + // Only export if we have backtraces enabled, in debug/test profiles the StackFrame is only used // to debug-print. #[cfg(feature="backtrace")] diff --git a/src/primitives.rs b/src/primitives.rs new file mode 100644 index 00000000..105f6a18 --- /dev/null +++ b/src/primitives.rs @@ -0,0 +1,129 @@ +//! Module used to construct fundamental parsers and combinators. +//! +//! # Primitive +//! +//! Only used by fundamental parsers and combinators. + +use types::Input; + +/// **Primitive:** Consumes self and reveals the inner state. +/// +/// # Primitive +/// +/// Only used by fundamental parsers and combinators. +pub trait IntoInner { + /// The inner type to be revealed. + type Inner; + + /// **Primitive:** Consumes self and reveals the inner state. + /// + /// # Primitive + /// + /// Only used by fundamental parsers and combinators. + #[inline(always)] + fn into_inner(self) -> Self::Inner; +} + +/// **Primitive:** Primitive inner type containing the parse-state. +/// +/// # Primitive +/// +/// Only used by fundamental parsers and combinators. +#[must_use] +#[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum State { + /// Successful parser state, first item is the input state and the second item is the contained + /// value. + Data(I, T), + /// Parse error state, first item is a slice from where the error occurred in the input buffer + /// to the end of the input buffer and the second item is the error value. + Error(I, E), +} + +/// This is a zero-sized type used by the `Primitives` trait implementation to guarantee that +/// access to primitive methods on `Input` only happens when the `Primitives` trait has been +/// imported. +/// +/// It cannot be instantiated outside of the `Primitives` trait blanket implementation. +pub struct Guard(()); + +/// **Primitive:** Trait enabling primitive actions on an `Input` type. +/// +/// # Primitive +/// +/// Only used by fundamental parsers and combinators. +/// +// FIXME: Rename and documentation +pub trait Primitives: Input { + #[inline(always)] + fn peek(&mut self) -> Option { + self._peek(Guard(())) + } + + #[inline(always)] + fn pop(&mut self) -> Option { + self._pop(Guard(())) + } + + #[inline(always)] + fn consume(&mut self, n: usize) -> Option { + self._consume(Guard(()), n) + } + + #[inline(always)] + fn consume_while(&mut self, f: F) -> Self::Buffer + where F: FnMut(Self::Token) -> bool { + self._consume_while(Guard(()), f) + } + + /// Returns the buffer from the marker `m` to the current position, discarding the + /// backtracking position carried by `m`. + #[inline(always)] + fn consume_from(&mut self, m: Self::Marker) -> Self::Buffer { + self._consume_from(Guard(()), m) + } + + #[inline(always)] + fn consume_remaining(&mut self) -> Self::Buffer { + self._consume_remaining(Guard(())) + } + + /// Marks the current position to be able to backtrack to it using `restore`. + /// + /// # Example + /// + /// ``` + /// use chomp::{take, Input}; + /// use chomp::primitives::Primitives; + /// use chomp::primitives::{InputBuffer, IntoInner, State}; + /// + /// let i = &b"Testing"; + /// + /// assert_eq!(i.buffer(), b"Testing"); + /// assert_eq!(i.is_end(), true); + /// + /// // mark and eat one token + /// let m = i.mark(); + /// let i = i.consume(1); + /// + /// assert_eq!(i.buffer(), b"esting"); + /// + /// // restore and continue parsing + /// let j = i.restore(m); + /// + /// let r = take(j, 4); + /// + /// assert_eq!(r.into_inner(), State::Data(input::new(input::END_OF_INPUT, b""), &b"Test"[..])); + /// ``` + #[inline(always)] + fn mark(&self) -> Self::Marker { + self._mark(Guard(())) + } + + #[inline(always)] + fn restore(self, m: Self::Marker) -> Self { + self._restore(Guard(()), m) + } +} + +impl Primitives for I {} diff --git a/src/input.rs b/src/types/input.rs similarity index 80% rename from src/input.rs rename to src/types/input.rs index b8ccf3b1..57ac6ed7 100644 --- a/src/input.rs +++ b/src/types/input.rs @@ -1,7 +1,7 @@ -use parse_result::{ParseResult, State}; -use parse_result; +use types::ParseResult; +use super::parse_result; -use primitives::Guard; +use primitives::{Guard, State}; pub trait U8Input: Input {} @@ -9,101 +9,6 @@ pub trait U8Input: Input {} impl U8Input for T where T: Input {} -/// Primitive operations on `Input` types. -/// -/// All parsers only require a few primitive operations to parse data: -pub mod primitives { - use Input; - - /// This is a zero-sized type used by the `Primitives` trait implementation to guarantee that - /// access to primitive methods on `Input` only happens when the `Primitives` trait has been - /// imported. - /// - /// It cannot be instantiated outside of the `Primitives` trait blanket implementation. - pub struct Guard(()); - - /// **Primitive:** Trait enabling primitive actions on an `Input` type. - /// - /// # Primitive - /// - /// Only used by fundamental parsers and combinators. - /// - // FIXME: Rename and documentation - pub trait Primitives: Input { - #[inline(always)] - fn peek(&mut self) -> Option { - self._peek(Guard(())) - } - - #[inline(always)] - fn pop(&mut self) -> Option { - self._pop(Guard(())) - } - - #[inline(always)] - fn consume(&mut self, n: usize) -> Option { - self._consume(Guard(()), n) - } - - #[inline(always)] - fn consume_while(&mut self, f: F) -> Self::Buffer - where F: FnMut(Self::Token) -> bool { - self._consume_while(Guard(()), f) - } - - /// Returns the buffer from the marker `m` to the current position, discarding the - /// backtracking position carried by `m`. - #[inline(always)] - fn consume_from(&mut self, m: Self::Marker) -> Self::Buffer { - self._consume_from(Guard(()), m) - } - - #[inline(always)] - fn consume_remaining(&mut self) -> Self::Buffer { - self._consume_remaining(Guard(())) - } - - /// Marks the current position to be able to backtrack to it using `restore`. - /// - /// # Example - /// - /// ``` - /// use chomp::{take, Input}; - /// use chomp::primitives::Primitives; - /// use chomp::primitives::{InputBuffer, IntoInner, State}; - /// - /// let i = &b"Testing"; - /// - /// assert_eq!(i.buffer(), b"Testing"); - /// assert_eq!(i.is_end(), true); - /// - /// // mark and eat one token - /// let m = i.mark(); - /// let i = i.consume(1); - /// - /// assert_eq!(i.buffer(), b"esting"); - /// - /// // restore and continue parsing - /// let j = i.restore(m); - /// - /// let r = take(j, 4); - /// - /// assert_eq!(r.into_inner(), State::Data(input::new(input::END_OF_INPUT, b""), &b"Test"[..])); - /// ``` - #[inline(always)] - fn mark(&self) -> Self::Marker { - self._mark(Guard(())) - } - - #[inline(always)] - fn restore(self, m: Self::Marker) -> Self { - self._restore(Guard(()), m) - } - } - - impl Primitives for I {} -} - // FIXME: Docs // TODO: More methods? pub trait Buffer { @@ -359,39 +264,29 @@ impl<'a, I: Copy + PartialEq> Input for &'a [I] { } } -// TODO: Replace with boolean flag -bitflags!{ - pub flags InputMode: u32 { - /// Default (empty) input state. - const DEFAULT = 0, - /// If a parser has attempted to read incomplete - const INCOMPLETE = 1, - } -} - // FIXME: Docs /// Input buffer type which contains a flag which tells if we might have more data to read. // TODO: Move to buffer module and make public? +// TODO: Replace mode with boolean #[must_use] #[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub struct InputBuf<'a, I: 'a>(InputMode, &'a [I]); - -/// **Primitive:** Creates a new input from the given state and buffer. -/// -/// # Primitive -/// -/// Only used by fundamental parsers and combinators. -// TODO: Remove and replace with new? -#[inline] -pub fn new_buf(state: InputMode, buffer: &[I]) -> InputBuf { - InputBuf(state, buffer) -} +pub struct InputBuf<'a, I: 'a>( + /// If this is set to true a parser has tried to read past the end of this buffer. + pub bool, + /// Current buffer slice + pub &'a [I], +); // FIXME: Docs impl<'a, I: 'a> InputBuf<'a, I> { + #[inline] + pub fn new(buf: &'a [I]) -> Self { + InputBuf(false, buf) + } + #[inline] pub fn is_incomplete(&self) -> bool { - self.0.contains(INCOMPLETE) + self.0 } #[inline] @@ -415,7 +310,7 @@ impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { match self.1.first() { Some(c) => Some(*c), None => { - self.0.insert(INCOMPLETE); + self.0 = true; None }, @@ -434,7 +329,7 @@ impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { #[inline] fn _consume(&mut self, _g: Guard, n: usize) -> Option { if n > self.1.len() { - self.0.insert(INCOMPLETE); + self.0 = true; None } else { @@ -458,7 +353,7 @@ impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { b }, None => { - self.0.insert(INCOMPLETE); + self.0 = true; let b = self.1; diff --git a/src/types/mod.rs b/src/types/mod.rs new file mode 100644 index 00000000..e572cb95 --- /dev/null +++ b/src/types/mod.rs @@ -0,0 +1,12 @@ +mod input; +mod parse_result; + +pub use self::parse_result::{ + ParseResult +}; +pub use self::input::{ + Buffer, + Input, + InputBuf, + U8Input, +}; diff --git a/src/parse_result.rs b/src/types/parse_result.rs similarity index 90% rename from src/parse_result.rs rename to src/types/parse_result.rs index e8448d2b..c602acbc 100644 --- a/src/parse_result.rs +++ b/src/types/parse_result.rs @@ -1,46 +1,5 @@ -use input::Input; -use parsers::Error; - -/// Result for dealing with the basic parsers when parsing a stream of `u8`. -// FIXME: Uncomment -//pub type U8Result = ParseResult<, T, Error>; -/// Result returned from the basic parsers. -// FIXME: Uncomment -pub type SimpleResult = ParseResult::Token>>; - -/// **Primitive:** Primitive inner type containing the parse-state. -/// -/// # Primitive -/// -/// Only used by fundamental parsers and combinators. -#[must_use] -#[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub enum State { - /// Successful parser state, first item is the input state and the second item is the contained - /// value. - Data(I, T), - /// Parse error state, first item is a slice from where the error occurred in the input buffer - /// to the end of the input buffer and the second item is the error value. - Error(I, E), -} - -/// **Primitive:** Consumes self and reveals the inner state. -/// -/// # Primitive -/// -/// Only used by fundamental parsers and combinators. -pub trait IntoInner { - /// The inner type to be revealed. - type Inner; - - /// **Primitive:** Consumes self and reveals the inner state. - /// - /// # Primitive - /// - /// Only used by fundamental parsers and combinators. - #[inline(always)] - fn into_inner(self) -> Self::Inner; -} +use types::Input; +use primitives::{IntoInner, State}; /// The basic return type of a parser. /// From 46a972ca2cca27dafcd27af944ce26d322d41123 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Mon, 28 Mar 2016 11:42:17 +0200 Subject: [PATCH 18/63] Fixed use statements with prelude move --- benches/combinators.rs | 10 +++++----- benches/http_bench.rs | 2 +- examples/http_parser.rs | 2 +- src/buffer/stateful.rs | 9 +++++++-- src/lib.rs | 4 ++++ src/parse.rs | 2 +- src/types/input.rs | 14 +++----------- 7 files changed, 22 insertions(+), 21 deletions(-) diff --git a/benches/combinators.rs b/benches/combinators.rs index 2244cc91..cd34e359 100644 --- a/benches/combinators.rs +++ b/benches/combinators.rs @@ -7,8 +7,8 @@ use test::Bencher; use std::iter; -use chomp::*; -use chomp::primitives::input::{new_buf, DEFAULT}; +use chomp::prelude::*; +use chomp::types::InputBuf; #[bench] fn count_vec_1k(b: &mut Bencher) { @@ -97,7 +97,7 @@ fn count_vec_10k_maybe_incomplete(b: &mut Bencher) { } b.iter(|| { - count_vec(new_buf(DEFAULT, &data)) + count_vec(InputBuf::new(&data)) }) } @@ -110,7 +110,7 @@ fn many_vec_10k_maybe_incomplete(b: &mut Bencher) { } b.iter(|| { - many_vec(new_buf(DEFAULT, &data)) + many_vec(InputBuf::new(&data)) }) } @@ -123,6 +123,6 @@ fn many1_vec_10k_maybe_incomplete(b: &mut Bencher) { } b.iter(|| { - many1_vec(new_buf(DEFAULT, &data)) + many1_vec(InputBuf::new(&data)) }) } diff --git a/benches/http_bench.rs b/benches/http_bench.rs index d9db76e3..1c30cb13 100644 --- a/benches/http_bench.rs +++ b/benches/http_bench.rs @@ -4,7 +4,7 @@ extern crate test; extern crate chomp; use test::Bencher; -use chomp::*; +use chomp::prelude::*; #[derive(Debug)] struct Request { diff --git a/examples/http_parser.rs b/examples/http_parser.rs index 254f14aa..478313ff 100644 --- a/examples/http_parser.rs +++ b/examples/http_parser.rs @@ -15,7 +15,7 @@ extern crate chomp; use std::fs::File; use std::env; -use chomp::*; +use chomp::prelude::*; use chomp::buffer::{Source, Stream, StreamError}; diff --git a/src/buffer/stateful.rs b/src/buffer/stateful.rs index 918a6410..92c0afbe 100644 --- a/src/buffer/stateful.rs +++ b/src/buffer/stateful.rs @@ -260,8 +260,13 @@ impl<'a, S: DataSource, B: Buffer> Stream<'a, 'a> for Source #[cfg(test)] mod test { use std::io; - use {Input, any, take}; - use Error; + + use types::Input; + use parsers::{ + Error, + any, + take, + }; use buffer::{ FixedSizeBuffer, StreamError, diff --git a/src/lib.rs b/src/lib.rs index 530387e3..6d51dc7b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -259,6 +259,10 @@ pub mod prelude { U8Input, ParseResult, }; + pub use parse::{ + ParseError, + parse_only, + }; } pub use parse::{ diff --git a/src/parse.rs b/src/parse.rs index 542125ed..de118079 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -62,7 +62,7 @@ pub fn parse_only<'a, I, T, E, F>(parser: F, input: &'a [I]) -> Result Self::Marker; + fn _mark(&self, Guard) -> Self::Marker; /// **Primitive:** See `Primitives::restore` for documentation. /// @@ -342,7 +342,7 @@ impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { } #[inline] - fn _consume_while(&mut self, _g: Guard, mut f: F) -> Self::Buffer + fn _consume_while(&mut self, g: Guard, mut f: F) -> Self::Buffer where F: FnMut(Self::Token) -> bool { match self.1.iter().position(|c| !f(*c)) { Some(n) => { @@ -352,15 +352,7 @@ impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { b }, - None => { - self.0 = true; - - let b = self.1; - - self.1 = &self.1[..0]; - - b - } + None => self._consume_remaining(g), } } From f9cbeddad823a7e2bb4baf84e57f30d61f9bb0a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Mon, 28 Mar 2016 12:11:36 +0200 Subject: [PATCH 19/63] Added Buffer::iter --- src/ascii.rs | 24 ++++++++++++------------ src/buffer/mod.rs | 2 +- src/types/input.rs | 27 +++++++++++++++++++++++++-- 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/src/ascii.rs b/src/ascii.rs index eeae2ed3..55a64571 100644 --- a/src/ascii.rs +++ b/src/ascii.rs @@ -8,7 +8,7 @@ use conv::{ }; use conv::errors::UnwrapOk; -use types::Input; +use types::{Buffer, Input}; use combinators::option; use parsers::{ SimpleResult, @@ -156,8 +156,8 @@ pub fn signed, T, F>(i: I, f: F) -> SimpleResult /// ``` // TODO: Use methods on `Buffer` to implement `to_decimal` #[inline] -pub fn decimal<'a, I: Input, T: Copy + ValueFrom + Add + Mul>(i: I) -> SimpleResult { - take_while1(i, is_digit).map(to_decimal) +pub fn decimal<'a, I: Input, T: Copy + ValueFrom + Add + Mul>(i: I) -> SimpleResult { + take_while1(i, is_digit).map(|b| to_decimal(b.iter())) } /// Internal function converting a `[u8]` to the given integer type `T`. @@ -167,8 +167,8 @@ pub fn decimal<'a, I: Input, T: Copy + ValueFrom + Add + Mul>(buf: &[u8]) -> T { - buf.iter().fold(T::value_from(0).unwrap_ok(), |a, n| a * T::value_from(10).unwrap_ok() + T::value_from(n - b'0').unwrap_ok()) +fn to_decimal + Add + Mul, I: Iterator>(iter: I) -> T { + iter.fold(T::value_from(0).unwrap_ok(), |a, n| a * T::value_from(10).unwrap_ok() + T::value_from(n - b'0').unwrap_ok()) } #[cfg(test)] @@ -177,13 +177,13 @@ mod test { macro_rules! test_to_decimal { ( $($n:ty),+ ) => { $( - assert_eq!(to_decimal::<$n>(b""), 0); - assert_eq!(to_decimal::<$n>(b"0"), 0); - assert_eq!(to_decimal::<$n>(b"1"), 1); - assert_eq!(to_decimal::<$n>(b"2"), 2); - assert_eq!(to_decimal::<$n>(b"10"), 10); - assert_eq!(to_decimal::<$n>(b"20"), 20); - assert_eq!(to_decimal::<$n>(b"25"), 25); + assert_eq!(to_decimal::<$n, _>(b"".iter().cloned()), 0); + assert_eq!(to_decimal::<$n, _>(b"0".iter().cloned()), 0); + assert_eq!(to_decimal::<$n, _>(b"1".iter().cloned()), 1); + assert_eq!(to_decimal::<$n, _>(b"2".iter().cloned()), 2); + assert_eq!(to_decimal::<$n, _>(b"10".iter().cloned()), 10); + assert_eq!(to_decimal::<$n, _>(b"20".iter().cloned()), 20); + assert_eq!(to_decimal::<$n, _>(b"25".iter().cloned()), 25); )+ } } diff --git a/src/buffer/mod.rs b/src/buffer/mod.rs index dbefac12..1b16f65b 100644 --- a/src/buffer/mod.rs +++ b/src/buffer/mod.rs @@ -79,7 +79,7 @@ impl<'a, I, E> PartialEq for StreamError<'a, I, E> } } -impl<'a, T, I: Input, E> From> for StreamError<'a, T, E> +impl<'a, T: Copy + PartialEq, I: Input, E> From> for StreamError<'a, T, E> where I: 'a { fn from(e: ParseError) -> Self { use primitives::Primitives; diff --git a/src/types/input.rs b/src/types/input.rs index f3cf1f95..d5b650ac 100644 --- a/src/types/input.rs +++ b/src/types/input.rs @@ -1,3 +1,7 @@ +use std::iter; +use std::slice; +use std::str; + use types::ParseResult; use super::parse_result; @@ -12,6 +16,10 @@ impl U8Input for T // FIXME: Docs // TODO: More methods? pub trait Buffer { + type Token: Copy + PartialEq; + type Iter: Iterator; + + fn iter(&self) -> Self::Iter; fn len(&self) -> usize; fn is_empty(&self) -> bool { @@ -42,7 +50,8 @@ pub trait Input: Sized { type Marker; // TODO: Bounds for common usage (eg. to iterator?) - type Buffer: Buffer; + // FIXME: Docs + type Buffer: Buffer; /// Returns `t` as a success value in the parsing context. /// @@ -179,7 +188,14 @@ pub trait Input: Sized { fn _restore(self, Guard, Self::Marker) -> Self; } -impl<'a, I> Buffer for &'a [I] { +impl<'a, I: Copy + PartialEq> Buffer for &'a [I] { + type Token = I; + type Iter = iter::Cloned>; + + fn iter(&self) -> Self::Iter { + (&self[..]).iter().cloned() + } + fn len(&self) -> usize { // Slice to reach inherent method to prevent infinite recursion (&self[..]).len() @@ -386,6 +402,13 @@ impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { } impl<'a> Buffer for &'a str { + type Token = char; + type Iter = str::Chars<'a>; + + fn iter(&self) -> Self::Iter { + self.chars() + } + fn len(&self) -> usize { // Slice to reach inherent method to prevent infinite recursion (&self[..]).len() From c58c51cf2dabf8f51515a9bfdfff708b2d13582f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Mon, 28 Mar 2016 12:23:40 +0200 Subject: [PATCH 20/63] Input for ByteTendril --- Cargo.toml | 1 + src/lib.rs | 3 ++ src/types/mod.rs | 2 + src/types/tendril.rs | 101 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 107 insertions(+) create mode 100644 src/types/tendril.rs diff --git a/Cargo.toml b/Cargo.toml index e3642946..91a0e3e6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ include = [ bitflags = "0.5.0" conv = { version = "0.3", default-features = false, features = ["std"] } debugtrace = { version = "0.1.0" } +tendril = { version = "0.2.2", optional = true } # Technically a dev-dependency, but dev-dependencies are not allowed to be optional, # compiletest_rs fails to compile on stable and beta diff --git a/src/lib.rs b/src/lib.rs index 6d51dc7b..aa64be61 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -201,6 +201,9 @@ //! The built-in `chomp::parsers::Error` type is zero-sized and carry no error-information. This //! increases performance somewhat. +#[cfg(feature = "tendril")] +extern crate tendril; + #[macro_use] extern crate bitflags; extern crate conv; diff --git a/src/types/mod.rs b/src/types/mod.rs index e572cb95..30965c5a 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -1,5 +1,7 @@ mod input; mod parse_result; +#[cfg(feature = "tendril")] +mod tendril; pub use self::parse_result::{ ParseResult diff --git a/src/types/tendril.rs b/src/types/tendril.rs new file mode 100644 index 00000000..c09c6b01 --- /dev/null +++ b/src/types/tendril.rs @@ -0,0 +1,101 @@ +use std::mem; + +use tendril::ByteTendril; +use primitives::Guard; +use types::{Buffer, Input}; + +// TODO: Impl for more than byte tendril +impl Input for ByteTendril { + type Token = u8; + type Marker = ByteTendril; + type Buffer = ByteTendril; + + #[inline] + fn _peek(&mut self, _g: Guard) -> Option { + self.as_ref().first().cloned() + } + + #[inline] + fn _pop(&mut self, _g: Guard) -> Option { + if self.len32() > 0 { + let t = self.as_ref()[0]; + + self.pop_front(1); + + Some(t) + } else { + None + } + } + + #[inline] + fn _consume(&mut self, _g: Guard, n: usize) -> Option { + if n > self.len32() as usize { + None + } else { + // TODO: How does this work with allocation in the tendril? + let b = self.subtendril(0, n as u32); + + self.pop_front(n as u32); + + Some(b) + } + } + + #[inline] + fn _consume_while(&mut self, g: Guard, mut f: F) -> Self::Buffer + where F: FnMut(Self::Token) -> bool { + match self.iter().position(|c| !f(*c)) { + Some(n) => { + // TODO: How does this work with allocation in the tendril? + let b = self.subtendril(0, n as u32); + + self.pop_front(n as u32); + + b + }, + None => self._consume_remaining(g), + } + } + + #[inline] + fn _consume_from(&mut self, _g: Guard, m: Self::Marker) -> Self::Buffer { + // Just the tendril analogue of the slice version: + m.subtendril(0, m.len32() - self.len32()) + } + + #[inline] + fn _consume_remaining(&mut self, _g: Guard) -> Self::Buffer { + let b = self.subtendril(0, 0); + + mem::replace(self, b) + } + + #[inline] + fn _mark(&self, _g: Guard) -> Self::Marker { + self.clone() + } + + #[inline] + fn _restore(self, _g: Guard, m: Self::Marker) -> Self { + m + } +} + +impl Buffer for ByteTendril { + fn len(&self) -> usize { + self.len32() as usize + } +} + +// FIXME: Tests +#[cfg(test)] +mod test { + fn basic() { + use prelude::*; + use ascii::decimal; + use primitives::{IntoInner, State}; + + assert_eq!(decimal(Tendril::from_slice(b"123")).into_inner(), State::Data(Tendril::from_slice(b""), 123i32)); + } +} From 0caabd796394d9946c10991a8e8ab7631ce57341 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Mon, 28 Mar 2016 12:48:18 +0200 Subject: [PATCH 21/63] Replaced inner State with Result --- src/buffer/slice.rs | 6 +++--- src/buffer/stateful.rs | 6 +++--- src/combinators/bounded.rs | 38 +++++++++++++++++++------------------- src/combinators/macros.rs | 12 ++++++------ src/combinators/mod.rs | 20 ++++++++------------ src/parse.rs | 5 ++--- src/primitives.rs | 16 ---------------- src/types/input.rs | 11 ++++------- src/types/parse_result.rs | 34 +++++++++++++++++----------------- 9 files changed, 62 insertions(+), 86 deletions(-) diff --git a/src/buffer/slice.rs b/src/buffer/slice.rs index 378e4900..45f3aa7b 100644 --- a/src/buffer/slice.rs +++ b/src/buffer/slice.rs @@ -1,4 +1,4 @@ -use primitives::{State, IntoInner}; +use primitives::IntoInner; use types::{InputBuf, ParseResult}; use buffer::{IntoStream, StreamError, Stream}; @@ -92,13 +92,13 @@ impl<'a, 'i, I: 'i + Copy + PartialEq> Stream<'a, 'i> for SliceStream<'i, I> { } match f(InputBuf::new(&self.slice[self.pos..])).into_inner() { - State::Data(remainder, data) => { + (remainder, Ok(data)) => { // TODO: Do something neater with the remainder self.pos += self.len() - remainder.len(); Ok(data) }, - State::Error(mut remainder, err) => { + (mut remainder, Err(err)) => { if remainder.is_incomplete() { // TODO: 1 is not correct, n is expected len but we can't obtain that right now Err(StreamError::Incomplete(self.len() + 1)) diff --git a/src/buffer/stateful.rs b/src/buffer/stateful.rs index 92c0afbe..61d61190 100644 --- a/src/buffer/stateful.rs +++ b/src/buffer/stateful.rs @@ -2,7 +2,7 @@ use std::io; use std::cmp; use types::{InputBuf, ParseResult}; -use primitives::{State, IntoInner}; +use primitives::IntoInner; use buffer::{ Buffer, @@ -218,7 +218,7 @@ impl<'a, S: DataSource, B: Buffer> Stream<'a, 'a> for Source } match f(InputBuf::new(&self.buffer)).into_inner() { - State::Data(remainder, data) => { + (remainder, Ok(data)) => { if remainder.is_incomplete() && self.state.contains(END_OF_INPUT) { // We can't accept this since we might have hit a premature end self.request = self.buffer.len() + 1; @@ -233,7 +233,7 @@ impl<'a, S: DataSource, B: Buffer> Stream<'a, 'a> for Source Ok(data) } }, - State::Error(mut remainder, err) => { + (mut remainder, Err(err)) => { if remainder.is_incomplete() { // TODO: How to deal with n, no longer present? self.request = self.buffer.len() + 1; diff --git a/src/combinators/bounded.rs b/src/combinators/bounded.rs index 6a0be65e..6ac3a081 100644 --- a/src/combinators/bounded.rs +++ b/src/combinators/bounded.rs @@ -26,7 +26,7 @@ use std::ops::{ use std::cmp::max; use types::{Input, ParseResult}; -use primitives::{Primitives, IntoInner, State}; +use primitives::{Primitives, IntoInner}; /// Trait for applying a parser multiple times based on a range. pub trait BoundedRange { @@ -159,13 +159,13 @@ impl BoundedRange for Range { let m = i.mark(); match f(i).into_inner() { - State::Data(b, _) => { + (b, Ok(_)) => { min = if min == 0 { 0 } else { min - 1 }; max -= 1; i = b }, - State::Error(b, e) => if min == 0 { + (b, Err(e)) => if min == 0 { i = b.restore(m); break; @@ -210,21 +210,21 @@ impl BoundedRange for Range { match (self.data.1, (self.end)(i).into_inner()) { // We can always end - (_, State::Data(b, _)) => { + (_, (b, Ok(_))) => { self.buf = Some(b); self.state = EndStateTill::EndSuccess; return None }, // We have reached end, end must match or it is an error - (0, State::Error(b, e)) => { + (0, (b, Err(e))) => { self.buf = Some(b); self.state = EndStateTill::Error(From::from(e)); return None; }, // Failed to end, restore and continue since we can parse more - (_, State::Error(b, _)) => self.buf = Some(b.restore(m)), + (_, (b, Err(_))) => self.buf = Some(b.restore(m)), } } } @@ -290,12 +290,12 @@ impl BoundedRange for RangeFrom { let m = i.mark(); match f(i).into_inner() { - State::Data(b, _) => { + (b, Ok(_)) => { min = if min == 0 { 0 } else { min - 1 }; i = b }, - State::Error(b, e) => if min == 0 { + (b, Err(e)) => if min == 0 { i = b.restore(m); break; @@ -372,8 +372,8 @@ impl BoundedRange for RangeFull { } => result : T { - (s, (), m, Some(_)) => s.restore(m).ret(result), - (_, _, _, None) => unreachable!(), + (s, (), m, Some(_)) => s.restore(m).ret(result), + (_, _, _, None) => unreachable!(), } } } @@ -385,8 +385,8 @@ impl BoundedRange for RangeFull { let m = i.mark(); match f(i).into_inner() { - State::Data(b, _) => i = b, - State::Error(b, _) => { + (b, Ok(_)) => i = b, + (b, Err(_)) => { i = b.restore(m); break; @@ -482,13 +482,13 @@ impl BoundedRange for RangeTo { let m = i.mark(); match f(i).into_inner() { - State::Data(b, _) => { + (b, Ok(_)) => { max -= 1; i = b }, // Always ok to end iteration - State::Error(b, _) => { + (b, Err(_)) => { i = b.restore(m); break; @@ -524,21 +524,21 @@ impl BoundedRange for RangeTo { match (self.data, (self.end)(i).into_inner()) { // We can always end - (_, State::Data(b, _)) => { + (_, (b, Ok(_))) => { self.buf = Some(b); self.state = EndStateTill::EndSuccess; return None }, // We have reached end, end must match or it is an error - (0, State::Error(b, e)) => { + (0, (b, Err(e))) => { self.buf = Some(b); self.state = EndStateTill::Error(From::from(e)); return None; }, // Failed to end, restore and continue since we can parse more - (_, State::Error(b, _)) => self.buf = Some(b.restore(m)), + (_, (b, Err(_))) => self.buf = Some(b.restore(m)), } } on { @@ -607,12 +607,12 @@ impl BoundedRange for usize { let m = i.mark(); match f(i).into_inner() { - State::Data(b, _) => { + (b, Ok(_)) => { n -= 1; i = b }, - State::Error(b, e) => if n == 0 { + (b, Err(e)) => if n == 0 { i = b.restore(m); break; diff --git a/src/combinators/macros.rs b/src/combinators/macros.rs index f85b0379..d0a121fa 100644 --- a/src/combinators/macros.rs +++ b/src/combinators/macros.rs @@ -65,14 +65,14 @@ macro_rules! run_iter { $next_self.mark = i.mark(); match ($next_self.parser)(i).into_inner() { - State::Data(b, v) => { + (b, Ok(v)) => { $next_self.buf = Some(b); $on_next Some(v) }, - State::Error(b, e) => { + (b, Err(e)) => { $next_self.buf = Some(b); $next_self.state = Some(e); @@ -172,14 +172,14 @@ macro_rules! run_iter_till { let i = $next_self.buf.take().expect("Iter.buf was None"); match ($next_self.parser)(i).into_inner() { - State::Data(b, v) => { + (b, Ok(v)) => { $next_self.buf = Some(b); $on_next Some(v) }, - State::Error(b, e) => { + (b, Err(e)) => { $next_self.buf = Some(b); $next_self.state = EndStateTill::Error(e); @@ -216,14 +216,14 @@ macro_rules! iter_till_end_test { let m = i.mark(); match ($the_self.end)(i).into_inner() { - State::Data(b, _) => { + (b, Ok(_)) => { $the_self.buf = Some(b); $the_self.state = EndStateTill::EndSuccess; return None }, // Failed to end, restore and continue - State::Error(b, _) => $the_self.buf = Some(b.restore(m)), + (b, Err(_)) => $the_self.buf = Some(b.restore(m)), } } } } diff --git a/src/combinators/mod.rs b/src/combinators/mod.rs index 2181e058..f6af5778 100644 --- a/src/combinators/mod.rs +++ b/src/combinators/mod.rs @@ -15,7 +15,6 @@ use types::{ use primitives::{ IntoInner, Primitives, - State, }; /// Applies the parser ``p`` exactly ``num`` times collecting all items into `T: FromIterator`. @@ -61,12 +60,9 @@ pub fn option(i: I, f: F, default: T) -> ParseResult where F: FnOnce(I) -> ParseResult { let m = i.mark(); - // TODO: Make macro? Shared with or match f(i).into_inner() { - State::Data(b, d) => b.ret(d), - State::Error(b, _) => { - b.restore(m).ret(default) - }, + (b, Ok(d)) => b.ret(d), + (b, Err(_)) => b.restore(m).ret(default), } } @@ -96,8 +92,8 @@ pub fn or(i: I, f: F, g: G) -> ParseResult let m = i.mark(); match f(i).into_inner() { - State::Data(b, d) => b.ret(d), - State::Error(b, _) => g(b.restore(m)), + (b, Ok(d)) => b.ret(d), + (b, Err(_)) => g(b.restore(m)), } } @@ -284,12 +280,12 @@ pub fn matched_by(i: I, f: F) -> ParseResult { + (mut b, Ok(t)) => { let n = b.consume_from(m); b.ret((n, t)) }, - State::Error(b, e) => b.err(e), + (b, Err(e)) => b.err(e), } } @@ -309,8 +305,8 @@ pub fn look_ahead(i: I, f: F) -> ParseResult let m = i.mark(); match f(i).into_inner() { - State::Data(b, t) => b.restore(m).ret(t), - State::Error(b, t) => b.restore(m).err(t), + (b, Ok(t)) => b.restore(m).ret(t), + (b, Err(t)) => b.restore(m).err(t), } } diff --git a/src/parse.rs b/src/parse.rs index de118079..e706e024 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -2,7 +2,6 @@ use types::ParseResult; use primitives::{ IntoInner, Primitives, - State, }; /// Simple error type returned from `parse_only`. @@ -55,8 +54,8 @@ pub fn parse_only<'a, I, T, E, F>(parser: F, input: &'a [I]) -> Result ParseResult<&'a [I], T, E> { match parser(input).into_inner() { - State::Data(_, t) => Ok(t), - State::Error(mut b, e) => Err(ParseError::Error(b.consume_remaining(), e)), + (_, Ok(t)) => Ok(t), + (mut b, Err(e)) => Err(ParseError::Error(b.consume_remaining(), e)), } } diff --git a/src/primitives.rs b/src/primitives.rs index 105f6a18..26dd74d0 100644 --- a/src/primitives.rs +++ b/src/primitives.rs @@ -24,22 +24,6 @@ pub trait IntoInner { fn into_inner(self) -> Self::Inner; } -/// **Primitive:** Primitive inner type containing the parse-state. -/// -/// # Primitive -/// -/// Only used by fundamental parsers and combinators. -#[must_use] -#[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub enum State { - /// Successful parser state, first item is the input state and the second item is the contained - /// value. - Data(I, T), - /// Parse error state, first item is a slice from where the error occurred in the input buffer - /// to the end of the input buffer and the second item is the error value. - Error(I, E), -} - /// This is a zero-sized type used by the `Primitives` trait implementation to guarantee that /// access to primitive methods on `Input` only happens when the `Primitives` trait has been /// imported. diff --git a/src/types/input.rs b/src/types/input.rs index d5b650ac..6a90a31a 100644 --- a/src/types/input.rs +++ b/src/types/input.rs @@ -5,7 +5,7 @@ use std::str; use types::ParseResult; use super::parse_result; -use primitives::{Guard, State}; +use primitives::Guard; pub trait U8Input: Input {} @@ -71,7 +71,7 @@ pub trait Input: Sized { /// ``` #[inline] fn ret(self, t: T) -> ParseResult { - parse_result::new(State::Data(self, t)) + parse_result::new(self, Ok(t)) } /// Returns `e` as an error value in the parsing context. @@ -92,7 +92,7 @@ pub trait Input: Sized { /// ``` #[inline] fn err(self, e: E) -> ParseResult { - parse_result::new(State::Error(self, e)) + parse_result::new(self, Err(e)) } /// Converts a `Result` into a `ParseResult`, preserving parser state. @@ -117,10 +117,7 @@ pub trait Input: Sized { /// ``` #[inline] fn from_result(self, r: Result) -> ParseResult { - match r { - Ok(t) => parse_result::new(State::Data(self, t)), - Err(e) => parse_result::new(State::Error(self, e)), - } + parse_result::new(self, r) } // Primitive methods diff --git a/src/types/parse_result.rs b/src/types/parse_result.rs index c602acbc..be17fd7e 100644 --- a/src/types/parse_result.rs +++ b/src/types/parse_result.rs @@ -1,5 +1,5 @@ use types::Input; -use primitives::{IntoInner, State}; +use primitives::IntoInner; /// The basic return type of a parser. /// @@ -33,7 +33,7 @@ use primitives::{IntoInner, State}; /// ``` #[must_use] #[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub struct ParseResult(State); +pub struct ParseResult(I, Result); /// **Primitive:** Creates a new `ParseResult`. /// @@ -44,8 +44,8 @@ pub struct ParseResult(State); /// # Note /// /// Prefer to use ``Input::ret``, ``Input::err`` or ``Input::incomplete`` instead of using -pub fn new(s: State) -> ParseResult { - ParseResult(s) +pub fn new(i: I, r: Result) -> ParseResult { + ParseResult(i, r) } impl ParseResult { @@ -95,9 +95,9 @@ impl ParseResult { pub fn bind(self, f: F) -> ParseResult where F: FnOnce(I, T) -> ParseResult, V: From { - match self.0 { - State::Data(i, t) => f(i, t).map_err(From::from), - State::Error(i, e) => ParseResult(State::Error(i, From::from(e))), + match self.1 { + Ok(t) => f(self.0, t).map_err(From::from), + Err(e) => ParseResult(self.0, Err(From::from(e))), } } @@ -148,9 +148,9 @@ impl ParseResult { #[inline] pub fn map(self, f: F) -> ParseResult where F: FnOnce(T) -> U { - match self.0 { - State::Data(i, t) => ParseResult(State::Data(i, f(t))), - State::Error(i, e) => ParseResult(State::Error(i, e)), + match self { + ParseResult(i, Ok(t)) => ParseResult(i, Ok(f(t))), + ParseResult(i, Err(e)) => ParseResult(i, Err(e)), } } @@ -170,9 +170,9 @@ impl ParseResult { #[inline] pub fn map_err(self, f: F) -> ParseResult where F: FnOnce(E) -> V { - match self.0 { - State::Data(i, t) => ParseResult(State::Data(i, t)), - State::Error(i, e) => ParseResult(State::Error(i, f(e))), + match self { + ParseResult(i, Ok(t)) => ParseResult(i, Ok(t)), + ParseResult(i, Err(e)) => ParseResult(i, Err(f(e))), } } @@ -193,7 +193,7 @@ impl ParseResult { #[inline] pub fn inspect(self, f: F) -> ParseResult where F: FnOnce(&T) { - if let State::Data(_, ref t) = self.0 { + if let Ok(ref t) = self.1 { f(t) } @@ -212,7 +212,7 @@ impl ParseResult { /// The `ParseResult` type is a semi-linear type, supposed to act like a linear type while used in /// a parsing context to carry the state. Normally it should be as restrictive as the `Input` type /// in terms of how much it exposes its internals, but the `IntoInner` trait implementation -/// allows fundamental parsers and combinators to expose the inner `State` of the `ParseResult` +/// allows fundamental parsers and combinators to expose the inner `Result` of the `ParseResult` /// and act on this. /// /// # Example @@ -237,11 +237,11 @@ impl ParseResult { /// assert_eq!(r, Ok(&b"test"[..])); /// ``` impl IntoInner for ParseResult { - type Inner = State; + type Inner = (I, Result); #[inline(always)] fn into_inner(self) -> Self::Inner { - self.0 + (self.0, self.1) } } From 6fb259f626237fa6cae9229eea547880b554c0b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 30 Mar 2016 16:55:44 +0200 Subject: [PATCH 22/63] Replaced Buffer::iter with Buffer::fold --- src/ascii.rs | 18 +++++----- src/buffer/mod.rs | 25 ++++++------- src/buffer/slice.rs | 2 +- src/buffer/stateful.rs | 2 +- src/types/input.rs | 82 ++++++++++++++++++++++-------------------- src/types/mod.rs | 2 +- src/types/tendril.rs | 18 ++++++++-- 7 files changed, 83 insertions(+), 66 deletions(-) diff --git a/src/ascii.rs b/src/ascii.rs index 55a64571..28338fb3 100644 --- a/src/ascii.rs +++ b/src/ascii.rs @@ -157,7 +157,7 @@ pub fn signed, T, F>(i: I, f: F) -> SimpleResult // TODO: Use methods on `Buffer` to implement `to_decimal` #[inline] pub fn decimal<'a, I: Input, T: Copy + ValueFrom + Add + Mul>(i: I) -> SimpleResult { - take_while1(i, is_digit).map(|b| to_decimal(b.iter())) + take_while1(i, is_digit).map(to_decimal) } /// Internal function converting a `[u8]` to the given integer type `T`. @@ -167,7 +167,7 @@ pub fn decimal<'a, I: Input, T: Copy + ValueFrom + Ad /// * The slice must not contain any other characters besides 0 to 9. /// * The `T` type must be larger than `u8` if it is signed. #[inline] -fn to_decimal + Add + Mul, I: Iterator>(iter: I) -> T { +fn to_decimal + Add + Mul, I: Buffer>(iter: I) -> T { iter.fold(T::value_from(0).unwrap_ok(), |a, n| a * T::value_from(10).unwrap_ok() + T::value_from(n - b'0').unwrap_ok()) } @@ -177,13 +177,13 @@ mod test { macro_rules! test_to_decimal { ( $($n:ty),+ ) => { $( - assert_eq!(to_decimal::<$n, _>(b"".iter().cloned()), 0); - assert_eq!(to_decimal::<$n, _>(b"0".iter().cloned()), 0); - assert_eq!(to_decimal::<$n, _>(b"1".iter().cloned()), 1); - assert_eq!(to_decimal::<$n, _>(b"2".iter().cloned()), 2); - assert_eq!(to_decimal::<$n, _>(b"10".iter().cloned()), 10); - assert_eq!(to_decimal::<$n, _>(b"20".iter().cloned()), 20); - assert_eq!(to_decimal::<$n, _>(b"25".iter().cloned()), 25); + assert_eq!(to_decimal::<$n, _>(&b""[..]), 0); + assert_eq!(to_decimal::<$n, _>(&b"0"[..]), 0); + assert_eq!(to_decimal::<$n, _>(&b"1"[..]), 1); + assert_eq!(to_decimal::<$n, _>(&b"2"[..]), 2); + assert_eq!(to_decimal::<$n, _>(&b"10"[..]), 10); + assert_eq!(to_decimal::<$n, _>(&b"20"[..]), 20); + assert_eq!(to_decimal::<$n, _>(&b"25"[..]), 25); )+ } } diff --git a/src/buffer/mod.rs b/src/buffer/mod.rs index 1b16f65b..35991f20 100644 --- a/src/buffer/mod.rs +++ b/src/buffer/mod.rs @@ -35,7 +35,8 @@ pub mod data_source; use std::io; -use types::{ParseResult, Input, InputBuf}; +use types::{ParseResult, InputBuf}; +use types::Buffer as InputBuffer; use parse::ParseError; pub use self::slice::SliceStream; @@ -49,10 +50,9 @@ pub use self::buffer::{ /// Error type for parsing using the `Stream` trait. #[derive(Debug)] -pub enum StreamError<'a, I, E> - where I: 'a { +pub enum StreamError { /// An error occurred in the parser, the given slice indicates the part which failed. - ParseError(&'a [I], E), + ParseError(B, E), /// Parser failed to complete with the available data. Incomplete(usize), /// An IO-error occurred while attempting to fill the buffer. @@ -65,10 +65,9 @@ pub enum StreamError<'a, I, E> Retry, } -impl<'a, I, E> PartialEq for StreamError<'a, I, E> - where E: PartialEq, I: PartialEq { +impl> PartialEq for StreamError { #[inline] - fn eq(&self, other: &StreamError<'a, I, E>) -> bool { + fn eq(&self, other: &StreamError) -> bool { match (self, other) { (&StreamError::ParseError(ref b1, ref e1), &StreamError::ParseError(ref b2, ref e2)) => b1 == b2 && e1 == e2, (&StreamError::Incomplete(n1), &StreamError::Incomplete(n2)) => n1 == n2, @@ -79,17 +78,19 @@ impl<'a, I, E> PartialEq for StreamError<'a, I, E> } } -impl<'a, T: Copy + PartialEq, I: Input, E> From> for StreamError<'a, T, E> - where I: 'a { +// FIXME: +/* +impl From> for StreamError { fn from(e: ParseError) -> Self { use primitives::Primitives; match e { - ParseError::Error(mut b, e) => StreamError::ParseError(b.consume_remaining(), e), - ParseError::Incomplete(n) => StreamError::Incomplete(n), + ParseError::Error(b, e) => StreamError::ParseError(b, e), + ParseError::Incomplete(n) => StreamError::Incomplete(n), } } } +*/ /// Trait wrapping the state management in reading from a data source while parsing. pub trait Stream<'a, 'i> { @@ -102,7 +103,7 @@ pub trait Stream<'a, 'i> { /// If a `StreamError::Retry` is returned the consuming code it should just retry the action /// (the implementation might require a separate call to refill the stream). #[inline] - fn parse(&'a mut self, f: F) -> Result> + fn parse(&'a mut self, f: F) -> Result> where F: FnOnce(InputBuf<'i, Self::Item>) -> ParseResult, T, E>, T: 'i, E: 'i; diff --git a/src/buffer/slice.rs b/src/buffer/slice.rs index 45f3aa7b..2079f722 100644 --- a/src/buffer/slice.rs +++ b/src/buffer/slice.rs @@ -81,7 +81,7 @@ impl<'a, 'i, I: 'i + Copy + PartialEq> Stream<'a, 'i> for SliceStream<'i, I> { type Item = I; #[inline] - fn parse(&'a mut self, f: F) -> Result> + fn parse(&'a mut self, f: F) -> Result> where F: FnOnce(InputBuf<'i, Self::Item>) -> ParseResult, T, E>, T: 'i, E: 'i { diff --git a/src/buffer/stateful.rs b/src/buffer/stateful.rs index 61d61190..e5a8489d 100644 --- a/src/buffer/stateful.rs +++ b/src/buffer/stateful.rs @@ -203,7 +203,7 @@ impl<'a, S: DataSource, B: Buffer> Stream<'a, 'a> for Source type Item = S::Item; #[inline] - fn parse(&'a mut self, f: F) -> Result> + fn parse(&'a mut self, f: F) -> Result> where F: FnOnce(InputBuf<'a, Self::Item>) -> ParseResult, T, E>, T: 'a, E: 'a { diff --git a/src/types/input.rs b/src/types/input.rs index 6a90a31a..773bdacb 100644 --- a/src/types/input.rs +++ b/src/types/input.rs @@ -1,5 +1,3 @@ -use std::iter; -use std::slice; use std::str; use types::ParseResult; @@ -15,18 +13,49 @@ impl U8Input for T // FIXME: Docs // TODO: More methods? -pub trait Buffer { +pub trait Buffer: PartialEq { + /// The token type of this buffer. type Token: Copy + PartialEq; - type Iter: Iterator; - fn iter(&self) -> Self::Iter; + fn fold(self, B, F) -> B + where F: FnMut(B, Self::Token) -> B; + fn len(&self) -> usize; + //fn to_vec(self) -> Vec; fn is_empty(&self) -> bool { self.len() == 0 } } +impl<'a, I: Copy + PartialEq> Buffer for &'a [I] { + type Token = I; + + fn fold(self, init: B, f: F) -> B + where F: FnMut(B, Self::Token) -> B { + (&self[..]).iter().cloned().fold(init, f) + } + + fn len(&self) -> usize { + // Slice to reach inherent method to prevent infinite recursion + (&self[..]).len() + } +} + +impl<'a> Buffer for &'a str { + type Token = char; + + fn fold(self, init: B, f: F) -> B + where F: FnMut(B, Self::Token) -> B { + self.chars().fold(init, f) + } + + fn len(&self) -> usize { + // Slice to reach inherent method to prevent infinite recursion + (&self[..]).len() + } +} + // FIXME: Update, do not refer to type or linear type /// Linear type containing the parser state, this type is threaded though `bind` and is also the /// initial type passed to a parser. @@ -41,16 +70,19 @@ pub trait Buffer { /// something like `u8`), `...` additional parameters to the parser, `T` the carried type and `E` /// the potential error type. pub trait Input: Sized { - /// The token type of the input + /// The token type of the input. // TODO: Maybe remove the copy bound at some point? type Token: Copy + PartialEq; - /// A marker type which is used to backtrack using `_mark` and `_restore`. It should also be - /// possible to use this type to consume the data from the marked position to the current - /// position. + + /// A marker type which is used to backtrack using `_mark` and `_restore`. + /// + /// It should also be possible to use this type to consume the data from the marked position to + /// the current position. type Marker; - // TODO: Bounds for common usage (eg. to iterator?) - // FIXME: Docs + /// The buffer type yielded by this input when multiple tokens are consumed in sequence. + /// + /// Can eg. provide zero-copy parsing if the input type is built to support it. type Buffer: Buffer; /// Returns `t` as a success value in the parsing context. @@ -185,20 +217,6 @@ pub trait Input: Sized { fn _restore(self, Guard, Self::Marker) -> Self; } -impl<'a, I: Copy + PartialEq> Buffer for &'a [I] { - type Token = I; - type Iter = iter::Cloned>; - - fn iter(&self) -> Self::Iter { - (&self[..]).iter().cloned() - } - - fn len(&self) -> usize { - // Slice to reach inherent method to prevent infinite recursion - (&self[..]).len() - } -} - impl<'a, I: Copy + PartialEq> Input for &'a [I] { type Token = I; type Marker = &'a [I]; @@ -398,20 +416,6 @@ impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { } } -impl<'a> Buffer for &'a str { - type Token = char; - type Iter = str::Chars<'a>; - - fn iter(&self) -> Self::Iter { - self.chars() - } - - fn len(&self) -> usize { - // Slice to reach inherent method to prevent infinite recursion - (&self[..]).len() - } -} - impl<'a> Input for &'a str { type Token = char; type Marker = &'a str; diff --git a/src/types/mod.rs b/src/types/mod.rs index 30965c5a..5f3c5560 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -1,7 +1,7 @@ mod input; mod parse_result; #[cfg(feature = "tendril")] -mod tendril; +pub mod tendril; pub use self::parse_result::{ ParseResult diff --git a/src/types/tendril.rs b/src/types/tendril.rs index c09c6b01..175d9cff 100644 --- a/src/types/tendril.rs +++ b/src/types/tendril.rs @@ -83,6 +83,13 @@ impl Input for ByteTendril { } impl Buffer for ByteTendril { + type Token = u8; + + fn fold(self, init: B, f: F) -> B + where F: FnMut(B, Self::Token) -> B { + (&self[..]).iter().cloned().fold(init, f) + } + fn len(&self) -> usize { self.len32() as usize } @@ -91,11 +98,16 @@ impl Buffer for ByteTendril { // FIXME: Tests #[cfg(test)] mod test { + use tendril::Tendril; + fn basic() { - use prelude::*; use ascii::decimal; - use primitives::{IntoInner, State}; + use primitives::IntoInner; + + assert_eq!(decimal(Tendril::from_slice(b"123")).into_inner(), (Tendril::from_slice(b""), Ok(123i32))); + } - assert_eq!(decimal(Tendril::from_slice(b"123")).into_inner(), State::Data(Tendril::from_slice(b""), 123i32)); + fn primitives() { + ::types::input::test::run_primitives_test(Tendril::from_slice(b"123"), |x| x); } } From 94297d939474c7031247b238e47a4bce965427fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 30 Mar 2016 17:00:24 +0200 Subject: [PATCH 23/63] Moved traits in types to solve name resolution issues --- src/types/input.rs | 199 +++--------------------------------------- src/types/mod.rs | 200 +++++++++++++++++++++++++++++++++++++++++-- src/types/tendril.rs | 4 + 3 files changed, 208 insertions(+), 195 deletions(-) diff --git a/src/types/input.rs b/src/types/input.rs index 773bdacb..0498c4c1 100644 --- a/src/types/input.rs +++ b/src/types/input.rs @@ -1,33 +1,6 @@ -use std::str; - -use types::ParseResult; -use super::parse_result; - +use types::{Buffer, Input}; use primitives::Guard; -pub trait U8Input: Input {} - -// TODO: More restrictions? Buffer=&[u8]? -impl U8Input for T - where T: Input {} - -// FIXME: Docs -// TODO: More methods? -pub trait Buffer: PartialEq { - /// The token type of this buffer. - type Token: Copy + PartialEq; - - fn fold(self, B, F) -> B - where F: FnMut(B, Self::Token) -> B; - - fn len(&self) -> usize; - //fn to_vec(self) -> Vec; - - fn is_empty(&self) -> bool { - self.len() == 0 - } -} - impl<'a, I: Copy + PartialEq> Buffer for &'a [I] { type Token = I; @@ -40,6 +13,10 @@ impl<'a, I: Copy + PartialEq> Buffer for &'a [I] { // Slice to reach inherent method to prevent infinite recursion (&self[..]).len() } + + fn to_vec(self) -> Vec { + (&self[..]).iter().cloned().collect() + } } impl<'a> Buffer for &'a str { @@ -51,170 +28,16 @@ impl<'a> Buffer for &'a str { } fn len(&self) -> usize { - // Slice to reach inherent method to prevent infinite recursion - (&self[..]).len() + self.chars().count() } -} -// FIXME: Update, do not refer to type or linear type -/// Linear type containing the parser state, this type is threaded though `bind` and is also the -/// initial type passed to a parser. -/// -/// Coupled with the `ParseResult` type it forms the parser monad: -/// -/// ```ignore -/// Fn*(Input, ...) -> ParseResult; -/// ``` -/// -/// where ``Fn*`` is the appropriate closure/function trait, `I` the input token type (usually -/// something like `u8`), `...` additional parameters to the parser, `T` the carried type and `E` -/// the potential error type. -pub trait Input: Sized { - /// The token type of the input. - // TODO: Maybe remove the copy bound at some point? - type Token: Copy + PartialEq; - - /// A marker type which is used to backtrack using `_mark` and `_restore`. - /// - /// It should also be possible to use this type to consume the data from the marked position to - /// the current position. - type Marker; - - /// The buffer type yielded by this input when multiple tokens are consumed in sequence. - /// - /// Can eg. provide zero-copy parsing if the input type is built to support it. - type Buffer: Buffer; - - /// Returns `t` as a success value in the parsing context. - /// - /// Equivalent to Haskell's `return` function in the `Monad` typeclass. - /// - /// # Example - /// - /// ``` - /// use chomp::parse_only; - /// - /// let r = parse_only(|i| - /// // Annotate the error type - /// i.ret::<_, ()>("Wohoo, success!"), - /// b"some input"); - /// - /// assert_eq!(r, Ok("Wohoo, success!")); - /// ``` - #[inline] - fn ret(self, t: T) -> ParseResult { - parse_result::new(self, Ok(t)) - } - - /// Returns `e` as an error value in the parsing context. - /// - /// A more general version of Haskell's `fail` function in the `Monad` typeclass. - /// - /// # Example - /// - /// ``` - /// use chomp::{ParseError, parse_only}; - /// - /// let r = parse_only(|i| - /// // Annotate the value type - /// i.err::<(), _>("Something went wrong"), - /// b"some input"); - /// - /// assert_eq!(r, Err(ParseError::Error(b"some input", "Something went wrong"))); - /// ``` - #[inline] - fn err(self, e: E) -> ParseResult { - parse_result::new(self, Err(e)) - } - - /// Converts a `Result` into a `ParseResult`, preserving parser state. - /// - /// To convert an `Option` into a `ParseResult` it is recommended to use - /// [`Option::ok_or`](https://doc.rust-lang.org/std/option/enum.Option.html#method.ok_or) - /// or [`Option::ok_or_else`](https://doc.rust-lang.org/std/option/enum.Option.html#method.ok_or_else) - /// in combination with this method. - /// - /// # Examples - /// - /// ``` - /// use chomp::{ParseError, parse_only}; - /// - /// let r = parse_only(|i| i.from_result::<_, ()>(Ok("foo")), b"test"); - /// - /// assert_eq!(r, Ok("foo")); - /// - /// let r = parse_only(|i| i.from_result::<(), _>(Err("error message")), b"test"); - /// - /// assert_eq!(r, Err(ParseError::Error(&b"test"[..], "error message"))); - /// ``` - #[inline] - fn from_result(self, r: Result) -> ParseResult { - parse_result::new(self, r) + fn is_empty(&self) -> bool { + (&self[..]).is_empty() } - // Primitive methods - - /// **Primitive:** See `Primitives::peek` for documentation. - /// - /// Peeks at the next token in the input without consuming it. `None` if no more input is - /// available. - /// - /// Note: Will refill automatically. - #[inline] - fn _peek(&mut self, Guard) -> Option; - - /// **Primitive:** See `Primitives::pop` for documentation. - /// - /// Pops a token off the start of the input. `None` if no more input is available. - /// - /// Note: Will refill automatically. - #[inline] - fn _pop(&mut self, Guard) -> Option; - - /// **Primitive:** See `Primitives::consume` for documentation. - /// - /// Attempt to consume `n` tokens, if it fails do not advance the position but return `None`. - #[inline] - fn _consume(&mut self, Guard, usize) -> Option; - - /// **Primitive:** See `Primitives::consume_while` for documentation. - /// - /// Runs the closure `F` on the tokens *in order* until it returns false, all tokens up to that - /// token will be returned as a buffer and discarded from the current input. MUST never run the - /// closure more than once on the exact same token. - /// - /// Note: Will refill automatically. - #[inline] - fn _consume_while(&mut self, Guard, F) -> Self::Buffer - where F: FnMut(Self::Token) -> bool; - - /// **Primitive:** See `Primitives::consume_from for documentation. - /// - /// Returns the buffer from the marker to the current position, discarding the - /// backtracking position carried by the marker. - #[inline] - fn _consume_from(&mut self, Guard, Self::Marker) -> Self::Buffer; - - /// **Primitive:** See `Primitives::consume_remaining` for documentation. - /// - /// Returns the remainder of the input in a buffer. - /// - /// Note: Will refill the intenal buffer until no more data is available if the underlying - /// implementation supports it. - #[inline] - fn _consume_remaining(&mut self, Guard) -> Self::Buffer; - - /// **Primitive:** See `Primitives::mark` for documentation. - /// - /// Marks a position for backtracking along with relevant state. - #[inline] - fn _mark(&self, Guard) -> Self::Marker; - - /// **Primitive:** See `Primitives::restore` for documentation. - /// - /// Resumes from a previously marked state. - #[inline] - fn _restore(self, Guard, Self::Marker) -> Self; + fn to_vec(self) -> Vec { + (&self[..]).chars().collect() + } } impl<'a, I: Copy + PartialEq> Input for &'a [I] { diff --git a/src/types/mod.rs b/src/types/mod.rs index 5f3c5560..06d2df29 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -3,12 +3,198 @@ mod parse_result; #[cfg(feature = "tendril")] pub mod tendril; -pub use self::parse_result::{ +use primitives::Guard; + +pub use types::parse_result::{ ParseResult }; -pub use self::input::{ - Buffer, - Input, - InputBuf, - U8Input, -}; +pub use types::input::InputBuf; + +pub trait U8Input: Input {} + +// TODO: More restrictions? Buffer=&[u8]? +impl U8Input for T + where T: Input {} + +// FIXME: Docs +// TODO: More methods? +pub trait Buffer: PartialEq { + /// The token type of this buffer. + type Token: Copy + PartialEq; + + fn fold(self, B, F) -> B + where F: FnMut(B, Self::Token) -> B; + + /// The number of tokens present in this + fn len(&self) -> usize; + + /// Consumes self to create an owned vector of tokens. + /// + /// Will allocate if the implementation borrows storage or does not use a `Vec` internally. + fn to_vec(self) -> Vec; + + fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +// FIXME: Update, do not refer to type or linear type +/// Linear type containing the parser state, this type is threaded though `bind` and is also the +/// initial type passed to a parser. +/// +/// Coupled with the `ParseResult` type it forms the parser monad: +/// +/// ```ignore +/// Fn*(Input, ...) -> ParseResult; +/// ``` +/// +/// where ``Fn*`` is the appropriate closure/function trait, `I` the input token type (usually +/// something like `u8`), `...` additional parameters to the parser, `T` the carried type and `E` +/// the potential error type. +pub trait Input: Sized { + /// The token type of the input. + // TODO: Maybe remove the copy bound at some point? + type Token: Copy + PartialEq; + + /// A marker type which is used to backtrack using `_mark` and `_restore`. + /// + /// It should also be possible to use this type to consume the data from the marked position to + /// the current position. + type Marker; + + /// The buffer type yielded by this input when multiple tokens are consumed in sequence. + /// + /// Can eg. provide zero-copy parsing if the input type is built to support it. + type Buffer: Buffer; + + /// Returns `t` as a success value in the parsing context. + /// + /// Equivalent to Haskell's `return` function in the `Monad` typeclass. + /// + /// # Example + /// + /// ``` + /// use chomp::parse_only; + /// + /// let r = parse_only(|i| + /// // Annotate the error type + /// i.ret::<_, ()>("Wohoo, success!"), + /// b"some input"); + /// + /// assert_eq!(r, Ok("Wohoo, success!")); + /// ``` + #[inline] + fn ret(self, t: T) -> ParseResult { + parse_result::new(self, Ok(t)) + } + + /// Returns `e` as an error value in the parsing context. + /// + /// A more general version of Haskell's `fail` function in the `Monad` typeclass. + /// + /// # Example + /// + /// ``` + /// use chomp::{ParseError, parse_only}; + /// + /// let r = parse_only(|i| + /// // Annotate the value type + /// i.err::<(), _>("Something went wrong"), + /// b"some input"); + /// + /// assert_eq!(r, Err(ParseError::Error(b"some input", "Something went wrong"))); + /// ``` + #[inline] + fn err(self, e: E) -> ParseResult { + parse_result::new(self, Err(e)) + } + + /// Converts a `Result` into a `ParseResult`, preserving parser state. + /// + /// To convert an `Option` into a `ParseResult` it is recommended to use + /// [`Option::ok_or`](https://doc.rust-lang.org/std/option/enum.Option.html#method.ok_or) + /// or [`Option::ok_or_else`](https://doc.rust-lang.org/std/option/enum.Option.html#method.ok_or_else) + /// in combination with this method. + /// + /// # Examples + /// + /// ``` + /// use chomp::{ParseError, parse_only}; + /// + /// let r = parse_only(|i| i.from_result::<_, ()>(Ok("foo")), b"test"); + /// + /// assert_eq!(r, Ok("foo")); + /// + /// let r = parse_only(|i| i.from_result::<(), _>(Err("error message")), b"test"); + /// + /// assert_eq!(r, Err(ParseError::Error(&b"test"[..], "error message"))); + /// ``` + #[inline] + fn from_result(self, r: Result) -> ParseResult { + parse_result::new(self, r) + } + + // Primitive methods + + /// **Primitive:** See `Primitives::peek` for documentation. + /// + /// Peeks at the next token in the input without consuming it. `None` if no more input is + /// available. + /// + /// Note: Will refill automatically. + #[inline] + fn _peek(&mut self, Guard) -> Option; + + /// **Primitive:** See `Primitives::pop` for documentation. + /// + /// Pops a token off the start of the input. `None` if no more input is available. + /// + /// Note: Will refill automatically. + #[inline] + fn _pop(&mut self, Guard) -> Option; + + /// **Primitive:** See `Primitives::consume` for documentation. + /// + /// Attempt to consume `n` tokens, if it fails do not advance the position but return `None`. + #[inline] + fn _consume(&mut self, Guard, usize) -> Option; + + /// **Primitive:** See `Primitives::consume_while` for documentation. + /// + /// Runs the closure `F` on the tokens *in order* until it returns false, all tokens up to that + /// token will be returned as a buffer and discarded from the current input. MUST never run the + /// closure more than once on the exact same token. + /// + /// Note: Will refill automatically. + #[inline] + fn _consume_while(&mut self, Guard, F) -> Self::Buffer + where F: FnMut(Self::Token) -> bool; + + /// **Primitive:** See `Primitives::consume_from for documentation. + /// + /// Returns the buffer from the marker to the current position, discarding the + /// backtracking position carried by the marker. + #[inline] + fn _consume_from(&mut self, Guard, Self::Marker) -> Self::Buffer; + + /// **Primitive:** See `Primitives::consume_remaining` for documentation. + /// + /// Returns the remainder of the input in a buffer. + /// + /// Note: Will refill the intenal buffer until no more data is available if the underlying + /// implementation supports it. + #[inline] + fn _consume_remaining(&mut self, Guard) -> Self::Buffer; + + /// **Primitive:** See `Primitives::mark` for documentation. + /// + /// Marks a position for backtracking along with relevant state. + #[inline] + fn _mark(&self, Guard) -> Self::Marker; + + /// **Primitive:** See `Primitives::restore` for documentation. + /// + /// Resumes from a previously marked state. + #[inline] + fn _restore(self, Guard, Self::Marker) -> Self; +} diff --git a/src/types/tendril.rs b/src/types/tendril.rs index 175d9cff..e7001d85 100644 --- a/src/types/tendril.rs +++ b/src/types/tendril.rs @@ -93,6 +93,10 @@ impl Buffer for ByteTendril { fn len(&self) -> usize { self.len32() as usize } + + fn to_vec(self) -> Vec { + (&self[..]).iter().cloned().collect() + } } // FIXME: Tests From 29d2855f2cbc5fbd98ca7339a3e9ae20e67ad005 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 30 Mar 2016 17:56:39 +0200 Subject: [PATCH 24/63] Input tests --- src/types/input.rs | 296 +++++++++++++++++++++++++++++++++------------ src/types/mod.rs | 42 +++++++ 2 files changed, 261 insertions(+), 77 deletions(-) diff --git a/src/types/input.rs b/src/types/input.rs index 0498c4c1..b2186ef0 100644 --- a/src/types/input.rs +++ b/src/types/input.rs @@ -119,9 +119,8 @@ impl<'a, I: Copy + PartialEq> Input for &'a [I] { } // FIXME: Docs -/// Input buffer type which contains a flag which tells if we might have more data to read. +/// Input buffer type which contains a flag which tells if we might need to read more data. // TODO: Move to buffer module and make public? -// TODO: Replace mode with boolean #[must_use] #[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] pub struct InputBuf<'a, I: 'a>( @@ -217,6 +216,8 @@ impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { #[inline] fn _consume_remaining(&mut self, _g: Guard) -> Self::Buffer { + self.0 = true; + let b = self.1; self.1 = &self.1[..0]; @@ -266,14 +267,22 @@ impl<'a> Input for &'a str { #[inline] fn _consume(&mut self, _g: Guard, n: usize) -> Option { match self.char_indices().enumerate().take(n + 1).last() { - // n always less than num if self contains more than n characters - Some((num, (pos, _))) if n < num => { + // num always equal to n if self contains more than n characters + Some((num, (pos, _))) if n == num => { let b = &self[..pos]; *self = &self[pos..]; Some(b) }, + // num always equal to n - 1 if self contains exactly n characters + Some((num, _)) if n == num + 1 => { + let b = &self[..]; + + *self = &self[..0]; + + Some(b) + }, _ => None, } } @@ -325,99 +334,232 @@ impl<'a> Input for &'a str { } } -/* #[cfg(test)] mod test { use std::fmt::Debug; - use super::{new_buf, Input, InputBuf, DEFAULT, END_OF_INPUT}; - use parse_result::ParseResult; - use primitives::{IntoInner, State}; + use types::{Buffer, Input, InputBuf}; #[test] - fn make_ret() { - let i1: InputBuf = new_buf(END_OF_INPUT, b"in1"); - let i2: InputBuf = new_buf(DEFAULT, b"in2"); - - let r1: ParseResult<_, u32, ()> = i1.ret::<_, ()>(23u32); - let r2: ParseResult<_, i32, ()> = i2.ret::<_, ()>(23i32); - - assert_eq!(r1.into_inner(), State::Data(InputBuf(END_OF_INPUT, b"in1"), 23u32)); - assert_eq!(r2.into_inner(), State::Data(InputBuf(DEFAULT, b"in2"), 23i32)); + fn test_slice() { + run_primitives_test(&b"abc"[..], |x| x); } #[test] - fn make_err() { - let i1: InputBuf = new_buf(END_OF_INPUT, b"in1"); - let i2: InputBuf = new_buf(DEFAULT, b"in2"); - - let r1: ParseResult<_, (), u32> = i1.err::<(), _>(23u32); - let r2: ParseResult<_, (), i32> = i2.err::<(), _>(23i32); - - assert_eq!(r1.into_inner(), State::Error(new_buf(END_OF_INPUT, b"in1"), 23u32)); - assert_eq!(r2.into_inner(), State::Error(new_buf(DEFAULT, b"in2"), 23i32)); + fn test_string() { + run_primitives_test(&"abc"[..], |c| c as char); } - /* #[test] - fn primitives_slice() { + fn test_input_buf() { use primitives::Primitives; - run_primitives_test(&b"abc"[..], true); - - let mut s = &b"abc"[..]; - let m = s.mark(); - s.discard(2); - assert_eq!(s.consume_from(m), &b"ab"[..]); - assert_eq!(s, &b"c"[..]); - } - #[test] - fn primitives_input_buf_default() { + run_primitives_test(InputBuf::new(b"abc"), |x| x); + + let mut b = InputBuf::new(b"a"); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 1); + assert_eq!(b.is_empty(), false); + b.peek(); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 1); + assert_eq!(b.is_empty(), false); + b.pop(); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + assert_eq!(b.peek(), None); + assert_eq!(b.is_incomplete(), true); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + assert_eq!(b.pop(), None); + assert_eq!(b.is_incomplete(), true); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + + let mut b = InputBuf::new(b"ab"); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 2); + assert_eq!(b.is_empty(), false); + b.consume(1); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 1); + assert_eq!(b.is_empty(), false); + b.consume(1); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + assert_eq!(b.consume(1), None); + assert_eq!(b.is_incomplete(), true); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + + let mut b = InputBuf::new(b"ab"); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 2); + assert_eq!(b.is_empty(), false); + assert_eq!(b.consume(3), None); + assert_eq!(b.is_incomplete(), true); + assert_eq!(b.len(), 2); + assert_eq!(b.is_empty(), false); + + let mut b = InputBuf::new(b"ab"); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 2); + assert_eq!(b.is_empty(), false); + assert_eq!(b.consume_while(|_| true), &b"ab"[..]); + assert_eq!(b.is_incomplete(), true); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + + let mut b = InputBuf::new(b"ab"); + assert_eq!(b.consume_while(|c| c == b'a'), &b"a"[..]); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 1); + assert_eq!(b.is_empty(), false); + assert_eq!(b.consume_while(|c| c == b'b'), &b"b"[..]); + assert_eq!(b.is_incomplete(), true); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + assert_eq!(b.consume_while(|c| c == b'b'), &b""[..]); + assert_eq!(b.is_incomplete(), true); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + + let mut b = InputBuf::new(b"abc"); + let m = b.mark(); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 3); + assert_eq!(b.is_empty(), false); + b.consume(3); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + assert_eq!(b.consume_from(m), &b"abc"[..]); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + + let mut b = InputBuf::new(b"abc"); + let m = b.mark(); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 3); + assert_eq!(b.is_empty(), false); + b.consume(2); + b.consume(2); + assert_eq!(b.is_incomplete(), true); + assert_eq!(b.len(), 1); + assert_eq!(b.is_empty(), false); + let mut b = b.restore(m); + assert_eq!(b.is_incomplete(), true); + assert_eq!(b.len(), 3); + assert_eq!(b.is_empty(), false); + + let mut b = InputBuf::new(b"abc"); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 3); + assert_eq!(b.is_empty(), false); + b.consume_remaining(); + assert_eq!(b.is_incomplete(), true); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + } + + /// Should recieve an Input with the tokens 'a', 'b' and 'c' remaining. + pub fn run_primitives_test I::Token>(mut s: I, f: F) + where I::Token: Debug, + I::Buffer: Clone { use primitives::Primitives; - run_primitives_test(new_buf(DEFAULT, b"abc"), false); - let mut s = new_buf(DEFAULT, b"abc"); - let m = s.mark(); - s.discard(2); - assert_eq!(s.consume_from(m), &b"ab"[..]); - assert_eq!(s, new_buf(DEFAULT, b"c")); - } + fn buffer_eq_slice B::Token>(b: B, s: &[u8], f: F) + where B::Token: Debug, { + assert_eq!(b.len(), s.len()); + assert_eq!(b.is_empty(), s.is_empty()); + assert_eq!(b.clone().fold(0, |n, c| { + assert_eq!(c, f(s[n])); - #[test] - fn primitives_input_buf_end() { - use primitives::Primitives; - run_primitives_test(new_buf(END_OF_INPUT, b"abc"), true); + n + 1 + }), s.iter().count()); + assert_eq!(b.to_vec(), s.iter().cloned().map(f).collect::>()); + } - let mut s = new_buf(END_OF_INPUT, b"abc"); let m = s.mark(); - s.discard(2); - assert_eq!(s.consume_from(m), &b"ab"[..]); - assert_eq!(s, new_buf(END_OF_INPUT, b"c")); - } + assert_eq!(s.peek(), Some(f(b'a'))); + assert_eq!(s.pop(), Some(f(b'a'))); + assert_eq!(s.peek(), Some(f(b'b'))); + assert_eq!(s.pop(), Some(f(b'b'))); + assert_eq!(s.peek(), Some(f(b'c'))); + assert_eq!(s.pop(), Some(f(b'c'))); + assert_eq!(s.peek(), None); + assert_eq!(s.pop(), None); + assert_eq!(s.peek(), None); + assert_eq!(s.pop(), None); + assert!(s.consume(1).is_none()); + buffer_eq_slice(s.consume_remaining(), &b""[..], &f); + + { + let mut n = 0; + + let b = s.consume_while(|_| { n += 1; true }); + + assert_eq!(n, 0); + buffer_eq_slice(b, &b""[..], &f); + } - fn run_primitives_test PartialEq<&'a [u8]>, I: Input>(mut s: I, last: bool) { - use primitives::Primitives; + let mut s = s.restore(m); + assert_eq!(s.peek(), Some(f(b'a'))); + let m = s.mark(); + buffer_eq_slice(s.consume_remaining(), &b"abc"[..], &f); + assert_eq!(s.peek(), None); + let mut s = s.restore(m); + assert_eq!(s.peek(), Some(f(b'a'))); + let m = s.mark(); + + { + let b = s.consume(2); + + assert_eq!(b.is_some(), true); + buffer_eq_slice(b.unwrap(), &b"ab"[..], &f); + } - assert_eq!(s.min_remaining(), 3); + assert_eq!(s.peek(), Some(f(b'c'))); + + let mut s = s.restore(m); + assert_eq!(s.peek(), Some(f(b'a'))); let m = s.mark(); - assert_eq!(s.min_remaining(), 3); - assert_eq!(s.first(), Some(b'a')); - assert_eq!(s.min_remaining(), 3); - assert_eq!(s.iter().collect::>(), vec![b'a', b'b', b'c']); - assert_eq!(s.min_remaining(), 3); - assert_eq!(s.consume(2), &b"ab"[..]); - assert_eq!(s.min_remaining(), 1); - assert_eq!(s.iter().collect::>(), vec![b'c']); - assert_eq!(s.consume(1), &b"c"[..]); - assert_eq!(s.min_remaining(), 0); - assert_eq!(s.iter().collect::>(), vec![]); + + { + let b = s.consume(3); + + assert_eq!(b.is_some(), true); + buffer_eq_slice(b.unwrap(), &b"abc"[..], &f); + } + + assert_eq!(s.peek(), None); let mut s = s.restore(m); - assert_eq!(s.min_remaining(), 3); - assert_eq!(s.iter().collect::>(), vec![b'a', b'b', b'c']); - s.discard(1); - assert_eq!(s.first(), Some(b'b')); - assert_eq!(s.min_remaining(), 2); - assert_eq!(s.iter().collect::>(), vec![b'b', b'c']); - } - */ -}*/ + assert_eq!(s.peek(), Some(f(b'a'))); + let m = s.mark(); + + { + let mut n = 0; + + let b = s.consume_while(|c| { + assert_eq!(c, f(b"abc"[n])); + + n += 1; + + n < 3 + }); + + assert_eq!(n, 3); + buffer_eq_slice(b, &b"ab"[..], &f); + } + + assert_eq!(s.peek(), Some(f(b'c'))); + assert_eq!(s.pop(), Some(f(b'c'))); + assert_eq!(s.peek(), None); + assert_eq!(s.pop(), None); + + buffer_eq_slice(s.consume_from(m), &b"abc"[..], &f); + } +} diff --git a/src/types/mod.rs b/src/types/mod.rs index 06d2df29..116c86fe 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -198,3 +198,45 @@ pub trait Input: Sized { #[inline] fn _restore(self, Guard, Self::Marker) -> Self; } + +#[cfg(test)] +mod test { + use super::{Input, InputBuf, ParseResult}; + use primitives::IntoInner; + + #[test] + fn ret() { + let i1: InputBuf = InputBuf::new(b"in1"); + let i2: InputBuf = InputBuf::new(b"in2"); + + let r1: ParseResult<_, u32, ()> = i1.ret::<_, ()>(23u32); + let r2: ParseResult<_, i32, &str> = i2.ret::<_, &str>(23i32); + + assert_eq!(r1.into_inner(), (InputBuf::new(b"in1"), Ok(23u32))); + assert_eq!(r2.into_inner(), (InputBuf::new(b"in2"), Ok(23i32))); + } + + #[test] + fn err() { + let i1: InputBuf = InputBuf::new(b"in1"); + let i2: InputBuf = InputBuf::new(b"in2"); + + let r1: ParseResult<_, (), u32> = i1.err::<(), _>(23u32); + let r2: ParseResult<_, &str, i32> = i2.err::<&str, _>(23i32); + + assert_eq!(r1.into_inner(), (InputBuf::new(b"in1"), Err(23u32))); + assert_eq!(r2.into_inner(), (InputBuf::new(b"in2"), Err(23i32))); + } + + #[test] + fn from_result() { + let i1: Result = Ok(23); + let i2: Result<&str, &str> = Err("foobar"); + + let r1 = InputBuf::new(b"in1").from_result(i1); + let r2 = InputBuf::new(b"in2").from_result(i2); + + assert_eq!(r1.into_inner(), (InputBuf::new(b"in1"), Ok(23u32))); + assert_eq!(r2.into_inner(), (InputBuf::new(b"in2"), Err("foobar"))); + } +} From 8b87bdadcdccb20a7ee5443759dc6391fcfbc862 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 30 Mar 2016 19:20:43 +0200 Subject: [PATCH 25/63] Removed incomplete amount from streams and parse_only --- src/buffer/buffer.rs | 1 - src/buffer/mod.rs | 19 ++----------------- src/buffer/slice.rs | 2 +- src/buffer/stateful.rs | 20 +++++++++----------- src/lib.rs | 10 ++-------- src/parse.rs | 22 ++++------------------ src/types/input.rs | 2 +- 7 files changed, 19 insertions(+), 57 deletions(-) diff --git a/src/buffer/buffer.rs b/src/buffer/buffer.rs index 60368e3d..35185771 100644 --- a/src/buffer/buffer.rs +++ b/src/buffer/buffer.rs @@ -28,7 +28,6 @@ pub trait Buffer: ops::Deref { #[inline] fn fill>(&mut self, &mut S) -> io::Result; - /// Buffer attempts to clear space for additional items. #[inline] fn request_space(&mut self, usize); diff --git a/src/buffer/mod.rs b/src/buffer/mod.rs index 35991f20..b878715e 100644 --- a/src/buffer/mod.rs +++ b/src/buffer/mod.rs @@ -37,7 +37,6 @@ use std::io; use types::{ParseResult, InputBuf}; use types::Buffer as InputBuffer; -use parse::ParseError; pub use self::slice::SliceStream; pub use self::data_source::DataSource; @@ -54,7 +53,7 @@ pub enum StreamError { /// An error occurred in the parser, the given slice indicates the part which failed. ParseError(B, E), /// Parser failed to complete with the available data. - Incomplete(usize), + Incomplete, /// An IO-error occurred while attempting to fill the buffer. IoError(io::Error), /// The last parser completed successfully and there is no more input to parse. @@ -70,7 +69,7 @@ impl> PartialEq for StreamError { fn eq(&self, other: &StreamError) -> bool { match (self, other) { (&StreamError::ParseError(ref b1, ref e1), &StreamError::ParseError(ref b2, ref e2)) => b1 == b2 && e1 == e2, - (&StreamError::Incomplete(n1), &StreamError::Incomplete(n2)) => n1 == n2, + (&StreamError::Incomplete, &StreamError::Incomplete) => true, (&StreamError::EndOfInput, &StreamError::EndOfInput) => true, (&StreamError::Retry, &StreamError::Retry) => true, _ => false, @@ -78,20 +77,6 @@ impl> PartialEq for StreamError { } } -// FIXME: -/* -impl From> for StreamError { - fn from(e: ParseError) -> Self { - use primitives::Primitives; - - match e { - ParseError::Error(b, e) => StreamError::ParseError(b, e), - ParseError::Incomplete(n) => StreamError::Incomplete(n), - } - } -} -*/ - /// Trait wrapping the state management in reading from a data source while parsing. pub trait Stream<'a, 'i> { /// The input item type, usually depending on which `DataSource` is used. diff --git a/src/buffer/slice.rs b/src/buffer/slice.rs index 2079f722..472ef212 100644 --- a/src/buffer/slice.rs +++ b/src/buffer/slice.rs @@ -101,7 +101,7 @@ impl<'a, 'i, I: 'i + Copy + PartialEq> Stream<'a, 'i> for SliceStream<'i, I> { (mut remainder, Err(err)) => { if remainder.is_incomplete() { // TODO: 1 is not correct, n is expected len but we can't obtain that right now - Err(StreamError::Incomplete(self.len() + 1)) + Err(StreamError::Incomplete) } else { // TODO: Do something neater with the remainder // TODO: Detail this behaviour, maybe make it configurable diff --git a/src/buffer/stateful.rs b/src/buffer/stateful.rs index e5a8489d..b88a62ba 100644 --- a/src/buffer/stateful.rs +++ b/src/buffer/stateful.rs @@ -103,13 +103,12 @@ impl> Source { /// Attempts to fill the buffer to satisfy the last call to `parse()`. #[inline] pub fn fill(&mut self) -> io::Result { - // Make sure we actually try to read something in case the buffer is empty - let req = cmp::max(1, self.request); + let req = self.buffer.len() + 1; self.fill_requested(req).map(|n| { self.state.remove(INCOMPLETE); - if self.buffer.len() >= req { + if n > 0 { self.state.remove(END_OF_INPUT); } else { self.state.insert(END_OF_INPUT); @@ -221,8 +220,6 @@ impl<'a, S: DataSource, B: Buffer> Stream<'a, 'a> for Source (remainder, Ok(data)) => { if remainder.is_incomplete() && self.state.contains(END_OF_INPUT) { // We can't accept this since we might have hit a premature end - self.request = self.buffer.len() + 1; - self.state.insert(INCOMPLETE); Err(StreamError::Retry) @@ -235,11 +232,9 @@ impl<'a, S: DataSource, B: Buffer> Stream<'a, 'a> for Source }, (mut remainder, Err(err)) => { if remainder.is_incomplete() { - // TODO: How to deal with n, no longer present? - self.request = self.buffer.len() + 1; - if self.state.contains(END_OF_INPUT) { - Err(StreamError::Incomplete(self.request)) + // TODO: Remove the 1? + Err(StreamError::Incomplete) } else { self.state.insert(INCOMPLETE); @@ -416,12 +411,15 @@ mod test { assert_eq!(b.parse(|i| { n += 1; take(i, 2).inspect(|_| m += 1) }), Err(StreamError::Retry)); assert_eq!(n, 2); assert_eq!(m, 1); - assert_eq!(b.parse(|i| { n += 1; take(i, 2).inspect(|_| m += 1) }), Err(StreamError::Incomplete(2))); + assert_eq!(b.parse(|i| { n += 1; take(i, 2).inspect(|_| m += 1) }), Err(StreamError::Retry)); assert_eq!(n, 3); assert_eq!(m, 1); - assert_eq!(b.parse(|i| { n += 1; take(i, 2).inspect(|_| m += 1) }), Err(StreamError::Incomplete(2))); + assert_eq!(b.parse(|i| { n += 1; take(i, 2).inspect(|_| m += 1) }), Err(StreamError::Incomplete)); assert_eq!(n, 4); assert_eq!(m, 1); + assert_eq!(b.parse(|i| { n += 1; take(i, 2).inspect(|_| m += 1) }), Err(StreamError::Incomplete)); + assert_eq!(n, 5); + assert_eq!(m, 1); } #[test] diff --git a/src/lib.rs b/src/lib.rs index aa64be61..d4e35082 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -262,13 +262,7 @@ pub mod prelude { U8Input, ParseResult, }; - pub use parse::{ - ParseError, - parse_only, - }; + pub use parse::parse_only; } -pub use parse::{ - ParseError, - parse_only, -}; +pub use parse::parse_only; diff --git a/src/parse.rs b/src/parse.rs index e706e024..67b934b4 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -4,15 +4,6 @@ use primitives::{ Primitives, }; -/// Simple error type returned from `parse_only`. -#[derive(Debug, Eq, PartialEq)] -pub enum ParseError { - /// A parse error occurred. - Error(I, E), - /// The parser attempted to read more data than available. - Incomplete(usize), -} - /// Runs the given parser on the supplied finite input. /// /// ``` @@ -50,12 +41,12 @@ pub enum ParseError { /// Err(ParseError::Error(&b" and more"[..], Error::new()))); /// # } /// ``` -pub fn parse_only<'a, I, T, E, F>(parser: F, input: &'a [I]) -> Result> +pub fn parse_only<'a, I, T, E, F>(parser: F, input: &'a [I]) -> Result where I: Copy + PartialEq, F: FnOnce(&'a [I]) -> ParseResult<&'a [I], T, E> { match parser(input).into_inner() { (_, Ok(t)) => Ok(t), - (mut b, Err(e)) => Err(ParseError::Error(b.consume_remaining(), e)), + (mut b, Err(e)) => Err((b.consume_remaining(), e)), } } @@ -64,27 +55,21 @@ mod test { use types::Input; use primitives::Primitives; use super::{ - ParseError, parse_only, }; - /* #[test] fn inspect_input() { - let mut state = None; let mut input = None; assert_eq!(parse_only(|i| { - state = Some(i.is_end()); input = Some(i.iter().cloned().collect()); i.ret::<_, ()>("the result") }, b"the input"), Ok("the result")); assert_eq!(input, Some(b"the input".to_vec())); - assert_eq!(state, Some(true)); } - */ #[test] fn err() { @@ -92,9 +77,10 @@ mod test { i.consume(4); i.err::<(), _>("my error") - }, b"the input"), Err(ParseError::Error(&b"input"[..], "my error"))); + }, b"the input"), Err((&b"input"[..], "my error"))); } + // FIXME: /* #[test] fn incomplete() { diff --git a/src/types/input.rs b/src/types/input.rs index b2186ef0..d2ef098e 100644 --- a/src/types/input.rs +++ b/src/types/input.rs @@ -450,7 +450,7 @@ mod test { assert_eq!(b.is_incomplete(), true); assert_eq!(b.len(), 1); assert_eq!(b.is_empty(), false); - let mut b = b.restore(m); + let b = b.restore(m); assert_eq!(b.is_incomplete(), true); assert_eq!(b.len(), 3); assert_eq!(b.is_empty(), false); From 8b0401283c0bec8c4dab60074d41c280dff75d13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 30 Mar 2016 20:43:48 +0200 Subject: [PATCH 26/63] Updated rustdoc --- src/ascii.rs | 2 +- src/buffer/mod.rs | 2 +- src/buffer/slice.rs | 5 ++- src/buffer/stateful.rs | 2 -- src/combinators/bounded.rs | 13 +++---- src/combinators/mod.rs | 37 +++++++++---------- src/lib.rs | 22 ++++++------ src/macros.rs | 74 +++++++++++++++++++++++--------------- src/parse.rs | 12 +++---- src/parsers.rs | 42 +++++++++++----------- src/primitives.rs | 26 -------------- src/types/mod.rs | 10 +++--- src/types/parse_result.rs | 40 +++++---------------- 13 files changed, 127 insertions(+), 160 deletions(-) diff --git a/src/ascii.rs b/src/ascii.rs index 28338fb3..3ca99292 100644 --- a/src/ascii.rs +++ b/src/ascii.rs @@ -150,7 +150,7 @@ pub fn signed, T, F>(i: I, f: F) -> SimpleResult /// use chomp::parse_only; /// use chomp::ascii::decimal; /// -/// let r = parse_only(decimal::, b"123"); +/// let r = parse_only(decimal::<_, u8>, b"123"); /// /// assert_eq!(r, Ok(123u8)); /// ``` diff --git a/src/buffer/mod.rs b/src/buffer/mod.rs index b878715e..a0f9d0a8 100644 --- a/src/buffer/mod.rs +++ b/src/buffer/mod.rs @@ -9,7 +9,7 @@ //! //! use chomp::buffer; //! use chomp::buffer::Stream; -//! use chomp::{token, take_while, take_while1}; +//! use chomp::prelude::{token, take_while, take_while1}; //! use chomp::ascii::is_whitespace; //! //! let f = File::open("./README.md").unwrap(); diff --git a/src/buffer/slice.rs b/src/buffer/slice.rs index 472ef212..e8778b2b 100644 --- a/src/buffer/slice.rs +++ b/src/buffer/slice.rs @@ -8,7 +8,7 @@ use buffer::{IntoStream, StreamError, Stream}; /// ``` /// # #[macro_use] extern crate chomp; /// # fn main() { -/// use chomp::{token, take}; +/// use chomp::parsers::{token, take}; /// use chomp::buffer::{IntoStream, Stream}; /// /// let i = b"foo"; @@ -25,7 +25,7 @@ use buffer::{IntoStream, StreamError, Stream}; /// ``` /// # #[macro_use] extern crate chomp; /// # fn main() { -/// use chomp::{token, many, take}; +/// use chomp::prelude::{token, many, take}; /// use chomp::buffer::{IntoStream, Stream}; /// /// let i = b"foofoo"; @@ -100,7 +100,6 @@ impl<'a, 'i, I: 'i + Copy + PartialEq> Stream<'a, 'i> for SliceStream<'i, I> { }, (mut remainder, Err(err)) => { if remainder.is_incomplete() { - // TODO: 1 is not correct, n is expected len but we can't obtain that right now Err(StreamError::Incomplete) } else { // TODO: Do something neater with the remainder diff --git a/src/buffer/stateful.rs b/src/buffer/stateful.rs index b88a62ba..f9c8f541 100644 --- a/src/buffer/stateful.rs +++ b/src/buffer/stateful.rs @@ -1,5 +1,4 @@ use std::io; -use std::cmp; use types::{InputBuf, ParseResult}; use primitives::IntoInner; @@ -233,7 +232,6 @@ impl<'a, S: DataSource, B: Buffer> Stream<'a, 'a> for Source (mut remainder, Err(err)) => { if remainder.is_incomplete() { if self.state.contains(END_OF_INPUT) { - // TODO: Remove the 1? Err(StreamError::Incomplete) } else { self.state.insert(INCOMPLETE); diff --git a/src/combinators/bounded.rs b/src/combinators/bounded.rs index 6ac3a081..4e3083b5 100644 --- a/src/combinators/bounded.rs +++ b/src/combinators/bounded.rs @@ -7,7 +7,8 @@ //! //! ``` //! use chomp::combinators::bounded::many; -//! use chomp::{parse_only, any}; +//! use chomp::parse_only; +//! use chomp::parsers::any; //! //! // Read any character 2 or 3 times //! let r: Result, _> = parse_only(|i| many(i, 2..4, any), b"abcd"); @@ -239,7 +240,7 @@ impl BoundedRange for Range { (s, (0, _), EndStateTill::EndSuccess) => s.ret(result), // Did not reach minimum or a failure, propagate (s, (_, _), EndStateTill::Error(e)) => s.err(e), - (s, (_, _), EndStateTill::Incomplete) => unreachable!(), + (_, (_, _), EndStateTill::Incomplete) => unreachable!(), // We cannot reach this since we only run the end test once we have reached the // minimum number of matches (_, (_, _), EndStateTill::EndSuccess) => unreachable!() @@ -343,7 +344,7 @@ impl BoundedRange for RangeFrom { (s, 0, EndStateTill::EndSuccess) => s.ret(result), // Did not reach minimum or a failure, propagate (s, _, EndStateTill::Error(e)) => s.err(e), - (s, _, EndStateTill::Incomplete) => unreachable!(), + (_, _, EndStateTill::Incomplete) => unreachable!(), // We cannot reach this since we only run the end test once we have reached the // minimum number of matches (_, _, EndStateTill::EndSuccess) => unreachable!() @@ -425,7 +426,7 @@ impl BoundedRange for RangeFull { (s, (), EndStateTill::EndSuccess) => s.ret(result), (s, (), EndStateTill::Error(e)) => s.err(e), // Nested parser incomplete, propagate if not at end - (s, (), EndStateTill::Incomplete) => unreachable!() + (_, (), EndStateTill::Incomplete) => unreachable!() } } } @@ -551,7 +552,7 @@ impl BoundedRange for RangeTo { (s, _, EndStateTill::EndSuccess) => s.ret(result), // Did not reach minimum or a failure, propagate (s, _, EndStateTill::Error(e)) => s.err(e), - (s, _, EndStateTill::Incomplete) => unreachable!(), + (_, _, EndStateTill::Incomplete) => unreachable!(), } } } @@ -661,7 +662,7 @@ impl BoundedRange for usize { (s, 0, EndStateTill::EndSuccess) => s.ret(result), // Did not reach minimum or a failure, propagate (s, _, EndStateTill::Error(e)) => s.err(e), - (s, _, EndStateTill::Incomplete) => unreachable!(), + (_, _, EndStateTill::Incomplete) => unreachable!(), // We cannot reach this since we only run the end test once we have reached the // minimum number of matches (_, _, EndStateTill::EndSuccess) => unreachable!() diff --git a/src/combinators/mod.rs b/src/combinators/mod.rs index f6af5778..47407b6f 100644 --- a/src/combinators/mod.rs +++ b/src/combinators/mod.rs @@ -20,13 +20,13 @@ use primitives::{ /// Applies the parser ``p`` exactly ``num`` times collecting all items into `T: FromIterator`. /// /// ``` -/// use chomp::{U8Result, ParseError, Error, Input, parse_only, count, token, take_remainder}; +/// use chomp::prelude::*; /// -/// fn parse(i: Input) -> U8Result> { +/// fn parse(i: I) -> SimpleResult> { /// count(i, 2, |i| token(i, b'a')) /// } /// -/// assert_eq!(parse_only(parse, b"a "), Err(ParseError::Error(b" ", Error::expected(b'a')))); +/// assert_eq!(parse_only(parse, b"a "), Err((&b" "[..], Error::expected(b'a')))); /// assert_eq!(parse_only(parse, b"aa "), Ok(vec![b'a', b'a'])); /// /// let with_remainder = |i| parse(i).bind(|i, d| take_remainder(i).map(|r| (r, d))); @@ -46,9 +46,9 @@ pub fn count(i: I, num: usize, p: F) -> ParseResult) -> U8Result { +/// fn f(i: I) -> SimpleResult { /// option(i, |i| token(i, b'a'), b'd') /// } /// @@ -75,7 +75,7 @@ pub fn option(i: I, f: F, default: T) -> ParseResult /// and its alternation operator (`<|>`). /// /// ``` -/// use chomp::{ParseError, Error, parse_only, or, token}; +/// use chomp::prelude::{Error, parse_only, or, token}; /// /// let p = |i| or(i, /// |i| token(i, b'a'), @@ -83,7 +83,7 @@ pub fn option(i: I, f: F, default: T) -> ParseResult /// /// assert_eq!(parse_only(&p, b"abc"), Ok(b'a')); /// assert_eq!(parse_only(&p, b"bbc"), Ok(b'b')); -/// assert_eq!(parse_only(&p, b"cbc"), Err(ParseError::Error(b"cbc", Error::expected(b'b')))); +/// assert_eq!(parse_only(&p, b"cbc"), Err((&b"cbc"[..], Error::expected(b'b')))); /// ``` #[inline] pub fn or(i: I, f: F, g: G) -> ParseResult @@ -106,7 +106,7 @@ pub fn or(i: I, f: F, g: G) -> ParseResult /// Note: Allocates data. /// /// ``` -/// use chomp::{parse_only, token, many, take_while1}; +/// use chomp::prelude::{parse_only, token, many, take_while1}; /// /// let r: Result, _> = parse_only(|i| many(i, /// |i| take_while1(i, |c| c != b',' && c != b' ') @@ -132,13 +132,13 @@ pub fn many(i: I, f: F) -> ParseResult /// Note: Allocates data. /// /// ``` -/// use chomp::{ParseError, Error, parse_only, token, many1, take_while1}; +/// use chomp::prelude::{Error, parse_only, token, many1, take_while1}; /// /// let p = |i| many1(i, |i| take_while1(i, |c| c != b',' && c != b' ') /// .bind(|i, c| token(i, b',') /// .map(|_| c))); /// -/// assert_eq!(parse_only(&p, b"a "), Err(ParseError::Error(b" ", Error::expected(b',')))); +/// assert_eq!(parse_only(&p, b"a "), Err((&b" "[..], Error::expected(b',')))); /// assert_eq!(parse_only(&p, b"a, "), Ok(vec![&b"a"[..]])); /// ``` #[inline] @@ -157,7 +157,7 @@ pub fn many1(i: I, f: F) -> ParseResult /// Incomplete will be propagated from `R` if end of input has not been read. /// /// ``` -/// use chomp::{parse_only, sep_by, token}; +/// use chomp::prelude::{parse_only, sep_by, token}; /// use chomp::ascii::decimal; /// /// let r: Result, _> = parse_only(|i| sep_by(i, decimal, |i| token(i, b';')), b"91;03;20"); @@ -183,7 +183,7 @@ pub fn sep_by(i: I, p: R, sep: F) -> ParseResult< /// Incomplete will be propagated from `R` if end of input has not been read. /// /// ``` -/// use chomp::{parse_only, sep_by1, token}; +/// use chomp::prelude::{parse_only, sep_by1, token}; /// use chomp::ascii::decimal; /// /// let r: Result, _> = parse_only(|i| sep_by1(i, decimal, |i| token(i, b';')), b"91;03;20"); @@ -207,7 +207,7 @@ pub fn sep_by1(i: I, p: R, sep: F) -> ParseResult /// Errors from `R` are propagated. /// /// ``` -/// use chomp::{parse_only, many_till, any, token}; +/// use chomp::prelude::{parse_only, many_till, any, token}; /// /// let r: Result, _> = parse_only(|i| many_till(i, any, |i| token(i, b';')), b"abc;def"); /// @@ -230,7 +230,7 @@ pub fn many_till(i: I, p: R, end: F) -> ParseResu /// ``many`` allocates a separate data structure to contain the data before proceeding. /// /// ``` -/// use chomp::{parse_only, skip_many, token}; +/// use chomp::prelude::{parse_only, skip_many, token}; /// /// let r = parse_only(|i| skip_many(i, |i| token(i, b'a')).then(|i| token(i, b'b')), b"aaaabc"); /// @@ -251,14 +251,14 @@ pub fn skip_many(i: I, f: F) -> ParseResult /// ``many1`` allocates a separate data structure to contain the data before proceeding. /// /// ``` -/// use chomp::{ParseError, Error, parse_only, skip_many1, token}; +/// use chomp::prelude::{Error, parse_only, skip_many1, token}; /// /// let p = |i| skip_many1(i, |i| token(i, b'a')).bind(|i, _| token(i, b'b')); /// /// assert_eq!(parse_only(&p, b"aaaabc"), Ok(b'b')); /// assert_eq!(parse_only(&p, b"abc"), Ok(b'b')); /// -/// assert_eq!(parse_only(&p, b"bc"), Err(ParseError::Error(b"bc", Error::expected(b'a')))); +/// assert_eq!(parse_only(&p, b"bc"), Err((&b"bc"[..], Error::expected(b'a')))); /// ``` #[inline] pub fn skip_many1(i: I, f: F) -> ParseResult @@ -269,7 +269,7 @@ pub fn skip_many1(i: I, f: F) -> ParseResult /// Returns the result of the given parser as well as the slice which matched it. /// /// ``` -/// use chomp::{parse_only, matched_by}; +/// use chomp::prelude::{parse_only, matched_by}; /// use chomp::ascii::decimal; /// /// assert_eq!(parse_only(|i| matched_by(i, decimal), b"123"), Ok((&b"123"[..], 123u32))); @@ -292,7 +292,7 @@ pub fn matched_by(i: I, f: F) -> ParseResult(i: I, f: F) -> ParseResult } } +// FIXME: /* #[cfg(test)] mod test { diff --git a/src/lib.rs b/src/lib.rs index d4e35082..64a24271 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,16 +16,15 @@ //! ``` //! # #[macro_use] extern crate chomp; //! # fn main() { -//! use chomp::{Input, U8Result, parse_only}; -//! use chomp::{take_while1, token}; +//! use chomp::prelude::*; //! //! #[derive(Debug, Eq, PartialEq)] -//! struct Name<'a> { -//! first: &'a [u8], -//! last: &'a [u8], +//! struct Name { +//! first: B, +//! last: B, //! } //! -//! fn name(i: Input) -> U8Result { +//! fn name(i: I) -> SimpleResult> { //! parse!{i; //! let first = take_while1(|c| c != b' '); //! token(b' '); // skipping this char @@ -39,7 +38,7 @@ //! } //! //! assert_eq!(parse_only(name, "Martin Wernstål\n".as_bytes()), Ok(Name{ -//! first: b"Martin", +//! first: &b"Martin"[..], //! last: "Wernstål".as_bytes() //! })); //! # } @@ -93,7 +92,7 @@ //! ``` //! # #[macro_use] extern crate chomp; //! # fn main() { -//! # use chomp::{satisfy, parse_only}; +//! # use chomp::prelude::*; //! # let r = parse_only(parser!{ //! satisfy(|c| { //! match c { @@ -144,8 +143,8 @@ //! //! ``` //! # #[macro_use] extern crate chomp; -//! # use chomp::{Input, parse_only, satisfy, string, U8Result}; -//! fn f(i: Input) -> U8Result<(u8, u8, u8)> { +//! # use chomp::prelude::*; +//! fn f(i: I) -> SimpleResult { //! parse!{i; //! let a = digit(); //! let b = digit(); @@ -154,7 +153,7 @@ //! } //! } //! -//! fn digit(i: Input) -> U8Result { +//! fn digit(i: I) -> SimpleResult { //! satisfy(i, |c| b'0' <= c && c <= b'9').map(|c| c - b'0') //! } //! # fn main() { @@ -258,6 +257,7 @@ pub mod prelude { matched_by, }; pub use types::{ + Buffer, Input, U8Input, ParseResult, diff --git a/src/macros.rs b/src/macros.rs index 4da176cc..ddcb22f2 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -1,16 +1,28 @@ /// Macro emulating `do`-notation for the parser monad, automatically threading the linear type. /// -/// ```ignore +/// ``` +/// # #[macro_use] extern crate chomp; +/// # fn main() { +/// # use chomp::prelude::*; +/// # fn parser(i: I, _s: &str) -> ParseResult { i.ret(()) } +/// # fn other_parser(i: I) -> ParseResult { i.ret(23) } +/// # fn do_something(_i: u8) -> u32 { 23 } +/// # let input = &b"foo"[..]; +/// # let _r: ParseResult<_, _, ()> = /// parse!{input; /// parser("parameter"); /// let value = other_parser(); /// -/// ret do_something(value); +/// ret do_something(value) /// } +/// # ; +/// # let _r: ParseResult<_, _, ()> = /// // is equivalent to: /// parser(input, "parameter").bind(|i, _| /// other_parser(i).bind(|i, value| /// i.ret(do_something(value)))) +/// # ; +/// # } /// ``` /// /// # Examples @@ -20,15 +32,20 @@ /// ``` /// # #[macro_use] extern crate chomp; /// # fn main() { -/// use chomp::{Error, parse_only}; -/// use chomp::{take_while1, token}; +/// use chomp::prelude::{Buffer, Error, Input, parse_only, take_while1, token}; /// /// #[derive(Debug, Eq, PartialEq)] -/// struct Name<'a> { -/// first: &'a [u8], -/// last: &'a [u8], +/// struct Name { +/// first: B, +/// last: B, /// } /// +/// let r = |i| take_while1(i, |c| c != b' ').bind(|i, first| +/// token(i, b' ').then(|i| +/// take_while1(i, |c| c != b'\n').bind(|i, last| +/// i.ret::<_, Error<_>>(Name { first: first, last: last })))); +/// +/// /* /// let r = |i| parse!{i; /// let first = take_while1(|c| c != b' '); /// token(b' '); @@ -39,9 +56,10 @@ /// last: last, /// } /// }; +/// */ /// /// assert_eq!(parse_only(r, "Martin Wernstål\n".as_bytes()), Ok(Name{ -/// first: b"Martin", +/// first: &b"Martin"[..], /// last: "Wernstål".as_bytes() /// })); /// # } @@ -53,10 +71,10 @@ /// ``` /// # #[macro_use] extern crate chomp; /// # fn main() { -/// use chomp::{Input, U8Result, parse_only, string, token}; +/// use chomp::prelude::{U8Input, SimpleResult, parse_only, string, token}; /// use chomp::ascii::decimal; /// -/// fn parse_ip(i: Input) -> U8Result<(u8, u8, u8, u8)> { +/// fn parse_ip(i: I) -> SimpleResult { /// parse!{i; /// string(b"ip:"); /// let a = decimal() <* token(b'.'); @@ -77,7 +95,7 @@ /// ``` /// # #[macro_use] extern crate chomp; /// # fn main() { -/// use chomp::{parse_only, string}; +/// use chomp::prelude::{parse_only, string}; /// /// #[derive(Debug, Eq, PartialEq)] /// enum Log { @@ -143,8 +161,8 @@ /// # #[macro_use] extern crate chomp; /// # fn main() { /// # use chomp::ascii::decimal; -/// # use chomp::{parse_only, Input, token, U8Result}; -/// # fn my_parser(i: Input) -> U8Result { +/// # use chomp::prelude::{parse_only, U8Input, token, SimpleResult}; +/// # fn my_parser(i: I) -> SimpleResult { /// parse!{i; /// token(b':'); /// let n: u32 = decimal(); @@ -186,9 +204,9 @@ /// ``` /// # #[macro_use] extern crate chomp; /// # fn main() { -/// # use chomp::{parse_only, Input, U8Result}; -/// # fn my_parser(i: Input) -> U8Result<&'static str> { -/// fn do_it<'i, 'a>(i: Input<'i, u8>, s: &'a str) -> U8Result<'i, &'a str> { i.ret(s) } +/// # use chomp::prelude::{parse_only, U8Input, SimpleResult}; +/// # fn my_parser(i: I) -> SimpleResult { +/// fn do_it<'a, I: U8Input>(i: I, s: &'a str) -> SimpleResult { i.ret(s) } /// /// parse!{i; /// do_it("second parameter") @@ -206,8 +224,8 @@ /// ``` /// # #[macro_use] extern crate chomp; /// # fn main() { -/// # use chomp::{parse_only, ParseError}; -/// let r: Result<_, ParseError<_, ()>> = parse_only( +/// # use chomp::prelude::{parse_only, Input}; +/// let r: Result<_, (_, ())> = parse_only( /// parser!{ ret "some success data" }, /// b"input data" /// ); @@ -216,7 +234,7 @@ /// # } /// ``` /// -/// In the example above the `Result<_, ParseError<_, ()>>` type-annotation is required since `ret` +/// In the example above the `Result<_, (_, ())>` type-annotation is required since `ret` /// leaves the error type `E` free which means that the `parser!` expression above cannot infer the /// error type without the annotation. `ret` and `end` both provide a mechanism to supply this /// information inline: @@ -224,10 +242,10 @@ /// ``` /// # #[macro_use] extern crate chomp; /// # fn main() { -/// # use chomp::{parse_only, ParseError}; +/// # use chomp::prelude::{parse_only, Input}; /// let r = parse_only(parser!{ err @ u32, _: "some error data" }, b"input data"); /// -/// assert_eq!(r, Err(ParseError::Error(b"input data", "some error data"))); +/// assert_eq!(r, Err((&b"input data"[..], "some error data"))); /// # } /// ``` /// @@ -243,8 +261,8 @@ /// ``` /// # #[macro_use] extern crate chomp; /// # fn main() { -/// # use chomp::{parse_only, Input, ParseResult}; -/// fn other_parser(i: Input) -> ParseResult { +/// # use chomp::prelude::{parse_only, Input, ParseResult}; +/// fn other_parser(i: I) -> ParseResult { /// i.ret("Success!") /// } /// @@ -278,7 +296,7 @@ /// ``` /// # #[macro_use] extern crate chomp; /// # fn main() { -/// # use chomp::ascii::decimal; use chomp::{parse_only, token}; +/// # use chomp::ascii::decimal; use chomp::prelude::{parse_only, token}; /// let p = parser!{ decimal() <* token(b';') }; /// /// assert_eq!(parse_only(p, b"123;"), Ok(123u32)); @@ -293,7 +311,7 @@ /// ``` /// # #[macro_use] extern crate chomp; /// # fn main() { -/// # use chomp::{parse_only, token}; +/// # use chomp::prelude::{parse_only, token}; /// let p = parser!{ token(b'a') <|> token(b'b') }; /// /// assert_eq!(parse_only(p, b"b"), Ok(b'b')); @@ -308,7 +326,7 @@ /// ``` /// # #[macro_use] extern crate chomp; /// # fn main() { -/// # use chomp::{parse_only, token}; +/// # use chomp::prelude::{parse_only, token}; /// let p = parser!{ token(b'a') >> token(b';') }; /// /// assert_eq!(parse_only(p, b"a;"), Ok(b';')); @@ -325,7 +343,7 @@ /// ``` /// # #[macro_use] extern crate chomp; /// # fn main() { -/// # use chomp::{parse_only}; +/// # use chomp::prelude::{Input, parse_only}; /// let p = parser!{ (i -> i.err("foo")) <|> (i -> i.ret("bar")) }; /// /// assert_eq!(parse_only(p, b"a;"), Ok("bar")); @@ -589,7 +607,7 @@ macro_rules! __parse_internal { /// ``` /// # #[macro_use] extern crate chomp; /// # fn main() { -/// use chomp::{parse_only, string}; +/// use chomp::prelude::{parse_only, string}; /// /// let r = parser!{ string(b"ab") <|> string(b"ac") }; /// diff --git a/src/parse.rs b/src/parse.rs index 67b934b4..34de42e9 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -7,15 +7,14 @@ use primitives::{ /// Runs the given parser on the supplied finite input. /// /// ``` -/// use chomp::{ParseError, Error}; -/// use chomp::parse_only; +/// use chomp::prelude::{parse_only, Error}; /// use chomp::ascii::decimal; /// /// assert_eq!(parse_only(decimal, b"123foobar"), Ok(123u32)); /// /// // Annotation because `decimal` is generic over number types /// let r: Result = parse_only(decimal, b"foobar"); -/// assert_eq!(r, Err(ParseError::Error(&b"foobar"[..], Error::new()))); +/// assert_eq!(r, Err((&b"foobar"[..], Error::new()))); /// ``` /// /// This will not force the parser to consume all available input, any remainder will be @@ -24,10 +23,9 @@ use primitives::{ /// ``` /// # #[macro_use] extern crate chomp; /// # fn main() { -/// use chomp::{Input, ParseError, Error, U8Result}; -/// use chomp::{parse_only, string, eof}; +/// use chomp::prelude::{U8Input, Error, SimpleResult, parse_only, string, eof}; /// -/// fn my_parser(i: Input) -> U8Result<&[u8]> { +/// fn my_parser(i: I) -> SimpleResult { /// parse!{i; /// let r = string(b"pattern"); /// eof(); @@ -38,7 +36,7 @@ use primitives::{ /// /// assert_eq!(parse_only(my_parser, b"pattern"), Ok(&b"pattern"[..])); /// assert_eq!(parse_only(my_parser, b"pattern and more"), -/// Err(ParseError::Error(&b" and more"[..], Error::new()))); +/// Err((&b" and more"[..], Error::new()))); /// # } /// ``` pub fn parse_only<'a, I, T, E, F>(parser: F, input: &'a [I]) -> Result diff --git a/src/parsers.rs b/src/parsers.rs index e334e108..31b767ab 100644 --- a/src/parsers.rs +++ b/src/parsers.rs @@ -24,7 +24,7 @@ pub use debugtrace::StackFrame; /// If the buffer length is 0 this parser is considered incomplete. /// /// ``` -/// use chomp::{parse_only, any}; +/// use chomp::prelude::{parse_only, any}; /// /// assert_eq!(parse_only(any, b"abc"), Ok(b'a')); /// ``` @@ -42,7 +42,7 @@ pub fn any(mut i: I) -> SimpleResult { /// If the buffer length is 0 this parser is considered incomplete. /// /// ``` -/// use chomp::{parse_only, satisfy}; +/// use chomp::prelude::{parse_only, satisfy}; /// /// assert_eq!(parse_only(|i| satisfy(i, |c| c == b'a'), b"abc"), Ok(b'a')); /// ``` @@ -61,7 +61,7 @@ pub fn satisfy(mut i: I, f: F) -> SimpleResult /// ``` /// use std::ascii::AsciiExt; /// -/// use chomp::{parse_only, satisfy_with}; +/// use chomp::prelude::{parse_only, satisfy_with}; /// /// let r = parse_only( /// |i| satisfy_with(i, |c| AsciiExt::to_ascii_uppercase(&c), |c| c == b'T'), @@ -92,7 +92,7 @@ pub fn satisfy_with(mut i: I, f: F, p: P) -> SimpleRes /// If the buffer length is 0 this parser is considered incomplete. /// /// ``` -/// use chomp::{parse_only, token}; +/// use chomp::prelude::{parse_only, token}; /// /// assert_eq!(parse_only(|i| token(i, b'a'), b"abc"), Ok(b'a')); /// ``` @@ -100,7 +100,7 @@ pub fn satisfy_with(mut i: I, f: F, p: P) -> SimpleRes pub fn token(mut i: I, t: I::Token) -> SimpleResult { match i.peek() { Some(c) if c == t => { i.pop(); i.ret(c) }, - _ => i.err(Error::unexpected()), + _ => i.err(Error::expected(t)), } } @@ -109,7 +109,7 @@ pub fn token(mut i: I, t: I::Token) -> SimpleResult { /// If the buffer length is 0 this parser is considered incomplete. /// /// ``` -/// use chomp::{parse_only, not_token}; +/// use chomp::prelude::{parse_only, not_token}; /// /// assert_eq!(parse_only(|i| not_token(i, b'b'), b"abc"), Ok(b'a')); /// ``` @@ -127,7 +127,7 @@ pub fn not_token(mut i: I, t: I::Token) -> SimpleResult { /// This parser is never considered incomplete. /// /// ``` -/// use chomp::{parse_only, peek}; +/// use chomp::prelude::{parse_only, peek}; /// /// assert_eq!(parse_only(peek, b"abc"), Ok(Some(b'a'))); /// @@ -145,7 +145,7 @@ pub fn peek(mut i: I) -> SimpleResult> { /// If the buffer length is 0 this parser is considered incomplete. /// /// ``` -/// use chomp::{parse_only, peek_next}; +/// use chomp::prelude::{parse_only, peek_next}; /// /// assert_eq!(parse_only(peek_next, b"abc"), Ok(b'a')); /// ``` @@ -162,7 +162,7 @@ pub fn peek_next(mut i: I) -> SimpleResult { /// If the buffer length is less than ``num`` this parser is considered incomplete. /// /// ``` -/// use chomp::{parse_only, take}; +/// use chomp::prelude::{parse_only, take}; /// /// assert_eq!(parse_only(|i| take(i, 3), b"abcd"), Ok(&b"abc"[..])); /// ``` @@ -181,7 +181,7 @@ pub fn take(mut i: I, num: usize) -> SimpleResult { /// more input which needs to be matched. /// /// ``` -/// use chomp::{parse_only, take_while}; +/// use chomp::prelude::{parse_only, take_while}; /// /// let r = parse_only(|i| take_while(i, |c| c == b'a' || c == b'b'), b"abcdcba"); /// @@ -191,7 +191,7 @@ pub fn take(mut i: I, num: usize) -> SimpleResult { /// Without managing to match anything: /// /// ``` -/// use chomp::{parse_only, take_while}; +/// use chomp::prelude::{parse_only, take_while}; /// /// let r = parse_only(|i| take_while(i, |c| c == b'z'), b"abcdcba"); /// @@ -212,7 +212,7 @@ pub fn take_while(mut i: I, f: F) -> SimpleResult /// more input which needs to be matched. If zero items were matched an error will be returned. /// /// ``` -/// use chomp::{parse_only, take_while1}; +/// use chomp::prelude::{parse_only, take_while1}; /// /// let r = parse_only(|i| take_while1(i, |c| c == b'a' || c == b'b'), b"abcdcba"); /// @@ -237,7 +237,7 @@ pub fn take_while1(mut i: I, f: F) -> SimpleResult /// more input which needs to be matched. /// /// ``` -/// use chomp::{parse_only, take_till}; +/// use chomp::prelude::{parse_only, take_till}; /// /// let r = parse_only(|i| take_till(i, |c| c == b'd'), b"abcdef"); /// @@ -272,7 +272,7 @@ pub fn take_till, F>(mut i: I, f: F) -> SimpleResult None, @@ -287,7 +287,7 @@ pub fn scan(mut i: I, s: S, mut f: F) -> SimpleResult Option { let mut state = Some(s); - let b = i.consume_while(|c| { state = f(mem::replace(&mut state, None).unwrap(), c); state.is_none()}); + let b = i.consume_while(|c| { state = f(mem::replace(&mut state, None).unwrap(), c); state.is_some() }); i.ret(b) } @@ -295,7 +295,7 @@ pub fn scan(mut i: I, s: S, mut f: F) -> SimpleResult None, @@ -313,8 +313,8 @@ pub fn run_scanner(mut i: I, s: S, mut f: F) -> SimpleResu let b = i.consume_while(|c| { let t = f(state, c); match t { - None => true, - Some(v) => { state = v; false } + None => false, + Some(v) => { state = v; true } } }); @@ -324,7 +324,7 @@ pub fn run_scanner(mut i: I, s: S, mut f: F) -> SimpleResu /// Matches the remainder of the buffer and returns it, always succeeds. /// /// ``` -/// use chomp::{parse_only, take_remainder}; +/// use chomp::prelude::{parse_only, take_remainder}; /// /// assert_eq!(parse_only(take_remainder, b"abcd"), Ok(&b"abcd"[..])); /// ``` @@ -341,7 +341,7 @@ pub fn take_remainder(mut i: I) -> SimpleResult { /// incomplete. /// /// ``` -/// use chomp::{parse_only, string}; +/// use chomp::prelude::{parse_only, string}; /// /// assert_eq!(parse_only(|i| string(i, b"abc"), b"abcdef"), Ok(&b"abc"[..])); /// ``` @@ -379,7 +379,7 @@ pub fn string<'b, T: Copy + PartialEq, I: Input>(mut i: I, s: &'b [T]) /// Matches the end of the input. /// /// ``` -/// use chomp::{parse_only, token, eof}; +/// use chomp::prelude::{parse_only, token, eof}; /// /// let r = parse_only(|i| token(i, b'a').then(eof), b"a"); /// diff --git a/src/primitives.rs b/src/primitives.rs index 26dd74d0..cc54f92c 100644 --- a/src/primitives.rs +++ b/src/primitives.rs @@ -73,32 +73,6 @@ pub trait Primitives: Input { } /// Marks the current position to be able to backtrack to it using `restore`. - /// - /// # Example - /// - /// ``` - /// use chomp::{take, Input}; - /// use chomp::primitives::Primitives; - /// use chomp::primitives::{InputBuffer, IntoInner, State}; - /// - /// let i = &b"Testing"; - /// - /// assert_eq!(i.buffer(), b"Testing"); - /// assert_eq!(i.is_end(), true); - /// - /// // mark and eat one token - /// let m = i.mark(); - /// let i = i.consume(1); - /// - /// assert_eq!(i.buffer(), b"esting"); - /// - /// // restore and continue parsing - /// let j = i.restore(m); - /// - /// let r = take(j, 4); - /// - /// assert_eq!(r.into_inner(), State::Data(input::new(input::END_OF_INPUT, b""), &b"Test"[..])); - /// ``` #[inline(always)] fn mark(&self) -> Self::Marker { self._mark(Guard(())) diff --git a/src/types/mod.rs b/src/types/mod.rs index 116c86fe..3f95d38f 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -74,7 +74,7 @@ pub trait Input: Sized { /// # Example /// /// ``` - /// use chomp::parse_only; + /// use chomp::prelude::{Input, parse_only}; /// /// let r = parse_only(|i| /// // Annotate the error type @@ -95,14 +95,14 @@ pub trait Input: Sized { /// # Example /// /// ``` - /// use chomp::{ParseError, parse_only}; + /// use chomp::prelude::{Input, parse_only}; /// /// let r = parse_only(|i| /// // Annotate the value type /// i.err::<(), _>("Something went wrong"), /// b"some input"); /// - /// assert_eq!(r, Err(ParseError::Error(b"some input", "Something went wrong"))); + /// assert_eq!(r, Err((&b"some input"[..], "Something went wrong"))); /// ``` #[inline] fn err(self, e: E) -> ParseResult { @@ -119,7 +119,7 @@ pub trait Input: Sized { /// # Examples /// /// ``` - /// use chomp::{ParseError, parse_only}; + /// use chomp::prelude::{Input, parse_only}; /// /// let r = parse_only(|i| i.from_result::<_, ()>(Ok("foo")), b"test"); /// @@ -127,7 +127,7 @@ pub trait Input: Sized { /// /// let r = parse_only(|i| i.from_result::<(), _>(Err("error message")), b"test"); /// - /// assert_eq!(r, Err(ParseError::Error(&b"test"[..], "error message"))); + /// assert_eq!(r, Err((&b"test"[..], "error message"))); /// ``` #[inline] fn from_result(self, r: Result) -> ParseResult { diff --git a/src/types/parse_result.rs b/src/types/parse_result.rs index be17fd7e..98448476 100644 --- a/src/types/parse_result.rs +++ b/src/types/parse_result.rs @@ -65,7 +65,7 @@ impl ParseResult { /// # Examples /// /// ``` - /// use chomp::parse_only; + /// use chomp::prelude::{Input, parse_only}; /// /// let r = parse_only(|i| { /// i.ret("data".to_owned()) @@ -81,9 +81,9 @@ impl ParseResult { /// the type-hint for the error in the function signature: /// /// ``` - /// use chomp::{Input, ParseResult, parse_only}; + /// use chomp::prelude::{Input, ParseResult, parse_only}; /// - /// fn parser(i: Input, n: i32) -> ParseResult { + /// fn parser(i: I, n: i32) -> ParseResult { /// i.ret(n + 10) /// } /// @@ -115,9 +115,9 @@ impl ParseResult { /// # Example /// /// ``` - /// use chomp::{Input, U8Result, parse_only}; + /// use chomp::prelude::{Input, SimpleResult, parse_only}; /// - /// fn g(i: Input) -> U8Result<&'static str> { + /// fn g(i: I) -> SimpleResult { /// i.ret("testing!") /// } /// @@ -139,7 +139,7 @@ impl ParseResult { /// # Example /// /// ``` - /// use chomp::{parse_only, any}; + /// use chomp::prelude::{parse_only, any}; /// /// let r = parse_only(|i| any(i).map(|c| c + 12), b"abc"); /// @@ -159,13 +159,13 @@ impl ParseResult { /// # Example /// /// ``` - /// use chomp::{ParseError, parse_only}; + /// use chomp::prelude::{Input, parse_only}; /// /// let r = parse_only(|i| i.err::<(), _>("this is") /// .map_err(|e| e.to_owned() + " an error"), /// b"foo"); /// - /// assert_eq!(r, Err(ParseError::Error(b"foo", "this is an error".to_owned()))); + /// assert_eq!(r, Err((&b"foo"[..], "this is an error".to_owned()))); /// ``` #[inline] pub fn map_err(self, f: F) -> ParseResult @@ -182,7 +182,7 @@ impl ParseResult { /// # Example /// /// ``` - /// use chomp::{parse_only, take_while}; + /// use chomp::prelude::{parse_only, take_while}; /// /// let r = parse_only(|i| take_while(i, |c| c != b' ').inspect(|b| { /// println!("{:?}", b); // Prints "test" @@ -214,28 +214,6 @@ impl ParseResult { /// in terms of how much it exposes its internals, but the `IntoInner` trait implementation /// allows fundamental parsers and combinators to expose the inner `Result` of the `ParseResult` /// and act on this. -/// -/// # Example -/// -/// ``` -/// use chomp::{Input, ParseResult, parse_only, take}; -/// use chomp::primitives::{InputClone, IntoInner, State}; -/// -/// // Version of option() which also catches incomplete -/// fn my_combinator<'a, I, T, E, F>(i: Input<'a, I>, f: F, default: T) -> ParseResult<'a, I, T, E> -/// where F: FnOnce(Input<'a, I>) -> ParseResult<'a, I, T, E> { -/// match f(i.clone()).into_inner() { -/// // Data, preserve the buffer and return -/// State::Data(b, d) => b.ret(d), -/// // Not data, use original buffer and return default -/// _ => i.ret(default), -/// } -/// } -/// -/// let r = parse_only(|i| my_combinator(i, |i| take(i, 10), &b"test"[..]), b"foo"); -/// -/// assert_eq!(r, Ok(&b"test"[..])); -/// ``` impl IntoInner for ParseResult { type Inner = (I, Result); From 7c5f000a8931942b4772e2956f564e72bacc54c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 30 Mar 2016 20:49:25 +0200 Subject: [PATCH 27/63] Use Input::* for parse! macro --- src/macros.rs | 23 ++++++++--------------- src/parse.rs | 2 +- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/macros.rs b/src/macros.rs index ddcb22f2..8655d97e 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -40,12 +40,6 @@ /// last: B, /// } /// -/// let r = |i| take_while1(i, |c| c != b' ').bind(|i, first| -/// token(i, b' ').then(|i| -/// take_while1(i, |c| c != b'\n').bind(|i, last| -/// i.ret::<_, Error<_>>(Name { first: first, last: last })))); -/// -/// /* /// let r = |i| parse!{i; /// let first = take_while1(|c| c != b' '); /// token(b' '); @@ -56,7 +50,6 @@ /// last: last, /// } /// }; -/// */ /// /// assert_eq!(parse_only(r, "Martin Wernstål\n".as_bytes()), Ok(Name{ /// first: &b"Martin"[..], @@ -71,7 +64,7 @@ /// ``` /// # #[macro_use] extern crate chomp; /// # fn main() { -/// use chomp::prelude::{U8Input, SimpleResult, parse_only, string, token}; +/// use chomp::prelude::{U8Input, Input, SimpleResult, parse_only, string, token}; /// use chomp::ascii::decimal; /// /// fn parse_ip(i: I) -> SimpleResult { @@ -161,7 +154,7 @@ /// # #[macro_use] extern crate chomp; /// # fn main() { /// # use chomp::ascii::decimal; -/// # use chomp::prelude::{parse_only, U8Input, token, SimpleResult}; +/// # use chomp::prelude::{parse_only, U8Input, Input, token, SimpleResult}; /// # fn my_parser(i: I) -> SimpleResult { /// parse!{i; /// token(b':'); @@ -271,7 +264,7 @@ /// let p = parser!{ /// state -> match condition { /// true => other_parser(state), -/// false => state.err("failure"), +/// false => Input::err(state, "failure"), /// } /// }; /// @@ -344,7 +337,7 @@ /// # #[macro_use] extern crate chomp; /// # fn main() { /// # use chomp::prelude::{Input, parse_only}; -/// let p = parser!{ (i -> i.err("foo")) <|> (i -> i.ret("bar")) }; +/// let p = parser!{ (i -> Input::err(i, "foo")) <|> (i -> Input::ret(i, "bar")) }; /// /// assert_eq!(parse_only(p, b"a;"), Ok("bar")); /// # } @@ -478,13 +471,13 @@ macro_rules! __parse_internal { // | Inline // | Named // Ret ::= "ret" Typed - ( @TERM($input:expr) ret @ $t_ty:ty , $e_ty:ty : $e:expr ) => { $input.ret::<$t_ty, $e_ty>($e) }; + ( @TERM($input:expr) ret @ $t_ty:ty , $e_ty:ty : $e:expr ) => { Input::ret::<$t_ty, $e_ty>($input, $e) }; // | "ret" $expr - ( @TERM($input:expr) ret $e:expr ) => { $input.ret($e) }; + ( @TERM($input:expr) ret $e:expr ) => { Input::ret($input, $e) }; // Err ::= "err" Typed - ( @TERM($input:expr) err @ $t_ty:ty , $e_ty:ty : $e:expr ) => { $input.err::<$t_ty, $e_ty>($e) }; + ( @TERM($input:expr) err @ $t_ty:ty , $e_ty:ty : $e:expr ) => { Input::err::<$t_ty, $e_ty>($input, $e) }; // | "err" $expr - ( @TERM($input:expr) err $e:expr ) => { $input.err($e) }; + ( @TERM($input:expr) err $e:expr ) => { Input::err($input, $e) }; // '(' Expr ')' ( @TERM($input:expr) ( $($inner:tt)* ) ) => { __parse_internal!{@EXPR($input;) $($inner)*} }; // Inline ::= $ident "->" $expr diff --git a/src/parse.rs b/src/parse.rs index 34de42e9..606eba74 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -23,7 +23,7 @@ use primitives::{ /// ``` /// # #[macro_use] extern crate chomp; /// # fn main() { -/// use chomp::prelude::{U8Input, Error, SimpleResult, parse_only, string, eof}; +/// use chomp::prelude::{U8Input, Input, Error, SimpleResult, parse_only, string, eof}; /// /// fn my_parser(i: I) -> SimpleResult { /// parse!{i; From a77456ace3b1a3699313f82f1b2a72f29b837310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 30 Mar 2016 21:38:58 +0200 Subject: [PATCH 28/63] Fixed compile tests --- tests/compile-fail/ascii_signed_unsigned_type.rs | 4 ++-- tests/compile-fail/ascii_signed_unsigned_type1.rs | 4 ++-- tests/compile-fail/ascii_signed_unsigned_type2.rs | 4 ++-- tests/compile-fail/ascii_signed_unsigned_type3.rs | 4 ++-- tests/compile-fail/ascii_signed_unsigned_type4.rs | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/compile-fail/ascii_signed_unsigned_type.rs b/tests/compile-fail/ascii_signed_unsigned_type.rs index dc745afa..0a1777a7 100644 --- a/tests/compile-fail/ascii_signed_unsigned_type.rs +++ b/tests/compile-fail/ascii_signed_unsigned_type.rs @@ -2,11 +2,11 @@ extern crate chomp; -use chomp::{Input, U8Result, parse_only}; +use chomp::prelude::{U8Input, SimpleResult, parse_only}; use chomp::ascii::{signed, decimal}; // Should not be possible to use unsigned integers with signed -fn parser(i: Input) -> U8Result { +fn parser(i: I) -> SimpleResult { signed(i, decimal) } diff --git a/tests/compile-fail/ascii_signed_unsigned_type1.rs b/tests/compile-fail/ascii_signed_unsigned_type1.rs index a383f985..64320b7f 100644 --- a/tests/compile-fail/ascii_signed_unsigned_type1.rs +++ b/tests/compile-fail/ascii_signed_unsigned_type1.rs @@ -2,11 +2,11 @@ extern crate chomp; -use chomp::{Input, U8Result, parse_only}; +use chomp::prelude::{U8Input, SimpleResult, parse_only}; use chomp::ascii::{signed, decimal}; // Should not be possible to use unsigned integers with signed -fn parser(i: Input) -> U8Result { +fn parser(i: I) -> SimpleResult { signed(i, decimal) } diff --git a/tests/compile-fail/ascii_signed_unsigned_type2.rs b/tests/compile-fail/ascii_signed_unsigned_type2.rs index 04d668ce..5ac7cad7 100644 --- a/tests/compile-fail/ascii_signed_unsigned_type2.rs +++ b/tests/compile-fail/ascii_signed_unsigned_type2.rs @@ -2,11 +2,11 @@ extern crate chomp; -use chomp::{Input, U8Result, parse_only}; +use chomp::prelude::{U8Input, SimpleResult, parse_only}; use chomp::ascii::{signed, decimal}; // Should not be possible to use unsigned integers with signed -fn parser(i: Input) -> U8Result { +fn parser(i: I) -> SimpleResult { signed(i, decimal) } diff --git a/tests/compile-fail/ascii_signed_unsigned_type3.rs b/tests/compile-fail/ascii_signed_unsigned_type3.rs index 728210b2..aac295c5 100644 --- a/tests/compile-fail/ascii_signed_unsigned_type3.rs +++ b/tests/compile-fail/ascii_signed_unsigned_type3.rs @@ -2,11 +2,11 @@ extern crate chomp; -use chomp::{Input, U8Result, parse_only}; +use chomp::prelude::{U8Input, SimpleResult, parse_only}; use chomp::ascii::{signed, decimal}; // Should not be possible to use unsigned integers with signed -fn parser(i: Input) -> U8Result { +fn parser(i: I) -> SimpleResult { signed(i, decimal) } diff --git a/tests/compile-fail/ascii_signed_unsigned_type4.rs b/tests/compile-fail/ascii_signed_unsigned_type4.rs index 5ca9aa86..004ea26d 100644 --- a/tests/compile-fail/ascii_signed_unsigned_type4.rs +++ b/tests/compile-fail/ascii_signed_unsigned_type4.rs @@ -2,11 +2,11 @@ extern crate chomp; -use chomp::{Input, U8Result, parse_only}; +use chomp::prelude::{U8Input, SimpleResult, parse_only}; use chomp::ascii::{signed, decimal}; // Should not be possible to use unsigned integers with signed -fn parser(i: Input) -> U8Result { +fn parser(i: I) -> SimpleResult { signed(i, decimal) } From 3289ce54c1fca3aea10bea62fe10dc815e918ef9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Thu, 31 Mar 2016 20:19:15 +0200 Subject: [PATCH 29/63] take* parsers now only require FnMut --- src/parsers.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/parsers.rs b/src/parsers.rs index 31b767ab..069924b9 100644 --- a/src/parsers.rs +++ b/src/parsers.rs @@ -199,7 +199,7 @@ pub fn take(mut i: I, num: usize) -> SimpleResult { /// ``` #[inline] pub fn take_while(mut i: I, f: F) -> SimpleResult - where F: Fn(I::Token) -> bool { + where F: FnMut(I::Token) -> bool { let b = i.consume_while(f); i.ret(b) @@ -220,7 +220,7 @@ pub fn take_while(mut i: I, f: F) -> SimpleResult /// ``` #[inline] pub fn take_while1(mut i: I, f: F) -> SimpleResult - where F: Fn(I::Token) -> bool { + where F: FnMut(I::Token) -> bool { let b = i.consume_while(f); if b.is_empty() { @@ -244,8 +244,8 @@ pub fn take_while1(mut i: I, f: F) -> SimpleResult /// assert_eq!(r, Ok(&b"abc"[..])); /// ``` #[inline] -pub fn take_till, F>(mut i: I, f: F) -> SimpleResult - where F: Fn(I::Token) -> bool, +pub fn take_till, F>(mut i: I, mut f: F) -> SimpleResult + where F: FnMut(I::Token) -> bool, I::Token: Clone { // TODO: How to check the last token? to make sure f succeeded on it? let mut ok = false; From e2a062b62c06a3a8c181a096f7204529afb7d522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Thu, 31 Mar 2016 20:22:35 +0200 Subject: [PATCH 30/63] Stateful: Fixed issue where incomplete was not handled correctly on success --- src/buffer/stateful.rs | 55 +++++++++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/src/buffer/stateful.rs b/src/buffer/stateful.rs index f9c8f541..ed848bcb 100644 --- a/src/buffer/stateful.rs +++ b/src/buffer/stateful.rs @@ -217,7 +217,7 @@ impl<'a, S: DataSource, B: Buffer> Stream<'a, 'a> for Source match f(InputBuf::new(&self.buffer)).into_inner() { (remainder, Ok(data)) => { - if remainder.is_incomplete() && self.state.contains(END_OF_INPUT) { + if remainder.is_incomplete() && !self.state.contains(END_OF_INPUT) { // We can't accept this since we might have hit a premature end self.state.insert(INCOMPLETE); @@ -230,20 +230,20 @@ impl<'a, S: DataSource, B: Buffer> Stream<'a, 'a> for Source } }, (mut remainder, Err(err)) => { - if remainder.is_incomplete() { - if self.state.contains(END_OF_INPUT) { - Err(StreamError::Incomplete) - } else { + match (remainder.is_incomplete(), self.state.contains(END_OF_INPUT)) { + (true, true) => Err(StreamError::Incomplete), + (true, false) => { self.state.insert(INCOMPLETE); Err(StreamError::Retry) - } - } else { - // TODO: Do something neater with the remainder - // TODO: Detail this behaviour, maybe make it configurable - self.buffer.consume(self.buffer.len() - remainder.len()); + }, + _ => { + // TODO: Do something neater with the remainder + // TODO: Detail this behaviour, maybe make it configurable + self.buffer.consume(self.buffer.len() - remainder.len()); - Err(StreamError::ParseError(remainder.consume_remaining(), err)) + Err(StreamError::ParseError(remainder.consume_remaining(), err)) + } } }, } @@ -259,6 +259,7 @@ mod test { Error, any, take, + take_while, }; use buffer::{ FixedSizeBuffer, @@ -420,6 +421,38 @@ mod test { assert_eq!(m, 1); } + #[test] + fn incomplete2() { + let mut o = 0; + let mut n = 0; // Times it has entered the parsing function + let mut m = 0; // Times it has managed to get past the request for data + let mut b = buf(&b"tes"[..], 2); + + assert_eq!(b.parse(|i| { n += 1; take_while(i, |_| { o += 1; o < 2 }).inspect(|_| m += 1) }), Ok(&b"t"[..])); + assert_eq!(n, 1); + assert_eq!(m, 1); + o = 0; + assert_eq!(b.parse(|i| { n += 1; take_while(i, |_| { o += 1; o < 2 }).inspect(|_| m += 1) }), Err(StreamError::Retry)); + assert_eq!(n, 2); + assert_eq!(m, 2); + o = 0; + assert_eq!(b.parse(|i| { n += 1; take_while(i, |_| { o += 1; o < 2 }).inspect(|_| m += 1) }), Ok(&b"e"[..])); + assert_eq!(n, 3); + assert_eq!(m, 3); + o = 0; + assert_eq!(b.parse(|i| { n += 1; take_while(i, |_| { o += 1; o < 2 }).inspect(|_| m += 1) }), Err(StreamError::Retry)); + assert_eq!(n, 4); + assert_eq!(m, 4); + o = 0; + assert_eq!(b.parse(|i| { n += 1; take_while(i, |_| { o += 1; o < 2 }).inspect(|_| m += 1) }), Ok(&b"s"[..])); + assert_eq!(n, 5); + assert_eq!(m, 5); + o = 0; + assert_eq!(b.parse(|i| { n += 1; take_while(i, |_| { o += 1; o < 2 }).inspect(|_| m += 1) }), Err(StreamError::EndOfInput)); + assert_eq!(n, 5); + assert_eq!(m, 5); + } + #[test] fn no_autofill() { let mut n = 0; // Times it has entered the parsing function From c6f320318d197fb26d3d3798ed3eda5abeb8daff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Thu, 31 Mar 2016 21:47:08 +0200 Subject: [PATCH 31/63] clippy --- src/ascii.rs | 3 +- src/buffer/mod.rs | 6 ++-- src/combinators/bounded.rs | 8 ----- src/combinators/macros.rs | 6 ++-- src/combinators/mod.rs | 21 ++++++------- src/parsers.rs | 15 ++++----- src/types/input.rs | 62 +++++++++++++++++--------------------- src/types/mod.rs | 15 +++++++-- src/types/parse_result.rs | 26 ++++++++-------- 9 files changed, 77 insertions(+), 85 deletions(-) diff --git a/src/ascii.rs b/src/ascii.rs index 3ca99292..6c59346e 100644 --- a/src/ascii.rs +++ b/src/ascii.rs @@ -154,9 +154,8 @@ pub fn signed, T, F>(i: I, f: F) -> SimpleResult /// /// assert_eq!(r, Ok(123u8)); /// ``` -// TODO: Use methods on `Buffer` to implement `to_decimal` #[inline] -pub fn decimal<'a, I: Input, T: Copy + ValueFrom + Add + Mul>(i: I) -> SimpleResult { +pub fn decimal, T: Copy + ValueFrom + Add + Mul>(i: I) -> SimpleResult { take_while1(i, is_digit).map(to_decimal) } diff --git a/src/buffer/mod.rs b/src/buffer/mod.rs index a0f9d0a8..147d82d7 100644 --- a/src/buffer/mod.rs +++ b/src/buffer/mod.rs @@ -69,9 +69,9 @@ impl> PartialEq for StreamError { fn eq(&self, other: &StreamError) -> bool { match (self, other) { (&StreamError::ParseError(ref b1, ref e1), &StreamError::ParseError(ref b2, ref e2)) => b1 == b2 && e1 == e2, - (&StreamError::Incomplete, &StreamError::Incomplete) => true, - (&StreamError::EndOfInput, &StreamError::EndOfInput) => true, - (&StreamError::Retry, &StreamError::Retry) => true, + (&StreamError::Incomplete, &StreamError::Incomplete) + | (&StreamError::EndOfInput, &StreamError::EndOfInput) + | (&StreamError::Retry, &StreamError::Retry) => true, _ => false, } } diff --git a/src/combinators/bounded.rs b/src/combinators/bounded.rs index 4e3083b5..8f6e09bb 100644 --- a/src/combinators/bounded.rs +++ b/src/combinators/bounded.rs @@ -684,8 +684,6 @@ impl BoundedRange for usize { /// /// * Will allocate depending on the `FromIterator` implementation. /// * Will never yield more items than the upper bound of the range. -/// * If the last parser succeeds on the last input item then this parser is still considered -/// incomplete if the input flag END_OF_INPUT is not set as there might be more data to fill. #[inline] pub fn many(i: I, r: R, f: F) -> ParseResult where R: BoundedRange, @@ -706,8 +704,6 @@ pub fn many(i: I, r: R, f: F) -> ParseResult /// # Notes /// /// * Will never yield more items than the upper bound of the range. -/// * If the last parser succeeds on the last input item then this parser is still considered -/// incomplete if the input flag END_OF_INPUT is not set as there might be more data to fill. #[inline] pub fn skip_many(i: I, r: R, f: F) -> ParseResult where R: BoundedRange, @@ -728,8 +724,6 @@ pub fn skip_many(i: I, r: R, f: F) -> ParseResult(i: I, r: R, p: P, end: F) -> ParseResult where R: BoundedRange, @@ -753,8 +747,6 @@ pub fn many_till(i: I, r: R, p: P, end: F) -> /// /// * Will allocate depending on the `FromIterator` implementation. /// * Will never yield more items than the upper bound of the range. -/// * If the last parser succeeds on the last input item then this combinator is still considered -/// incomplete unless the parser `F` matches or the lower bound has not been met. #[inline] pub fn sep_by(i: I, r: R, mut p: P, mut sep: F) -> ParseResult where T: FromIterator, diff --git a/src/combinators/macros.rs b/src/combinators/macros.rs index d0a121fa..11c2722f 100644 --- a/src/combinators/macros.rs +++ b/src/combinators/macros.rs @@ -83,13 +83,13 @@ macro_rules! run_iter { } // TODO: Not always used - let m = $input.mark(); + let mark = $input.mark(); let mut iter = Iter { state: None, parser: $parser, buf: Some($input), - mark: m, + mark: mark, data: $data, _t: PhantomData, }; @@ -127,7 +127,7 @@ macro_rules! run_iter_till { EndSuccess, } - /// Iterator used by ``many_till`` and ``many1``. + /// Iterator used by `many_till` and `many1`. struct IterTill where E: From, P: FnMut(I) -> ParseResult, diff --git a/src/combinators/mod.rs b/src/combinators/mod.rs index 47407b6f..23fcd1b3 100644 --- a/src/combinators/mod.rs +++ b/src/combinators/mod.rs @@ -17,7 +17,7 @@ use primitives::{ Primitives, }; -/// Applies the parser ``p`` exactly ``num`` times collecting all items into `T: FromIterator`. +/// Applies the parser `p` exactly `num` times collecting all items into `T: FromIterator`. /// /// ``` /// use chomp::prelude::*; @@ -40,7 +40,7 @@ pub fn count(i: I, num: usize, p: F) -> ParseResult(i: I, f: F, default: T) -> ParseResult } } -/// Tries to match the parser ``f``, if ``f`` fails it tries ``g``. Returns the success value of +/// Tries to match the parser `f`, if `f` fails it tries `g`. Returns the success value of /// the first match, otherwise the error of the last one if both fail. /// /// Incomplete state is propagated from the first one to report incomplete. @@ -97,12 +97,9 @@ pub fn or(i: I, f: F, g: G) -> ParseResult } } -/// Parses many instances of ``f`` until it does no longer match, collecting all matches into the +/// Parses many instances of `f` until it does no longer match, collecting all matches into the /// type `T: FromIterator`. /// -/// Note: If the last parser succeeds on the last input item then this parser is still considered -/// incomplete if the input flag END_OF_INPUT is not set as there might be more data to fill. -/// /// Note: Allocates data. /// /// ``` @@ -123,7 +120,7 @@ pub fn many(i: I, f: F) -> ParseResult bounded::many(i, .., f) } -/// Parses at least one instance of ``f`` and continues until it does no longer match, collecting +/// Parses at least one instance of `f` and continues until it does no longer match, collecting /// all matches into the type `T: FromIterator`. /// /// Note: If the last parser succeeds on the last input item then this parser is still considered @@ -226,8 +223,8 @@ pub fn many_till(i: I, p: R, end: F) -> ParseResu /// /// Incomplete state will be propagated. /// -/// This is more efficient compared to using ``many`` and then just discarding the result as -/// ``many`` allocates a separate data structure to contain the data before proceeding. +/// This is more efficient compared to using `many` and then just discarding the result as +/// `many` allocates a separate data structure to contain the data before proceeding. /// /// ``` /// use chomp::prelude::{parse_only, skip_many, token}; @@ -247,8 +244,8 @@ pub fn skip_many(i: I, f: F) -> ParseResult /// Incomplete state will be propagated. Will propagate the error if it occurs during the first /// attempt. /// -/// This is more efficient compared to using ``many1`` and then just discarding the result as -/// ``many1`` allocates a separate data structure to contain the data before proceeding. +/// This is more efficient compared to using `many1` and then just discarding the result as +/// `many1` allocates a separate data structure to contain the data before proceeding. /// /// ``` /// use chomp::prelude::{Error, parse_only, skip_many1, token}; diff --git a/src/parsers.rs b/src/parsers.rs index 069924b9..3307f2e7 100644 --- a/src/parsers.rs +++ b/src/parsers.rs @@ -251,12 +251,12 @@ pub fn take_till, F>(mut i: I, mut f: F) -> SimpleResult(mut i: I, s: S, mut f: F) -> SimpleResult Option { let mut state = Some(s); - let b = i.consume_while(|c| { state = f(mem::replace(&mut state, None).unwrap(), c); state.is_some() }); + let b = i.consume_while(|c| { state = f(mem::replace(&mut state, None).expect("scan: Failed to obtain state, consume_while most likely called closure after end"), c); state.is_some() }); i.ret(b) } @@ -347,7 +347,7 @@ pub fn take_remainder(mut i: I) -> SimpleResult { /// ``` // TODO: Does not actually work with &str yet #[inline] -pub fn string<'b, T: Copy + PartialEq, I: Input>(mut i: I, s: &'b [T]) +pub fn string>(mut i: I, s: &[T]) -> SimpleResult { let mut n = 0; let len = s.len(); @@ -521,8 +521,9 @@ mod error { /// /// Should be used when a specific token was expected. #[inline(always)] - pub fn expected(_i: I) -> Self { - create_error!(Some(_i)) + #[allow(unused_variables)] + pub fn expected(i: I) -> Self { + create_error!(Some(i)) } /// Returns `Some(&I)` if a specific token was expected, `None` otherwise. diff --git a/src/types/input.rs b/src/types/input.rs index d2ef098e..736b8c6e 100644 --- a/src/types/input.rs +++ b/src/types/input.rs @@ -75,21 +75,18 @@ impl<'a, I: Copy + PartialEq> Input for &'a [I] { #[inline] fn _consume_while(&mut self, _g: Guard, mut f: F) -> Self::Buffer where F: FnMut(Self::Token) -> bool { - match self.iter().position(|c| !f(*c)) { - Some(n) => { - let b = &self[..n]; + if let Some(n) = self.iter().position(|c| !f(*c)) { + let b = &self[..n]; - *self = &self[n..]; + *self = &self[n..]; - b - }, - None => { - let b = &self[..]; + b + } else { + let b = &self[..]; - *self = &self[..0]; + *self = &self[..0]; - b - } + b } } @@ -160,13 +157,12 @@ impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { #[inline] fn _peek(&mut self, _g: Guard) -> Option { - match self.1.first() { - Some(c) => Some(*c), - None => { - self.0 = true; + if let Some(c) = self.1.first() { + Some(*c) + } else { + self.0 = true; - None - }, + None } } @@ -197,15 +193,14 @@ impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { #[inline] fn _consume_while(&mut self, g: Guard, mut f: F) -> Self::Buffer where F: FnMut(Self::Token) -> bool { - match self.1.iter().position(|c| !f(*c)) { - Some(n) => { - let b = &self.1[..n]; + if let Some(n) = self.1.iter().position(|c| !f(*c)) { + let b = &self.1[..n]; - self.1 = &self.1[n..]; + self.1 = &self.1[n..]; - b - }, - None => self._consume_remaining(g), + b + } else { + self._consume_remaining(g) } } @@ -291,21 +286,18 @@ impl<'a> Input for &'a str { fn _consume_while(&mut self, _g: Guard, mut f: F) -> Self::Buffer where F: FnMut(Self::Token) -> bool { // We need to find the character following the one which did not match - match self.char_indices().skip_while(|&(_, c)| f(c)).next() { - Some((pos, _)) => { - let b = &self[..pos]; + if let Some((pos, _)) = self.char_indices().skip_while(|&(_, c)| f(c)).next() { + let b = &self[..pos]; - *self = &self[pos..]; + *self = &self[pos..]; - b - }, - None => { - let b = &self[..]; + b + } else { + let b = &self[..]; - *self = &self[..0]; + *self = &self[..0]; - b - } + b } } diff --git a/src/types/mod.rs b/src/types/mod.rs index 3f95d38f..a060e76c 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -22,17 +22,28 @@ pub trait Buffer: PartialEq { /// The token type of this buffer. type Token: Copy + PartialEq; + /// Applies a function in order on all tokens present in the buffer carrying an accumulator + /// value `B` between invocations. The buffer is consumed as part of the folding and the last + /// value of the accumulator is returned. + // Would be prefereable if there was a &self -> Iterator method, but that does not work for + // owned or maybe owned since the lifetimes will be wrong for one or the other. Higher Ranked + // Trait Bounds (HRTB) does not seem to work either since it is not possible to later + // instantiate the type in a function signature with a concrete lifetime without running into + // an "expected bound lifetime but found concrete lifetime" error. Instantiation for HRTBs seem + // to only take place in the actual code, not when a type is used in eg. a where clause. fn fold(self, B, F) -> B where F: FnMut(B, Self::Token) -> B; - /// The number of tokens present in this + /// The number of tokens present in this buffer. fn len(&self) -> usize; /// Consumes self to create an owned vector of tokens. /// - /// Will allocate if the implementation borrows storage or does not use a `Vec` internally. + /// Will allocate if the implementation borrows storage or does not use an owned type + /// compatible with `Vec` internally. fn to_vec(self) -> Vec; + /// Returns true if this buffer is empty. fn is_empty(&self) -> bool { self.len() == 0 } diff --git a/src/types/parse_result.rs b/src/types/parse_result.rs index 98448476..f471602e 100644 --- a/src/types/parse_result.rs +++ b/src/types/parse_result.rs @@ -3,7 +3,7 @@ use primitives::IntoInner; /// The basic return type of a parser. /// -/// This type satisfies a variant of the ``Monad`` typeclass. Due to the limitations of Rust's +/// This type satisfies a variant of the `Monad` typeclass. Due to the limitations of Rust's /// return types closures cannot be returned without boxing which has an unacceptable performance /// impact. /// @@ -11,11 +11,11 @@ use primitives::IntoInner; /// violate the monad laws or the assumptions taken by the parser type) an `Input` wrapper is /// provided which ensures that the parser state is carried properly through every call to `bind`. /// This is also known as a Linear Type (emulated through hiding destructors and using the -/// annotation ``#[must_use]``). +/// annotation `#[must_use]`). /// -/// Do-notation is provided by the macro ``parse!``. +/// Do-notation is provided by the macro `parse!`. /// -/// # Equivalence with Haskell's ``Monad`` typeclass: +/// # Equivalence with Haskell's `Monad` typeclass: /// /// ```text /// f >>= g ≡ f(m).bind(g) @@ -43,23 +43,23 @@ pub struct ParseResult(I, Result); /// /// # Note /// -/// Prefer to use ``Input::ret``, ``Input::err`` or ``Input::incomplete`` instead of using +/// Prefer to use `Input::ret`, `Input::err` or `Input::from_result` instead of using pub fn new(i: I, r: Result) -> ParseResult { ParseResult(i, r) } impl ParseResult { - /// Sequentially composes the result with a parse action ``f``, passing any produced value as + /// Sequentially composes the result with a parse action `f`, passing any produced value as /// the second parameter. /// - /// The first parameter to the supplied function ``f`` is the parser state (``Input``). This + /// The first parameter to the supplied function `f` is the parser state (`Input`). This /// state is then passed on to other parsers or used to return a value or an error. /// - /// # Automatic conversion of ``E`` + /// # Automatic conversion of `E` /// - /// The error value ``E`` will automatically be converted using the ``From`` trait to the + /// The error value `E` will automatically be converted using the `From` trait to the /// desired type. The downside with this using the current stable version of Rust (1.4) is that - /// the type inferrence will currently not use the default value for the generic ``V`` and will + /// the type inferrence will currently not use the default value for the generic `V` and will /// therefore require extra type hint for the error. /// /// # Examples @@ -101,12 +101,12 @@ impl ParseResult { } } - /// Sequentially composes the result with a parse action ``f``, discarding any produced value. + /// Sequentially composes the result with a parse action `f`, discarding any produced value. /// - /// The first parameter to the supplied function ``f`` is the parser state (``Input``). This + /// The first parameter to the supplied function `f` is the parser state (`Input`). This /// state is then passed on to other parsers or used to return a value or an error. /// - /// # Relation to ``bind`` + /// # Relation to `bind` /// /// ```text /// ParseResult::then(g) ≡ ParseResult::bind(|i, _| g(i)) From 5256a5037835f4ce6ad5c8db9fcb80a519996d95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Thu, 31 Mar 2016 21:49:49 +0200 Subject: [PATCH 32/63] Clippy as feature --- Cargo.toml | 1 + src/lib.rs | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 91a0e3e6..058c1553 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ tendril = { version = "0.2.2", optional = true } # Technically a dev-dependency, but dev-dependencies are not allowed to be optional, # compiletest_rs fails to compile on stable and beta compiletest_rs = { version = "0.2.1", optional = true } +clippy = { version = ">0.0.1", optional = true } [features] # Feature for running extra (compiletime fail) tests on nightly diff --git a/src/lib.rs b/src/lib.rs index 64a24271..65dcb82d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -200,6 +200,19 @@ //! The built-in `chomp::parsers::Error` type is zero-sized and carry no error-information. This //! increases performance somewhat. +#![cfg_attr(feature="clippy", feature(plugin))] +#![cfg_attr(feature="clippy", plugin(clippy))] +#![cfg_attr(feature="clippy", warn( + nonminimal_bool, + option_unwrap_used, + print_stdout, + result_unwrap_used, + shadow_reuse, + shadow_same, + shadow_unrelated, + single_match_else))] +#![cfg_attr(feature="clippy", allow(inline_always))] + #[cfg(feature = "tendril")] extern crate tendril; From 578492dfde10948cfc96f806be261e552e3b1352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Tue, 16 Aug 2016 06:08:39 +0200 Subject: [PATCH 33/63] Macros: Reversed use of Universal Function Call syntax for Input This caused issues due to `Input` inside of the function body not being considered the same as the actual type supplied to the function. --- src/macros.rs | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/macros.rs b/src/macros.rs index 8655d97e..45066cae 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -32,7 +32,7 @@ /// ``` /// # #[macro_use] extern crate chomp; /// # fn main() { -/// use chomp::prelude::{Buffer, Error, Input, parse_only, take_while1, token}; +/// use chomp::prelude::{Buffer, Error, Input, ParseResult, parse_only, take_while1, token}; /// /// #[derive(Debug, Eq, PartialEq)] /// struct Name { @@ -40,18 +40,20 @@ /// last: B, /// } /// -/// let r = |i| parse!{i; -/// let first = take_while1(|c| c != b' '); -/// token(b' '); -/// let last = take_while1(|c| c != b'\n'); -/// -/// ret @ _, Error<_>: Name{ -/// first: first, -/// last: last, +/// fn parser>(i: I) -> ParseResult, Error> { +/// parse!{i; +/// let first = take_while1(|c| c != b' '); +/// token(b' '); +/// let last = take_while1(|c| c != b'\n'); +/// +/// ret @ _, Error: Name{ +/// first: first, +/// last: last, +/// } /// } -/// }; +/// } /// -/// assert_eq!(parse_only(r, "Martin Wernstål\n".as_bytes()), Ok(Name{ +/// assert_eq!(parse_only(parser, "Martin Wernstål\n".as_bytes()), Ok(Name{ /// first: &b"Martin"[..], /// last: "Wernstål".as_bytes() /// })); @@ -471,13 +473,13 @@ macro_rules! __parse_internal { // | Inline // | Named // Ret ::= "ret" Typed - ( @TERM($input:expr) ret @ $t_ty:ty , $e_ty:ty : $e:expr ) => { Input::ret::<$t_ty, $e_ty>($input, $e) }; + ( @TERM($input:expr) ret @ $t_ty:ty , $e_ty:ty : $e:expr ) => { $input.ret::<$t_ty, $e_ty>($e) }; // | "ret" $expr - ( @TERM($input:expr) ret $e:expr ) => { Input::ret($input, $e) }; + ( @TERM($input:expr) ret $e:expr ) => { $input.ret($e) }; // Err ::= "err" Typed - ( @TERM($input:expr) err @ $t_ty:ty , $e_ty:ty : $e:expr ) => { Input::err::<$t_ty, $e_ty>($input, $e) }; + ( @TERM($input:expr) err @ $t_ty:ty , $e_ty:ty : $e:expr ) => { $input.err::<$t_ty, $e_ty>($e) }; // | "err" $expr - ( @TERM($input:expr) err $e:expr ) => { Input::err($input, $e) }; + ( @TERM($input:expr) err $e:expr ) => { $input.err($e) }; // '(' Expr ')' ( @TERM($input:expr) ( $($inner:tt)* ) ) => { __parse_internal!{@EXPR($input;) $($inner)*} }; // Inline ::= $ident "->" $expr From b4c209a895e0985e4bd01389195eb18441e54f3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Tue, 16 Aug 2016 06:10:14 +0200 Subject: [PATCH 34/63] Macros: Fixed syntax error caused by missing parenthesis --- src/macros.rs | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/macros.rs b/src/macros.rs index 45066cae..3bc8042f 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -495,21 +495,21 @@ macro_rules! __parse_internal { // recurse until >> or end // unrolled: // ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $($tail:tt)* ) => { __parse_internal!{@EXPR($input; $($lhs)* $t1) $($tail)*} }; - ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt ) => { __parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1} }; - ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt >> $($tail:tt)* ) => { __parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1}.bind(|i, _| __parse_internal!{@EXPR(i;) $($tail)*}) }; - ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt ) => { __parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2} }; - ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt >> $($tail:tt)* ) => { __parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2}.bind(|i, _| __parse_internal!{@EXPR(i;) $($tail)*}) }; - ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt ) => { __parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2 $t3} }; - ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt >> $($tail:tt)* ) => { __parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2 $t3}.bind(|i, _| __parse_internal!{@EXPR(i;) $($tail)*}) }; - ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt $t4:tt ) => { __parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2 $t3 $t4} }; - ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt $t4:tt >> $($tail:tt)* ) => { __parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2 $t3 $t4}.bind(|i, _| __parse_internal!{@EXPR(i;) $($tail)*}) }; - ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt ) => { __parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2 $t3 $t4 $t5} }; - ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt >> $($tail:tt)* ) => { __parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2 $t3 $t4 $t5}.bind(|i, _| __parse_internal!{@EXPR(i;) $($tail)*}) }; - ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt ) => { __parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2 $t3 $t4 $t5 $t6} }; - ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt >> $($tail:tt)* ) => { __parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2 $t3 $t4 $t5 $t6}.bind(|i, _| __parse_internal!{@EXPR(i;) $($tail)*}) }; - ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt ) => { __parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2 $t3 $t4 $t5 $t6 $t7} }; - ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt >> $($tail:tt)* ) => { __parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2 $t3 $t4 $t5 $t6 $t7}.bind(|i, _| __parse_internal!{@EXPR(i;) $($tail)*}) }; - ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $($tail:tt)* ) => { __parse_internal!{@EXPR($input; $($lhs)* $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8) $($tail)*} }; + ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt ) => { __parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1} }; + ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt >> $($tail:tt)* ) => { (__parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1}).bind(|i, _| __parse_internal!{@EXPR(i;) $($tail)*}) }; + ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt ) => { __parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2} }; + ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt >> $($tail:tt)* ) => { (__parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2}).bind(|i, _| __parse_internal!{@EXPR(i;) $($tail)*}) }; + ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt ) => { __parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2 $t3} }; + ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt >> $($tail:tt)* ) => { (__parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2 $t3}).bind(|i, _| __parse_internal!{@EXPR(i;) $($tail)*}) }; + ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt $t4:tt ) => { __parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2 $t3 $t4} }; + ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt $t4:tt >> $($tail:tt)* ) => { (__parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2 $t3 $t4}).bind(|i, _| __parse_internal!{@EXPR(i;) $($tail)*}) }; + ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt ) => { __parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2 $t3 $t4 $t5} }; + ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt >> $($tail:tt)* ) => { (__parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2 $t3 $t4 $t5}).bind(|i, _| __parse_internal!{@EXPR(i;) $($tail)*}) }; + ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt ) => { __parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2 $t3 $t4 $t5 $t6} }; + ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt >> $($tail:tt)* ) => { (__parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2 $t3 $t4 $t5 $t6}).bind(|i, _| __parse_internal!{@EXPR(i;) $($tail)*}) }; + ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt ) => { __parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2 $t3 $t4 $t5 $t6 $t7} }; + ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt >> $($tail:tt)* ) => { (__parse_internal!{@EXPR_ALT($input;) $($lhs)* $t1 $t2 $t3 $t4 $t5 $t6 $t7}).bind(|i, _| __parse_internal!{@EXPR(i;) $($tail)*}) }; + ( @EXPR($input:expr; $($lhs:tt)*) $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $($tail:tt)* ) => { __parse_internal!{@EXPR($input; $($lhs)* $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8) $($tail)*} }; // ExprAlt ::= ExprSkip ( @EXPR_ALT($input:expr; $($lhs:tt)*) ) => { __parse_internal!{@EXPR_SKIP($input;) $($lhs)*} }; From f6c783409c0d32df7882e417f837159049a0bf29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Tue, 16 Aug 2016 06:11:34 +0200 Subject: [PATCH 35/63] Primitives: Slight doc updates --- src/primitives.rs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/primitives.rs b/src/primitives.rs index cc54f92c..287ea5df 100644 --- a/src/primitives.rs +++ b/src/primitives.rs @@ -1,8 +1,8 @@ //! Module used to construct fundamental parsers and combinators. //! -//! # Primitive -//! -//! Only used by fundamental parsers and combinators. +//! The `Primitives` trait allows access to the primitive methods of the `Input` trait. These +//! methods are hidden in the documentation to make it easier to read the documentation since the +//! methods are not useful when using the library or writing primitive parsers. use types::Input; @@ -31,12 +31,7 @@ pub trait IntoInner { /// It cannot be instantiated outside of the `Primitives` trait blanket implementation. pub struct Guard(()); -/// **Primitive:** Trait enabling primitive actions on an `Input` type. -/// -/// # Primitive -/// -/// Only used by fundamental parsers and combinators. -/// +/// Trait enabling primitive actions on an `Input` type. // FIXME: Rename and documentation pub trait Primitives: Input { #[inline(always)] From f1696b6086798a505693f2b709cc683c150cb2a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 17 Aug 2016 16:54:06 +0200 Subject: [PATCH 36/63] Tests: Combinators --- src/combinators/mod.rs | 319 ++++++++++++++++++----------------------- src/parsers.rs | 7 +- 2 files changed, 145 insertions(+), 181 deletions(-) diff --git a/src/combinators/mod.rs b/src/combinators/mod.rs index 23fcd1b3..6dfd2c4e 100644 --- a/src/combinators/mod.rs +++ b/src/combinators/mod.rs @@ -308,253 +308,220 @@ pub fn look_ahead(i: I, f: F) -> ParseResult } // FIXME: -/* #[cfg(test)] mod test { - use {Input, Error, ParseResult}; - use primitives::State; - use primitives::input::{new_buf, DEFAULT, END_OF_INPUT}; + use types::{Input, ParseResult}; use primitives::IntoInner; use super::*; - use parsers::{any, take, token, string}; + use parsers::{Error, any, take, token, string}; #[test] fn option_test() { - assert_eq!(option(new_buf(DEFAULT, b""), any, b'-').into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - assert_eq!(option(new_buf(DEFAULT, b"a"), any, b'-').into_inner(), State::Data(new_buf(DEFAULT, b""), b'a')); - assert_eq!(option(new_buf(DEFAULT, b""), |i| take(i, 2).map(ToOwned::to_owned), vec![]).into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); - assert_eq!(option(new_buf(DEFAULT, b"a"), |i| take(i, 2).map(ToOwned::to_owned), vec![]).into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); - assert_eq!(option(new_buf(DEFAULT, b"ab"), |i| take(i, 2).map(ToOwned::to_owned), vec![]).into_inner(), State::Data(new_buf(DEFAULT, b""), vec![b'a', b'b'])); + assert_eq!(option(&b""[..], any, b'-').into_inner(), (&b""[..], Ok(b'-'))); + assert_eq!(option(&b"a"[..], any, b'-').into_inner(), (&b""[..], Ok(b'a'))); + assert_eq!(option(&b""[..], |i| take(i, 2).map(ToOwned::to_owned), vec![]).into_inner(), (&b""[..], Ok(vec![]))); + assert_eq!(option(&b"a"[..], |i| take(i, 2).map(ToOwned::to_owned), vec![]).into_inner(), (&b"a"[..], Ok(vec![]))); + assert_eq!(option(&b"ab"[..], |i| take(i, 2).map(ToOwned::to_owned), vec![]).into_inner(), (&b""[..], Ok(vec![b'a', b'b']))); - assert_eq!(option(new_buf(DEFAULT, b"a"), |i| token(i, b' ').map_err(|_| "token_err"), b'-').into_inner(), State::Data(new_buf(DEFAULT, b"a"), b'-')); - - assert_eq!(option(new_buf(END_OF_INPUT, b""), any, b'-').into_inner(), State::Data(new_buf(END_OF_INPUT, b""), b'-')); - assert_eq!(option(new_buf(END_OF_INPUT, b"a"), any, b'-').into_inner(), State::Data(new_buf(END_OF_INPUT, b""), b'a')); - assert_eq!(option(new_buf(END_OF_INPUT, b""), |i| take(i, 2).map(ToOwned::to_owned), vec![]).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); - assert_eq!(option(new_buf(END_OF_INPUT, b"a"), |i| take(i, 2).map(ToOwned::to_owned), vec![]).into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![])); - assert_eq!(option(new_buf(END_OF_INPUT, b"ab"), |i| take(i, 2).map(ToOwned::to_owned), vec![]).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'b'])); - - assert_eq!(option(new_buf(END_OF_INPUT, b"a"), |i| token(i, b' ').map_err(|_| "token_err"), b'-').into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), b'-')); + assert_eq!(option(&b"a"[..], |i| token(i, b' ').map_err(|_| "token_err"), b'-').into_inner(), (&b"a"[..], Ok(b'-'))); } #[test] fn or_test() { - assert_eq!(or(new_buf(DEFAULT, b""), any, any).into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - assert_eq!(or(new_buf(DEFAULT, b"a"), any, any).into_inner(), State::Data(new_buf(DEFAULT, b""), b'a')); - assert_eq!(or(new_buf(DEFAULT, b"a"), |i| take(i, 2), |i| take(i, 1)).into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); - assert_eq!(or(new_buf(DEFAULT, b"ab"), |i| take(i, 2), |i| take(i, 1)).into_inner(), State::Data(new_buf(DEFAULT, b""), &b"ab"[..])); - assert_eq!(or(new_buf(DEFAULT, b"a"), |i| token(i, b'a'), |i| token(i, b'b')).into_inner(), State::Data(new_buf(DEFAULT, b""), b'a')); - assert_eq!(or(new_buf(DEFAULT, b"b"), |i| token(i, b'a'), |i| token(i, b'b')).into_inner(), State::Data(new_buf(DEFAULT, b""), b'b')); - assert_eq!(or(new_buf(DEFAULT, b"c"), |i| token(i, b'a').map_err(|_| "a err"), |i| token(i, b'b').map_err(|_| "b err")).into_inner(), State::Error(new_buf(DEFAULT, b"c"), "b err")); - - assert_eq!(or(new_buf(END_OF_INPUT, b""), any, any).into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); - assert_eq!(or(new_buf(END_OF_INPUT, b"a"), any, any).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), b'a')); - assert_eq!(or(new_buf(END_OF_INPUT, b"a"), |i| take(i, 2), |i| take(i, 1)).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), &b"a"[..])); - assert_eq!(or(new_buf(END_OF_INPUT, b"ab"), |i| take(i, 2), |i| take(i, 1)).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), &b"ab"[..])); - assert_eq!(or(new_buf(END_OF_INPUT, b"a"), |i| token(i, b'a'), |i| token(i, b'b')).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), b'a')); - assert_eq!(or(new_buf(END_OF_INPUT, b"b"), |i| token(i, b'a'), |i| token(i, b'b')).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), b'b')); - assert_eq!(or(new_buf(END_OF_INPUT, b"c"), |i| token(i, b'a').map_err(|_| "a err"), |i| token(i, b'b').map_err(|_| "b err")).into_inner(), State::Error(new_buf(END_OF_INPUT, b"c"), "b err")); + assert_eq!(or(&b""[..], any, any).into_inner(), (&b""[..], Err(Error::unexpected()))); + assert_eq!(or(&b"a"[..], any, any).into_inner(), (&b""[..], Ok(b'a'))); + assert_eq!(or(&b"a"[..], |i| take(i, 2), |i| take(i, 1)).into_inner(), (&b""[..], Ok(&b"a"[..]))); + assert_eq!(or(&b"ab"[..], |i| take(i, 2), |i| take(i, 1)).into_inner(), (&b""[..], Ok(&b"ab"[..]))); + assert_eq!(or(&b"a"[..], |i| token(i, b'a'), |i| token(i, b'b')).into_inner(), (&b""[..], Ok(b'a'))); + assert_eq!(or(&b"b"[..], |i| token(i, b'a'), |i| token(i, b'b')).into_inner(), (&b""[..], Ok(b'b'))); + assert_eq!(or(&b"c"[..], |i| token(i, b'a').map_err(|_| "a err"), |i| token(i, b'b').map_err(|_| "b err")).into_inner(), (&b"c"[..], Err("b err"))); } #[test] fn many_test() { - let r: State<_, Vec<_>, _> = many(new_buf(DEFAULT, b""), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: State<_, Vec<_>, _> = many(new_buf(DEFAULT, b"a"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: State<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aa"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b""), 1)); - - let r: State<_, Vec<_>, _> = many(new_buf(DEFAULT, b"bbb"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new_buf(DEFAULT, b"bbb"), vec![])); - let r: State<_, Vec<_>, _> = many(new_buf(DEFAULT, b"abb"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new_buf(DEFAULT, b"bb"), vec![b'a'])); - let r: State<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aab"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new_buf(DEFAULT, b"b"), vec![b'a', b'a'])); - - let r: State<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b""), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new_buf(END_OF_INPUT, b""), vec![])); - let r: State<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"a"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new_buf(END_OF_INPUT, b""), vec![b'a'])); - let r: State<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aa"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'a'])); - - let r: State<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aab"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a', b'a'])); + let r: (_, Result, _>) = many(&b""[..], |i| i.err("the error")).into_inner(); + assert_eq!(r, (&b""[..], Ok(vec![]))); + let r: (_, Result, _>) = many(&b"abc"[..], |i| i.err("the error")).into_inner(); + assert_eq!(r, (&b"abc"[..], Ok(vec![]))); + + let r: (_, Result, _>) = many(&b""[..], |i| token(i, b'a')).into_inner(); + assert_eq!(r, (&b""[..], Ok(vec![]))); + let r: (_, Result, _>) = many(&b"a"[..], |i| token(i, b'a')).into_inner(); + assert_eq!(r, (&b""[..], Ok(vec![b'a']))); + let r: (_, Result, _>) = many(&b"aa"[..], |i| token(i, b'a')).into_inner(); + assert_eq!(r, (&b""[..], Ok(vec![b'a', b'a']))); + + let r: (_, Result, _>) = many(&b"bbb"[..], |i| token(i, b'a')).into_inner(); + assert_eq!(r, (&b"bbb"[..], Ok(vec![]))); + let r: (_, Result, _>) = many(&b"abb"[..], |i| token(i, b'a')).into_inner(); + assert_eq!(r, (&b"bb"[..], Ok(vec![b'a']))); + let r: (_, Result, _>) = many(&b"aab"[..], |i| token(i, b'a')).into_inner(); + assert_eq!(r, (&b"b"[..], Ok(vec![b'a', b'a']))); } #[test] fn many1_test() { - let r: State<_, Vec<_>, _> = many1(new_buf(DEFAULT, b""), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: State<_, Vec<_>, _> = many1(new_buf(DEFAULT, b"a"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: State<_, Vec<_>, _> = many1(new_buf(DEFAULT, b"aa"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b""), 1)); - - let r: State<_, Vec<_>, _> = many1(new_buf(DEFAULT, b"bbb"), |i| token(i, b'a').map_err(|_| "token_error")).into_inner(); - assert_eq!(r, State::Error(new_buf(DEFAULT, b"bbb"), "token_error")); - let r: State<_, Vec<_>, _> = many1(new_buf(DEFAULT, b"abb"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new_buf(DEFAULT, b"bb"), vec![b'a'])); - let r: State<_, Vec<_>, _> = many1(new_buf(DEFAULT, b"aab"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new_buf(DEFAULT, b"b"), vec![b'a', b'a'])); - - let r: State<_, Vec<_>, _> = many1(new_buf(END_OF_INPUT, b""), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); - let r: State<_, Vec<_>, _> = many1(new_buf(END_OF_INPUT, b"a"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new_buf(END_OF_INPUT, b""), vec![b'a'])); - let r: State<_, Vec<_>, _> = many1(new_buf(END_OF_INPUT, b"aa"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'a'])); - - let r: State<_, Vec<_>, _> = many1(new_buf(END_OF_INPUT, b"aab"), |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a', b'a'])); + let r: (_, Result, _>) = many1(&b""[..], |i| i.err("the error")).into_inner(); + assert_eq!(r, (&b""[..], Err("the error"))); + let r: (_, Result, _>) = many1(&b"abc"[..], |i| i.err("the error")).into_inner(); + assert_eq!(r, (&b"abc"[..], Err("the error"))); + + let r: (_, Result, _>) = many1(&b""[..], |i| token(i, b'a')).into_inner(); + assert_eq!(r, (&b""[..], Err(Error::expected(b'a')))); + let r: (_, Result, _>) = many1(&b"a"[..], |i| token(i, b'a')).into_inner(); + assert_eq!(r, (&b""[..], Ok(vec![b'a']))); + let r: (_, Result, _>) = many1(&b"aa"[..], |i| token(i, b'a')).into_inner(); + assert_eq!(r, (&b""[..], Ok(vec![b'a', b'a']))); + + let r: (_, Result, _>) = many1(&b"bbb"[..], |i| token(i, b'a')).into_inner(); + assert_eq!(r, (&b"bbb"[..], Err(Error::expected(b'a')))); + let r: (_, Result, _>) = many1(&b"abb"[..], |i| token(i, b'a')).into_inner(); + assert_eq!(r, (&b"bb"[..], Ok(vec![b'a']))); + let r: (_, Result, _>) = many1(&b"aab"[..], |i| token(i, b'a')).into_inner(); + assert_eq!(r, (&b"b"[..], Ok(vec![b'a', b'a']))); } #[test] fn count_test() { - let r: State<_, Vec<_>, _> = count(new_buf(DEFAULT, b""), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: State<_, Vec<_>, _> = count(new_buf(DEFAULT, b"a"), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: State<_, Vec<_>, _> = count(new_buf(DEFAULT, b"aa"), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: State<_, Vec<_>, _> = count(new_buf(DEFAULT, b"aaa"), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new_buf(DEFAULT, b""), vec![b'a', b'a', b'a'])); - let r: State<_, Vec<_>, _> = count(new_buf(DEFAULT, b"aaaa"), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new_buf(DEFAULT, b"a"), vec![b'a', b'a', b'a'])); - - let r: State<_, Vec<_>, _> = count(new_buf(END_OF_INPUT, b""), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); - let r: State<_, Vec<_>, _> = count(new_buf(END_OF_INPUT, b"a"), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); - let r: State<_, Vec<_>, _> = count(new_buf(END_OF_INPUT, b"aa"), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); - let r: State<_, Vec<_>, _> = count(new_buf(END_OF_INPUT, b"aaa"), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'a', b'a'])); - let r: State<_, Vec<_>, _> = count(new_buf(END_OF_INPUT, b"aaaa"), 3, |i| token(i, b'a')).into_inner(); - assert_eq!(r, State::Data(new_buf(END_OF_INPUT, b"a"), vec![b'a', b'a', b'a'])); + let r: (_, Result, _>) = count(&b""[..], 3, |i| i.err("the error")).into_inner(); + assert_eq!(r, (&b""[..], Err("the error"))); + + let r: (_, Result, _>) = count(&b""[..], 3, |i| token(i, b'a')).into_inner(); + assert_eq!(r, (&b""[..], Err(Error::expected(b'a')))); + let r: (_, Result, _>) = count(&b"a"[..], 3, |i| token(i, b'a')).into_inner(); + assert_eq!(r, (&b""[..], Err(Error::expected(b'a')))); + let r: (_, Result, _>) = count(&b"aa"[..], 3, |i| token(i, b'a')).into_inner(); + assert_eq!(r, (&b""[..], Err(Error::expected(b'a')))); + let r: (_, Result, _>) = count(&b"aaa"[..], 3, |i| token(i, b'a')).into_inner(); + assert_eq!(r, (&b""[..], Ok(vec![b'a', b'a', b'a']))); + let r: (_, Result, _>) = count(&b"aaaa"[..], 3, |i| token(i, b'a')).into_inner(); + assert_eq!(r, (&b"a"[..], Ok(vec![b'a', b'a', b'a']))); } #[test] fn skip_many1_test() { - assert_eq!(skip_many1(new_buf(DEFAULT, b"aabc"), |i| token(i, b'a')).into_inner(), State::Data(new_buf(DEFAULT, b"bc"), ())); - assert_eq!(skip_many1(new_buf(DEFAULT, b"abc"), |i| token(i, b'a')).into_inner(), State::Data(new_buf(DEFAULT, b"bc"), ())); - assert_eq!(skip_many1(new_buf(DEFAULT, b"bc"), |i| i.err::<(), _>("error")).into_inner(), State::Error(new_buf(DEFAULT, b"bc"), "error")); - assert_eq!(skip_many1(new_buf(DEFAULT, b"aaa"), |i| token(i, b'a')).into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - assert_eq!(skip_many1(new_buf(END_OF_INPUT, b"aabc"), |i| token(i, b'a')).into_inner(), State::Data(new_buf(END_OF_INPUT, b"bc"), ())); - assert_eq!(skip_many1(new_buf(END_OF_INPUT, b"abc"), |i| token(i, b'a')).into_inner(), State::Data(new_buf(END_OF_INPUT, b"bc"), ())); - assert_eq!(skip_many1(new_buf(END_OF_INPUT, b"bc"), |i| i.err::<(), _>("error")).into_inner(), State::Error(new_buf(END_OF_INPUT, b"bc"), "error")); - assert_eq!(skip_many1(new_buf(END_OF_INPUT, b"aaa"), |i| token(i, b'a')).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); + assert_eq!(skip_many1(&b"bc"[..], |i| i.err::<(), _>("error")).into_inner(), (&b"bc"[..], Err("error"))); + + assert_eq!(skip_many1(&b"aabc"[..], |i| token(i, b'a')).into_inner(), (&b"bc"[..], Ok(()))); + assert_eq!(skip_many1(&b"abc"[..], |i| token(i, b'a')).into_inner(), (&b"bc"[..], Ok(()))); + assert_eq!(skip_many1(&b"bc"[..], |i| token(i, b'a')).into_inner(), (&b"bc"[..], Err(Error::expected(b'a')))); + assert_eq!(skip_many1(&b""[..], |i| token(i, b'a')).into_inner(), (&b""[..], Err(Error::expected(b'a')))); + assert_eq!(skip_many1(&b"aaa"[..], |i| token(i, b'a')).into_inner(), (&b""[..], Ok(()))); } #[test] fn many_till_test() { - assert_eq!(many_till(new_buf(DEFAULT, b"abcd"), any, |i| token(i, b'c')).into_inner(), State::Data(new_buf(DEFAULT, b"d"), vec![b'a', b'b'])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abd"), any, |i| token(i, b'c')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + assert_eq!(many_till(&b"abcd"[..], any, |i| token(i, b'c')).into_inner(), (&b"d"[..], Ok(vec![b'a', b'b']))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abd"[..], any, |i| token(i, b'c')); + assert_eq!(r.into_inner(), (&b""[..], Err(Error::unexpected()))); - let r: ParseResult<_, Vec, _> = many_till(new_buf(DEFAULT, b"abcd"), |i| i.err(Error::expected(b'@')), |i| token(i, b'c')); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"abcd"), Error::expected(b'@'))); + let r: ParseResult<_, Vec, _> = many_till(&b"abcd"[..], |i| i.err(Error::expected(b'@')), |i| token(i, b'c')); + assert_eq!(r.into_inner(), (&b"abcd"[..], Err(Error::expected(b'@')))); // Variant to make sure error slice is propagated let mut n = 0; - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abcd"), |i| if n == 0 { n += 1; any(i).map_err(|_| Error::expected(b'i')) } else { i.err(Error::expected(b'@')) }, |i| token(i, b'c')); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"bcd"), Error::expected(b'@'))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abcd"[..], |i| if n == 0 { n += 1; any(i).map_err(|_| Error::expected(b'i')) } else { i.err(Error::expected(b'@')) }, |i| token(i, b'c')); + assert_eq!(r.into_inner(), (&b"bcd"[..], Err(Error::expected(b'@')))); } #[test] fn matched_by_test() { - assert_eq!(matched_by(new_buf(DEFAULT, b"abc"), any).into_inner(), State::Data(new_buf(DEFAULT, b"bc"), (&b"a"[..], b'a'))); - assert_eq!(matched_by(new_buf(DEFAULT, b"abc"), |i| i.err::<(), _>("my error")).into_inner(), State::Error(new_buf(DEFAULT, b"abc"), "my error")); - assert_eq!(matched_by(new_buf(DEFAULT, b"abc"), |i| any(i).map_err(|_| "any error").then(|i| i.err::<(), _>("my error"))).into_inner(), State::Error(new_buf(DEFAULT, b"bc"), "my error")); - assert_eq!(matched_by(new_buf(DEFAULT, b""), any).into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + assert_eq!(matched_by(&b"abc"[..], any).into_inner(), (&b"bc"[..], Ok((&b"a"[..], b'a')))); + assert_eq!(matched_by(&b"abc"[..], |i| i.err::<(), _>("my error")).into_inner(), (&b"abc"[..], Err("my error"))); + assert_eq!(matched_by(&b"abc"[..], |i| any(i).map_err(|_| "any error").then(|i| i.err::<(), _>("my error"))).into_inner(), (&b"bc"[..], Err("my error"))); + assert_eq!(matched_by(&b""[..], any).into_inner(), (&b""[..], Err(Error::unexpected()))); } #[test] fn sep_by_test() { - assert_eq!(sep_by(new_buf(END_OF_INPUT, b""), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); - assert_eq!(sep_by(new_buf(END_OF_INPUT, b"b"), |i| token(i, b'a'), |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![])); - assert_eq!(sep_by(new_buf(END_OF_INPUT, b"a"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a'])); - assert_eq!(sep_by(new_buf(END_OF_INPUT, b"a;c"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'c'])); - assert_eq!(sep_by(new_buf(END_OF_INPUT, b"a;c;"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b";"), vec![b'a', b'c'])); - assert_eq!(sep_by(new_buf(END_OF_INPUT, b"a--c-"), any, |i| string(i, b"--")).into_inner(), State::Data(new_buf(END_OF_INPUT, b"-"), vec![b'a', b'c'])); - assert_eq!(sep_by(new_buf(END_OF_INPUT, b"abc"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b"bc"), vec![b'a'])); - assert_eq!(sep_by(new_buf(END_OF_INPUT, b"a;bc"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b"c"), vec![b'a', b'b'])); - - assert_eq!(sep_by(new_buf(DEFAULT, b"abc"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(DEFAULT, b"bc"), vec![b'a'])); - assert_eq!(sep_by(new_buf(DEFAULT, b"a;bc"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(DEFAULT, b"c"), vec![b'a', b'b'])); + assert_eq!(sep_by(&b""[..], any, |i| token(i, b';')).into_inner(), (&b""[..], Ok(vec![]))); + assert_eq!(sep_by(&b"a"[..], any, |i| token(i, b';')).into_inner(), (&b""[..], Ok(vec![b'a']))); + assert_eq!(sep_by(&b"a;c"[..], any, |i| token(i, b';')).into_inner(), (&b""[..], Ok(vec![b'a', b'c']))); + assert_eq!(sep_by(&b"a;c;"[..], any, |i| token(i, b';')).into_inner(), (&b";"[..], Ok(vec![b'a', b'c']))); + assert_eq!(sep_by(&b"abc"[..], any, |i| token(i, b';')).into_inner(), (&b"bc"[..], Ok(vec![b'a']))); + assert_eq!(sep_by(&b"a;bc"[..], any, |i| token(i, b';')).into_inner(), (&b"c"[..], Ok(vec![b'a', b'b']))); + assert_eq!(sep_by(&b"abc"[..], any, |i| token(i, b';')).into_inner(), (&b"bc"[..], Ok(vec![b'a']))); + assert_eq!(sep_by(&b"a;bc"[..], any, |i| token(i, b';')).into_inner(), (&b"c"[..], Ok(vec![b'a', b'b']))); + + assert_eq!(sep_by(&b"b"[..], |i| token(i, b'a'), |i| token(i, b';')).into_inner(), (&b"b"[..], Ok(vec![]))); + assert_eq!(sep_by(&b"a--c-"[..], any, |i| string(i, b"--")).into_inner(), (&b"-"[..], Ok(vec![b'a', b'c']))); // Incomplete becasue there might be another separator or item to be read - let r: ParseResult<_, Vec<_>, _> = sep_by(new_buf(DEFAULT, b""), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = sep_by(&b""[..], any, |i| token(i, b';')); + assert_eq!(r.into_inner(), (&b""[..], Ok(vec![]))); - let r: ParseResult<_, Vec<_>, _> = sep_by(new_buf(DEFAULT, b"a"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = sep_by(&b"a"[..], any, |i| token(i, b';')); + assert_eq!(r.into_inner(), (&b""[..], Ok(vec![b'a']))); - let r: ParseResult<_, Vec<_>, _> = sep_by(new_buf(DEFAULT, b"a;"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = sep_by(&b"a;"[..], any, |i| token(i, b';')); + assert_eq!(r.into_inner(), (&b";"[..], Ok(vec![b'a']))); - let r: ParseResult<_, Vec<_>, _> = sep_by(new_buf(DEFAULT, b"a;c"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = sep_by(&b"a;c"[..], any, |i| token(i, b';')); + assert_eq!(r.into_inner(), (&b""[..], Ok(vec![b'a', b'c']))); - let r: ParseResult<_, Vec<_>, _> = sep_by(new_buf(DEFAULT, b"a;c;"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = sep_by(&b"a;c;"[..], any, |i| token(i, b';')); + assert_eq!(r.into_inner(), (&b";"[..], Ok(vec![b'a', b'c']))); - let r: ParseResult<_, Vec<_>, _> = sep_by(new_buf(DEFAULT, b"a--c-"), any, |i| string(i, b"--")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"-"), 1)); - - let r: ParseResult<_, Vec<_>, _> = sep_by(new_buf(DEFAULT, b"aaa--a"), |i| string(i, b"aaa"), |i| string(i, b"--")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 2)); + let r: ParseResult<_, Vec<_>, _> = sep_by(&b"a--c-"[..], any, |i| string(i, b"--")); + assert_eq!(r.into_inner(), (&b"-"[..], Ok(vec![b'a', b'c']))); + // FIXME: Indefinite execution on line below + let r: ParseResult<_, Vec<_>, _> = sep_by(&b"aaa--a"[..], |i| string(i, b"aaa"), |i| string(i, b"--")); + assert_eq!(r.into_inner(), (&b"--a"[..], Ok(vec![&b"aaa"[..]]))); } #[test] fn sep_by1_test() { - let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(END_OF_INPUT, b""), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = sep_by1(&b""[..], any, |i| token(i, b';')); + assert_eq!(r.into_inner(), (&b""[..], Err(Error::unexpected()))); + + let r: ParseResult<_, Vec<()>, _> = sep_by1(&b"b"[..], |i| i.err("my err"), |i| token(i, b';').map_err(|_| "token_err")); + assert_eq!(r.into_inner(), (&b"b"[..], Err("my err"))); + + let r: ParseResult<_, Vec<_>, _> = sep_by1(&b""[..], any, |i| token(i, b';')); + assert_eq!(r.into_inner(), (&b""[..], Err(Error::unexpected()))); - let r: ParseResult<_, Vec<()>, _> = sep_by1(new_buf(END_OF_INPUT, b"b"), |i| i.err("my err"), |i| token(i, b';').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "my err")); + let r: ParseResult<_, Vec<_>, _> = sep_by1(&b"b"[..], |i| token(i, b'a'), |i| token(i, b';')); + assert_eq!(r.into_inner(), (&b"b"[..], Err(Error::expected(b'a')))); - assert_eq!(sep_by1(new_buf(END_OF_INPUT, b"a"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a'])); - assert_eq!(sep_by1(new_buf(END_OF_INPUT, b"a;c"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'c'])); - assert_eq!(sep_by1(new_buf(END_OF_INPUT, b"a;c;"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b";"), vec![b'a', b'c'])); - assert_eq!(sep_by1(new_buf(END_OF_INPUT, b"a--c-"), any, |i| string(i, b"--")).into_inner(), State::Data(new_buf(END_OF_INPUT, b"-"), vec![b'a', b'c'])); - assert_eq!(sep_by1(new_buf(END_OF_INPUT, b"abc"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b"bc"), vec![b'a'])); - assert_eq!(sep_by1(new_buf(END_OF_INPUT, b"a;bc"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(END_OF_INPUT, b"c"), vec![b'a', b'b'])); + assert_eq!(sep_by1(&b"a"[..], any, |i| token(i, b';')).into_inner(), (&b""[..], Ok(vec![b'a']))); + assert_eq!(sep_by1(&b"a;c"[..], any, |i| token(i, b';')).into_inner(), (&b""[..], Ok(vec![b'a', b'c']))); + assert_eq!(sep_by1(&b"a;c;"[..], any, |i| token(i, b';')).into_inner(), (&b";"[..], Ok(vec![b'a', b'c']))); + assert_eq!(sep_by1(&b"a--c-"[..], any, |i| string(i, b"--")).into_inner(), (&b"-"[..], Ok(vec![b'a', b'c']))); + assert_eq!(sep_by1(&b"abc"[..], any, |i| token(i, b';')).into_inner(), (&b"bc"[..], Ok(vec![b'a']))); + assert_eq!(sep_by1(&b"a;bc"[..], any, |i| token(i, b';')).into_inner(), (&b"c"[..], Ok(vec![b'a', b'b']))); - assert_eq!(sep_by1(new_buf(DEFAULT, b"abc"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(DEFAULT, b"bc"), vec![b'a'])); - assert_eq!(sep_by1(new_buf(DEFAULT, b"a;bc"), any, |i| token(i, b';')).into_inner(), State::Data(new_buf(DEFAULT, b"c"), vec![b'a', b'b'])); + assert_eq!(sep_by1(&b"abc"[..], any, |i| token(i, b';')).into_inner(), (&b"bc"[..], Ok(vec![b'a']))); + assert_eq!(sep_by1(&b"a;bc"[..], any, |i| token(i, b';')).into_inner(), (&b"c"[..], Ok(vec![b'a', b'b']))); // Incomplete becasue there might be another separator or item to be read - let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(DEFAULT, b""), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = sep_by1(&b""[..], any, |i| token(i, b';')); + assert_eq!(r.into_inner(), (&b""[..], Err(Error::unexpected()))); - let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(DEFAULT, b"a"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = sep_by1(&b"a"[..], any, |i| token(i, b';')); + assert_eq!(r.into_inner(), (&b""[..], Ok(vec![b'a']))); - let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(DEFAULT, b"a;"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = sep_by1(&b"a;"[..], any, |i| token(i, b';')); + assert_eq!(r.into_inner(), (&b";"[..], Ok(vec![b'a']))); - let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(DEFAULT, b"a;c"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = sep_by1(&b"a;c"[..], any, |i| token(i, b';')); + assert_eq!(r.into_inner(), (&b""[..], Ok(vec![b'a', b'c']))); - let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(DEFAULT, b"a;c;"), any, |i| token(i, b';')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + let r: ParseResult<_, Vec<_>, _> = sep_by1(&b"a;c;"[..], any, |i| token(i, b';')); + assert_eq!(r.into_inner(), (&b";"[..], Ok(vec![b'a', b'c']))); - let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(DEFAULT, b"a--c-"), any, |i| string(i, b"--")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"-"), 1)); + let r: ParseResult<_, Vec<_>, _> = sep_by1(&b"a--c-"[..], any, |i| string(i, b"--")); + assert_eq!(r.into_inner(), (&b"-"[..], Ok(vec![b'a', b'c']))); - let r: ParseResult<_, Vec<_>, _> = sep_by1(new_buf(DEFAULT, b"aaa--a"), |i| string(i, b"aaa"), |i| string(i, b"--")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 2)); + let r: ParseResult<_, Vec<_>, _> = sep_by1(&b"aaa--a"[..], |i| string(i, b"aaa"), |i| string(i, b"--")); + assert_eq!(r.into_inner(), (&b"--a"[..], Ok(vec![&b"aaa"[..]]))); } #[test] fn look_ahead_test() { - assert_eq!(look_ahead(new_buf(DEFAULT, b"abc"), any).into_inner(), State::Data(new_buf(DEFAULT, b"abc"), b'a')); - assert_eq!(look_ahead(new_buf(DEFAULT, b"a"), |i| string(i, b"abc")).into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 2)); - assert_eq!(look_ahead(new_buf(DEFAULT, b"aa"), |i| token(i, b'a').then(|i| token(i, b'b')).map_err(|_| "err")).into_inner(), State::Error(new_buf(DEFAULT, b"aa"), "err")); + assert_eq!(look_ahead(&b"abc"[..], any).into_inner(), (&b"abc"[..], Ok(b'a'))); + assert_eq!(look_ahead(&b"a"[..], |i| string(i, b"abc")).into_inner(), (&b"a"[..], Err(Error::expected(b'b')))); + assert_eq!(look_ahead(&b"aa"[..], |i| token(i, b'a').then(|i| token(i, b'b')).map_err(|_| "err")).into_inner(), (&b"aa"[..], Err("err"))); } } -*/ diff --git a/src/parsers.rs b/src/parsers.rs index 3307f2e7..e600d17c 100644 --- a/src/parsers.rs +++ b/src/parsers.rs @@ -244,10 +244,8 @@ pub fn take_while1(mut i: I, f: F) -> SimpleResult /// assert_eq!(r, Ok(&b"abc"[..])); /// ``` #[inline] -pub fn take_till, F>(mut i: I, mut f: F) -> SimpleResult - where F: FnMut(I::Token) -> bool, - I::Token: Clone { - // TODO: How to check the last token? to make sure f succeeded on it? +pub fn take_till(mut i: I, mut f: F) -> SimpleResult + where F: FnMut(I::Token) -> bool { let mut ok = false; let b = i.consume_while(|c| { @@ -263,7 +261,6 @@ pub fn take_till, F>(mut i: I, mut f: F) -> SimpleResult Date: Wed, 17 Aug 2016 17:20:30 +0200 Subject: [PATCH 37/63] Tests: parsers --- src/parsers.rs | 115 ++++++++++++++++++++++--------------------------- 1 file changed, 51 insertions(+), 64 deletions(-) diff --git a/src/parsers.rs b/src/parsers.rs index e600d17c..35c83b31 100644 --- a/src/parsers.rs +++ b/src/parsers.rs @@ -346,9 +346,8 @@ pub fn take_remainder(mut i: I) -> SimpleResult { #[inline] pub fn string>(mut i: I, s: &[T]) -> SimpleResult { - let mut n = 0; - let len = s.len(); - let mut ok = true; + let mut n = 0; + let len = s.len(); // TODO: There has to be some more efficient way here let b = i.consume_while(|c| { @@ -360,16 +359,14 @@ pub fn string>(mut i: I, s: &[T]) true } else { - ok = false; - false } }); - if ok { + if n >= len { i.ret(b) } else { - i.err(Error::expected(s[b.len()])) + i.err(Error::expected(s[n])) } } @@ -548,14 +545,11 @@ mod error { } // FIXME: Uncomment and fix -/* #[cfg(test)] mod test { - use primitives::input::{new_buf, DEFAULT}; use primitives::IntoInner; - use primitives::State; use super::*; - use {Input, SimpleResult, ParseResult}; + use types::{Input, ParseResult}; #[test] fn parse_decimal() { @@ -580,103 +574,97 @@ mod test { let d: ParseResult<_, _, Error> = p.bind(|i, num| take_remainder(i) .bind(|i, r| i.ret((r, num)))); - assert_eq!(d.into_inner(), State::Data(&b""[..], (&b" "[..], (123, 4567)))); + assert_eq!(d.into_inner(), (&b""[..], Ok((&b" "[..], (123, 4567))))); } #[test] fn parse_remainder_empty() { - let i = &b""[..]; - - let r = take_remainder(i); - - assert_eq!(r.into_inner(), State::Data(&b""[..], &b""[..])); + assert_eq!(take_remainder(&b""[..]).into_inner(), (&b""[..], Ok(&b""[..]))); } #[test] fn take_while1_empty() { - let n = &b""[..]; - - let r = take_while1(n, |_| true); - - assert_eq!(r.into_inner(), State::Incomplete(&b""[..], 1)); + assert_eq!(take_while1(&b""[..], |_| true).into_inner(), (&b""[..], Err(Error::unexpected()))); } #[test] fn token_test() { - assert_eq!(token(new_buf(DEFAULT, b""), b'a').into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - assert_eq!(token(new_buf(DEFAULT, b"ab"), b'a').into_inner(), State::Data(new_buf(DEFAULT, b"b"), b'a')); - assert_eq!(token(new_buf(DEFAULT, b"bb"), b'a').into_inner(), State::Error(new_buf(DEFAULT, b"bb"), Error::expected(b'a'))); + assert_eq!(token(&b""[..], b'a').into_inner(), (&b""[..], Err(Error::expected(b'a')))); + assert_eq!(token(&b"ab"[..], b'a').into_inner(), (&b"b"[..], Ok(b'a'))); + assert_eq!(token(&b"bb"[..], b'a').into_inner(), (&b"bb"[..], Err(Error::expected(b'a')))); } #[test] fn take_test() { - assert_eq!(take(new_buf(DEFAULT, b"a"), 1).into_inner(), State::Data(new_buf(DEFAULT, b""), &b"a"[..])); - assert_eq!(take(new_buf(DEFAULT, b"a"), 2).into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); - assert_eq!(take(new_buf(DEFAULT, b"a"), 3).into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 2)); - assert_eq!(take(new_buf(DEFAULT, b"ab"), 1).into_inner(), State::Data(new_buf(DEFAULT, b"b"), &b"a"[..])); - assert_eq!(take(new_buf(DEFAULT, b"ab"), 2).into_inner(), State::Data(new_buf(DEFAULT, b""), &b"ab"[..])); + assert_eq!(take(&b""[..], 0).into_inner(), (&b""[..], Ok(&b""[..]))); + assert_eq!(take(&b"a"[..], 0).into_inner(), (&b"a"[..], Ok(&b""[..]))); + assert_eq!(take(&b"a"[..], 1).into_inner(), (&b""[..], Ok(&b"a"[..]))); + assert_eq!(take(&b"a"[..], 2).into_inner(), (&b"a"[..], Err(Error::unexpected()))); + assert_eq!(take(&b"a"[..], 3).into_inner(), (&b"a"[..], Err(Error::unexpected()))); + assert_eq!(take(&b"ab"[..], 1).into_inner(), (&b"b"[..], Ok(&b"a"[..]))); + assert_eq!(take(&b"ab"[..], 2).into_inner(), (&b""[..], Ok(&b"ab"[..]))); } #[test] fn take_while_test() { - assert_eq!(take_while(new_buf(DEFAULT, b"abc"), |c| c != b'b').into_inner(), State::Data(new_buf(DEFAULT, b"bc"), &b"a"[..])); - assert_eq!(take_while(new_buf(DEFAULT, b"bbc"), |c| c != b'b').into_inner(), State::Data(new_buf(DEFAULT, b"bbc"), &b""[..])); - assert_eq!(take_while(&b"bbc"[..], |c| c != b'b').into_inner(), State::Data(&b"bbc"[..], &b""[..])); - assert_eq!(take_while(&b"abc"[..], |c| c != b'b').into_inner(), State::Data(&b"bc"[..], &b"a"[..])); - // TODO: Update when the incomplete type has been updated - assert_eq!(take_while(new_buf(DEFAULT, b"acc"), |c| c != b'b').into_inner(), State::Incomplete(new_buf(DEFAULT, b"acc"), 1)); - assert_eq!(take_while(&b"acc"[..], |c| c != b'b').into_inner(), State::Data(&b""[..], &b"acc"[..])); + assert_eq!(take_while(&b""[..], |c| c != b'b').into_inner(), (&b""[..], Ok(&b""[..]))); + assert_eq!(take_while(&b"a"[..], |c| c != b'b').into_inner(), (&b""[..], Ok(&b"a"[..]))); + assert_eq!(take_while(&b"b"[..], |c| c != b'b').into_inner(), (&b"b"[..], Ok(&b""[..]))); + assert_eq!(take_while(&b"abc"[..], |c| c != b'b').into_inner(), (&b"bc"[..], Ok(&b"a"[..]))); + assert_eq!(take_while(&b"bbc"[..], |c| c != b'b').into_inner(), (&b"bbc"[..], Ok(&b""[..]))); + assert_eq!(take_while(&b"bbc"[..], |c| c != b'b').into_inner(), (&b"bbc"[..], Ok(&b""[..]))); + assert_eq!(take_while(&b"abc"[..], |c| c != b'b').into_inner(), (&b"bc"[..], Ok(&b"a"[..]))); + assert_eq!(take_while(&b"acc"[..], |c| c != b'b').into_inner(), (&b""[..], Ok(&b"acc"[..]))); } #[test] fn take_while1_test() { - assert_eq!(take_while1(new_buf(DEFAULT, b"abc"), |c| c != b'b').into_inner(), State::Data(new_buf(DEFAULT, b"bc"), &b"a"[..])); - assert_eq!(take_while1(new_buf(DEFAULT, b"bbc"), |c| c != b'b').into_inner(), State::Error(new_buf(DEFAULT, b"bbc"), Error::unexpected())); - assert_eq!(take_while1(&b"bbc"[..], |c| c != b'b').into_inner(), State::Error(&b"bbc"[..], Error::unexpected())); - assert_eq!(take_while1(&b"abc"[..], |c| c != b'b').into_inner(), State::Data(&b"bc"[..], &b"a"[..])); - // TODO: Update when the incomplete type has been updated - assert_eq!(take_while1(new_buf(DEFAULT, b"acc"), |c| c != b'b').into_inner(), State::Incomplete(new_buf(DEFAULT, b"acc"), 1)); - assert_eq!(take_while1(&b"acc"[..], |c| c != b'b').into_inner(), State::Data(&b""[..], &b"acc"[..])); + assert_eq!(take_while1(&b""[..], |c| c != b'b').into_inner(), (&b""[..], Err(Error::unexpected()))); + assert_eq!(take_while1(&b"a"[..], |c| c != b'b').into_inner(), (&b""[..], Ok(&b"a"[..]))); + assert_eq!(take_while1(&b"b"[..], |c| c != b'b').into_inner(), (&b"b"[..], Err(Error::unexpected()))); + assert_eq!(take_while1(&b"ab"[..], |c| c != b'b').into_inner(), (&b"b"[..], Ok(&b"a"[..]))); + assert_eq!(take_while1(&b"abc"[..], |c| c != b'b').into_inner(), (&b"bc"[..], Ok(&b"a"[..]))); + assert_eq!(take_while1(&b"bbc"[..], |c| c != b'b').into_inner(), (&b"bbc"[..], Err(Error::unexpected()))); + assert_eq!(take_while1(&b"bbc"[..], |c| c != b'b').into_inner(), (&b"bbc"[..], Err(Error::unexpected()))); + assert_eq!(take_while1(&b"abc"[..], |c| c != b'b').into_inner(), (&b"bc"[..], Ok(&b"a"[..]))); + assert_eq!(take_while1(&b"acc"[..], |c| c != b'b').into_inner(), (&b""[..], Ok(&b"acc"[..]))); } #[test] fn peek_next_test() { - assert_eq!(peek_next(new_buf(DEFAULT, b"abc")).into_inner(), State::Data(new_buf(DEFAULT, b"abc"), b'a')); - assert_eq!(peek_next(&b"abc"[..]).into_inner(), State::Data(&b"abc"[..], b'a')); - assert_eq!(peek_next(new_buf(DEFAULT, b"")).into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - assert_eq!(peek_next(&b""[..]).into_inner(), State::Incomplete(&b""[..], 1)); + assert_eq!(peek_next(&b"abc"[..]).into_inner(), (&b"abc"[..], Ok(b'a'))); + assert_eq!(peek_next(&b"abc"[..]).into_inner(), (&b"abc"[..], Ok(b'a'))); + assert_eq!(peek_next(&b""[..]).into_inner(), (&b""[..], Err(Error::unexpected()))); + assert_eq!(peek_next(&b""[..]).into_inner(), (&b""[..], Err(Error::unexpected()))); } #[test] fn satisfy_with_test() { let mut m1 = 0; let mut n1 = 0; - assert_eq!(satisfy_with(new_buf(DEFAULT, b"abc"), |m| { m1 += 1; m % 8 }, |n| { n1 += 1; n == 1 }).into_inner(), State::Data(new_buf(DEFAULT, b"bc"), 1)); + assert_eq!(satisfy_with(&b"abc"[..], |m| { m1 += 1; m % 8 }, |n| { n1 += 1; n == 1 }).into_inner(), (&b"bc"[..], Ok(1))); assert_eq!(m1, 1); assert_eq!(n1, 1); let mut m2 = 0; let mut n2 = 0; - assert_eq!(satisfy_with(new_buf(DEFAULT, b""), |m| { m2 += 1; m % 8 }, |n| { n2 += 1; n == 1 }).into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); + assert_eq!(satisfy_with(&b""[..], |m| { m2 += 1; m % 8 }, |n| { n2 += 1; n == 1 }).into_inner(), (&b""[..], Err(Error::unexpected()))); assert_eq!(m2, 0); assert_eq!(n2, 0); } #[test] fn string_test() { - assert_eq!(string(new_buf(DEFAULT, b"abc"), b"a").into_inner(), State::Data(new_buf(DEFAULT, b"bc"), &b"a"[..])); - assert_eq!(string(new_buf(DEFAULT, b"abc"), b"ab").into_inner(), State::Data(new_buf(DEFAULT, b"c"), &b"ab"[..])); - assert_eq!(string(new_buf(DEFAULT, b"abc"), b"abc").into_inner(), State::Data(new_buf(DEFAULT, b""), &b"abc"[..])); - assert_eq!(string(new_buf(DEFAULT, b"abc"), b"abcd").into_inner(), State::Incomplete(new_buf(DEFAULT, b"abc"), 1)); - assert_eq!(string(new_buf(DEFAULT, b"abc"), b"abcde").into_inner(), State::Incomplete(new_buf(DEFAULT, b"abc"), 2)); - assert_eq!(string(new_buf(DEFAULT, b"abc"), b"ac").into_inner(), State::Error(new_buf(DEFAULT, b"bc"), Error::expected(b'c'))); - - assert_eq!(string(&b"abc"[..], b"a").into_inner(), State::Data(&b"bc"[..], &b"a"[..])); - assert_eq!(string(&b"abc"[..], b"ab").into_inner(), State::Data(&b"c"[..], &b"ab"[..])); - assert_eq!(string(&b"abc"[..], b"abc").into_inner(), State::Data(&b""[..], &b"abc"[..])); - assert_eq!(string(&b"abc"[..], b"abcd").into_inner(), State::Incomplete(&b"abc"[..], 1)); - assert_eq!(string(&b"abc"[..], b"abcde").into_inner(), State::Incomplete(&b"abc"[..], 2)); - assert_eq!(string(&b"abc"[..], b"ac").into_inner(), State::Error(&b"bc"[..], Error::expected(b'c'))); + assert_eq!(string(&b""[..], b"").into_inner(), (&b""[..], Ok(&b""[..]))); + assert_eq!(string(&b""[..], b"a").into_inner(), (&b""[..], Err(Error::expected(b'a')))); + assert_eq!(string(&b"a"[..], b"a").into_inner(), (&b""[..], Ok(&b"a"[..]))); + assert_eq!(string(&b"b"[..], b"a").into_inner(), (&b"b"[..], Err(Error::expected(b'a')))); + assert_eq!(string(&b"abc"[..], b"a").into_inner(), (&b"bc"[..], Ok(&b"a"[..]))); + assert_eq!(string(&b"abc"[..], b"ab").into_inner(), (&b"c"[..], Ok(&b"ab"[..]))); + assert_eq!(string(&b"abc"[..], b"abc").into_inner(), (&b""[..], Ok(&b"abc"[..]))); + assert_eq!(string(&b"abc"[..], b"abcd").into_inner(), (&b""[..], Err(Error::expected(b'd')))); + assert_eq!(string(&b"abc"[..], b"abcde").into_inner(), (&b""[..], Err(Error::expected(b'd')))); + assert_eq!(string(&b"abc"[..], b"ac").into_inner(), (&b"bc"[..], Err(Error::expected(b'c')))); } #[test] @@ -712,4 +700,3 @@ mod test { assert!(this.name.as_ref().map(|n| n.contains("parsers::test::backtrace_test")).unwrap_or(false), "Expected trace to contain \"parsers::test::backtrace_test\", got: {:?}", this.name.as_ref()); } } -*/ From af07b428080d704df3b1fbd89ac730b779a3b5df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 17 Aug 2016 17:21:25 +0200 Subject: [PATCH 38/63] Tests: ParseResult --- src/types/parse_result.rs | 78 ++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 43 deletions(-) diff --git a/src/types/parse_result.rs b/src/types/parse_result.rs index f471602e..833c8b13 100644 --- a/src/types/parse_result.rs +++ b/src/types/parse_result.rs @@ -223,12 +223,9 @@ impl IntoInner for ParseResult { } } -/* #[cfg(test)] mod test { - use input; - use input::{Input, DEFAULT, END_OF_INPUT}; - use primitives::State; + use types::{Input, InputBuf}; use super::ParseResult; @@ -238,8 +235,8 @@ mod test { i.ret(n + 1) } - let m1 = input::new_buf(END_OF_INPUT, b"test"); - let m2 = input::new_buf(END_OF_INPUT, b"test"); + let m1 = InputBuf::new(b"test"); + let m2 = InputBuf::new(b"test"); let a = 123; // return a >>= f @@ -247,21 +244,21 @@ mod test { // f a let rhs = f(m2, a); - assert_eq!(lhs.0, State::Data(input::new_buf(END_OF_INPUT, b"test"), 124)); - assert_eq!(rhs.0, State::Data(input::new_buf(END_OF_INPUT, b"test"), 124)); + assert_eq!((lhs.0, lhs.1), (InputBuf::new(b"test"), Ok(124))); + assert_eq!((rhs.0, rhs.1), (InputBuf::new(b"test"), Ok(124))); } #[test] fn monad_right_identity() { - let m1 = input::new_buf(END_OF_INPUT, b"test").ret::<_, ()>(1); - let m2 = input::new_buf(END_OF_INPUT, b"test").ret::<_, ()>(1); + let m1 = InputBuf::new(b"test").ret::<_, ()>(1); + let m2 = InputBuf::new(b"test").ret::<_, ()>(1); // m1 >>= ret === m2 let lhs = m1.bind::<_, _, ()>(Input::ret); let rhs = m2; - assert_eq!(lhs.0, State::Data(input::new_buf(END_OF_INPUT, b"test"), 1)); - assert_eq!(rhs.0, State::Data(input::new_buf(END_OF_INPUT, b"test"), 1)); + assert_eq!((lhs.0, rhs.1), (InputBuf::new(b"test"), Ok(1))); + assert_eq!((rhs.0, lhs.1), (InputBuf::new(b"test"), Ok(1))); } #[test] @@ -274,16 +271,16 @@ mod test { i.ret(num * 2) } - let lhs_m = input::new_buf(END_OF_INPUT, b"test").ret::<_, ()>(2); - let rhs_m = input::new_buf(END_OF_INPUT, b"test").ret::<_, ()>(2); + let lhs_m = InputBuf::new(b"test").ret::<_, ()>(2); + let rhs_m = InputBuf::new(b"test").ret::<_, ()>(2); // (m >>= f) >>= g let lhs = lhs_m.bind(f).bind(g); // m >>= (\x -> f x >>= g) let rhs = rhs_m.bind(|i, x| f(i, x).bind(g)); - assert_eq!(lhs.0, State::Data(input::new_buf(END_OF_INPUT, b"test"), 6)); - assert_eq!(rhs.0, State::Data(input::new_buf(END_OF_INPUT, b"test"), 6)); + assert_eq!((lhs.0, lhs.1), (InputBuf::new(b"test"), Ok(6))); + assert_eq!((rhs.0, rhs.1), (InputBuf::new(b"test"), Ok(6))); } #[test] @@ -292,8 +289,8 @@ mod test { let mut n1 = 0; let mut n2 = 0; - let i1 = input::new_buf(DEFAULT, b"test ").ret::(23); - let i2 = input::new_buf(END_OF_INPUT, b"test ").ret::(23); + let i1 = InputBuf::new(b"test ").ret::(23); + let i2 = InputBuf::new(b"test ").ret::(23); let r1 = i1.inspect(|d: &u32| { assert_eq!(d, &23); @@ -306,44 +303,40 @@ mod test { n2 += 1; }); - assert_eq!(r1.into_inner(), State::Data(input::new_buf(DEFAULT, b"test "), 23)); + assert_eq!(r1.into_inner(), (InputBuf::new(b"test "), Ok(23))); assert_eq!(n1, 1); - assert_eq!(r2.into_inner(), State::Data(input::new_buf(END_OF_INPUT, b"test "), 23)); + assert_eq!(r2.into_inner(), (InputBuf::new(b"test "), Ok(23))); assert_eq!(n2, 1); } #[test] fn input_propagation() { - let mut n1_calls = 0; - let mut n2_calls = 0; + let mut n_calls = 0; - let i1 = input::new_buf(DEFAULT, b"test1").ret::<_, ()>(23); - let i2 = input::new_buf(END_OF_INPUT, b"test2").ret::<_, ()>(24); + let i = InputBuf::new(b"test1").ret::<_, ()>(23); - let r1: ParseResult<_, _, ()> = i1.bind(|i, t| { n1_calls += 1; i.ret(t) }); - let r2: ParseResult<_, _, ()> = i2.bind(|i, t| { n2_calls += 1; i.ret(t) }); + assert_eq!(i.0, InputBuf::new(b"test1")); + assert_eq!(i.1, Ok(23)); - assert_eq!(r1.0, State::Data(input::new_buf(DEFAULT, b"test1"), 23)); - assert_eq!(r2.0, State::Data(input::new_buf(END_OF_INPUT, b"test2"), 24)); - assert_eq!(n1_calls, 1); - assert_eq!(n2_calls, 1); + let r: ParseResult<_, _, ()> = i.bind(|i, t| { n_calls += 1; i.ret(t) }); + + assert_eq!((r.0, r.1), (InputBuf::new(b"test1"), Ok(23))); + assert_eq!(n_calls, 1); } #[test] fn error_propagation() { - let mut n1_calls = 0; - let mut n2_calls = 0; + let mut n_calls = 0; + + let i = InputBuf::new(b"test1").err::<(), _>(23); - let i1 = input::new_buf(DEFAULT, b"test1").err::<(), _>(23); - let i2 = input::new_buf(END_OF_INPUT, b"test2").err::<(), _>(24); + assert_eq!(i.0, InputBuf::new(b"test1")); + assert_eq!(i.1, Err(23)); - let r1 = i1.bind(|i, t| { n1_calls += 1; i.ret(t) }); - let r2 = i2.bind(|i, t| { n2_calls += 1; i.ret(t) }); + let r = i.bind(|i, t| { n_calls += 1; i.ret(t) }); - assert_eq!(r1.0, State::Error(input::new_buf(DEFAULT, b"test1"), 23)); - assert_eq!(r2.0, State::Error(input::new_buf(END_OF_INPUT, b"test2"), 24)); - assert_eq!(n1_calls, 0); - assert_eq!(n2_calls, 0); + assert_eq!((r.0, r.1), (InputBuf::new(b"test1"), Err(23))); + assert_eq!(n_calls, 0); } #[test] @@ -353,8 +346,7 @@ mod test { let lhs = (&b"test"[..]).ret(123).bind(f); let rhs = f(&b"test"[..], 123); - assert_eq!(lhs.0, State::Data(&b"test"[..], 124)); - assert_eq!(rhs.0, State::Data(&b"test"[..], 124)); + assert_eq!((lhs.0, lhs.1), (&b"test"[..], Ok(124))); + assert_eq!((rhs.0, rhs.1), (&b"test"[..], Ok(124))); } } -*/ From c9c0d54e854494d6f2fbc5f47aeaadcf7fa7eebe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 17 Aug 2016 17:40:37 +0200 Subject: [PATCH 39/63] Moved ParseResult to types --- src/types/mod.rs | 357 +++++++++++++++++++++++++++++++++++++- src/types/parse_result.rs | 352 ------------------------------------- 2 files changed, 349 insertions(+), 360 deletions(-) delete mode 100644 src/types/parse_result.rs diff --git a/src/types/mod.rs b/src/types/mod.rs index a060e76c..7dc73d25 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -1,13 +1,9 @@ mod input; -mod parse_result; #[cfg(feature = "tendril")] pub mod tendril; -use primitives::Guard; +use primitives::{Guard, IntoInner}; -pub use types::parse_result::{ - ParseResult -}; pub use types::input::InputBuf; pub trait U8Input: Input {} @@ -96,7 +92,7 @@ pub trait Input: Sized { /// ``` #[inline] fn ret(self, t: T) -> ParseResult { - parse_result::new(self, Ok(t)) + ParseResult(self, Ok(t)) } /// Returns `e` as an error value in the parsing context. @@ -117,7 +113,7 @@ pub trait Input: Sized { /// ``` #[inline] fn err(self, e: E) -> ParseResult { - parse_result::new(self, Err(e)) + ParseResult(self, Err(e)) } /// Converts a `Result` into a `ParseResult`, preserving parser state. @@ -142,7 +138,7 @@ pub trait Input: Sized { /// ``` #[inline] fn from_result(self, r: Result) -> ParseResult { - parse_result::new(self, r) + ParseResult(self, r) } // Primitive methods @@ -176,6 +172,8 @@ pub trait Input: Sized { /// token will be returned as a buffer and discarded from the current input. MUST never run the /// closure more than once on the exact same token. /// + /// If the end of the input is reached, the whole input is returned. + /// /// Note: Will refill automatically. #[inline] fn _consume_while(&mut self, Guard, F) -> Self::Buffer @@ -210,6 +208,228 @@ pub trait Input: Sized { fn _restore(self, Guard, Self::Marker) -> Self; } +/// The basic return type of a parser. +/// +/// This type satisfies a variant of the `Monad` typeclass. Due to the limitations of Rust's +/// return types closures cannot be returned without boxing which has an unacceptable performance +/// impact. +/// +/// To get around this issue and still provide a simple to use and safe (as in hard to accidentally +/// violate the monad laws or the assumptions taken by the parser type) an `Input` wrapper is +/// provided which ensures that the parser state is carried properly through every call to `bind`. +/// This is also known as a Linear Type (emulated through hiding destructors and using the +/// annotation `#[must_use]`). +/// +/// Do-notation is provided by the macro `parse!`. +/// +/// # Equivalence with Haskell's `Monad` typeclass: +/// +/// ```text +/// f >>= g ≡ f(m).bind(g) +/// f >> g ≡ f(m).then(g) +/// return a ≡ m.ret(a) +/// fail a ≡ m.err(a) +/// ``` +/// +/// It also satisfies the monad laws: +/// +/// ```text +/// return a >>= f ≡ f a +/// m >>= return ≡ m +/// (m >>= f) >>= g ≡ m >>= (\x -> f x >>= g) +/// ``` +#[must_use] +#[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct ParseResult(I, Result); + +/// **Primitive:** Creates a new `ParseResult`. +/// +/// # Primitive +/// +/// Only used by fundamental parsers and combinators. +/// +/// # Note +/// +/// Prefer to use `Input::ret`, `Input::err` or `Input::from_result` instead of using +pub fn new(i: I, r: Result) -> ParseResult { + ParseResult(i, r) +} + +impl ParseResult { + /// Sequentially composes the result with a parse action `f`, passing any produced value as + /// the second parameter. + /// + /// The first parameter to the supplied function `f` is the parser state (`Input`). This + /// state is then passed on to other parsers or used to return a value or an error. + /// + /// # Automatic conversion of `E` + /// + /// The error value `E` will automatically be converted using the `From` trait to the + /// desired type. The downside with this using the current stable version of Rust (1.4) is that + /// the type inferrence will currently not use the default value for the generic `V` and will + /// therefore require extra type hint for the error. + /// + /// # Examples + /// + /// ``` + /// use chomp::prelude::{Input, parse_only}; + /// + /// let r = parse_only(|i| { + /// i.ret("data".to_owned()) + /// // Explicitly state the error type + /// .bind::<_, _, ()>(|i, x| i.ret(x + " here!")) + /// }, + /// b"test"); + /// + /// assert_eq!(r, Ok("data here!".to_owned())); + /// ``` + /// + /// Wrapping the expression in a function will both make it easier to compose and also provides + /// the type-hint for the error in the function signature: + /// + /// ``` + /// use chomp::prelude::{Input, ParseResult, parse_only}; + /// + /// fn parser(i: I, n: i32) -> ParseResult { + /// i.ret(n + 10) + /// } + /// + /// let r = parse_only(|i| i.ret(23).bind(parser), b"test"); + /// + /// assert_eq!(r, Ok(33)); + /// ``` + #[inline] + pub fn bind(self, f: F) -> ParseResult + where F: FnOnce(I, T) -> ParseResult, + V: From { + match self.1 { + Ok(t) => f(self.0, t).map_err(From::from), + Err(e) => ParseResult(self.0, Err(From::from(e))), + } + } + + /// Sequentially composes the result with a parse action `f`, discarding any produced value. + /// + /// The first parameter to the supplied function `f` is the parser state (`Input`). This + /// state is then passed on to other parsers or used to return a value or an error. + /// + /// # Relation to `bind` + /// + /// ```text + /// ParseResult::then(g) ≡ ParseResult::bind(|i, _| g(i)) + /// ``` + /// + /// # Example + /// + /// ``` + /// use chomp::prelude::{Input, SimpleResult, parse_only}; + /// + /// fn g(i: I) -> SimpleResult { + /// i.ret("testing!") + /// } + /// + /// let r1 = parse_only(|i| i.ret("initial state").bind(|i, _| g(i)), b"data"); + /// let r2 = parse_only(|i| i.ret("initial state").then(g), b"data"); + /// + /// assert_eq!(r1, Ok("testing!")); + /// assert_eq!(r2, Ok("testing!")); + /// ``` + #[inline] + pub fn then(self, f: F) -> ParseResult + where F: FnOnce(I) -> ParseResult, + V: From { + self.bind(|i, _| f(i)) + } + + /// Applies the function `f` on the contained data if the parser is in a success state. + /// + /// # Example + /// + /// ``` + /// use chomp::prelude::{parse_only, any}; + /// + /// let r = parse_only(|i| any(i).map(|c| c + 12), b"abc"); + /// + /// assert_eq!(r, Ok(b'm')); + /// ``` + #[inline] + pub fn map(self, f: F) -> ParseResult + where F: FnOnce(T) -> U { + match self { + ParseResult(i, Ok(t)) => ParseResult(i, Ok(f(t))), + ParseResult(i, Err(e)) => ParseResult(i, Err(e)), + } + } + + /// Applies the function `f` on the contained error if the parser is in an error state. + /// + /// # Example + /// + /// ``` + /// use chomp::prelude::{Input, parse_only}; + /// + /// let r = parse_only(|i| i.err::<(), _>("this is") + /// .map_err(|e| e.to_owned() + " an error"), + /// b"foo"); + /// + /// assert_eq!(r, Err((&b"foo"[..], "this is an error".to_owned()))); + /// ``` + #[inline] + pub fn map_err(self, f: F) -> ParseResult + where F: FnOnce(E) -> V { + match self { + ParseResult(i, Ok(t)) => ParseResult(i, Ok(t)), + ParseResult(i, Err(e)) => ParseResult(i, Err(f(e))), + } + } + + /// Calls the function `f` with a reference of the contained data if the parser is in a success + /// state. + /// + /// # Example + /// + /// ``` + /// use chomp::prelude::{parse_only, take_while}; + /// + /// let r = parse_only(|i| take_while(i, |c| c != b' ').inspect(|b| { + /// println!("{:?}", b); // Prints "test" + /// }), b"test and more"); + /// + /// assert_eq!(r, Ok(&b"test"[..])); + /// ``` + #[inline] + pub fn inspect(self, f: F) -> ParseResult + where F: FnOnce(&T) { + if let Ok(ref t) = self.1 { + f(t) + } + + self + } +} + +/// **Primitive:** Consumes the `ParseResult` and exposes the internal state. +/// +/// # Primitive +/// +/// Only used by fundamental parsers and combinators. +/// +/// # Motivation +/// +/// The `ParseResult` type is a semi-linear type, supposed to act like a linear type while used in +/// a parsing context to carry the state. Normally it should be as restrictive as the `Input` type +/// in terms of how much it exposes its internals, but the `IntoInner` trait implementation +/// allows fundamental parsers and combinators to expose the inner `Result` of the `ParseResult` +/// and act on this. +impl IntoInner for ParseResult { + type Inner = (I, Result); + + #[inline(always)] + fn into_inner(self) -> Self::Inner { + (self.0, self.1) + } +} + #[cfg(test)] mod test { use super::{Input, InputBuf, ParseResult}; @@ -250,4 +470,125 @@ mod test { assert_eq!(r1.into_inner(), (InputBuf::new(b"in1"), Ok(23u32))); assert_eq!(r2.into_inner(), (InputBuf::new(b"in2"), Err("foobar"))); } + + #[test] + fn monad_left_identity() { + fn f(i: I, n: u32) -> ParseResult { + i.ret(n + 1) + } + + let m1 = InputBuf::new(b"test"); + let m2 = InputBuf::new(b"test"); + + let a = 123; + // return a >>= f + let lhs = m1.ret(a).bind(f); + // f a + let rhs = f(m2, a); + + assert_eq!((lhs.0, lhs.1), (InputBuf::new(b"test"), Ok(124))); + assert_eq!((rhs.0, rhs.1), (InputBuf::new(b"test"), Ok(124))); + } + + #[test] + fn monad_right_identity() { + let m1 = InputBuf::new(b"test").ret::<_, ()>(1); + let m2 = InputBuf::new(b"test").ret::<_, ()>(1); + + // m1 >>= ret === m2 + let lhs = m1.bind::<_, _, ()>(Input::ret); + let rhs = m2; + + assert_eq!((lhs.0, rhs.1), (InputBuf::new(b"test"), Ok(1))); + assert_eq!((rhs.0, lhs.1), (InputBuf::new(b"test"), Ok(1))); + } + + #[test] + fn monad_associativity() { + fn f(i: I, num: u32) -> ParseResult { + i.ret((num + 1) as u64) + } + + fn g(i: I, num: u64) -> ParseResult { + i.ret(num * 2) + } + + let lhs_m = InputBuf::new(b"test").ret::<_, ()>(2); + let rhs_m = InputBuf::new(b"test").ret::<_, ()>(2); + + // (m >>= f) >>= g + let lhs = lhs_m.bind(f).bind(g); + // m >>= (\x -> f x >>= g) + let rhs = rhs_m.bind(|i, x| f(i, x).bind(g)); + + assert_eq!((lhs.0, lhs.1), (InputBuf::new(b"test"), Ok(6))); + assert_eq!((rhs.0, rhs.1), (InputBuf::new(b"test"), Ok(6))); + } + + #[test] + fn parse_result_inspect() { + use primitives::IntoInner; + + let mut n1 = 0; + let mut n2 = 0; + let i1 = InputBuf::new(b"test ").ret::(23); + let i2 = InputBuf::new(b"test ").ret::(23); + + let r1 = i1.inspect(|d: &u32| { + assert_eq!(d, &23); + + n1 += 1; + }); + let r2 = i2.inspect(|d: &u32| { + assert_eq!(d, &23); + + n2 += 1; + }); + + assert_eq!(r1.into_inner(), (InputBuf::new(b"test "), Ok(23))); + assert_eq!(n1, 1); + assert_eq!(r2.into_inner(), (InputBuf::new(b"test "), Ok(23))); + assert_eq!(n2, 1); + } + + #[test] + fn input_propagation() { + let mut n_calls = 0; + + let i = InputBuf::new(b"test1").ret::<_, ()>(23); + + assert_eq!(i.0, InputBuf::new(b"test1")); + assert_eq!(i.1, Ok(23)); + + let r: ParseResult<_, _, ()> = i.bind(|i, t| { n_calls += 1; i.ret(t) }); + + assert_eq!((r.0, r.1), (InputBuf::new(b"test1"), Ok(23))); + assert_eq!(n_calls, 1); + } + + #[test] + fn error_propagation() { + let mut n_calls = 0; + + let i = InputBuf::new(b"test1").err::<(), _>(23); + + assert_eq!(i.0, InputBuf::new(b"test1")); + assert_eq!(i.1, Err(23)); + + let r = i.bind(|i, t| { n_calls += 1; i.ret(t) }); + + assert_eq!((r.0, r.1), (InputBuf::new(b"test1"), Err(23))); + assert_eq!(n_calls, 0); + } + + #[test] + fn slice() { + fn f(i: I, n: u32) -> ParseResult { i.ret(n + 1) } + + let lhs = (&b"test"[..]).ret(123).bind(f); + let rhs = f(&b"test"[..], 123); + + assert_eq!((lhs.0, lhs.1), (&b"test"[..], Ok(124))); + assert_eq!((rhs.0, rhs.1), (&b"test"[..], Ok(124))); + } } diff --git a/src/types/parse_result.rs b/src/types/parse_result.rs deleted file mode 100644 index 833c8b13..00000000 --- a/src/types/parse_result.rs +++ /dev/null @@ -1,352 +0,0 @@ -use types::Input; -use primitives::IntoInner; - -/// The basic return type of a parser. -/// -/// This type satisfies a variant of the `Monad` typeclass. Due to the limitations of Rust's -/// return types closures cannot be returned without boxing which has an unacceptable performance -/// impact. -/// -/// To get around this issue and still provide a simple to use and safe (as in hard to accidentally -/// violate the monad laws or the assumptions taken by the parser type) an `Input` wrapper is -/// provided which ensures that the parser state is carried properly through every call to `bind`. -/// This is also known as a Linear Type (emulated through hiding destructors and using the -/// annotation `#[must_use]`). -/// -/// Do-notation is provided by the macro `parse!`. -/// -/// # Equivalence with Haskell's `Monad` typeclass: -/// -/// ```text -/// f >>= g ≡ f(m).bind(g) -/// f >> g ≡ f(m).then(g) -/// return a ≡ m.ret(a) -/// fail a ≡ m.err(a) -/// ``` -/// -/// It also satisfies the monad laws: -/// -/// ```text -/// return a >>= f ≡ f a -/// m >>= return ≡ m -/// (m >>= f) >>= g ≡ m >>= (\x -> f x >>= g) -/// ``` -#[must_use] -#[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub struct ParseResult(I, Result); - -/// **Primitive:** Creates a new `ParseResult`. -/// -/// # Primitive -/// -/// Only used by fundamental parsers and combinators. -/// -/// # Note -/// -/// Prefer to use `Input::ret`, `Input::err` or `Input::from_result` instead of using -pub fn new(i: I, r: Result) -> ParseResult { - ParseResult(i, r) -} - -impl ParseResult { - /// Sequentially composes the result with a parse action `f`, passing any produced value as - /// the second parameter. - /// - /// The first parameter to the supplied function `f` is the parser state (`Input`). This - /// state is then passed on to other parsers or used to return a value or an error. - /// - /// # Automatic conversion of `E` - /// - /// The error value `E` will automatically be converted using the `From` trait to the - /// desired type. The downside with this using the current stable version of Rust (1.4) is that - /// the type inferrence will currently not use the default value for the generic `V` and will - /// therefore require extra type hint for the error. - /// - /// # Examples - /// - /// ``` - /// use chomp::prelude::{Input, parse_only}; - /// - /// let r = parse_only(|i| { - /// i.ret("data".to_owned()) - /// // Explicitly state the error type - /// .bind::<_, _, ()>(|i, x| i.ret(x + " here!")) - /// }, - /// b"test"); - /// - /// assert_eq!(r, Ok("data here!".to_owned())); - /// ``` - /// - /// Wrapping the expression in a function will both make it easier to compose and also provides - /// the type-hint for the error in the function signature: - /// - /// ``` - /// use chomp::prelude::{Input, ParseResult, parse_only}; - /// - /// fn parser(i: I, n: i32) -> ParseResult { - /// i.ret(n + 10) - /// } - /// - /// let r = parse_only(|i| i.ret(23).bind(parser), b"test"); - /// - /// assert_eq!(r, Ok(33)); - /// ``` - #[inline] - pub fn bind(self, f: F) -> ParseResult - where F: FnOnce(I, T) -> ParseResult, - V: From { - match self.1 { - Ok(t) => f(self.0, t).map_err(From::from), - Err(e) => ParseResult(self.0, Err(From::from(e))), - } - } - - /// Sequentially composes the result with a parse action `f`, discarding any produced value. - /// - /// The first parameter to the supplied function `f` is the parser state (`Input`). This - /// state is then passed on to other parsers or used to return a value or an error. - /// - /// # Relation to `bind` - /// - /// ```text - /// ParseResult::then(g) ≡ ParseResult::bind(|i, _| g(i)) - /// ``` - /// - /// # Example - /// - /// ``` - /// use chomp::prelude::{Input, SimpleResult, parse_only}; - /// - /// fn g(i: I) -> SimpleResult { - /// i.ret("testing!") - /// } - /// - /// let r1 = parse_only(|i| i.ret("initial state").bind(|i, _| g(i)), b"data"); - /// let r2 = parse_only(|i| i.ret("initial state").then(g), b"data"); - /// - /// assert_eq!(r1, Ok("testing!")); - /// assert_eq!(r2, Ok("testing!")); - /// ``` - #[inline] - pub fn then(self, f: F) -> ParseResult - where F: FnOnce(I) -> ParseResult, - V: From { - self.bind(|i, _| f(i)) - } - - /// Applies the function `f` on the contained data if the parser is in a success state. - /// - /// # Example - /// - /// ``` - /// use chomp::prelude::{parse_only, any}; - /// - /// let r = parse_only(|i| any(i).map(|c| c + 12), b"abc"); - /// - /// assert_eq!(r, Ok(b'm')); - /// ``` - #[inline] - pub fn map(self, f: F) -> ParseResult - where F: FnOnce(T) -> U { - match self { - ParseResult(i, Ok(t)) => ParseResult(i, Ok(f(t))), - ParseResult(i, Err(e)) => ParseResult(i, Err(e)), - } - } - - /// Applies the function `f` on the contained error if the parser is in an error state. - /// - /// # Example - /// - /// ``` - /// use chomp::prelude::{Input, parse_only}; - /// - /// let r = parse_only(|i| i.err::<(), _>("this is") - /// .map_err(|e| e.to_owned() + " an error"), - /// b"foo"); - /// - /// assert_eq!(r, Err((&b"foo"[..], "this is an error".to_owned()))); - /// ``` - #[inline] - pub fn map_err(self, f: F) -> ParseResult - where F: FnOnce(E) -> V { - match self { - ParseResult(i, Ok(t)) => ParseResult(i, Ok(t)), - ParseResult(i, Err(e)) => ParseResult(i, Err(f(e))), - } - } - - /// Calls the function `f` with a reference of the contained data if the parser is in a success - /// state. - /// - /// # Example - /// - /// ``` - /// use chomp::prelude::{parse_only, take_while}; - /// - /// let r = parse_only(|i| take_while(i, |c| c != b' ').inspect(|b| { - /// println!("{:?}", b); // Prints "test" - /// }), b"test and more"); - /// - /// assert_eq!(r, Ok(&b"test"[..])); - /// ``` - #[inline] - pub fn inspect(self, f: F) -> ParseResult - where F: FnOnce(&T) { - if let Ok(ref t) = self.1 { - f(t) - } - - self - } -} - -/// **Primitive:** Consumes the `ParseResult` and exposes the internal state. -/// -/// # Primitive -/// -/// Only used by fundamental parsers and combinators. -/// -/// # Motivation -/// -/// The `ParseResult` type is a semi-linear type, supposed to act like a linear type while used in -/// a parsing context to carry the state. Normally it should be as restrictive as the `Input` type -/// in terms of how much it exposes its internals, but the `IntoInner` trait implementation -/// allows fundamental parsers and combinators to expose the inner `Result` of the `ParseResult` -/// and act on this. -impl IntoInner for ParseResult { - type Inner = (I, Result); - - #[inline(always)] - fn into_inner(self) -> Self::Inner { - (self.0, self.1) - } -} - -#[cfg(test)] -mod test { - use types::{Input, InputBuf}; - - use super::ParseResult; - - #[test] - fn monad_left_identity() { - fn f(i: I, n: u32) -> ParseResult { - i.ret(n + 1) - } - - let m1 = InputBuf::new(b"test"); - let m2 = InputBuf::new(b"test"); - - let a = 123; - // return a >>= f - let lhs = m1.ret(a).bind(f); - // f a - let rhs = f(m2, a); - - assert_eq!((lhs.0, lhs.1), (InputBuf::new(b"test"), Ok(124))); - assert_eq!((rhs.0, rhs.1), (InputBuf::new(b"test"), Ok(124))); - } - - #[test] - fn monad_right_identity() { - let m1 = InputBuf::new(b"test").ret::<_, ()>(1); - let m2 = InputBuf::new(b"test").ret::<_, ()>(1); - - // m1 >>= ret === m2 - let lhs = m1.bind::<_, _, ()>(Input::ret); - let rhs = m2; - - assert_eq!((lhs.0, rhs.1), (InputBuf::new(b"test"), Ok(1))); - assert_eq!((rhs.0, lhs.1), (InputBuf::new(b"test"), Ok(1))); - } - - #[test] - fn monad_associativity() { - fn f(i: I, num: u32) -> ParseResult { - i.ret((num + 1) as u64) - } - - fn g(i: I, num: u64) -> ParseResult { - i.ret(num * 2) - } - - let lhs_m = InputBuf::new(b"test").ret::<_, ()>(2); - let rhs_m = InputBuf::new(b"test").ret::<_, ()>(2); - - // (m >>= f) >>= g - let lhs = lhs_m.bind(f).bind(g); - // m >>= (\x -> f x >>= g) - let rhs = rhs_m.bind(|i, x| f(i, x).bind(g)); - - assert_eq!((lhs.0, lhs.1), (InputBuf::new(b"test"), Ok(6))); - assert_eq!((rhs.0, rhs.1), (InputBuf::new(b"test"), Ok(6))); - } - - #[test] - fn parse_result_inspect() { - use primitives::IntoInner; - - let mut n1 = 0; - let mut n2 = 0; - let i1 = InputBuf::new(b"test ").ret::(23); - let i2 = InputBuf::new(b"test ").ret::(23); - - let r1 = i1.inspect(|d: &u32| { - assert_eq!(d, &23); - - n1 += 1; - }); - let r2 = i2.inspect(|d: &u32| { - assert_eq!(d, &23); - - n2 += 1; - }); - - assert_eq!(r1.into_inner(), (InputBuf::new(b"test "), Ok(23))); - assert_eq!(n1, 1); - assert_eq!(r2.into_inner(), (InputBuf::new(b"test "), Ok(23))); - assert_eq!(n2, 1); - } - - #[test] - fn input_propagation() { - let mut n_calls = 0; - - let i = InputBuf::new(b"test1").ret::<_, ()>(23); - - assert_eq!(i.0, InputBuf::new(b"test1")); - assert_eq!(i.1, Ok(23)); - - let r: ParseResult<_, _, ()> = i.bind(|i, t| { n_calls += 1; i.ret(t) }); - - assert_eq!((r.0, r.1), (InputBuf::new(b"test1"), Ok(23))); - assert_eq!(n_calls, 1); - } - - #[test] - fn error_propagation() { - let mut n_calls = 0; - - let i = InputBuf::new(b"test1").err::<(), _>(23); - - assert_eq!(i.0, InputBuf::new(b"test1")); - assert_eq!(i.1, Err(23)); - - let r = i.bind(|i, t| { n_calls += 1; i.ret(t) }); - - assert_eq!((r.0, r.1), (InputBuf::new(b"test1"), Err(23))); - assert_eq!(n_calls, 0); - } - - #[test] - fn slice() { - fn f(i: I, n: u32) -> ParseResult { i.ret(n + 1) } - - let lhs = (&b"test"[..]).ret(123).bind(f); - let rhs = f(&b"test"[..], 123); - - assert_eq!((lhs.0, lhs.1), (&b"test"[..], Ok(124))); - assert_eq!((rhs.0, rhs.1), (&b"test"[..], Ok(124))); - } -} From 47e8512eb14704dc6b49635ceaea04415d1f8c2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 17 Aug 2016 18:20:30 +0200 Subject: [PATCH 40/63] Buffer: Followed convention for to_vec() and into_vec() --- src/types/input.rs | 14 +++++++++++--- src/types/mod.rs | 5 ++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/types/input.rs b/src/types/input.rs index 736b8c6e..60e8f785 100644 --- a/src/types/input.rs +++ b/src/types/input.rs @@ -14,8 +14,12 @@ impl<'a, I: Copy + PartialEq> Buffer for &'a [I] { (&self[..]).len() } - fn to_vec(self) -> Vec { - (&self[..]).iter().cloned().collect() + fn to_vec(&self) -> Vec { + (&self[..]).to_vec() + } + + fn into_vec(self) -> Vec { + (&self[..]).to_vec() } } @@ -35,7 +39,11 @@ impl<'a> Buffer for &'a str { (&self[..]).is_empty() } - fn to_vec(self) -> Vec { + fn to_vec(&self) -> Vec { + (&self[..]).chars().collect() + } + + fn into_vec(self) -> Vec { (&self[..]).chars().collect() } } diff --git a/src/types/mod.rs b/src/types/mod.rs index 7dc73d25..b77cced1 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -33,11 +33,14 @@ pub trait Buffer: PartialEq { /// The number of tokens present in this buffer. fn len(&self) -> usize; + /// Copies all the tokens in this buffer to a new `Vec`. + fn to_vec(&self) -> Vec; + /// Consumes self to create an owned vector of tokens. /// /// Will allocate if the implementation borrows storage or does not use an owned type /// compatible with `Vec` internally. - fn to_vec(self) -> Vec; + fn into_vec(self) -> Vec; /// Returns true if this buffer is empty. fn is_empty(&self) -> bool { From d642e76b2d8d2f31b93be4dcc290aeced9ea04ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 17 Aug 2016 18:47:42 +0200 Subject: [PATCH 41/63] Removed unused ParseResult constructor function --- src/types/mod.rs | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/types/mod.rs b/src/types/mod.rs index b77cced1..5baf13b7 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -245,19 +245,6 @@ pub trait Input: Sized { #[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] pub struct ParseResult(I, Result); -/// **Primitive:** Creates a new `ParseResult`. -/// -/// # Primitive -/// -/// Only used by fundamental parsers and combinators. -/// -/// # Note -/// -/// Prefer to use `Input::ret`, `Input::err` or `Input::from_result` instead of using -pub fn new(i: I, r: Result) -> ParseResult { - ParseResult(i, r) -} - impl ParseResult { /// Sequentially composes the result with a parse action `f`, passing any produced value as /// the second parameter. From 107c0cfa54a5d5ac0928f181a3d7afea5f00c746 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 17 Aug 2016 18:51:38 +0200 Subject: [PATCH 42/63] moved InputBuf to buffer module --- src/buffer/mod.rs | 269 +++++++++++++++++++- src/buffer/slice.rs | 4 +- src/buffer/stateful.rs | 5 +- src/types/input.rs | 565 ----------------------------------------- src/types/mod.rs | 426 +++++++++++++++++++++++++++---- 5 files changed, 643 insertions(+), 626 deletions(-) delete mode 100644 src/types/input.rs diff --git a/src/buffer/mod.rs b/src/buffer/mod.rs index 147d82d7..8863aa95 100644 --- a/src/buffer/mod.rs +++ b/src/buffer/mod.rs @@ -35,8 +35,9 @@ pub mod data_source; use std::io; -use types::{ParseResult, InputBuf}; +use types::{Input, ParseResult}; use types::Buffer as InputBuffer; +use primitives::Guard; pub use self::slice::SliceStream; pub use self::data_source::DataSource; @@ -105,3 +106,269 @@ pub trait IntoStream<'a, 'i> { #[inline] fn into_stream(self) -> Self::Into; } + +// FIXME: Docs +/// Input buffer type which contains a flag which tells if we might need to read more data. +#[must_use] +#[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct InputBuf<'a, I: 'a>( + /// If this is set to true a parser has tried to read past the end of this buffer. + pub bool, + /// Current buffer slice + pub &'a [I], +); + +// FIXME: Docs +impl<'a, I: 'a> InputBuf<'a, I> { + #[inline] + pub fn new(buf: &'a [I]) -> Self { + InputBuf(false, buf) + } + + #[inline] + pub fn is_incomplete(&self) -> bool { + self.0 + } + + #[inline] + pub fn len(&self) -> usize { + self.1.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { + type Token = I; + type Marker = &'a [I]; + type Buffer = &'a [I]; + + #[inline] + fn _peek(&mut self, _g: Guard) -> Option { + if let Some(c) = self.1.first() { + Some(*c) + } else { + self.0 = true; + + None + } + } + + #[inline] + fn _pop(&mut self, g: Guard) -> Option { + self._peek(g).map(|c| { + self.1 = &self.1[1..]; + + c + }) + } + + #[inline] + fn _consume(&mut self, _g: Guard, n: usize) -> Option { + if n > self.1.len() { + self.0 = true; + + None + } else { + let b = &self.1[..n]; + + self.1 = &self.1[n..]; + + Some(b) + } + } + + #[inline] + fn _consume_while(&mut self, g: Guard, mut f: F) -> Self::Buffer + where F: FnMut(Self::Token) -> bool { + if let Some(n) = self.1.iter().position(|c| !f(*c)) { + let b = &self.1[..n]; + + self.1 = &self.1[n..]; + + b + } else { + self._consume_remaining(g) + } + } + + #[inline] + fn _consume_from(&mut self, _g: Guard, m: Self::Marker) -> Self::Buffer { + &m[..m.len() - self.1.len()] + } + + #[inline] + fn _consume_remaining(&mut self, _g: Guard) -> Self::Buffer { + self.0 = true; + + let b = self.1; + + self.1 = &self.1[..0]; + + b + } + + #[inline] + fn _mark(&self, _g: Guard) -> Self::Marker { + // Incomplete state is separate from the parsed state, no matter how we hit incomplete we + // want to keep it. + &self.1 + } + + #[inline] + fn _restore(mut self, _g: Guard, m: Self::Marker) -> Self { + self.1 = m; + + self + } +} + +#[cfg(test)] +mod test { + use super::InputBuf; + use types::{Input, ParseResult}; + use primitives::{IntoInner, Primitives}; + + use types::test::run_primitives_test; + + #[test] + fn ret() { + let i1: InputBuf = InputBuf::new(b"in1"); + let i2: InputBuf = InputBuf::new(b"in2"); + + let r1: ParseResult<_, u32, ()> = i1.ret::<_, ()>(23u32); + let r2: ParseResult<_, i32, &str> = i2.ret::<_, &str>(23i32); + + assert_eq!(r1.into_inner(), (InputBuf::new(b"in1"), Ok(23u32))); + assert_eq!(r2.into_inner(), (InputBuf::new(b"in2"), Ok(23i32))); + } + + #[test] + fn err() { + let i1: InputBuf = InputBuf::new(b"in1"); + let i2: InputBuf = InputBuf::new(b"in2"); + + let r1: ParseResult<_, (), u32> = i1.err::<(), _>(23u32); + let r2: ParseResult<_, &str, i32> = i2.err::<&str, _>(23i32); + + assert_eq!(r1.into_inner(), (InputBuf::new(b"in1"), Err(23u32))); + assert_eq!(r2.into_inner(), (InputBuf::new(b"in2"), Err(23i32))); + } + + #[test] + fn test_input_buf() { + + run_primitives_test(InputBuf::new(b"abc"), |x| x); + + let mut b = InputBuf::new(b"a"); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 1); + assert_eq!(b.is_empty(), false); + b.peek(); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 1); + assert_eq!(b.is_empty(), false); + b.pop(); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + assert_eq!(b.peek(), None); + assert_eq!(b.is_incomplete(), true); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + assert_eq!(b.pop(), None); + assert_eq!(b.is_incomplete(), true); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + + let mut b = InputBuf::new(b"ab"); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 2); + assert_eq!(b.is_empty(), false); + b.consume(1); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 1); + assert_eq!(b.is_empty(), false); + b.consume(1); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + assert_eq!(b.consume(1), None); + assert_eq!(b.is_incomplete(), true); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + + let mut b = InputBuf::new(b"ab"); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 2); + assert_eq!(b.is_empty(), false); + assert_eq!(b.consume(3), None); + assert_eq!(b.is_incomplete(), true); + assert_eq!(b.len(), 2); + assert_eq!(b.is_empty(), false); + + let mut b = InputBuf::new(b"ab"); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 2); + assert_eq!(b.is_empty(), false); + assert_eq!(b.consume_while(|_| true), &b"ab"[..]); + assert_eq!(b.is_incomplete(), true); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + + let mut b = InputBuf::new(b"ab"); + assert_eq!(b.consume_while(|c| c == b'a'), &b"a"[..]); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 1); + assert_eq!(b.is_empty(), false); + assert_eq!(b.consume_while(|c| c == b'b'), &b"b"[..]); + assert_eq!(b.is_incomplete(), true); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + assert_eq!(b.consume_while(|c| c == b'b'), &b""[..]); + assert_eq!(b.is_incomplete(), true); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + + let mut b = InputBuf::new(b"abc"); + let m = b.mark(); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 3); + assert_eq!(b.is_empty(), false); + b.consume(3); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + assert_eq!(b.consume_from(m), &b"abc"[..]); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + + let mut b = InputBuf::new(b"abc"); + let m = b.mark(); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 3); + assert_eq!(b.is_empty(), false); + b.consume(2); + b.consume(2); + assert_eq!(b.is_incomplete(), true); + assert_eq!(b.len(), 1); + assert_eq!(b.is_empty(), false); + let b = b.restore(m); + assert_eq!(b.is_incomplete(), true); + assert_eq!(b.len(), 3); + assert_eq!(b.is_empty(), false); + + let mut b = InputBuf::new(b"abc"); + assert_eq!(b.is_incomplete(), false); + assert_eq!(b.len(), 3); + assert_eq!(b.is_empty(), false); + b.consume_remaining(); + assert_eq!(b.is_incomplete(), true); + assert_eq!(b.len(), 0); + assert_eq!(b.is_empty(), true); + } +} diff --git a/src/buffer/slice.rs b/src/buffer/slice.rs index e8778b2b..a8b091a3 100644 --- a/src/buffer/slice.rs +++ b/src/buffer/slice.rs @@ -1,7 +1,7 @@ use primitives::IntoInner; -use types::{InputBuf, ParseResult}; -use buffer::{IntoStream, StreamError, Stream}; +use types::ParseResult; +use buffer::{InputBuf, IntoStream, StreamError, Stream}; /// Stream implementation for immutable slices. /// diff --git a/src/buffer/stateful.rs b/src/buffer/stateful.rs index ed848bcb..046d7201 100644 --- a/src/buffer/stateful.rs +++ b/src/buffer/stateful.rs @@ -1,14 +1,15 @@ use std::io; -use types::{InputBuf, ParseResult}; +use types::ParseResult; use primitives::IntoInner; use buffer::{ Buffer, DataSource, FixedSizeBuffer, - StreamError, + InputBuf, Stream, + StreamError, }; use buffer::data_source::{IteratorDataSource, ReadDataSource}; diff --git a/src/types/input.rs b/src/types/input.rs deleted file mode 100644 index 60e8f785..00000000 --- a/src/types/input.rs +++ /dev/null @@ -1,565 +0,0 @@ -use types::{Buffer, Input}; -use primitives::Guard; - -impl<'a, I: Copy + PartialEq> Buffer for &'a [I] { - type Token = I; - - fn fold(self, init: B, f: F) -> B - where F: FnMut(B, Self::Token) -> B { - (&self[..]).iter().cloned().fold(init, f) - } - - fn len(&self) -> usize { - // Slice to reach inherent method to prevent infinite recursion - (&self[..]).len() - } - - fn to_vec(&self) -> Vec { - (&self[..]).to_vec() - } - - fn into_vec(self) -> Vec { - (&self[..]).to_vec() - } -} - -impl<'a> Buffer for &'a str { - type Token = char; - - fn fold(self, init: B, f: F) -> B - where F: FnMut(B, Self::Token) -> B { - self.chars().fold(init, f) - } - - fn len(&self) -> usize { - self.chars().count() - } - - fn is_empty(&self) -> bool { - (&self[..]).is_empty() - } - - fn to_vec(&self) -> Vec { - (&self[..]).chars().collect() - } - - fn into_vec(self) -> Vec { - (&self[..]).chars().collect() - } -} - -impl<'a, I: Copy + PartialEq> Input for &'a [I] { - type Token = I; - type Marker = &'a [I]; - type Buffer = &'a [I]; - - #[inline] - fn _peek(&mut self, _g: Guard) -> Option { - self.first().cloned() - } - - #[inline] - fn _pop(&mut self, _g: Guard) -> Option { - self.first().cloned().map(|c| { - *self = &self[1..]; - - c - }) - } - - #[inline] - fn _consume(&mut self, _g: Guard, n: usize) -> Option { - if n > self.len() { - None - } else { - let b = &self[..n]; - - *self = &self[n..]; - - Some(b) - } - } - - #[inline] - fn _consume_while(&mut self, _g: Guard, mut f: F) -> Self::Buffer - where F: FnMut(Self::Token) -> bool { - if let Some(n) = self.iter().position(|c| !f(*c)) { - let b = &self[..n]; - - *self = &self[n..]; - - b - } else { - let b = &self[..]; - - *self = &self[..0]; - - b - } - } - - #[inline] - fn _consume_from(&mut self, _g: Guard, m: Self::Marker) -> Self::Buffer { - &m[..m.len() - self.len()] - } - - #[inline] - fn _consume_remaining(&mut self, _g: Guard) -> Self::Buffer { - let b = &self[..]; - - *self = &self[..0]; - - b - } - - #[inline] - fn _mark(&self, _g: Guard) -> Self::Marker { - &self - } - - #[inline] - fn _restore(self, _g: Guard, m: Self::Marker) -> Self { - m - } -} - -// FIXME: Docs -/// Input buffer type which contains a flag which tells if we might need to read more data. -// TODO: Move to buffer module and make public? -#[must_use] -#[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub struct InputBuf<'a, I: 'a>( - /// If this is set to true a parser has tried to read past the end of this buffer. - pub bool, - /// Current buffer slice - pub &'a [I], -); - -// FIXME: Docs -impl<'a, I: 'a> InputBuf<'a, I> { - #[inline] - pub fn new(buf: &'a [I]) -> Self { - InputBuf(false, buf) - } - - #[inline] - pub fn is_incomplete(&self) -> bool { - self.0 - } - - #[inline] - pub fn len(&self) -> usize { - self.1.len() - } - - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } -} - -impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { - type Token = I; - type Marker = &'a [I]; - type Buffer = &'a [I]; - - #[inline] - fn _peek(&mut self, _g: Guard) -> Option { - if let Some(c) = self.1.first() { - Some(*c) - } else { - self.0 = true; - - None - } - } - - #[inline] - fn _pop(&mut self, g: Guard) -> Option { - self._peek(g).map(|c| { - self.1 = &self.1[1..]; - - c - }) - } - - #[inline] - fn _consume(&mut self, _g: Guard, n: usize) -> Option { - if n > self.1.len() { - self.0 = true; - - None - } else { - let b = &self.1[..n]; - - self.1 = &self.1[n..]; - - Some(b) - } - } - - #[inline] - fn _consume_while(&mut self, g: Guard, mut f: F) -> Self::Buffer - where F: FnMut(Self::Token) -> bool { - if let Some(n) = self.1.iter().position(|c| !f(*c)) { - let b = &self.1[..n]; - - self.1 = &self.1[n..]; - - b - } else { - self._consume_remaining(g) - } - } - - #[inline] - fn _consume_from(&mut self, _g: Guard, m: Self::Marker) -> Self::Buffer { - &m[..m.len() - self.1.len()] - } - - #[inline] - fn _consume_remaining(&mut self, _g: Guard) -> Self::Buffer { - self.0 = true; - - let b = self.1; - - self.1 = &self.1[..0]; - - b - } - - #[inline] - fn _mark(&self, _g: Guard) -> Self::Marker { - // Incomplete state is separate from the parsed state, no matter how we hit incomplete we - // want to keep it. - &self.1 - } - - #[inline] - fn _restore(mut self, _g: Guard, m: Self::Marker) -> Self { - self.1 = m; - - self - } -} - -impl<'a> Input for &'a str { - type Token = char; - type Marker = &'a str; - type Buffer = &'a str; - - #[inline] - fn _peek(&mut self, _g: Guard) -> Option { - self.chars().next() - } - - #[inline] - fn _pop(&mut self, _g: Guard) -> Option { - let mut iter = self.char_indices(); - - iter.next().map(|(_, c)| { - match iter.next().map(|(p, _)| p) { - Some(n) => *self = &self[n..], - None => *self = &self[..0], - } - - c - }) - } - - #[inline] - fn _consume(&mut self, _g: Guard, n: usize) -> Option { - match self.char_indices().enumerate().take(n + 1).last() { - // num always equal to n if self contains more than n characters - Some((num, (pos, _))) if n == num => { - let b = &self[..pos]; - - *self = &self[pos..]; - - Some(b) - }, - // num always equal to n - 1 if self contains exactly n characters - Some((num, _)) if n == num + 1 => { - let b = &self[..]; - - *self = &self[..0]; - - Some(b) - }, - _ => None, - } - } - - #[inline] - fn _consume_while(&mut self, _g: Guard, mut f: F) -> Self::Buffer - where F: FnMut(Self::Token) -> bool { - // We need to find the character following the one which did not match - if let Some((pos, _)) = self.char_indices().skip_while(|&(_, c)| f(c)).next() { - let b = &self[..pos]; - - *self = &self[pos..]; - - b - } else { - let b = &self[..]; - - *self = &self[..0]; - - b - } - } - - #[inline] - fn _consume_from(&mut self, _g: Guard, m: Self::Marker) -> Self::Buffer { - &m[..m.len() - self.len()] - } - - #[inline] - fn _consume_remaining(&mut self, _g: Guard) -> Self::Buffer { - let b = &self[..]; - - *self = &self[..0]; - - b - } - - #[inline] - fn _mark(&self, _g: Guard) -> Self::Marker { - &self - } - - #[inline] - fn _restore(self, _g: Guard, m: Self::Marker) -> Self { - m - } -} - -#[cfg(test)] -mod test { - use std::fmt::Debug; - - use types::{Buffer, Input, InputBuf}; - - #[test] - fn test_slice() { - run_primitives_test(&b"abc"[..], |x| x); - } - - #[test] - fn test_string() { - run_primitives_test(&"abc"[..], |c| c as char); - } - - #[test] - fn test_input_buf() { - use primitives::Primitives; - - run_primitives_test(InputBuf::new(b"abc"), |x| x); - - let mut b = InputBuf::new(b"a"); - assert_eq!(b.is_incomplete(), false); - assert_eq!(b.len(), 1); - assert_eq!(b.is_empty(), false); - b.peek(); - assert_eq!(b.is_incomplete(), false); - assert_eq!(b.len(), 1); - assert_eq!(b.is_empty(), false); - b.pop(); - assert_eq!(b.is_incomplete(), false); - assert_eq!(b.len(), 0); - assert_eq!(b.is_empty(), true); - assert_eq!(b.peek(), None); - assert_eq!(b.is_incomplete(), true); - assert_eq!(b.len(), 0); - assert_eq!(b.is_empty(), true); - assert_eq!(b.pop(), None); - assert_eq!(b.is_incomplete(), true); - assert_eq!(b.len(), 0); - assert_eq!(b.is_empty(), true); - - let mut b = InputBuf::new(b"ab"); - assert_eq!(b.is_incomplete(), false); - assert_eq!(b.len(), 2); - assert_eq!(b.is_empty(), false); - b.consume(1); - assert_eq!(b.is_incomplete(), false); - assert_eq!(b.len(), 1); - assert_eq!(b.is_empty(), false); - b.consume(1); - assert_eq!(b.is_incomplete(), false); - assert_eq!(b.len(), 0); - assert_eq!(b.is_empty(), true); - assert_eq!(b.consume(1), None); - assert_eq!(b.is_incomplete(), true); - assert_eq!(b.len(), 0); - assert_eq!(b.is_empty(), true); - - let mut b = InputBuf::new(b"ab"); - assert_eq!(b.is_incomplete(), false); - assert_eq!(b.len(), 2); - assert_eq!(b.is_empty(), false); - assert_eq!(b.consume(3), None); - assert_eq!(b.is_incomplete(), true); - assert_eq!(b.len(), 2); - assert_eq!(b.is_empty(), false); - - let mut b = InputBuf::new(b"ab"); - assert_eq!(b.is_incomplete(), false); - assert_eq!(b.len(), 2); - assert_eq!(b.is_empty(), false); - assert_eq!(b.consume_while(|_| true), &b"ab"[..]); - assert_eq!(b.is_incomplete(), true); - assert_eq!(b.len(), 0); - assert_eq!(b.is_empty(), true); - - let mut b = InputBuf::new(b"ab"); - assert_eq!(b.consume_while(|c| c == b'a'), &b"a"[..]); - assert_eq!(b.is_incomplete(), false); - assert_eq!(b.len(), 1); - assert_eq!(b.is_empty(), false); - assert_eq!(b.consume_while(|c| c == b'b'), &b"b"[..]); - assert_eq!(b.is_incomplete(), true); - assert_eq!(b.len(), 0); - assert_eq!(b.is_empty(), true); - assert_eq!(b.consume_while(|c| c == b'b'), &b""[..]); - assert_eq!(b.is_incomplete(), true); - assert_eq!(b.len(), 0); - assert_eq!(b.is_empty(), true); - - let mut b = InputBuf::new(b"abc"); - let m = b.mark(); - assert_eq!(b.is_incomplete(), false); - assert_eq!(b.len(), 3); - assert_eq!(b.is_empty(), false); - b.consume(3); - assert_eq!(b.is_incomplete(), false); - assert_eq!(b.len(), 0); - assert_eq!(b.is_empty(), true); - assert_eq!(b.consume_from(m), &b"abc"[..]); - assert_eq!(b.is_incomplete(), false); - assert_eq!(b.len(), 0); - assert_eq!(b.is_empty(), true); - - let mut b = InputBuf::new(b"abc"); - let m = b.mark(); - assert_eq!(b.is_incomplete(), false); - assert_eq!(b.len(), 3); - assert_eq!(b.is_empty(), false); - b.consume(2); - b.consume(2); - assert_eq!(b.is_incomplete(), true); - assert_eq!(b.len(), 1); - assert_eq!(b.is_empty(), false); - let b = b.restore(m); - assert_eq!(b.is_incomplete(), true); - assert_eq!(b.len(), 3); - assert_eq!(b.is_empty(), false); - - let mut b = InputBuf::new(b"abc"); - assert_eq!(b.is_incomplete(), false); - assert_eq!(b.len(), 3); - assert_eq!(b.is_empty(), false); - b.consume_remaining(); - assert_eq!(b.is_incomplete(), true); - assert_eq!(b.len(), 0); - assert_eq!(b.is_empty(), true); - } - - /// Should recieve an Input with the tokens 'a', 'b' and 'c' remaining. - pub fn run_primitives_test I::Token>(mut s: I, f: F) - where I::Token: Debug, - I::Buffer: Clone { - use primitives::Primitives; - - fn buffer_eq_slice B::Token>(b: B, s: &[u8], f: F) - where B::Token: Debug, { - assert_eq!(b.len(), s.len()); - assert_eq!(b.is_empty(), s.is_empty()); - assert_eq!(b.clone().fold(0, |n, c| { - assert_eq!(c, f(s[n])); - - n + 1 - }), s.iter().count()); - assert_eq!(b.to_vec(), s.iter().cloned().map(f).collect::>()); - } - - let m = s.mark(); - assert_eq!(s.peek(), Some(f(b'a'))); - assert_eq!(s.pop(), Some(f(b'a'))); - assert_eq!(s.peek(), Some(f(b'b'))); - assert_eq!(s.pop(), Some(f(b'b'))); - assert_eq!(s.peek(), Some(f(b'c'))); - assert_eq!(s.pop(), Some(f(b'c'))); - assert_eq!(s.peek(), None); - assert_eq!(s.pop(), None); - assert_eq!(s.peek(), None); - assert_eq!(s.pop(), None); - assert!(s.consume(1).is_none()); - buffer_eq_slice(s.consume_remaining(), &b""[..], &f); - - { - let mut n = 0; - - let b = s.consume_while(|_| { n += 1; true }); - - assert_eq!(n, 0); - buffer_eq_slice(b, &b""[..], &f); - } - - let mut s = s.restore(m); - assert_eq!(s.peek(), Some(f(b'a'))); - let m = s.mark(); - buffer_eq_slice(s.consume_remaining(), &b"abc"[..], &f); - assert_eq!(s.peek(), None); - let mut s = s.restore(m); - assert_eq!(s.peek(), Some(f(b'a'))); - let m = s.mark(); - - { - let b = s.consume(2); - - assert_eq!(b.is_some(), true); - buffer_eq_slice(b.unwrap(), &b"ab"[..], &f); - } - - assert_eq!(s.peek(), Some(f(b'c'))); - - let mut s = s.restore(m); - assert_eq!(s.peek(), Some(f(b'a'))); - let m = s.mark(); - - { - let b = s.consume(3); - - assert_eq!(b.is_some(), true); - buffer_eq_slice(b.unwrap(), &b"abc"[..], &f); - } - - assert_eq!(s.peek(), None); - let mut s = s.restore(m); - assert_eq!(s.peek(), Some(f(b'a'))); - let m = s.mark(); - - { - let mut n = 0; - - let b = s.consume_while(|c| { - assert_eq!(c, f(b"abc"[n])); - - n += 1; - - n < 3 - }); - - assert_eq!(n, 3); - buffer_eq_slice(b, &b"ab"[..], &f); - } - - assert_eq!(s.peek(), Some(f(b'c'))); - assert_eq!(s.pop(), Some(f(b'c'))); - assert_eq!(s.peek(), None); - assert_eq!(s.pop(), None); - - buffer_eq_slice(s.consume_from(m), &b"abc"[..], &f); - } -} diff --git a/src/types/mod.rs b/src/types/mod.rs index 5baf13b7..142b581c 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -1,19 +1,14 @@ -mod input; +//! Types which facillitates the chaining of parsers and their results. + #[cfg(feature = "tendril")] pub mod tendril; use primitives::{Guard, IntoInner}; -pub use types::input::InputBuf; - -pub trait U8Input: Input {} - -// TODO: More restrictions? Buffer=&[u8]? -impl U8Input for T - where T: Input {} - -// FIXME: Docs -// TODO: More methods? +/// The buffers yielded parsers consuming a sequence of the input. +/// +/// This could either be an owned type or a slice reference depending on the `Input` +/// implementation. pub trait Buffer: PartialEq { /// The token type of this buffer. type Token: Copy + PartialEq; @@ -48,6 +43,53 @@ pub trait Buffer: PartialEq { } } +impl<'a, I: Copy + PartialEq> Buffer for &'a [I] { + type Token = I; + + fn fold(self, init: B, f: F) -> B + where F: FnMut(B, Self::Token) -> B { + (&self[..]).iter().cloned().fold(init, f) + } + + fn len(&self) -> usize { + // Slice to reach inherent method to prevent infinite recursion + (&self[..]).len() + } + + fn to_vec(&self) -> Vec { + (&self[..]).to_vec() + } + + fn into_vec(self) -> Vec { + (&self[..]).to_vec() + } +} + +impl<'a> Buffer for &'a str { + type Token = char; + + fn fold(self, init: B, f: F) -> B + where F: FnMut(B, Self::Token) -> B { + self.chars().fold(init, f) + } + + fn len(&self) -> usize { + self.chars().count() + } + + fn is_empty(&self) -> bool { + (&self[..]).is_empty() + } + + fn to_vec(&self) -> Vec { + (&self[..]).chars().collect() + } + + fn into_vec(self) -> Vec { + (&self[..]).chars().collect() + } +} + // FIXME: Update, do not refer to type or linear type /// Linear type containing the parser state, this type is threaded though `bind` and is also the /// initial type passed to a parser. @@ -211,6 +253,178 @@ pub trait Input: Sized { fn _restore(self, Guard, Self::Marker) -> Self; } +impl<'a, I: Copy + PartialEq> Input for &'a [I] { + type Token = I; + type Marker = &'a [I]; + type Buffer = &'a [I]; + + #[inline] + fn _peek(&mut self, _g: Guard) -> Option { + self.first().cloned() + } + + #[inline] + fn _pop(&mut self, _g: Guard) -> Option { + self.first().cloned().map(|c| { + *self = &self[1..]; + + c + }) + } + + #[inline] + fn _consume(&mut self, _g: Guard, n: usize) -> Option { + if n > self.len() { + None + } else { + let b = &self[..n]; + + *self = &self[n..]; + + Some(b) + } + } + + #[inline] + fn _consume_while(&mut self, _g: Guard, mut f: F) -> Self::Buffer + where F: FnMut(Self::Token) -> bool { + if let Some(n) = self.iter().position(|c| !f(*c)) { + let b = &self[..n]; + + *self = &self[n..]; + + b + } else { + let b = &self[..]; + + *self = &self[..0]; + + b + } + } + + #[inline] + fn _consume_from(&mut self, _g: Guard, m: Self::Marker) -> Self::Buffer { + &m[..m.len() - self.len()] + } + + #[inline] + fn _consume_remaining(&mut self, _g: Guard) -> Self::Buffer { + let b = &self[..]; + + *self = &self[..0]; + + b + } + + #[inline] + fn _mark(&self, _g: Guard) -> Self::Marker { + &self + } + + #[inline] + fn _restore(self, _g: Guard, m: Self::Marker) -> Self { + m + } +} + +impl<'a> Input for &'a str { + type Token = char; + type Marker = &'a str; + type Buffer = &'a str; + + #[inline] + fn _peek(&mut self, _g: Guard) -> Option { + self.chars().next() + } + + #[inline] + fn _pop(&mut self, _g: Guard) -> Option { + let mut iter = self.char_indices(); + + iter.next().map(|(_, c)| { + match iter.next().map(|(p, _)| p) { + Some(n) => *self = &self[n..], + None => *self = &self[..0], + } + + c + }) + } + + #[inline] + fn _consume(&mut self, _g: Guard, n: usize) -> Option { + match self.char_indices().enumerate().take(n + 1).last() { + // num always equal to n if self contains more than n characters + Some((num, (pos, _))) if n == num => { + let b = &self[..pos]; + + *self = &self[pos..]; + + Some(b) + }, + // num always equal to n - 1 if self contains exactly n characters + Some((num, _)) if n == num + 1 => { + let b = &self[..]; + + *self = &self[..0]; + + Some(b) + }, + _ => None, + } + } + + #[inline] + fn _consume_while(&mut self, _g: Guard, mut f: F) -> Self::Buffer + where F: FnMut(Self::Token) -> bool { + // We need to find the character following the one which did not match + if let Some((pos, _)) = self.char_indices().skip_while(|&(_, c)| f(c)).next() { + let b = &self[..pos]; + + *self = &self[pos..]; + + b + } else { + let b = &self[..]; + + *self = &self[..0]; + + b + } + } + + #[inline] + fn _consume_from(&mut self, _g: Guard, m: Self::Marker) -> Self::Buffer { + &m[..m.len() - self.len()] + } + + #[inline] + fn _consume_remaining(&mut self, _g: Guard) -> Self::Buffer { + let b = &self[..]; + + *self = &self[..0]; + + b + } + + #[inline] + fn _mark(&self, _g: Guard) -> Self::Marker { + &self + } + + #[inline] + fn _restore(self, _g: Guard, m: Self::Marker) -> Self { + m + } +} + +/// A type alias for an `Input` with a token type of `u8`. +pub trait U8Input: Input {} + +impl U8Input for T + where T: Input {} + /// The basic return type of a parser. /// /// This type satisfies a variant of the `Monad` typeclass. Due to the limitations of Rust's @@ -421,32 +635,27 @@ impl IntoInner for ParseResult { } #[cfg(test)] -mod test { - use super::{Input, InputBuf, ParseResult}; +pub mod test { + use super::{Buffer, Input, ParseResult}; use primitives::IntoInner; + use std::fmt::Debug; #[test] fn ret() { - let i1: InputBuf = InputBuf::new(b"in1"); - let i2: InputBuf = InputBuf::new(b"in2"); - - let r1: ParseResult<_, u32, ()> = i1.ret::<_, ()>(23u32); - let r2: ParseResult<_, i32, &str> = i2.ret::<_, &str>(23i32); + let r1: ParseResult<_, u32, ()> = b"in1".ret::<_, ()>(23u32); + let r2: ParseResult<_, i32, &str> = b"in2".ret::<_, &str>(23i32); - assert_eq!(r1.into_inner(), (InputBuf::new(b"in1"), Ok(23u32))); - assert_eq!(r2.into_inner(), (InputBuf::new(b"in2"), Ok(23i32))); + assert_eq!(r1.into_inner(), (&b"in1"[..], Ok(23u32))); + assert_eq!(r2.into_inner(), (&b"in2"[..], Ok(23i32))); } #[test] fn err() { - let i1: InputBuf = InputBuf::new(b"in1"); - let i2: InputBuf = InputBuf::new(b"in2"); + let r1: ParseResult<_, (), u32> = b"in1".err::<(), _>(23u32); + let r2: ParseResult<_, &str, i32> = b"in2".err::<&str, _>(23i32); - let r1: ParseResult<_, (), u32> = i1.err::<(), _>(23u32); - let r2: ParseResult<_, &str, i32> = i2.err::<&str, _>(23i32); - - assert_eq!(r1.into_inner(), (InputBuf::new(b"in1"), Err(23u32))); - assert_eq!(r2.into_inner(), (InputBuf::new(b"in2"), Err(23i32))); + assert_eq!(r1.into_inner(), (&b"in1"[..], Err(23u32))); + assert_eq!(r2.into_inner(), (&b"in2"[..], Err(23i32))); } #[test] @@ -454,11 +663,11 @@ mod test { let i1: Result = Ok(23); let i2: Result<&str, &str> = Err("foobar"); - let r1 = InputBuf::new(b"in1").from_result(i1); - let r2 = InputBuf::new(b"in2").from_result(i2); + let r1 = b"in1".from_result(i1); + let r2 = b"in2".from_result(i2); - assert_eq!(r1.into_inner(), (InputBuf::new(b"in1"), Ok(23u32))); - assert_eq!(r2.into_inner(), (InputBuf::new(b"in2"), Err("foobar"))); + assert_eq!(r1.into_inner(), (&b"in1"[..], Ok(23u32))); + assert_eq!(r2.into_inner(), (&b"in2"[..], Err("foobar"))); } #[test] @@ -467,30 +676,27 @@ mod test { i.ret(n + 1) } - let m1 = InputBuf::new(b"test"); - let m2 = InputBuf::new(b"test"); - let a = 123; // return a >>= f - let lhs = m1.ret(a).bind(f); + let lhs = b"test".ret(a).bind(f); // f a - let rhs = f(m2, a); + let rhs = f(&b"test"[..], a); - assert_eq!((lhs.0, lhs.1), (InputBuf::new(b"test"), Ok(124))); - assert_eq!((rhs.0, rhs.1), (InputBuf::new(b"test"), Ok(124))); + assert_eq!((lhs.0, lhs.1), (&b"test"[..], Ok(124))); + assert_eq!((rhs.0, rhs.1), (&b"test"[..], Ok(124))); } #[test] fn monad_right_identity() { - let m1 = InputBuf::new(b"test").ret::<_, ()>(1); - let m2 = InputBuf::new(b"test").ret::<_, ()>(1); + let m1 = b"test".ret::<_, ()>(1); + let m2 = b"test".ret::<_, ()>(1); // m1 >>= ret === m2 let lhs = m1.bind::<_, _, ()>(Input::ret); let rhs = m2; - assert_eq!((lhs.0, rhs.1), (InputBuf::new(b"test"), Ok(1))); - assert_eq!((rhs.0, lhs.1), (InputBuf::new(b"test"), Ok(1))); + assert_eq!((lhs.0, rhs.1), (&b"test"[..], Ok(1))); + assert_eq!((rhs.0, lhs.1), (&b"test"[..], Ok(1))); } #[test] @@ -503,16 +709,16 @@ mod test { i.ret(num * 2) } - let lhs_m = InputBuf::new(b"test").ret::<_, ()>(2); - let rhs_m = InputBuf::new(b"test").ret::<_, ()>(2); + let lhs_m = b"test".ret::<_, ()>(2); + let rhs_m = b"test".ret::<_, ()>(2); // (m >>= f) >>= g let lhs = lhs_m.bind(f).bind(g); // m >>= (\x -> f x >>= g) let rhs = rhs_m.bind(|i, x| f(i, x).bind(g)); - assert_eq!((lhs.0, lhs.1), (InputBuf::new(b"test"), Ok(6))); - assert_eq!((rhs.0, rhs.1), (InputBuf::new(b"test"), Ok(6))); + assert_eq!((lhs.0, lhs.1), (&b"test"[..], Ok(6))); + assert_eq!((rhs.0, rhs.1), (&b"test"[..], Ok(6))); } #[test] @@ -521,8 +727,8 @@ mod test { let mut n1 = 0; let mut n2 = 0; - let i1 = InputBuf::new(b"test ").ret::(23); - let i2 = InputBuf::new(b"test ").ret::(23); + let i1 = b"test ".ret::(23); + let i2 = b"test ".ret::(23); let r1 = i1.inspect(|d: &u32| { assert_eq!(d, &23); @@ -535,9 +741,9 @@ mod test { n2 += 1; }); - assert_eq!(r1.into_inner(), (InputBuf::new(b"test "), Ok(23))); + assert_eq!(r1.into_inner(), (&b"test "[..], Ok(23))); assert_eq!(n1, 1); - assert_eq!(r2.into_inner(), (InputBuf::new(b"test "), Ok(23))); + assert_eq!(r2.into_inner(), (&b"test "[..], Ok(23))); assert_eq!(n2, 1); } @@ -545,14 +751,14 @@ mod test { fn input_propagation() { let mut n_calls = 0; - let i = InputBuf::new(b"test1").ret::<_, ()>(23); + let i = b"test1".ret::<_, ()>(23); - assert_eq!(i.0, InputBuf::new(b"test1")); + assert_eq!(i.0, b"test1"); assert_eq!(i.1, Ok(23)); let r: ParseResult<_, _, ()> = i.bind(|i, t| { n_calls += 1; i.ret(t) }); - assert_eq!((r.0, r.1), (InputBuf::new(b"test1"), Ok(23))); + assert_eq!((r.0, r.1), (&b"test1"[..], Ok(23))); assert_eq!(n_calls, 1); } @@ -560,14 +766,14 @@ mod test { fn error_propagation() { let mut n_calls = 0; - let i = InputBuf::new(b"test1").err::<(), _>(23); + let i = b"test1".err::<(), _>(23); - assert_eq!(i.0, InputBuf::new(b"test1")); + assert_eq!(i.0, b"test1"); assert_eq!(i.1, Err(23)); let r = i.bind(|i, t| { n_calls += 1; i.ret(t) }); - assert_eq!((r.0, r.1), (InputBuf::new(b"test1"), Err(23))); + assert_eq!((r.0, r.1), (&b"test1"[..], Err(23))); assert_eq!(n_calls, 0); } @@ -581,4 +787,112 @@ mod test { assert_eq!((lhs.0, lhs.1), (&b"test"[..], Ok(124))); assert_eq!((rhs.0, rhs.1), (&b"test"[..], Ok(124))); } + + #[test] + fn test_slice() { + run_primitives_test(&b"abc"[..], |x| x); + } + + #[test] + fn test_string() { + run_primitives_test(&"abc"[..], |c| c as char); + } + + /// Should recieve an Input with the tokens 'a', 'b' and 'c' remaining. + pub fn run_primitives_test I::Token>(mut s: I, f: F) + where I::Token: Debug, + I::Buffer: Clone { + use primitives::Primitives; + + fn buffer_eq_slice B::Token>(b: B, s: &[u8], f: F) + where B::Token: Debug, { + assert_eq!(b.len(), s.len()); + assert_eq!(b.is_empty(), s.is_empty()); + assert_eq!(b.clone().fold(0, |n, c| { + assert_eq!(c, f(s[n])); + + n + 1 + }), s.iter().count()); + assert_eq!(b.to_vec(), s.iter().cloned().map(f).collect::>()); + } + + let m = s.mark(); + assert_eq!(s.peek(), Some(f(b'a'))); + assert_eq!(s.pop(), Some(f(b'a'))); + assert_eq!(s.peek(), Some(f(b'b'))); + assert_eq!(s.pop(), Some(f(b'b'))); + assert_eq!(s.peek(), Some(f(b'c'))); + assert_eq!(s.pop(), Some(f(b'c'))); + assert_eq!(s.peek(), None); + assert_eq!(s.pop(), None); + assert_eq!(s.peek(), None); + assert_eq!(s.pop(), None); + assert!(s.consume(1).is_none()); + buffer_eq_slice(s.consume_remaining(), &b""[..], &f); + + { + let mut n = 0; + + let b = s.consume_while(|_| { n += 1; true }); + + assert_eq!(n, 0); + buffer_eq_slice(b, &b""[..], &f); + } + + let mut s = s.restore(m); + assert_eq!(s.peek(), Some(f(b'a'))); + let m = s.mark(); + buffer_eq_slice(s.consume_remaining(), &b"abc"[..], &f); + assert_eq!(s.peek(), None); + let mut s = s.restore(m); + assert_eq!(s.peek(), Some(f(b'a'))); + let m = s.mark(); + + { + let b = s.consume(2); + + assert_eq!(b.is_some(), true); + buffer_eq_slice(b.unwrap(), &b"ab"[..], &f); + } + + assert_eq!(s.peek(), Some(f(b'c'))); + + let mut s = s.restore(m); + assert_eq!(s.peek(), Some(f(b'a'))); + let m = s.mark(); + + { + let b = s.consume(3); + + assert_eq!(b.is_some(), true); + buffer_eq_slice(b.unwrap(), &b"abc"[..], &f); + } + + assert_eq!(s.peek(), None); + let mut s = s.restore(m); + assert_eq!(s.peek(), Some(f(b'a'))); + let m = s.mark(); + + { + let mut n = 0; + + let b = s.consume_while(|c| { + assert_eq!(c, f(b"abc"[n])); + + n += 1; + + n < 3 + }); + + assert_eq!(n, 3); + buffer_eq_slice(b, &b"ab"[..], &f); + } + + assert_eq!(s.peek(), Some(f(b'c'))); + assert_eq!(s.pop(), Some(f(b'c'))); + assert_eq!(s.peek(), None); + assert_eq!(s.pop(), None); + + buffer_eq_slice(s.consume_from(m), &b"abc"[..], &f); + } } From 4000dbf5f4b42c08544edb93bd083232fa0161e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 17 Aug 2016 18:59:23 +0200 Subject: [PATCH 43/63] Updated http parser bench and example to use U8Input --- benches/combinators.rs | 2 +- benches/http_bench.rs | 20 +++++++++----------- examples/http_parser.rs | 22 ++++++++++------------ 3 files changed, 20 insertions(+), 24 deletions(-) diff --git a/benches/combinators.rs b/benches/combinators.rs index cd34e359..f3a6f55a 100644 --- a/benches/combinators.rs +++ b/benches/combinators.rs @@ -8,7 +8,7 @@ use test::Bencher; use std::iter; use chomp::prelude::*; -use chomp::types::InputBuf; +use chomp::buffer::InputBuf; #[bench] fn count_vec_1k(b: &mut Bencher) { diff --git a/benches/http_bench.rs b/benches/http_bench.rs index 1c30cb13..5918825f 100644 --- a/benches/http_bench.rs +++ b/benches/http_bench.rs @@ -5,6 +5,7 @@ extern crate chomp; use test::Bencher; use chomp::prelude::*; +use chomp::buffer::InputBuf; #[derive(Debug)] struct Request { @@ -51,22 +52,18 @@ fn is_not_space(c: u8) -> bool { c != b' ' } fn is_end_of_line(c: u8) -> bool { c == b'\r' || c == b'\n' } fn is_http_version(c: u8) -> bool { c >= b'0' && c <= b'9' || c == b'.' } -fn end_of_line>(i: I) -> SimpleResult { - or(i, |i| parse!{i; - token(b'\r'); - token(b'\n'); - ret b'\r'}, - |i| token(i, b'\n')) +fn end_of_line(i: I) -> SimpleResult { + parse!{i; (token(b'\r') <|> ret b'\0') >> token(b'\n')} } -fn http_version>(i: I) -> SimpleResult { +fn http_version(i: I) -> SimpleResult { parse!{i; string(b"HTTP/"); take_while1(is_http_version) } } -fn request_line>(i: I) -> SimpleResult> { +fn request_line(i: I) -> SimpleResult> { parse!{i; let method = take_while1(is_token); take_while1(is_space); @@ -82,7 +79,7 @@ fn request_line>(i: I) -> SimpleResult> } } -fn message_header_line>(i: I) -> SimpleResult { +fn message_header_line(i: I) -> SimpleResult { parse!{i; take_while1(is_horizontal_space); let line = take_till(is_end_of_line); @@ -92,7 +89,7 @@ fn message_header_line>(i: I) -> SimpleResult { } } -fn message_header>(i: I) -> SimpleResult> { +fn message_header(i: I) -> SimpleResult> { parse!{i; let name = take_while1(is_token); token(b':'); @@ -105,7 +102,8 @@ fn message_header>(i: I) -> SimpleResult } } -fn request>(i: I) -> SimpleResult, Vec>)> { +#[inline(never)] +fn request(i: I) -> SimpleResult, Vec>)> { parse!{i; let r = request_line(); end_of_line(); diff --git a/examples/http_parser.rs b/examples/http_parser.rs index 478313ff..4deec9cd 100644 --- a/examples/http_parser.rs +++ b/examples/http_parser.rs @@ -64,22 +64,18 @@ fn is_not_space(c: u8) -> bool { c != b' ' } fn is_end_of_line(c: u8) -> bool { c == b'\r' || c == b'\n' } fn is_http_version(c: u8) -> bool { c >= b'0' && c <= b'9' || c == b'.' } -fn end_of_line>(i: I) -> SimpleResult { - or(i, |i| parse!{i; - token(b'\r'); - token(b'\n'); - ret b'\r'}, - |i| token(i, b'\n')) +fn end_of_line(i: I) -> SimpleResult { + parse!{i; (token(b'\r') <|> ret b'\0') >> token(b'\n')} } -fn http_version>(i: I) -> SimpleResult { +fn http_version(i: I) -> SimpleResult { parse!{i; string(b"HTTP/"); take_while1(is_http_version) } } -fn request_line>(i: I) -> SimpleResult> { +fn request_line(i: I) -> SimpleResult> { parse!{i; let method = take_while1(is_token); take_while1(is_space); @@ -95,7 +91,7 @@ fn request_line>(i: I) -> SimpleResult> } } -fn message_header_line>(i: I) -> SimpleResult { +fn message_header_line(i: I) -> SimpleResult { parse!{i; take_while1(is_horizontal_space); let line = take_till(is_end_of_line); @@ -105,7 +101,7 @@ fn message_header_line>(i: I) -> SimpleResult { } } -fn message_header>(i: I) -> SimpleResult> { +fn message_header(i: I) -> SimpleResult> { parse!{i; let name = take_while1(is_token); token(b':'); @@ -118,7 +114,9 @@ fn message_header>(i: I) -> SimpleResult } } -fn request>(i: I) -> SimpleResult, Vec>)> { +#[inline(never)] +fn request(i: I) -> SimpleResult, Vec>)> + where I::Buffer: ::std::ops::Deref { parse!{i; let r = request_line(); end_of_line(); @@ -140,7 +138,7 @@ fn main() { Ok(_) => n += 1, Err(StreamError::Retry) => {}, // Needed to refill buffer when necessary Err(StreamError::EndOfInput) => break, - Err(e) => { panic!("{:?}", e); } + Err(e) => { panic!("{}: {:?}", n, e); } } } From 4ca331267160f5147ef2e1c0d2ffb0e2e6856796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 17 Aug 2016 19:24:09 +0200 Subject: [PATCH 44/63] buffer: Stream now uses Input as an associated type, removed IntoStream --- src/buffer/mod.rs | 22 +++++----------------- src/buffer/slice.rs | 32 +++++++++----------------------- src/buffer/stateful.rs | 8 ++++---- 3 files changed, 18 insertions(+), 44 deletions(-) diff --git a/src/buffer/mod.rs b/src/buffer/mod.rs index 8863aa95..a2b920ac 100644 --- a/src/buffer/mod.rs +++ b/src/buffer/mod.rs @@ -81,7 +81,7 @@ impl> PartialEq for StreamError { /// Trait wrapping the state management in reading from a data source while parsing. pub trait Stream<'a, 'i> { /// The input item type, usually depending on which `DataSource` is used. - type Item: 'i + Copy + PartialEq; + type Input: Input + 'i; /// Attempts to run the supplied parser `F` once on the currently populated data in this /// stream, providing a borrow of the inner data storage. @@ -89,33 +89,21 @@ pub trait Stream<'a, 'i> { /// If a `StreamError::Retry` is returned the consuming code it should just retry the action /// (the implementation might require a separate call to refill the stream). #[inline] - fn parse(&'a mut self, f: F) -> Result> - where F: FnOnce(InputBuf<'i, Self::Item>) -> ParseResult, T, E>, + fn parse(&'a mut self, f: F) -> Result::Buffer, E>> + where F: FnOnce(Self::Input) -> ParseResult, T: 'i, E: 'i; } -/// Trait for conversion into a `Stream`. -pub trait IntoStream<'a, 'i> { - /// The input item type provided by the stream. - type Item: 'i + Copy + PartialEq; - /// The `Stream` instance type. - type Into: Stream<'a, 'i, Item=Self::Item>; - - /// Convert self into a `Stream`. - #[inline] - fn into_stream(self) -> Self::Into; -} - // FIXME: Docs /// Input buffer type which contains a flag which tells if we might need to read more data. #[must_use] #[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] pub struct InputBuf<'a, I: 'a>( /// If this is set to true a parser has tried to read past the end of this buffer. - pub bool, + bool, /// Current buffer slice - pub &'a [I], + &'a [I], ); // FIXME: Docs diff --git a/src/buffer/slice.rs b/src/buffer/slice.rs index a8b091a3..35966e40 100644 --- a/src/buffer/slice.rs +++ b/src/buffer/slice.rs @@ -1,7 +1,7 @@ use primitives::IntoInner; -use types::ParseResult; -use buffer::{InputBuf, IntoStream, StreamError, Stream}; +use types::{Input, ParseResult}; +use buffer::{InputBuf, StreamError, Stream}; /// Stream implementation for immutable slices. /// @@ -9,11 +9,9 @@ use buffer::{InputBuf, IntoStream, StreamError, Stream}; /// # #[macro_use] extern crate chomp; /// # fn main() { /// use chomp::parsers::{token, take}; -/// use chomp::buffer::{IntoStream, Stream}; +/// use chomp::buffer::{SliceStream, Stream}; /// -/// let i = b"foo"; -/// -/// let r = i.into_stream().parse(parser!{ +/// let r = SliceStream::new(b"foo").parse(parser!{ /// token(b'f'); /// take(2) /// }); @@ -26,11 +24,9 @@ use buffer::{InputBuf, IntoStream, StreamError, Stream}; /// # #[macro_use] extern crate chomp; /// # fn main() { /// use chomp::prelude::{token, many, take}; -/// use chomp::buffer::{IntoStream, Stream}; -/// -/// let i = b"foofoo"; +/// use chomp::buffer::{SliceStream, Stream}; /// -/// let r = i.into_stream().parse(parser!{many(parser!{ +/// let r = SliceStream::new(b"foofoo").parse(parser!{many(parser!{ /// token(b'f'); /// take(2) /// })}); @@ -67,22 +63,12 @@ impl<'i, I: 'i> SliceStream<'i, I> { } } -impl<'a, 'i, I: 'i + Copy + PartialEq> IntoStream<'a, 'i> for &'i [I] { - type Item = I; - type Into = SliceStream<'i, I>; - - #[inline] - fn into_stream(self) -> SliceStream<'i, I> { - SliceStream::new(self) - } -} - impl<'a, 'i, I: 'i + Copy + PartialEq> Stream<'a, 'i> for SliceStream<'i, I> { - type Item = I; + type Input = InputBuf<'i, I>; #[inline] - fn parse(&'a mut self, f: F) -> Result> - where F: FnOnce(InputBuf<'i, Self::Item>) -> ParseResult, T, E>, + fn parse(&'a mut self, f: F) -> Result::Buffer, E>> + where F: FnOnce(Self::Input) -> ParseResult, T: 'i, E: 'i { use primitives::Primitives; diff --git a/src/buffer/stateful.rs b/src/buffer/stateful.rs index 046d7201..b9b1af76 100644 --- a/src/buffer/stateful.rs +++ b/src/buffer/stateful.rs @@ -1,6 +1,6 @@ use std::io; -use types::ParseResult; +use types::{Input, ParseResult}; use primitives::IntoInner; use buffer::{ @@ -199,11 +199,11 @@ impl, B: Buffer> io::BufRead for Source { impl<'a, S: DataSource, B: Buffer> Stream<'a, 'a> for Source where S::Item: 'a { - type Item = S::Item; + type Input = InputBuf<'a, S::Item>; #[inline] - fn parse(&'a mut self, f: F) -> Result> - where F: FnOnce(InputBuf<'a, Self::Item>) -> ParseResult, T, E>, + fn parse(&'a mut self, f: F) -> Result::Buffer, E>> + where F: FnOnce(Self::Input) -> ParseResult, T: 'a, E: 'a { use primitives::Primitives; From 701ab9d17c8d12a21d7f39beebff65c4e0d18495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 17 Aug 2016 19:39:52 +0200 Subject: [PATCH 45/63] doc --- src/primitives.rs | 36 ++++++++++++++++++++++++++++++++---- src/types/mod.rs | 47 ++++++++++++----------------------------------- 2 files changed, 44 insertions(+), 39 deletions(-) diff --git a/src/primitives.rs b/src/primitives.rs index 287ea5df..03e5dcb7 100644 --- a/src/primitives.rs +++ b/src/primitives.rs @@ -6,7 +6,7 @@ use types::Input; -/// **Primitive:** Consumes self and reveals the inner state. +/// Consumes self and reveals the inner state. /// /// # Primitive /// @@ -32,36 +32,63 @@ pub trait IntoInner { pub struct Guard(()); /// Trait enabling primitive actions on an `Input` type. -// FIXME: Rename and documentation +/// +/// This trait is automatically implemented for all types implementing `Input` and acts as a +/// gatekeeper to the state-modifying methods of `Input`. pub trait Primitives: Input { + /// Peeks at the next token in the input without consuming it. `None` if no more input is + /// available. + /// + /// Note: Is allowed to refill automatically or any other appropriate action if the input does + /// not contain any more data. #[inline(always)] fn peek(&mut self) -> Option { self._peek(Guard(())) } + /// Pops a token off the start of the input. `None` if no more input is available. + /// + /// Note: Is allowed to refill automatically or any other appropriate action if the input does + /// not contain any more data. #[inline(always)] fn pop(&mut self) -> Option { self._pop(Guard(())) } + /// Attempt to consume `n` tokens, if it fails do not advance the position but return `None`. + /// + /// Note: Is allowed to refill automatically or any other appropriate action if the input does + /// not contain any more data. #[inline(always)] fn consume(&mut self, n: usize) -> Option { self._consume(Guard(()), n) } + /// Runs the closure `F` on the tokens *in order* until it returns false, all tokens up to that + /// token will be returned as a buffer and discarded from the current input. MUST never run the + /// closure more than once on the exact same token. + /// + /// If the end of the input is reached, the whole input is returned. + /// + /// Note: Is allowed to refill automatically or any other appropriate action if the input does + /// not contain any more data. #[inline(always)] fn consume_while(&mut self, f: F) -> Self::Buffer where F: FnMut(Self::Token) -> bool { self._consume_while(Guard(()), f) } - /// Returns the buffer from the marker `m` to the current position, discarding the - /// backtracking position carried by `m`. + /// Returns the buffer from the marker to the current position, discarding the + /// backtracking position carried by the marker. #[inline(always)] fn consume_from(&mut self, m: Self::Marker) -> Self::Buffer { self._consume_from(Guard(()), m) } + /// Returns the remainder of the input in a buffer. + /// + /// Note: Will refill the intenal buffer until no more data is available if the underlying + /// implementation supports it. #[inline(always)] fn consume_remaining(&mut self) -> Self::Buffer { self._consume_remaining(Guard(())) @@ -73,6 +100,7 @@ pub trait Primitives: Input { self._mark(Guard(())) } + /// Resumes from a previously marked state. #[inline(always)] fn restore(self, m: Self::Marker) -> Self { self._restore(Guard(()), m) diff --git a/src/types/mod.rs b/src/types/mod.rs index 142b581c..8505e5ef 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -97,21 +97,21 @@ impl<'a> Buffer for &'a str { /// Coupled with the `ParseResult` type it forms the parser monad: /// /// ```ignore -/// Fn*(Input, ...) -> ParseResult; +/// Fn*(I, ...) -> ParseResult; /// ``` /// -/// where ``Fn*`` is the appropriate closure/function trait, `I` the input token type (usually -/// something like `u8`), `...` additional parameters to the parser, `T` the carried type and `E` +/// where ``Fn*`` is the appropriate closure/function trait, `I` the input type (can be something +/// like `[u8]`), `...` additional parameters to the parser, `T` the carried success type and `E` /// the potential error type. pub trait Input: Sized { /// The token type of the input. - // TODO: Maybe remove the copy bound at some point? type Token: Copy + PartialEq; /// A marker type which is used to backtrack using `_mark` and `_restore`. /// /// It should also be possible to use this type to consume the data from the marked position to /// the current position. + #[doc(hidden)] type Marker; /// The buffer type yielded by this input when multiple tokens are consumed in sequence. @@ -189,67 +189,44 @@ pub trait Input: Sized { // Primitive methods /// **Primitive:** See `Primitives::peek` for documentation. - /// - /// Peeks at the next token in the input without consuming it. `None` if no more input is - /// available. - /// - /// Note: Will refill automatically. #[inline] + #[doc(hidden)] fn _peek(&mut self, Guard) -> Option; /// **Primitive:** See `Primitives::pop` for documentation. - /// - /// Pops a token off the start of the input. `None` if no more input is available. - /// - /// Note: Will refill automatically. #[inline] + #[doc(hidden)] fn _pop(&mut self, Guard) -> Option; /// **Primitive:** See `Primitives::consume` for documentation. - /// - /// Attempt to consume `n` tokens, if it fails do not advance the position but return `None`. #[inline] + #[doc(hidden)] fn _consume(&mut self, Guard, usize) -> Option; /// **Primitive:** See `Primitives::consume_while` for documentation. - /// - /// Runs the closure `F` on the tokens *in order* until it returns false, all tokens up to that - /// token will be returned as a buffer and discarded from the current input. MUST never run the - /// closure more than once on the exact same token. - /// - /// If the end of the input is reached, the whole input is returned. - /// - /// Note: Will refill automatically. #[inline] + #[doc(hidden)] fn _consume_while(&mut self, Guard, F) -> Self::Buffer where F: FnMut(Self::Token) -> bool; /// **Primitive:** See `Primitives::consume_from for documentation. - /// - /// Returns the buffer from the marker to the current position, discarding the - /// backtracking position carried by the marker. #[inline] + #[doc(hidden)] fn _consume_from(&mut self, Guard, Self::Marker) -> Self::Buffer; /// **Primitive:** See `Primitives::consume_remaining` for documentation. - /// - /// Returns the remainder of the input in a buffer. - /// - /// Note: Will refill the intenal buffer until no more data is available if the underlying - /// implementation supports it. #[inline] + #[doc(hidden)] fn _consume_remaining(&mut self, Guard) -> Self::Buffer; /// **Primitive:** See `Primitives::mark` for documentation. - /// - /// Marks a position for backtracking along with relevant state. #[inline] + #[doc(hidden)] fn _mark(&self, Guard) -> Self::Marker; /// **Primitive:** See `Primitives::restore` for documentation. - /// - /// Resumes from a previously marked state. #[inline] + #[doc(hidden)] fn _restore(self, Guard, Self::Marker) -> Self; } From b93d3b42660735fcb01ecde19cbb3de21c9f1ee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 17 Aug 2016 19:52:49 +0200 Subject: [PATCH 46/63] parse: Added parse_only_str --- src/lib.rs | 8 +++++--- src/parse.rs | 37 +++++++++++++++++++++++++++++-------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 65dcb82d..575aa127 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -232,6 +232,9 @@ pub mod parsers; pub mod primitives; pub mod types; +pub use parse::parse_only; +pub use parse::parse_only_str; + pub mod prelude { pub use parsers::{ any, @@ -275,7 +278,6 @@ pub mod prelude { U8Input, ParseResult, }; - pub use parse::parse_only; + pub use parse_only; + pub use parse_only_str; } - -pub use parse::parse_only; diff --git a/src/parse.rs b/src/parse.rs index 606eba74..a920c5a5 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -48,13 +48,20 @@ pub fn parse_only<'a, I, T, E, F>(parser: F, input: &'a [I]) -> Result(parser: F, input: &'a str) -> Result + where F: FnOnce(&'a str) -> ParseResult<&'a str, T, E> { + match parser(input).into_inner() { + (_, Ok(t)) => Ok(t), + (mut b, Err(e)) => Err((b.consume_remaining(), e)), + } +} + #[cfg(test)] mod test { use types::Input; use primitives::Primitives; - use super::{ - parse_only, - }; + use super::*; #[test] fn inspect_input() { @@ -78,11 +85,25 @@ mod test { }, b"the input"), Err((&b"input"[..], "my error"))); } - // FIXME: - /* #[test] - fn incomplete() { - assert_eq!(parse_only(|i| i.incomplete::<(), ()>(23), b"the input"), Err(ParseError::Incomplete(23))); + fn inspect_input_str() { + let mut input = None; + + assert_eq!(parse_only_str(|i| { + input = Some(i.to_owned()); + + i.ret::<_, ()>("the result") + }, "the input"), Ok("the result")); + + assert_eq!(input, Some("the input".to_owned())); + } + + #[test] + fn err_str() { + assert_eq!(parse_only_str(|mut i| { + i.consume(4); + + i.err::<(), _>("my error") + }, "the input"), Err(("input", "my error"))); } - */ } From a7e650302e1f1d664ac9814cc1f367ebdbec565c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 17 Aug 2016 23:49:37 +0200 Subject: [PATCH 47/63] Tests: combinators --- src/combinators/bounded.rs | 831 +++++++++---------------------------- 1 file changed, 187 insertions(+), 644 deletions(-) diff --git a/src/combinators/bounded.rs b/src/combinators/bounded.rs index 8f6e09bb..7d73f695 100644 --- a/src/combinators/bounded.rs +++ b/src/combinators/bounded.rs @@ -605,8 +605,6 @@ impl BoundedRange for usize { break; } - let m = i.mark(); - match f(i).into_inner() { (b, Ok(_)) => { n -= 1; @@ -614,9 +612,7 @@ impl BoundedRange for usize { i = b }, (b, Err(e)) => if n == 0 { - i = b.restore(m); - - break; + unreachable!(); } else { // Not enough iterations, propagate return b.err(e); @@ -768,13 +764,11 @@ pub fn sep_by(i: I, r: R, mut p: P, mut sep: F BoundedRange::parse_many(r, i, parser) } -/* #[cfg(test)] mod test { - use {Error, ParseResult}; - use parsers::{any, token, string}; - use primitives::input::*; - use primitives::{IntoInner, State}; + use types::ParseResult; + use parsers::{Error, any, token, string}; + use primitives::IntoInner; use super::{ many, @@ -784,715 +778,264 @@ mod test { #[test] fn many_range_full() { - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b""), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"a"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aa"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"b"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"ab"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), vec![b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aab"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), vec![b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b""), .., any); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"a"), .., any); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aa"), .., any); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"b"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"ab"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aab"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(&b"b"[..], .., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Ok(vec![]))); + let r: ParseResult<_, Vec<_>, _> = many(&b"ab"[..], .., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Ok(vec![b'a']))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aab"[..], .., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Ok(vec![b'a', b'a']))); + + let r: ParseResult<_, Vec<_>, _> = many(&b""[..], .., any); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![]))); + let r: ParseResult<_, Vec<_>, _> = many(&b"a"[..], .., any); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![b'a']))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aa"[..], .., any); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![b'a', b'a']))); // Test where we error inside of the inner parser - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"abac"), .., |i| string(i, b"ab")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ac"), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"abac"), .., |i| string(i, b"ab")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ac"), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aba"), .., |i| string(i, b"ab")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many(&b"abac"[..], .., |i| string(i, b"ab")); assert_eq!(r.into_inner(), (&b"ac"[..], Ok(vec![&b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many(&b"abac"[..], .., |i| string(i, b"ab")); assert_eq!(r.into_inner(), (&b"ac"[..], Ok(vec![&b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aba"[..], .., |i| string(i, b"ab")); assert_eq!(r.into_inner(), (&b"a"[..], Ok(vec![&b"ab"[..]]))); } #[test] fn many_range_to() { - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b""), ..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"a"), ..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b""), ..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"a"), ..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![])); - - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b""), ..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"a"), ..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b""), ..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"a"), ..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![])); - - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b""), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"a"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aa"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aaa"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), vec![b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"b"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"ab"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), vec![b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aab"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), vec![b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b""), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"a"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aa"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"b"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"ab"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aab"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aaab"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ab"), vec![b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(&b""[..], ..0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![]))); + let r: ParseResult<_, Vec<_>, _> = many(&b"a"[..], ..0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"a"[..], Ok(vec![]))); + + let r: ParseResult<_, Vec<_>, _> = many(&b""[..], ..1, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![]))); + let r: ParseResult<_, Vec<_>, _> = many(&b"a"[..], ..1, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"a"[..], Ok(vec![]))); + + let r: ParseResult<_, Vec<_>, _> = many(&b""[..], ..3, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![]))); + let r: ParseResult<_, Vec<_>, _> = many(&b"a"[..], ..3, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![b'a']))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aa"[..], ..3, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![b'a', b'a']))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aaa"[..], ..3, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"a"[..], Ok(vec![b'a', b'a']))); // Test where we error inside of the inner parser - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"abac"), ..3, |i| string(i, b"ab")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ac"), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"abac"), ..3, |i| string(i, b"ab")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ac"), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aba"), ..3, |i| string(i, b"ab")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![&b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many(&b"abac"[..], ..3, |i| string(i, b"ab")); assert_eq!(r.into_inner(), (&b"ac"[..], Ok(vec![&b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many(&b"abac"[..], ..3, |i| string(i, b"ab")); assert_eq!(r.into_inner(), (&b"ac"[..], Ok(vec![&b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aba"[..], ..3, |i| string(i, b"ab")); assert_eq!(r.into_inner(), (&b"a"[..], Ok(vec![&b"ab"[..]]))); } #[test] fn many_range_from() { - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b""), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"a"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aa"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aaa"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"b"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"ab"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aab"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), vec![b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b""), 2.., any); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"a"), 2.., any); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aa"), 2.., any); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aaa"), 2.., any); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"b"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"ab"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aab"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aaab"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a', b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(&b""[..], 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many(&b"a"[..], 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aa"[..], 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![b'a', b'a']))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aaa"[..], 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![b'a', b'a', b'a']))); + + let r: ParseResult<_, Vec<_>, _> = many(&b"b"[..], 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many(&b"ab"[..], 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aab"[..], 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Ok(vec![b'a', b'a']))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aaab"[..], 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Ok(vec![b'a', b'a', b'a']))); // Test where we error inside of the inner parser - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"ababac"), 2.., |i| string(i, b"ab")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ac"), vec![&b"ab"[..], &b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"ababac"), 2.., |i| string(i, b"ab")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ac"), vec![&b"ab"[..], &b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"ababa"), 2.., |i| string(i, b"ab")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![&b"ab"[..], &b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many(&b"ababac"[..], 2.., |i| string(i, b"ab")); assert_eq!(r.into_inner(), (&b"ac"[..], Ok(vec![&b"ab"[..], &b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many(&b"ababac"[..], 2.., |i| string(i, b"ab")); assert_eq!(r.into_inner(), (&b"ac"[..], Ok(vec![&b"ab"[..], &b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many(&b"ababa"[..], 2.., |i| string(i, b"ab")); assert_eq!(r.into_inner(), (&b"a"[..], Ok(vec![&b"ab"[..], &b"ab"[..]]))); } #[test] fn many_range() { - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b""), 0..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"a"), 0..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b""), 0..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"a"), 0..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![])); - - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b""), 0..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"a"), 0..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b""), 0..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"a"), 0..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![])); - - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b""), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"a"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aa"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aaa"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![b'a', b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aaaa"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), vec![b'a', b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"b"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"ab"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aaab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), vec![b'a', b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aaaab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ab"), vec![b'a', b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b""), 2..4, any); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"a"), 2..4, any); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aa"), 2..4, any); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aaa"), 2..4, any); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aaaa"), 2..4, any); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![b'a', b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"b"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"ab"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aaab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a', b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aaaab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ab"), vec![b'a', b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(&b""[..], 0..0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![]))); + let r: ParseResult<_, Vec<_>, _> = many(&b"a"[..], 0..0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"a"[..], Ok(vec![]))); + let r: ParseResult<_, Vec<_>, _> = many(&b""[..], 0..0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![]))); + let r: ParseResult<_, Vec<_>, _> = many(&b"a"[..], 0..0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"a"[..], Ok(vec![]))); + + let r: ParseResult<_, Vec<_>, _> = many(&b""[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many(&b"a"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aa"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![b'a', b'a']))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aaa"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![b'a', b'a', b'a']))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aaaa"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"a"[..], Ok(vec![b'a', b'a', b'a']))); + + let r: ParseResult<_, Vec<_>, _> = many(&b"b"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many(&b"ab"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aab"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Ok(vec![b'a', b'a']))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aaab"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Ok(vec![b'a', b'a', b'a']))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aaaab"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"ab"[..], Ok(vec![b'a', b'a', b'a']))); // Test where we error inside of the inner parser - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"abac"), 1..3, |i| string(i, b"ab")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ac"), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"ababac"), 1..3, |i| string(i, b"ab")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ac"), vec![&b"ab"[..], &b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"abac"), 1..3, |i| string(i, b"ab")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ac"), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"ababac"), 1..3, |i| string(i, b"ab")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ac"), vec![&b"ab"[..], &b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many(&b"abac"[..], 1..3, |i| string(i, b"ab")); assert_eq!(r.into_inner(), (&b"ac"[..], Ok(vec![&b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many(&b"ababac"[..], 1..3, |i| string(i, b"ab")); assert_eq!(r.into_inner(), (&b"ac"[..], Ok(vec![&b"ab"[..], &b"ab"[..]]))); } #[test] fn many_exact() { - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b""), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"a"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aa"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aaa"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), vec![b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"b"), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"ab"), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aab"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aaab"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ab"), vec![b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b""), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"a"), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aa"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aaa"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![b'a', b'a'])); - - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"b"), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"ab"), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aab"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), vec![b'a', b'a'])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"aaab"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ab"), vec![b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(&b""[..], 0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![]))); + let r: ParseResult<_, Vec<_>, _> = many(&b"a"[..], 0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"a"[..], Ok(vec![]))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aa"[..], 0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"aa"[..], Ok(vec![]))); + + let r: ParseResult<_, Vec<_>, _> = many(&b""[..], 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many(&b"a"[..], 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aa"[..], 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![b'a', b'a']))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aaa"[..], 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"a"[..], Ok(vec![b'a', b'a']))); + + let r: ParseResult<_, Vec<_>, _> = many(&b"b"[..], 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many(&b"ab"[..], 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aab"[..], 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Ok(vec![b'a', b'a']))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aaab"[..], 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"ab"[..], Ok(vec![b'a', b'a']))); // Test where we error inside of the inner parser - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"abac"), 2, |i| string(i, b"ab")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"c"), Error::expected(b'b'))); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"ababa"), 2, |i| string(i, b"ab")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), vec![&b"ab"[..], &b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"abac"), 2, |i| string(i, b"ab")); - assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"c"), Error::expected(b'b'))); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"ababac"), 2, |i| string(i, b"ab")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ac"), vec![&b"ab"[..], &b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many(new_buf(END_OF_INPUT, b"ababa"), 2, |i| string(i, b"ab")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), vec![&b"ab"[..], &b"ab"[..]])); + let r: ParseResult<_, Vec<_>, _> = many(&b"abac"[..], 2, |i| string(i, b"ab")); assert_eq!(r.into_inner(), (&b"c"[..], Err(Error::expected(b'b')))); + let r: ParseResult<_, Vec<_>, _> = many(&b"ababa"[..], 2, |i| string(i, b"ab")); assert_eq!(r.into_inner(), (&b"a"[..], Ok(vec![&b"ab"[..], &b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many(&b"abac"[..], 2, |i| string(i, b"ab")); assert_eq!(r.into_inner(), (&b"c"[..], Err(Error::expected(b'b')))); + let r: ParseResult<_, Vec<_>, _> = many(&b"ababac"[..], 2, |i| string(i, b"ab")); assert_eq!(r.into_inner(), (&b"ac"[..], Ok(vec![&b"ab"[..], &b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many(&b"ababa"[..], 2, |i| string(i, b"ab")); assert_eq!(r.into_inner(), (&b"a"[..], Ok(vec![&b"ab"[..], &b"ab"[..]]))); } #[test] fn many_till_range_full() { - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b""), .., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), .., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), .., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ababac"), .., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..], &b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ababab"), .., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abababa"), .., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); - - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b""), .., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 2)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), .., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abac"), .., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ababac"), .., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..], &b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ababab"), .., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 2)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abababa"), .., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b"a"), 1)); + let r: ParseResult<_, Vec<_>, _> = many_till(&b""[..], .., |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ac"[..], .., |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![]))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abac"[..], .., |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![&b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ababac"[..], .., |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![&b"ab"[..], &b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ababab"[..], .., |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abababa"[..], .., |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'b')))); } #[test] fn many_till_range_from() { - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b""), 0.., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), 0.., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), 1.., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"c"), Error::expected(b'b'))); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), 0.., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), 1.., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), 2.., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"c"), Error::expected(b'b'))); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ababac"), 2.., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..], &b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ababab"), 2.., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abababa"), 2.., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); - - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b""), 0.., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 2)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), 0.., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), 1.., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"c"), Error::expected(b'b'))); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abac"), 0.., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abac"), 1.., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abac"), 2.., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"c"), Error::expected(b'b'))); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ababac"), 2.., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..], &b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ababab"), 2.., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 2)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abababa"), 2.., |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b"a"), 1)); + let r: ParseResult<_, Vec<_>, _> = many_till(&b""[..], 0.., |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"a"[..], 0.., |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'b')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ab"[..], 0.., |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ac"[..], 0.., |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![]))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ac"[..], 1.., |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"c"[..], Err(Error::expected(b'b')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abac"[..], 0.., |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![&b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abac"[..], 1.., |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![&b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abac"[..], 2.., |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"c"[..], Err(Error::expected(b'b')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ababac"[..], 2.., |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![&b"ab"[..], &b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ababab"[..], 2.., |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abababa"[..], 2.., |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'b')))); } #[test] fn many_till_range_to() { - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b""), ..0, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"a"), ..0, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), ..0, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b""), ..1, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), ..1, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), ..2, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b""), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ababac"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..], &b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abababac"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"bac"), Error::expected(b'c'))); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ababab"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), Error::expected(b'c'))); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ababa"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); - - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b""), ..0, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 2)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"a"), ..0, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b"a"), 1)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), ..0, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b""), ..1, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 2)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), ..1, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abac"), ..2, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abac"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ababac"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..], &b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abababac"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"bac"), Error::expected(b'c'))); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ababa"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b"a"), 1)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abababa"), ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"ba"), Error::expected(b'c'))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b""[..], ..0, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"b"[..], ..0, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"b"[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"a"[..], ..0, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'c')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ac"[..], ..0, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![]))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b""[..], ..1, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ac"[..], ..1, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![]))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abac"[..], ..2, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![&b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b""[..], ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ac"[..], ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![]))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abac"[..], ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![&b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ababac"[..], ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![&b"ab"[..], &b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abababac"[..], ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"bac"[..], Err(Error::expected(b'c')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ababab"[..], ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"b"[..], Err(Error::expected(b'c')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ababa"[..], ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'c')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abababa"[..], ..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"ba"[..], Err(Error::expected(b'c')))); } #[test] fn many_till_range() { - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b""), 0..0, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"a"), 0..0, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), 0..0, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b""), 0..1, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), 0..1, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), 0..2, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), 0..2, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b""), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 2)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ababac"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..], &b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abababac"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"bac"), Error::expected(b'c'))); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ababab"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), Error::expected(b'c'))); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ababa"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b"a"), 1)); - - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"ac"), 1..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"c"), Error::expected(b'b'))); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), 1..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), 2..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"c"), Error::expected(b'b'))); - - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b""), 0..0, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 2)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"a"), 0..0, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b"a"), 1)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), 0..0, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b""), 0..1, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 2)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), 0..1, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), 0..2, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abac"), 0..2, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abac"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ababac"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..], &b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abababac"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"bac"), Error::expected(b'c'))); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ababa"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b"a"), 1)); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abababa"), 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"ba"), Error::expected(b'c'))); - - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"ac"), 1..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"c"), Error::expected(b'b'))); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(END_OF_INPUT, b"abac"), 1..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), vec![&b"ab"[..]])); - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"abac"), 2..3, |i| string(i, b"ab"), |i| string(i, b"ac")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"c"), Error::expected(b'b'))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b""[..], 0..0, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b""[..], 0..0, |i| string(i, b"ab"), |i| string(i, b"cd")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'c')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"a"[..], 0..0, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'c')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ac"[..], 0..0, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![]))); + + let r: ParseResult<_, Vec<_>, _> = many_till(&b""[..], 0..1, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ac"[..], 0..1, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![]))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ab"[..], 0..1, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"b"[..], Err(Error::expected(b'c')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abac"[..], 0..1, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"bac"[..], Err(Error::expected(b'c')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ac"[..], 0..2, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![]))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abac"[..], 0..2, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![&b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b""[..], 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"a"[..], 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'b')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ac"[..], 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![]))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abc"[..], 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"c"[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abac"[..], 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![&b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ababac"[..], 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![&b"ab"[..], &b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abababac"[..], 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"bac"[..], Err(Error::expected(b'c')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ababa"[..], 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'c')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abababa"[..], 0..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"ba"[..], Err(Error::expected(b'c')))); + + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ac"[..], 1..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"c"[..], Err(Error::expected(b'b')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abac"[..], 1..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![&b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abac"[..], 2..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"c"[..], Err(Error::expected(b'b')))); } #[test] fn skip_range_full() { - let r = skip_many(new_buf(DEFAULT, b""), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r = skip_many(new_buf(DEFAULT, b"a"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r = skip_many(new_buf(DEFAULT, b"aa"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - - let r = skip_many(new_buf(DEFAULT, b"b"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); - let r = skip_many(new_buf(DEFAULT, b"ab"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); - let r = skip_many(new_buf(DEFAULT, b"aab"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); - - let r = skip_many(new_buf(END_OF_INPUT, b""), .., any); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); - let r = skip_many(new_buf(END_OF_INPUT, b"a"), .., any); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); - let r = skip_many(new_buf(END_OF_INPUT, b"aa"), .., any); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); - - let r = skip_many(new_buf(END_OF_INPUT, b"b"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); - let r = skip_many(new_buf(END_OF_INPUT, b"ab"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); - let r = skip_many(new_buf(END_OF_INPUT, b"aab"), .., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); + let r = skip_many(&b""[..], .., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + let r = skip_many(&b"a"[..], .., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + let r = skip_many(&b"aa"[..], .., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + + let r = skip_many(&b"b"[..], .., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Ok(()))); + let r = skip_many(&b"ab"[..], .., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Ok(()))); + let r = skip_many(&b"aab"[..], .., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Ok(()))); } #[test] fn skip_range_to() { - let r = skip_many(new_buf(DEFAULT, b""), ..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), ())); - let r = skip_many(new_buf(DEFAULT, b"a"), ..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), ())); - let r = skip_many(new_buf(END_OF_INPUT, b""), ..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); - let r = skip_many(new_buf(END_OF_INPUT, b"a"), ..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), ())); - - let r = skip_many(new_buf(DEFAULT, b""), ..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), ())); - let r = skip_many(new_buf(DEFAULT, b"a"), ..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), ())); - let r = skip_many(new_buf(END_OF_INPUT, b""), ..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); - let r = skip_many(new_buf(END_OF_INPUT, b"a"), ..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), ())); - - let r = skip_many(new_buf(DEFAULT, b""), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r = skip_many(new_buf(DEFAULT, b"a"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r = skip_many(new_buf(DEFAULT, b"aa"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), ())); - let r = skip_many(new_buf(DEFAULT, b"aaa"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), ())); - - let r = skip_many(new_buf(DEFAULT, b"b"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); - let r = skip_many(new_buf(DEFAULT, b"ab"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); - let r = skip_many(new_buf(DEFAULT, b"aab"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); - - let r = skip_many(new_buf(END_OF_INPUT, b""), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); - let r = skip_many(new_buf(END_OF_INPUT, b"a"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); - let r = skip_many(new_buf(END_OF_INPUT, b"aa"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); - - let r = skip_many(new_buf(END_OF_INPUT, b"b"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); - let r = skip_many(new_buf(END_OF_INPUT, b"ab"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); - let r = skip_many(new_buf(END_OF_INPUT, b"aab"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); - let r = skip_many(new_buf(END_OF_INPUT, b"aaab"), ..3, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ab"), ())); + let r = skip_many(&b""[..], ..0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + let r = skip_many(&b"a"[..], ..0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"a"[..], Ok(()))); + + let r = skip_many(&b""[..], ..3, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + let r = skip_many(&b"a"[..], ..3, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + let r = skip_many(&b"aa"[..], ..3, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + let r = skip_many(&b"aaa"[..], ..3, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"a"[..], Ok(()))); + + let r = skip_many(&b"b"[..], ..3, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Ok(()))); + let r = skip_many(&b"ab"[..], ..3, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Ok(()))); + let r = skip_many(&b"aab"[..], ..3, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Ok(()))); + let r = skip_many(&b"aaab"[..], ..3, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"ab"[..], Ok(()))); } #[test] fn skip_range_from() { - let r = skip_many(new_buf(DEFAULT, b""), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r = skip_many(new_buf(DEFAULT, b"a"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r = skip_many(new_buf(DEFAULT, b"aa"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r = skip_many(new_buf(DEFAULT, b"aaa"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - - let r = skip_many(new_buf(DEFAULT, b"b"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); - let r = skip_many(new_buf(DEFAULT, b"ab"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); - let r = skip_many(new_buf(DEFAULT, b"aab"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); - - let r = skip_many(new_buf(END_OF_INPUT, b""), 2.., any); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); - let r = skip_many(new_buf(END_OF_INPUT, b"a"), 2.., any); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); - let r = skip_many(new_buf(END_OF_INPUT, b"aa"), 2.., any); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); - let r = skip_many(new_buf(END_OF_INPUT, b"aaa"), 2.., any); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); - - let r = skip_many(new_buf(END_OF_INPUT, b"b"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); - let r = skip_many(new_buf(END_OF_INPUT, b"ab"), 2.., |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); - let r = skip_many(new_buf(END_OF_INPUT, b"aab"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); - let r = skip_many(new_buf(END_OF_INPUT, b"aaab"), 2.., |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); + let r = skip_many(&b""[..], 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r = skip_many(&b"a"[..], 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r = skip_many(&b"aa"[..], 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + let r = skip_many(&b"aaa"[..], 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + + let r = skip_many(&b"b"[..], 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Err(Error::expected(b'a')))); + let r = skip_many(&b"ab"[..], 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Err(Error::expected(b'a')))); + let r = skip_many(&b"aab"[..], 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Ok(()))); + let r = skip_many(&b"aaab"[..], 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Ok(()))); } #[test] fn skip_range() { - let r = skip_many(new_buf(DEFAULT, b""), 0..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), ())); - let r = skip_many(new_buf(DEFAULT, b"a"), 0..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), ())); - let r = skip_many(new_buf(END_OF_INPUT, b""), 0..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); - let r = skip_many(new_buf(END_OF_INPUT, b"a"), 0..0, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), ())); - - let r = skip_many(new_buf(DEFAULT, b""), 0..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), ())); - let r = skip_many(new_buf(DEFAULT, b"a"), 0..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), ())); - let r = skip_many(new_buf(END_OF_INPUT, b""), 0..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); - let r = skip_many(new_buf(END_OF_INPUT, b"a"), 0..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), ())); - - let r = skip_many(new_buf(DEFAULT, b""), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r = skip_many(new_buf(DEFAULT, b"a"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r = skip_many(new_buf(DEFAULT, b"aa"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r = skip_many(new_buf(DEFAULT, b"aaa"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), ())); - let r = skip_many(new_buf(DEFAULT, b"aaaa"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), ())); - - let r = skip_many(new_buf(DEFAULT, b"b"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); - let r = skip_many(new_buf(DEFAULT, b"ab"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); - let r = skip_many(new_buf(DEFAULT, b"aab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); - let r = skip_many(new_buf(DEFAULT, b"aaab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); - let r = skip_many(new_buf(DEFAULT, b"aaaab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ab"), ())); - - let r = skip_many(new_buf(END_OF_INPUT, b""), 2..4, any); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); - let r = skip_many(new_buf(END_OF_INPUT, b"a"), 2..4, any); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); - let r = skip_many(new_buf(END_OF_INPUT, b"aa"), 2..4, any); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); - let r = skip_many(new_buf(END_OF_INPUT, b"aaa"), 2..4, any); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); - let r = skip_many(new_buf(END_OF_INPUT, b"aaaa"), 2..4, any); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), ())); - - let r = skip_many(new_buf(END_OF_INPUT, b"b"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); - let r = skip_many(new_buf(END_OF_INPUT, b"ab"), 2..4, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); - let r = skip_many(new_buf(END_OF_INPUT, b"aab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); - let r = skip_many(new_buf(END_OF_INPUT, b"aaab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); - let r = skip_many(new_buf(END_OF_INPUT, b"aaaab"), 2..4, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ab"), ())); + let r = skip_many(&b""[..], 0..0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + let r = skip_many(&b"a"[..], 0..0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"a"[..], Ok(()))); + + let r = skip_many(&b""[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r = skip_many(&b"a"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r = skip_many(&b"aa"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + let r = skip_many(&b"aaa"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + let r = skip_many(&b"aaaa"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"a"[..], Ok(()))); + + let r = skip_many(&b"b"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Err(Error::expected(b'a')))); + let r = skip_many(&b"ab"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Err(Error::expected(b'a')))); + let r = skip_many(&b"aab"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Ok(()))); + let r = skip_many(&b"aaab"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Ok(()))); + let r = skip_many(&b"aaaab"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"ab"[..], Ok(()))); } #[test] fn skip_exact() { - let r = skip_many(new_buf(DEFAULT, b""), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r = skip_many(new_buf(DEFAULT, b"a"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(DEFAULT, b""), 1)); - let r = skip_many(new_buf(DEFAULT, b"aa"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b""), ())); - let r = skip_many(new_buf(DEFAULT, b"aaa"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"a"), ())); - - let r = skip_many(new_buf(DEFAULT, b"b"), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); - let r = skip_many(new_buf(DEFAULT, b"ab"), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(DEFAULT, b"b"), "token_err")); - let r = skip_many(new_buf(DEFAULT, b"aab"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"b"), ())); - let r = skip_many(new_buf(DEFAULT, b"aaab"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ab"), ())); - - let r = skip_many(new_buf(END_OF_INPUT, b""), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); - let r = skip_many(new_buf(END_OF_INPUT, b"a"), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Incomplete(new_buf(END_OF_INPUT, b""), 1)); - let r = skip_many(new_buf(END_OF_INPUT, b"aa"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b""), ())); - let r = skip_many(new_buf(END_OF_INPUT, b"aaa"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"a"), ())); - - let r = skip_many(new_buf(END_OF_INPUT, b"b"), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); - let r = skip_many(new_buf(END_OF_INPUT, b"ab"), 2, |i| token(i, b'a').map_err(|_| "token_err")); - assert_eq!(r.into_inner(), State::Error(new_buf(END_OF_INPUT, b"b"), "token_err")); - let r = skip_many(new_buf(END_OF_INPUT, b"aab"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"b"), ())); - let r = skip_many(new_buf(END_OF_INPUT, b"aaab"), 2, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(END_OF_INPUT, b"ab"), ())); + let r = skip_many(&b""[..], 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r = skip_many(&b"a"[..], 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r = skip_many(&b"aa"[..], 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + let r = skip_many(&b"aaa"[..], 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"a"[..], Ok(()))); + let r = skip_many(&b"aaab"[..], 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"ab"[..], Ok(()))); + let r = skip_many(&b"aab"[..], 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Ok(()))); + let r = skip_many(&b"ab"[..], 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Err(Error::expected(b'a')))); + let r = skip_many(&b"b"[..], 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Err(Error::expected(b'a')))); } #[test] #[should_panic] fn panic_many_range_lt() { - let r: ParseResult<_, Vec<_>, _> = many(new_buf(DEFAULT, b"aaaab"), 2..1, |i| token(i, b'a')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ab"), vec![b'a', b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many(&b"aaaab"[..], 2..1, |i| token(i, b'a')); + assert_eq!(r.into_inner(), (&b"ab"[..], Ok(vec![b'a', b'a', b'a']))); } #[test] #[should_panic] fn panic_skip_many_range_lt() { - assert_eq!(skip_many(new_buf(DEFAULT, b"aaaab"), 2..1, |i| token(i, b'a')).into_inner(), State::Data(new_buf(DEFAULT, b"ab"), ())); + assert_eq!(skip_many(&b"aaaab"[..], 2..1, |i| token(i, b'a')).into_inner(), (&b"ab"[..], Ok(()))); } #[test] #[should_panic] fn panic_many_till_range_lt() { - let r: ParseResult<_, Vec<_>, _> = many_till(new_buf(DEFAULT, b"aaaab"), 2..1, |i| token(i, b'a'), |i| token(i, b'b')); - assert_eq!(r.into_inner(), State::Data(new_buf(DEFAULT, b"ab"), vec![b'a', b'a', b'a'])); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"aaaab"[..], 2..1, |i| token(i, b'a'), |i| token(i, b'b')); + assert_eq!(r.into_inner(), (&b"ab"[..], Ok(vec![b'a', b'a', b'a']))); } -}*/ +} From a058d69a5f538210d7917e08cf86924310927756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 17 Aug 2016 23:49:55 +0200 Subject: [PATCH 48/63] Doc --- src/buffer/mod.rs | 8 ++++++-- src/combinators/mod.rs | 2 -- src/lib.rs | 7 +++++++ src/parsers.rs | 1 - src/types/mod.rs | 1 - src/types/tendril.rs | 1 - 6 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/buffer/mod.rs b/src/buffer/mod.rs index a2b920ac..53e29254 100644 --- a/src/buffer/mod.rs +++ b/src/buffer/mod.rs @@ -95,7 +95,6 @@ pub trait Stream<'a, 'i> { E: 'i; } -// FIXME: Docs /// Input buffer type which contains a flag which tells if we might need to read more data. #[must_use] #[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] @@ -106,23 +105,28 @@ pub struct InputBuf<'a, I: 'a>( &'a [I], ); -// FIXME: Docs impl<'a, I: 'a> InputBuf<'a, I> { + /// Creates a new input buffer with incomplete set to false. #[inline] pub fn new(buf: &'a [I]) -> Self { InputBuf(false, buf) } + /// Returns true if parsers want to obtain more data. + /// + /// The result of the parsing is only accurate if this is false after completed parsing. #[inline] pub fn is_incomplete(&self) -> bool { self.0 } + /// Returns the length of the contained buffer, may be an incomplete buffer. #[inline] pub fn len(&self) -> usize { self.1.len() } + /// Returns true if the contained buffer is empty, may return true even when incomplete. #[inline] pub fn is_empty(&self) -> bool { self.len() == 0 diff --git a/src/combinators/mod.rs b/src/combinators/mod.rs index 6dfd2c4e..6f52ccbf 100644 --- a/src/combinators/mod.rs +++ b/src/combinators/mod.rs @@ -307,7 +307,6 @@ pub fn look_ahead(i: I, f: F) -> ParseResult } } -// FIXME: #[cfg(test)] mod test { use types::{Input, ParseResult}; @@ -466,7 +465,6 @@ mod test { let r: ParseResult<_, Vec<_>, _> = sep_by(&b"a--c-"[..], any, |i| string(i, b"--")); assert_eq!(r.into_inner(), (&b"-"[..], Ok(vec![b'a', b'c']))); - // FIXME: Indefinite execution on line below let r: ParseResult<_, Vec<_>, _> = sep_by(&b"aaa--a"[..], |i| string(i, b"aaa"), |i| string(i, b"--")); assert_eq!(r.into_inner(), (&b"--a"[..], Ok(vec![&b"aaa"[..]]))); } diff --git a/src/lib.rs b/src/lib.rs index 575aa127..ceb857ee 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -200,6 +200,12 @@ //! The built-in `chomp::parsers::Error` type is zero-sized and carry no error-information. This //! increases performance somewhat. +#![warn(missing_docs, + trivial_casts, + trivial_numeric_casts, + unused_import_braces, + unused_qualifications)] + #![cfg_attr(feature="clippy", feature(plugin))] #![cfg_attr(feature="clippy", plugin(clippy))] #![cfg_attr(feature="clippy", warn( @@ -235,6 +241,7 @@ pub mod types; pub use parse::parse_only; pub use parse::parse_only_str; +/// Basic prelude. pub mod prelude { pub use parsers::{ any, diff --git a/src/parsers.rs b/src/parsers.rs index 35c83b31..50735f7c 100644 --- a/src/parsers.rs +++ b/src/parsers.rs @@ -544,7 +544,6 @@ mod error { } } -// FIXME: Uncomment and fix #[cfg(test)] mod test { use primitives::IntoInner; diff --git a/src/types/mod.rs b/src/types/mod.rs index 8505e5ef..d30f539d 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -90,7 +90,6 @@ impl<'a> Buffer for &'a str { } } -// FIXME: Update, do not refer to type or linear type /// Linear type containing the parser state, this type is threaded though `bind` and is also the /// initial type passed to a parser. /// diff --git a/src/types/tendril.rs b/src/types/tendril.rs index e7001d85..dc4ebe11 100644 --- a/src/types/tendril.rs +++ b/src/types/tendril.rs @@ -99,7 +99,6 @@ impl Buffer for ByteTendril { } } -// FIXME: Tests #[cfg(test)] mod test { use tendril::Tendril; From 30744722358176411ab5bca1c925d3fceb118f36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Thu, 18 Aug 2016 00:34:45 +0200 Subject: [PATCH 49/63] Moved things around to satisfy Clippy lints --- src/buffer/buffer.rs | 288 ---------------------------------------- src/buffer/mod.rs | 293 ++++++++++++++++++++++++++++++++++++++++- src/buffer/stateful.rs | 2 +- src/lib.rs | 4 +- src/parsers.rs | 10 +- src/types/mod.rs | 4 +- 6 files changed, 295 insertions(+), 306 deletions(-) delete mode 100644 src/buffer/buffer.rs diff --git a/src/buffer/buffer.rs b/src/buffer/buffer.rs deleted file mode 100644 index 35185771..00000000 --- a/src/buffer/buffer.rs +++ /dev/null @@ -1,288 +0,0 @@ -use std::ops; -use std::ptr; -use std::io; - -use std::cell::Cell; - -use buffer::DataSource; - -const DEFAULT_BUFFER_SIZE: usize = 6 * 1024; - -/// Trait all parser buffers implement. -/// -/// Enables the consumer to request specific amounts of data and only consume partial parts of the -/// buffer. -pub trait Buffer: ops::Deref { - /// Attempt to fill the buffer using the closure `F`. - /// - /// The successful return from `F` should contain the number of items successfully written to - /// the slice. - /// - /// # Notes - /// - /// * The returned value must *NOT* be larger than the length of the given slice. - /// - /// * Return `0` if no more data is available or if the slice is of zero length. - /// - /// * The slice might contain uninitialized memory, do not read from the slice. - #[inline] - fn fill>(&mut self, &mut S) -> io::Result; - - /// Buffer attempts to clear space for additional items. - #[inline] - fn request_space(&mut self, usize); - - /// Consumes the given amount of bytes, must be less than or equal to `len()`. - /// - /// Does not invalidate any borrow of data from self. - #[inline] - fn consume(&self, items: usize); - - /// Returns the number of bytes left in the buffer. - #[inline] - fn len(&self) -> usize; - - /// If the buffer has no more data. - #[inline] - fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Returns the maximum amount of data which can be stored - #[inline] - fn capacity(&self) -> usize; -} - -/// A fixed size buffer. -/// -/// Only allocates when created. -// TODO: Tests -#[derive(Debug, Eq, PartialEq)] -pub struct FixedSizeBuffer { - /// Backing memory. - buffer: Vec, - /// Number of items of `buffer` which contain actual data. - populated: usize, - /// The number of bytes from the start of the buffer which are used. - /// - /// As long as used <= populated it is safe. - used: Cell, -} - -impl FixedSizeBuffer { - /// Creates a fixed-size buffer with the default buffer size. - #[inline] - pub fn new() -> Self { - Self::with_size(DEFAULT_BUFFER_SIZE) - } - - /// Creates a fixed-size buffer with the supplied buffer size. - #[inline] - pub fn with_size(size: usize) -> Self { - assert!(size > 0); - - let mut buf = Vec::with_capacity(size); - - // TODO: Would it be better with a Default requirement on I? - // We set the length here to allow fill() to hand out a slice of uninitialized memory - // to be populated. - // NOTE: We cannot actually expose this memory to the parser since self.populated will - // be the upper limit for the deref to slice. - unsafe { - buf.set_len(size); - } - - FixedSizeBuffer { - buffer: buf, - populated: 0, - used: Cell::new(0), - } - } -} - -impl ops::Deref for FixedSizeBuffer { - type Target = [I]; - - #[inline] - fn deref(&self) -> &[I] { - &self.buffer[self.used.get()..self.populated] - } -} - -impl ops::DerefMut for FixedSizeBuffer { - #[inline] - fn deref_mut(&mut self) -> &mut [I] { - &mut self.buffer[self.used.get()..self.populated] - } -} - -impl Buffer for FixedSizeBuffer { - #[inline] - fn fill>(&mut self, s: &mut S) -> io::Result { - s.read(&mut self.buffer[self.populated..]).map(|n| { - debug_assert!(self.populated + n <= self.buffer.len()); - - self.populated += n; - - n - }) - } - - #[inline] - fn request_space(&mut self, items: usize) { - use std::ptr; - - assert!(self.populated >= self.used.get()); - - // Only copy if we actually need to free the space - if self.buffer.len() - self.populated < items { - unsafe { - ptr::copy(self.buffer.as_ptr().offset(self.used.get() as isize), self.buffer.as_mut_ptr(), self.populated - self.used.get()); - } - - self.populated -= self.used.get(); - self.used.set(0); - } - } - - #[inline] - fn consume(&self, items: usize) { - debug_assert!(self.used.get() + items <= self.populated); - - self.used.set(self.used.get() + items) - } - - #[inline] - fn len(&self) -> usize { - self.populated - self.used.get() - } - - #[inline] - fn capacity(&self) -> usize { - self.buffer.len() - } -} - -/// A buffer which will reallocate to fit the requested amount of data. -/// -/// # Note: -/// -/// Will not decrease in size. -// TODO: Tests -#[derive(Debug)] -pub struct GrowingBuffer { - /// Backing memory. - buffer: Vec, - /// Number of items of `buffer` which contain actual data. - populated: usize, - /// Maximal size of the buffer, 0 means infinity. - limit: usize, - /// The number of bytes from the start of the buffer which are used. - /// - /// As long as used <= populated it is safe. - used: Cell, -} - -impl GrowingBuffer { - /// Creates a new unlimited `GrowingBuffer`. - #[inline] - pub fn new() -> Self { - Self::with_limit(0) - } - - /// Creates a new `GrowingBuffer` with the specified limit. - /// - /// # Note - /// - /// The actual amount of allocated memory might be larger than the specified limit, depends on - /// the allocator. - #[inline] - pub fn with_limit(limit: usize) -> Self { - GrowingBuffer { - buffer: Vec::new(), - populated: 0, - limit: limit, - used: Cell::new(0), - } - } -} - -impl ops::Deref for GrowingBuffer { - type Target = [I]; - - #[inline] - fn deref(&self) -> &[I] { - &self.buffer[self.used.get()..self.populated] - } -} - -impl ops::DerefMut for GrowingBuffer { - #[inline] - fn deref_mut(&mut self) -> &mut [I] { - &mut self.buffer[self.used.get()..self.populated] - } -} - -impl Buffer for GrowingBuffer { - #[inline] - fn fill>(&mut self, s: &mut S) -> io::Result { - s.read(&mut self.buffer[self.populated..]).map(|n| { - debug_assert!(self.populated + n <= self.buffer.len()); - - self.populated += n; - - n - }) - } - - #[inline] - fn request_space(&mut self, items: usize) { - // If we are over the limit, refuse - if self.limit != 0 && self.buffer.capacity() > self.limit { - return; - } - - if items + self.len() > self.buffer.capacity() { - // We do not have enough space for the new items, reallocate - self.buffer.reserve(items); - - let cap = self.buffer.capacity(); - - // TODO: Would it be better with a Default requirement on I? - // We set the length here to allow fill() to hand out a slice of uninitialized memory - // to be populated. - // NOTE: We cannot actually expose this memory to the parser since self.populated will - // be the upper limit for the deref to slice. - unsafe { - self.buffer.set_len(cap); - } - } - - // Only copy if we actually need to free the space - if self.buffer.len() - self.populated < items { - unsafe { - ptr::copy(self.buffer.as_ptr().offset(self.used.get() as isize), self.buffer.as_mut_ptr(), self.populated - self.used.get()); - } - - self.populated -= self.used.get(); - self.used.set(0); - } - } - - #[inline] - fn consume(&self, items: usize) { - debug_assert!(self.used.get() + items <= self.populated); - - self.used.set(self.used.get() + items) - } - - #[inline] - fn len(&self) -> usize { - self.populated - self.used.get() - } - - #[inline] - fn capacity(&self) -> usize { - self.buffer.len() - } -} diff --git a/src/buffer/mod.rs b/src/buffer/mod.rs index 53e29254..be21b2c3 100644 --- a/src/buffer/mod.rs +++ b/src/buffer/mod.rs @@ -28,12 +28,15 @@ //! ``` mod stateful; -mod buffer; mod slice; pub mod data_source; use std::io; +use std::ops; +use std::ptr; + +use std::cell::Cell; use types::{Input, ParseResult}; use types::Buffer as InputBuffer; @@ -42,11 +45,8 @@ use primitives::Guard; pub use self::slice::SliceStream; pub use self::data_source::DataSource; pub use self::stateful::Source; -pub use self::buffer::{ - Buffer, - FixedSizeBuffer, - GrowingBuffer, -}; + +const DEFAULT_BUFFER_SIZE: usize = 6 * 1024; /// Error type for parsing using the `Stream` trait. #[derive(Debug)] @@ -207,7 +207,7 @@ impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { fn _mark(&self, _g: Guard) -> Self::Marker { // Incomplete state is separate from the parsed state, no matter how we hit incomplete we // want to keep it. - &self.1 + self.1 } #[inline] @@ -218,6 +218,285 @@ impl<'a, I: Copy + PartialEq> Input for InputBuf<'a, I> { } } +/// Trait all parser buffers implement. +/// +/// Enables the consumer to request specific amounts of data and only consume partial parts of the +/// buffer. +pub trait Buffer: ops::Deref { + /// Attempt to fill the buffer using the closure `F`. + /// + /// The successful return from `F` should contain the number of items successfully written to + /// the slice. + /// + /// # Notes + /// + /// * The returned value must *NOT* be larger than the length of the given slice. + /// + /// * Return `0` if no more data is available or if the slice is of zero length. + /// + /// * The slice might contain uninitialized memory, do not read from the slice. + #[inline] + fn fill>(&mut self, &mut S) -> io::Result; + + /// Buffer attempts to clear space for additional items. + #[inline] + fn request_space(&mut self, usize); + + /// Consumes the given amount of bytes, must be less than or equal to `len()`. + /// + /// Does not invalidate any borrow of data from self. + #[inline] + fn consume(&self, items: usize); + + /// Returns the number of bytes left in the buffer. + #[inline] + fn len(&self) -> usize; + + /// If the buffer has no more data. + #[inline] + fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns the maximum amount of data which can be stored + #[inline] + fn capacity(&self) -> usize; +} + +/// A fixed size buffer. +/// +/// Only allocates when created. +// TODO: Tests +#[derive(Debug, Eq, PartialEq)] +pub struct FixedSizeBuffer { + /// Backing memory. + buffer: Vec, + /// Number of items of `buffer` which contain actual data. + populated: usize, + /// The number of bytes from the start of the buffer which are used. + /// + /// As long as used <= populated it is safe. + used: Cell, +} + +impl FixedSizeBuffer { + /// Creates a fixed-size buffer with the default buffer size. + #[inline] + pub fn new() -> Self { + Self::with_size(DEFAULT_BUFFER_SIZE) + } + + /// Creates a fixed-size buffer with the supplied buffer size. + #[inline] + pub fn with_size(size: usize) -> Self { + assert!(size > 0); + + let mut buf = Vec::with_capacity(size); + + // TODO: Would it be better with a Default requirement on I? + // We set the length here to allow fill() to hand out a slice of uninitialized memory + // to be populated. + // NOTE: We cannot actually expose this memory to the parser since self.populated will + // be the upper limit for the deref to slice. + unsafe { + buf.set_len(size); + } + + FixedSizeBuffer { + buffer: buf, + populated: 0, + used: Cell::new(0), + } + } +} + +impl ops::Deref for FixedSizeBuffer { + type Target = [I]; + + #[inline] + fn deref(&self) -> &[I] { + &self.buffer[self.used.get()..self.populated] + } +} + +impl ops::DerefMut for FixedSizeBuffer { + #[inline] + fn deref_mut(&mut self) -> &mut [I] { + &mut self.buffer[self.used.get()..self.populated] + } +} + +impl Buffer for FixedSizeBuffer { + #[inline] + fn fill>(&mut self, s: &mut S) -> io::Result { + s.read(&mut self.buffer[self.populated..]).map(|n| { + debug_assert!(self.populated + n <= self.buffer.len()); + + self.populated += n; + + n + }) + } + + #[inline] + fn request_space(&mut self, items: usize) { + use std::ptr; + + assert!(self.populated >= self.used.get()); + + // Only copy if we actually need to free the space + if self.buffer.len() - self.populated < items { + unsafe { + ptr::copy(self.buffer.as_ptr().offset(self.used.get() as isize), self.buffer.as_mut_ptr(), self.populated - self.used.get()); + } + + self.populated -= self.used.get(); + self.used.set(0); + } + } + + #[inline] + fn consume(&self, items: usize) { + debug_assert!(self.used.get() + items <= self.populated); + + self.used.set(self.used.get() + items) + } + + #[inline] + fn len(&self) -> usize { + self.populated - self.used.get() + } + + #[inline] + fn capacity(&self) -> usize { + self.buffer.len() + } +} + +/// A buffer which will reallocate to fit the requested amount of data. +/// +/// # Note: +/// +/// Will not decrease in size. +// TODO: Tests +#[derive(Debug)] +pub struct GrowingBuffer { + /// Backing memory. + buffer: Vec, + /// Number of items of `buffer` which contain actual data. + populated: usize, + /// Maximal size of the buffer, 0 means infinity. + limit: usize, + /// The number of bytes from the start of the buffer which are used. + /// + /// As long as used <= populated it is safe. + used: Cell, +} + +impl GrowingBuffer { + /// Creates a new unlimited `GrowingBuffer`. + #[inline] + pub fn new() -> Self { + Self::with_limit(0) + } + + /// Creates a new `GrowingBuffer` with the specified limit. + /// + /// # Note + /// + /// The actual amount of allocated memory might be larger than the specified limit, depends on + /// the allocator. + #[inline] + pub fn with_limit(limit: usize) -> Self { + GrowingBuffer { + buffer: Vec::new(), + populated: 0, + limit: limit, + used: Cell::new(0), + } + } +} + +impl ops::Deref for GrowingBuffer { + type Target = [I]; + + #[inline] + fn deref(&self) -> &[I] { + &self.buffer[self.used.get()..self.populated] + } +} + +impl ops::DerefMut for GrowingBuffer { + #[inline] + fn deref_mut(&mut self) -> &mut [I] { + &mut self.buffer[self.used.get()..self.populated] + } +} + +impl Buffer for GrowingBuffer { + #[inline] + fn fill>(&mut self, s: &mut S) -> io::Result { + s.read(&mut self.buffer[self.populated..]).map(|n| { + debug_assert!(self.populated + n <= self.buffer.len()); + + self.populated += n; + + n + }) + } + + #[inline] + fn request_space(&mut self, items: usize) { + // If we are over the limit, refuse + if self.limit != 0 && self.buffer.capacity() > self.limit { + return; + } + + if items + self.len() > self.buffer.capacity() { + // We do not have enough space for the new items, reallocate + self.buffer.reserve(items); + + let cap = self.buffer.capacity(); + + // TODO: Would it be better with a Default requirement on I? + // We set the length here to allow fill() to hand out a slice of uninitialized memory + // to be populated. + // NOTE: We cannot actually expose this memory to the parser since self.populated will + // be the upper limit for the deref to slice. + unsafe { + self.buffer.set_len(cap); + } + } + + // Only copy if we actually need to free the space + if self.buffer.len() - self.populated < items { + unsafe { + ptr::copy(self.buffer.as_ptr().offset(self.used.get() as isize), self.buffer.as_mut_ptr(), self.populated - self.used.get()); + } + + self.populated -= self.used.get(); + self.used.set(0); + } + } + + #[inline] + fn consume(&self, items: usize) { + debug_assert!(self.used.get() + items <= self.populated); + + self.used.set(self.used.get() + items) + } + + #[inline] + fn len(&self) -> usize { + self.populated - self.used.get() + } + + #[inline] + fn capacity(&self) -> usize { + self.buffer.len() + } +} + #[cfg(test)] mod test { use super::InputBuf; diff --git a/src/buffer/stateful.rs b/src/buffer/stateful.rs index b9b1af76..0f80cf09 100644 --- a/src/buffer/stateful.rs +++ b/src/buffer/stateful.rs @@ -92,7 +92,7 @@ impl> Source { while buffer.len() < request { match try!(buffer.fill(source)) { 0 => break, - n => read = read + n, + n => read += n, } } } diff --git a/src/lib.rs b/src/lib.rs index ceb857ee..e11eb6ed 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,7 +9,7 @@ //! For its current capabilities, you will find that Chomp performs consistently as well, if not //! better, than optimized C parsers, while being vastly more expressive. For an example that //! builds a performant HTTP parser out of smaller parsers, see -//! [http_parser.rs](examples/http_parser.rs). +//! [`http_parser.rs`](examples/http_parser.rs). //! //! # Example //! @@ -217,7 +217,7 @@ shadow_same, shadow_unrelated, single_match_else))] -#![cfg_attr(feature="clippy", allow(inline_always))] +#![cfg_attr(feature="clippy", allow(inline_always, many_single_char_names))] #[cfg(feature = "tendril")] extern crate tendril; diff --git a/src/parsers.rs b/src/parsers.rs index 50735f7c..1220d2f8 100644 --- a/src/parsers.rs +++ b/src/parsers.rs @@ -351,15 +351,13 @@ pub fn string>(mut i: I, s: &[T]) // TODO: There has to be some more efficient way here let b = i.consume_while(|c| { - if n >= len { + if n >= len || c != s[n] { false } - else if c == s[n] { + else { n += 1; true - } else { - false } }); @@ -460,8 +458,8 @@ mod error { where I: fmt::Debug { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self.0.as_ref() { - Some(ref c) => write!(f, "expected {:?}", *c), - None => write!(f, "unexpected"), + Some(c) => write!(f, "expected {:?}", *c), + None => write!(f, "unexpected"), } } } diff --git a/src/types/mod.rs b/src/types/mod.rs index d30f539d..90034c8e 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -295,7 +295,7 @@ impl<'a, I: Copy + PartialEq> Input for &'a [I] { #[inline] fn _mark(&self, _g: Guard) -> Self::Marker { - &self + self } #[inline] @@ -386,7 +386,7 @@ impl<'a> Input for &'a str { #[inline] fn _mark(&self, _g: Guard) -> Self::Marker { - &self + self } #[inline] From 42dd656cacf80de13c82766b1b14373aed682205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Thu, 18 Aug 2016 00:35:02 +0200 Subject: [PATCH 50/63] Updated readme and intro docs to use new syntax --- README.md | 14 +++++++------- src/lib.rs | 12 +++++------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 6579a70f..ecff757e 100644 --- a/README.md +++ b/README.md @@ -47,14 +47,14 @@ fn f() -> (u8, u8, u8) { A Chomp parser with a similar structure looks like this: ```rust -fn f(i: Input) -> U8Result<(u8, u8, u8)> { +fn f(i: I) -> SimpleResult { parse!{i; - let a = read_digit(); - let b = read_digit(); - string(b"missiles"); - ret (a, b, a + b); + let a = digit(); + let b = digit(); + string(b"missiles"); + ret (a, b, a + b) } -} +} ``` And to implement `read_digit` we can utilize the `map` function to manipulate any success value while preserving any error or incomplete state: @@ -68,7 +68,7 @@ fn read_digit() -> u8 { } // Chomp, error handling built in, and we make sure we only get a number: -fn read_digit(i: Input) -> U8Result { +fn read_digit(i: I) -> SimpleResult { satisfy(i, |c| b'0' <= c && c <= b'9').map(|c| c - b'0') } ``` diff --git a/src/lib.rs b/src/lib.rs index e11eb6ed..9e66c565 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -60,7 +60,7 @@ //! `token` parser, which matches a particular input. //! //! ```ignore -//! fn token(i: Input, t: I) -> SimpleResult {...} +//! fn token(i: I, t: I::Token) -> ParseResult> { ... } //! ``` //! //! Notice that the first argument is an `Input`, and the second argument is some `I`. @@ -81,8 +81,8 @@ //! A very useful parser is the `satisfy` parser: //! //! ```ignore -//! fn satisfy(i: Input, f: F) -> SimpleResult -//! where F: FnOnce(I) -> bool { ... } +//! fn satisfy(mut i: I, f: F) -> ParseResult> +//! where F: FnOnce(I::Token) -> bool { ... } //! ``` //! //! Besides the input state, satisfy's only parameter is a predicate function and will succeed only @@ -111,10 +111,8 @@ //! times on its input. //! //! ```ignore -//! pub fn count<'a, I, T, E, F, U>(i: Input<'a, I>, num: usize, p: F) -> ParseResult<'a, I, T, E> -//! where I: Copy, -//! U: 'a, -//! F: FnMut(Input<'a, I>) -> ParseResult<'a, I, U, E>, +//! pub fn count(i: I, num: usize, p: F) -> ParseResult +//! where F: FnMut(I) -> ParseResult, //! T: FromIterator { ... } //! ``` //! From ae9b9d0a3d0f1e1657140f92f418189b88470ece Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Thu, 18 Aug 2016 01:13:02 +0200 Subject: [PATCH 51/63] Updated changelog --- CHANGELOG.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddf0215d..920e5ffd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Added +- `buffer::InputBuf` which contains a slice and an incomplete flag, much as the old `Input` struct. + +- `Input` implementation for `&[T]` where `T: Copy + PartialEq`. + +- `Input` implementation for `&str`. + +- `types::Buffer` trait which is implemented for all buffers providing common logic to perform the + final parsing on a buffer without knowing the exact buffer implementation. + +- `types::U8Input` trait alias for `Input`. + +- `primitives::Primitives` trait providing access to the primitive methods of the `Input` trait. + + This is used for building fundamental parsers/combinators. + +- `ParseResult::inspect` allowing code to observe the success value. + - `chomp::Error` now includes a backtrace in `test` and `debug` build profiles thanks to the [debugtrace crate](https://github.com/m4rw3r/debugtrace_rs). Backtraces can also be activated permanently using the `backtrace` feature but this will incur the significant cost of allocating @@ -17,6 +34,27 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Changes +- **Backwards-incompatible:** `Input` is now a trait with associated types `Token` and `Buffer`. + + This removes all incomplete logic from the parsers themselves and moves it into the `InputBuf` + type. This `InputBuf` is used if a partial buffer is in memory. It also allows the parsers to + operate directly on slices or use more effective means of storing buffers depending on the + `Input` implementation. + + To upgrade you replace the previous concrete `Input` type with a generic, use its associated + type if required, and refer to the `Buffer` associated type to allow for zero-copy parsing:: + + -fn http_version(i: Input) -> U8Result<&[u8]>; + +fn http_version>(i: I) -> SimpleResult; + + The associated types can be restricted if requried: + + fn request(i: I) -> SimpleResult, Vec>)> + where I::Buffer: ::std::ops::Deref; + +- **Backwards-incompatible:** Moved types into a more logical module structure, prelude now + exists as a `prelude` module. + - **Backwards-incompatible:** `chomp::Error` is no longer an enum, this is to facillitate the support of backtraces while keeping code compatible between the different build profiles. @@ -27,6 +65,10 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Removed - `Input::new` +- `Input::incomplete` +- `buffer::IntoStream` +- `primitives::State` +- `primitives::InputClone` - `ParseResult::expect` - `ParseResult::unwrap` - `ParseResult::unwrap_err` From 21257edc21d7f5c37be40305c76ca3f434514203 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Thu, 18 Aug 2016 21:08:48 +0200 Subject: [PATCH 52/63] Implemented basic numbering wrapper for Input --- src/types/mod.rs | 21 ++++ src/types/numbering.rs | 235 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 256 insertions(+) create mode 100644 src/types/numbering.rs diff --git a/src/types/mod.rs b/src/types/mod.rs index 90034c8e..32706f34 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -1,5 +1,6 @@ //! Types which facillitates the chaining of parsers and their results. +pub mod numbering; #[cfg(feature = "tendril")] pub mod tendril; @@ -25,6 +26,12 @@ pub trait Buffer: PartialEq { fn fold(self, B, F) -> B where F: FnMut(B, Self::Token) -> B; + /// Runs the supplied function on a borrow of each token present in the buffer. Invoked in + /// order and once per token. + // Same reason for above for not returning an iterator. + fn iterate(&self, F) + where F: FnMut(Self::Token); + /// The number of tokens present in this buffer. fn len(&self) -> usize; @@ -51,6 +58,13 @@ impl<'a, I: Copy + PartialEq> Buffer for &'a [I] { (&self[..]).iter().cloned().fold(init, f) } + fn iterate(&self, mut f: F) + where F: FnMut(Self::Token) { + for c in (&self[..]).iter().cloned() { + f(c) + } + } + fn len(&self) -> usize { // Slice to reach inherent method to prevent infinite recursion (&self[..]).len() @@ -73,6 +87,13 @@ impl<'a> Buffer for &'a str { self.chars().fold(init, f) } + fn iterate(&self, mut f: F) + where F: FnMut(Self::Token) { + for c in self.chars() { + f(c) + } + } + fn len(&self) -> usize { self.chars().count() } diff --git a/src/types/numbering.rs b/src/types/numbering.rs new file mode 100644 index 00000000..3a429a0c --- /dev/null +++ b/src/types/numbering.rs @@ -0,0 +1,235 @@ +//! Module containing tools for working with position aware parsers. +//! +//! ``` +//! # #[macro_use] extern crate chomp; +//! use chomp::types::{Input, ParseResult}; +//! # fn run_parser(i: I, f: F) -> Result +//! # where F: FnOnce(I) -> ParseResult { +//! # use chomp::primitives::IntoInner; +//! # f(i).into_inner().1 +//! # } +//! # fn main() { +//! use chomp::types::numbering::{InputPosition, LineNumber, Numbering}; +//! use chomp::combinators::many; +//! use chomp::parsers::{Error, any, take_while1, string}; +//! +//! // Let's count some lines +//! let i = InputPosition::new(&b"test a\ntest b\n\ntest c\n"[..], LineNumber::new()); +//! +//! // We could use a concrete type P too if we want to restrict to a +//! // certain position-aware implementation +//! fn parser, P: Numbering>(i: InputPosition) +//! -> ParseResult, (char, P), Error> { +//! parse!{i; +//! string(b"test"); +//! take_while1(|c| c == b' ' || c == b'\t'); +//! let t_name = any(); +//! i -> { +//! // Obtain current parser position +//! let p = i.position(); +//! +//! i.ret((t_name as char, p)) +//! // We skip the take while below, because we want to determine the line right +//! // after reading the t_name +//! } <* take_while1(|c| c == b'\n'); +//! } +//! } +//! +//! let r = run_parser(i, |i| many(i, parser)); +//! +//! assert_eq!(r, Ok(vec![('a', LineNumber(0)), +//! ('b', LineNumber(1)), +//! // Note the two linebreaks in a row +//! ('c', LineNumber(3))])); +//! # } +//! ``` + +use primitives::Guard; +use types::{Buffer, Input}; + +/// Trait for managing some kind of numbering over the parsed data. +pub trait Numbering: Clone { + /// The token type accepted by the numbering. + type Token: Copy + PartialEq; + + /// Updates the numbering based on the contents of the buffer, adding it to the current + /// numbering. + fn update<'a, B>(&mut self, &'a B) + where B: Buffer; + + /// Adds the token to the numbering. + fn add(&mut self, Self::Token); +} + +/// Struct counting the number of newlines (`b'\n'`). +#[derive(Debug, PartialEq, Eq, Ord, PartialOrd, Hash)] +pub struct LineNumber( + /// The current line, zero-indexed. + pub u64, +); + +impl Clone for LineNumber { + fn clone(&self) -> Self { + LineNumber(self.0) + } +} + +impl LineNumber { + /// Creates a new line-number counter with zero. + pub fn new() -> Self { + LineNumber(0) + } +} + +impl Default for LineNumber { + fn default() -> Self { + LineNumber::new() + } +} + +impl Numbering for LineNumber { + type Token = u8; + + fn update<'a, B>(&mut self, b: &'a B) + where B: Buffer { + let mut n = 0; + + b.iterate(|c| if c == b'\n' { + n += 1; + }); + + self.0 += n + } + + fn add(&mut self, t: Self::Token) { + if t == b'\n' { + self.0 += 1 + } + } +} + +/// Wrapper around an `Input` implementation providing numbering support. +#[derive(Debug)] +pub struct InputPosition> { + input: I, + num: N, +} + +impl> InputPosition { + /// Creates a new input position instance. + pub fn new(i: I, n: N) -> Self { + InputPosition { + input: i, + num: n, + } + } + + /// Obtains the current position of the numbering. + pub fn position(&self) -> N { + self.num.clone() + } +} + +impl> Input for InputPosition { + type Token = I::Token; + type Marker = (N, I::Marker); + type Buffer = I::Buffer; + + #[inline] + fn _peek(&mut self, g: Guard) -> Option { + self.input._peek(g) + } + + #[inline] + fn _pop(&mut self, g: Guard) -> Option { + self.input._pop(g).map(|t| { + self.num.add(t); + + t + }) + } + + #[inline] + fn _consume(&mut self, g: Guard, n: usize) -> Option { + self.input._consume(g, n).map(|b| { + self.num.update(&b); + + b + }) + } + + #[inline] + fn _consume_while(&mut self, g: Guard, f: F) -> Self::Buffer + where F: FnMut(Self::Token) -> bool { + let b = self.input._consume_while(g, f); + + self.num.update(&b); + + b + } + + #[inline] + fn _consume_from(&mut self, g: Guard, m: Self::Marker) -> Self::Buffer { + // We have already counted to current position, no need to update + self.input._consume_from(g, m.1) + } + + #[inline] + fn _consume_remaining(&mut self, g: Guard) -> Self::Buffer { + let b = self.input._consume_remaining(g); + + self.num.update(&b); + + b + } + + #[inline] + fn _mark(&self, g: Guard) -> Self::Marker { + (self.num.clone(), self.input._mark(g)) + } + + #[inline] + fn _restore(self, g: Guard, m: Self::Marker) -> Self { + InputPosition { + input: self.input._restore(g, m.1), + num: m.0, + } + } +} + +#[cfg(test)] +mod test { + use types::{Input, ParseResult}; + use super::{InputPosition, LineNumber}; + use primitives::IntoInner; + + #[test] + fn basic_test() { + use combinators::many; + use parsers::{Error, any, take_while1, string}; + + let i = InputPosition::new(&b"test a\ntest b\n\ntest c\n"[..], Default::default()); + + fn parser>(i: InputPosition) + -> ParseResult, (char, LineNumber), Error> { + parse!{i; + string(b"test"); + take_while1(|c| c == b' ' || c == b'\t'); + let t_name = any(); + i -> { + let p = i.position(); + + i.ret((t_name as char, p)) + // We skip the take while below, because we want to determine the line right + // after reading the t_name + } <* take_while1(|c| c == b'\n'); + } + } + + assert_eq!(many(i, parser).into_inner().1, Ok(vec![ + ('a', LineNumber(0)), + ('b', LineNumber(1)), + // Note the two linebreaks in a row + ('c', LineNumber(3))])); + } +} From 5641525e187b1db97248ff2b50d138beec7089a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Fri, 19 Aug 2016 00:15:58 +0200 Subject: [PATCH 53/63] Implemented run_parser --- src/lib.rs | 1 + src/parse.rs | 9 ++++++++- src/types/numbering.rs | 10 +++------- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 9e66c565..8d27c588 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -238,6 +238,7 @@ pub mod types; pub use parse::parse_only; pub use parse::parse_only_str; +pub use parse::run_parser; /// Basic prelude. pub mod prelude { diff --git a/src/parse.rs b/src/parse.rs index a920c5a5..4eacea0a 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -1,9 +1,16 @@ -use types::ParseResult; +use types::{Input, ParseResult}; use primitives::{ IntoInner, Primitives, }; +/// Runs the supplied parser over the input. +pub fn run_parser(input: I, parser: F) -> (I, Result) + where I: Input, + F: FnOnce(I) -> ParseResult { + parser(input).into_inner() +} + /// Runs the given parser on the supplied finite input. /// /// ``` diff --git a/src/types/numbering.rs b/src/types/numbering.rs index 3a429a0c..053ffe9d 100644 --- a/src/types/numbering.rs +++ b/src/types/numbering.rs @@ -2,16 +2,12 @@ //! //! ``` //! # #[macro_use] extern crate chomp; -//! use chomp::types::{Input, ParseResult}; -//! # fn run_parser(i: I, f: F) -> Result -//! # where F: FnOnce(I) -> ParseResult { -//! # use chomp::primitives::IntoInner; -//! # f(i).into_inner().1 -//! # } //! # fn main() { +//! use chomp::types::{Input, ParseResult}; //! use chomp::types::numbering::{InputPosition, LineNumber, Numbering}; //! use chomp::combinators::many; //! use chomp::parsers::{Error, any, take_while1, string}; +//! use chomp::run_parser; //! //! // Let's count some lines //! let i = InputPosition::new(&b"test a\ntest b\n\ntest c\n"[..], LineNumber::new()); @@ -35,7 +31,7 @@ //! } //! } //! -//! let r = run_parser(i, |i| many(i, parser)); +//! let r = run_parser(i, |i| many(i, parser)).1; //! //! assert_eq!(r, Ok(vec![('a', LineNumber(0)), //! ('b', LineNumber(1)), From 2b5f519e64874f94511da15367d2d24d9bc0aae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Fri, 19 Aug 2016 00:27:43 +0200 Subject: [PATCH 54/63] Updated changelog --- CHANGELOG.md | 16 +++++++++++----- src/types/tendril.rs | 3 +++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 920e5ffd..cff972a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Added +- `prelude` module containing basic types, parsers and combinators. + + This is supposed to be the equivalent of Attoparsec's main package. + +- `run_parser` which executes a parser on any given `Input` type. + - `buffer::InputBuf` which contains a slice and an incomplete flag, much as the old `Input` struct. - `Input` implementation for `&[T]` where `T: Copy + PartialEq`. @@ -64,14 +70,14 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Removed -- `Input::new` - `Input::incomplete` -- `buffer::IntoStream` -- `primitives::State` -- `primitives::InputClone` +- `Input::new` - `ParseResult::expect` -- `ParseResult::unwrap` - `ParseResult::unwrap_err` +- `ParseResult::unwrap` +- `buffer::IntoStream` +- `primitives::InputClone` +- `primitives::State` ## [0.2.6] - 2016-07-07 diff --git a/src/types/tendril.rs b/src/types/tendril.rs index dc4ebe11..d3fb45c7 100644 --- a/src/types/tendril.rs +++ b/src/types/tendril.rs @@ -1,3 +1,6 @@ +//! Support for the tendril type, this is probably a bad idea since it is actually not a rope and +//! will probably cause excessive reallocations while parsing. + use std::mem; use tendril::ByteTendril; From f10131238e4142bf9c0d3c56f2e2f03b649f04bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Sun, 28 Aug 2016 23:20:09 +0200 Subject: [PATCH 55/63] Parsers: skip_while --- src/lib.rs | 3 ++- src/parsers.rs | 15 +++++++++++++++ src/primitives.rs | 15 +++++++++++++++ src/types/mod.rs | 8 ++++++++ 4 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 8d27c588..b990494c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -248,11 +248,12 @@ pub mod prelude { not_token, peek, peek_next, + run_scanner, satisfy, satisfy_with, scan, + skip_while, string, - run_scanner, take, take_remainder, take_till, diff --git a/src/parsers.rs b/src/parsers.rs index 1220d2f8..10f6f962 100644 --- a/src/parsers.rs +++ b/src/parsers.rs @@ -230,6 +230,21 @@ pub fn take_while1(mut i: I, f: F) -> SimpleResult } } +/// Skips over tokens in the input until `f` returns false. +/// +/// ``` +/// use chomp::prelude::{parse_only, skip_while}; +/// +/// assert_eq!(parse_only(|i| skip_while(i, |c| c == b'a'), &b"aaabc"[..]), Ok(())); +/// ``` +#[inline] +pub fn skip_while(mut i: I, f: F) -> SimpleResult + where F: FnMut(I::Token) -> bool { + i.skip_while(f); + + i.ret(()) +} + /// Matches all items until ``f`` returns true, all items to that point will be returned as a slice /// upon success. /// diff --git a/src/primitives.rs b/src/primitives.rs index 03e5dcb7..23879928 100644 --- a/src/primitives.rs +++ b/src/primitives.rs @@ -94,6 +94,21 @@ pub trait Primitives: Input { self._consume_remaining(Guard(())) } + /// Runs the closure `F` on the tokens *in order* until it returns false, all tokens up to that + /// token will be discarded from the current input. + /// + /// MUST never run the closure more than once on the exact same token. + /// + /// If the end of the input is reached, the whole input is discarded. + /// + /// Note: Default implementation uses `consume_while` and makes the assumption that it will + /// optimize away the resulting `Self::Buffer`. + #[inline(always)] + fn skip_while(&mut self, f: F) + where F: FnMut(Self::Token) -> bool { + self._skip_while(Guard(()), f) + } + /// Marks the current position to be able to backtrack to it using `restore`. #[inline(always)] fn mark(&self) -> Self::Marker { diff --git a/src/types/mod.rs b/src/types/mod.rs index 32706f34..ca333554 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -239,6 +239,14 @@ pub trait Input: Sized { #[doc(hidden)] fn _consume_remaining(&mut self, Guard) -> Self::Buffer; + /// **Primitive:** See `Primitives::skip_while` for documentation. + #[inline] + #[doc(hidden)] + fn _skip_while(&mut self, g: Guard, f: F) + where F: FnMut(Self::Token) -> bool { + self._consume_while(g, f); + } + /// **Primitive:** See `Primitives::mark` for documentation. #[inline] #[doc(hidden)] From bf5acc50bcc0ec371d08cbb5d29efa00a0b35445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Tue, 23 Aug 2016 00:48:32 +0200 Subject: [PATCH 56/63] Updated bitflags and conv --- Cargo.toml | 4 ++-- src/lib.rs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 058c1553..f7c5b291 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,8 +22,8 @@ include = [ ] [dependencies] -bitflags = "0.5.0" -conv = { version = "0.3", default-features = false, features = ["std"] } +bitflags = "0.7.0" +conv = { version = "0.3.3", default-features = false, features = ["std"] } debugtrace = { version = "0.1.0" } tendril = { version = "0.2.2", optional = true } diff --git a/src/lib.rs b/src/lib.rs index b990494c..b6f3b2da 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -200,7 +200,6 @@ #![warn(missing_docs, trivial_casts, - trivial_numeric_casts, unused_import_braces, unused_qualifications)] From a882292652b169f13130d4eeaf2d1659def791b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 31 Aug 2016 15:16:32 +0200 Subject: [PATCH 57/63] Updated minimum stable rust version to 1.6.0 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2c932076..1df1b81d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,8 @@ rust: - stable - beta - nightly - # Minimum supported version (bitflags fails on 1.1): - - 1.2.0 + # Minimum supported version (bitflags fails on 1.5 due to core module not being stable): + - 1.6.0 # necessary for `travis-cargo coveralls --no-sudo` addons: From 8bbbb83315390eb986a2817af604a1718f082c90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 31 Aug 2016 15:50:46 +0200 Subject: [PATCH 58/63] Increased minimum verison requirement due to bitflags crate --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1df1b81d..c835defa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,8 @@ rust: - stable - beta - nightly - # Minimum supported version (bitflags fails on 1.5 due to core module not being stable): - - 1.6.0 + # Minimum supported version (bitflags fails on 1.7 due to op_assign_traits not being stable): + - 1.8.0 # necessary for `travis-cargo coveralls --no-sudo` addons: From 5e4fbd15d6ae196dbccd2a2d8b2058870f63ae51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Wed, 31 Aug 2016 16:11:57 +0200 Subject: [PATCH 59/63] bench: Fixed compiler warning --- benches/http_bench.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/benches/http_bench.rs b/benches/http_bench.rs index 5918825f..e157990e 100644 --- a/benches/http_bench.rs +++ b/benches/http_bench.rs @@ -5,7 +5,6 @@ extern crate chomp; use test::Bencher; use chomp::prelude::*; -use chomp::buffer::InputBuf; #[derive(Debug)] struct Request { From 08704f3eec5b3b759715a43c131085b4b0fa590b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Thu, 1 Sep 2016 00:07:31 +0200 Subject: [PATCH 60/63] combinators::bounded::many_till: Fixed bug for exact number of iterations --- src/combinators/bounded.rs | 47 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/src/combinators/bounded.rs b/src/combinators/bounded.rs index 7d73f695..7de025c8 100644 --- a/src/combinators/bounded.rs +++ b/src/combinators/bounded.rs @@ -642,8 +642,20 @@ impl BoundedRange for usize { next(self) { pre { if self.data == 0 { - // Attempt to make a successful end - iter_till_end_test!(self); + // TODO: Remove the branches here (ie. take + unwrap) + let i = self.buf.take().expect("Iter.buf was None"); + + match (self.end)(i).into_inner() { + (b, Ok(_)) => { + self.buf = Some(b); + self.state = EndStateTill::EndSuccess; + }, + // Failed to end, restore and continue + (b, Err(e)) => { + self.buf = Some(b); + self.state = EndStateTill::Error(From::from(e)); + }, + } return None; } @@ -949,6 +961,37 @@ mod test { let r: ParseResult<_, Vec<_>, _> = many_till(&b"abac"[..], 2..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"c"[..], Err(Error::expected(b'b')))); } + #[test] + fn many_till_exact() { + let r: ParseResult<_, Vec<_>, _> = many_till(&b""[..] , 0, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a'))) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"a"[..] , 0, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'c'))) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ac"[..] , 0, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![])) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"aca"[..] , 0, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"a"[..], Ok(vec![])) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"acab"[..] , 0, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"ab"[..], Ok(vec![])) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ab"[..] , 0, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"b"[..], Err(Error::expected(b'c'))) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abac"[..] , 0, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"bac"[..], Err(Error::expected(b'c'))) ); + + let r: ParseResult<_, Vec<_>, _> = many_till(&b""[..] , 1, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a'))) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"a"[..] , 1, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'b'))) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ac"[..] , 1, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"c"[..], Err(Error::expected(b'b'))) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ab"[..] , 1, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a'))) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"aba"[..] , 1, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'c'))) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abab"[..] , 1, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"b"[..], Err(Error::expected(b'c'))) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abac"[..] , 1, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![&b"ab"[..]])) ); + + let r: ParseResult<_, Vec<_>, _> = many_till(&b""[..] , 2, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a'))) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"a"[..] , 2, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'b'))) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ac"[..] , 2, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"c"[..], Err(Error::expected(b'b'))) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ab"[..] , 2, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a'))) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"aba"[..] , 2, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'b'))) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abab"[..] , 2, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a'))) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abac"[..] , 2, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"c"[..], Err(Error::expected(b'b'))) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ababa"[..] , 2, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'c'))) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ababac"[..] , 2, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![&b"ab"[..], &b"ab"[..]])) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ababab"[..] , 2, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"b"[..], Err(Error::expected(b'c'))) ); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"abababac"[..], 2, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"bac"[..], Err(Error::expected(b'c'))) ); + } + #[test] fn skip_range_full() { let r = skip_many(&b""[..], .., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); From b23d182b1b8e9f50398a625a119d120d26fc5823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Thu, 1 Sep 2016 00:14:01 +0200 Subject: [PATCH 61/63] combinators::bounded: More tests for edge-cases --- src/combinators/bounded.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/combinators/bounded.rs b/src/combinators/bounded.rs index 7de025c8..4c739759 100644 --- a/src/combinators/bounded.rs +++ b/src/combinators/bounded.rs @@ -848,6 +848,11 @@ mod test { let r: ParseResult<_, Vec<_>, _> = many(&b""[..], 0..0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![]))); let r: ParseResult<_, Vec<_>, _> = many(&b"a"[..], 0..0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"a"[..], Ok(vec![]))); + let r: ParseResult<_, Vec<_>, _> = many(&b""[..], 2..2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many(&b"a"[..], 2..2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aa"[..], 2..2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![b'a', b'a']))); + let r: ParseResult<_, Vec<_>, _> = many(&b"aaa"[..], 2..2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"a"[..], Ok(vec![b'a', b'a']))); + let r: ParseResult<_, Vec<_>, _> = many(&b""[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); let r: ParseResult<_, Vec<_>, _> = many(&b"a"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); let r: ParseResult<_, Vec<_>, _> = many(&b"aa"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![b'a', b'a']))); @@ -959,6 +964,8 @@ mod test { let r: ParseResult<_, Vec<_>, _> = many_till(&b"ac"[..], 1..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"c"[..], Err(Error::expected(b'b')))); let r: ParseResult<_, Vec<_>, _> = many_till(&b"abac"[..], 1..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![&b"ab"[..]]))); let r: ParseResult<_, Vec<_>, _> = many_till(&b"abac"[..], 2..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b"c"[..], Err(Error::expected(b'b')))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ababac"[..], 2..3, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![&b"ab"[..], &b"ab"[..]]))); + let r: ParseResult<_, Vec<_>, _> = many_till(&b"ababac"[..], 2..2, |i| string(i, b"ab"), |i| string(i, b"ac")); assert_eq!(r.into_inner(), (&b""[..], Ok(vec![&b"ab"[..], &b"ab"[..]]))); } #[test] @@ -1006,6 +1013,7 @@ mod test { #[test] fn skip_range_to() { let r = skip_many(&b""[..], ..0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + let r = skip_many(&b"b"[..], ..0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"b"[..], Ok(()))); let r = skip_many(&b"a"[..], ..0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"a"[..], Ok(()))); let r = skip_many(&b""[..], ..3, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); @@ -1021,6 +1029,13 @@ mod test { #[test] fn skip_range_from() { + let r = skip_many(&b""[..], 0.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + let r = skip_many(&b"a"[..], 0.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + let r = skip_many(&b"aa"[..], 0.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + + let r = skip_many(&b""[..], 1.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r = skip_many(&b"a"[..], 0.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + let r = skip_many(&b""[..], 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); let r = skip_many(&b"a"[..], 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); let r = skip_many(&b"aa"[..], 2.., |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); @@ -1037,6 +1052,11 @@ mod test { let r = skip_many(&b""[..], 0..0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); let r = skip_many(&b"a"[..], 0..0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"a"[..], Ok(()))); + let r = skip_many(&b""[..], 2..2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r = skip_many(&b"a"[..], 2..2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r = skip_many(&b"aa"[..], 2..2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + let r = skip_many(&b"aaa"[..], 2..2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + let r = skip_many(&b""[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); let r = skip_many(&b"a"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); let r = skip_many(&b"aa"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); @@ -1052,6 +1072,10 @@ mod test { #[test] fn skip_exact() { + let r = skip_many(&b""[..], 0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + let r = skip_many(&b"a"[..], 0, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"a"[..], Ok(()))); + let r = skip_many(&b""[..], 1, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); + let r = skip_many(&b"a"[..], 1, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); let r = skip_many(&b""[..], 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); let r = skip_many(&b"a"[..], 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); let r = skip_many(&b"aa"[..], 2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); From de60bd71b5ab67207a13863a1b1eab4209554d04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Thu, 1 Sep 2016 00:30:11 +0200 Subject: [PATCH 62/63] combinators::bounded: Defined behavior for Range when start == end and fixed overflow errors --- src/combinators/bounded.rs | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/combinators/bounded.rs b/src/combinators/bounded.rs index 4c739759..18901b27 100644 --- a/src/combinators/bounded.rs +++ b/src/combinators/bounded.rs @@ -109,7 +109,7 @@ impl BoundedRange for Range { input: i, parser: f, // Range is closed on left side, open on right, ie. [self.start, self.end) - state: (usize, usize): (self.start, max(self.end, 1) - 1), + state: (usize, usize): (self.start, max(self.start, self.end.saturating_sub(1))), size_hint(self) { (self.data.0, Some(self.data.1)) @@ -122,8 +122,8 @@ impl BoundedRange for Range { } } on { - // TODO: Saturating sub? - self.data.0 = if self.data.0 == 0 { 0 } else { self.data.0 - 1 }; + self.data.0 = self.data.0.saturating_sub(1); + // Can't overflow unless we do not quit when self.data.1 == 0 self.data.1 -= 1; } } @@ -132,13 +132,13 @@ impl BoundedRange for Range { // Got all occurrences of the parser // First state or reached max => do not restore to mark since it is from last // iteration - (s, (0, 0), _, _) => s.ret(result), + (s, (0, 0), _, _) => s.ret(result), // Ok, last parser failed and we have reached minimum, we have iterated all. // Return remainder of buffer and the collected result - (s, (0, _), m, Some(_)) => s.restore(m).ret(result), + (s, (0, _), m, Some(_)) => s.restore(m).ret(result), // Did not reach minimum, propagate - (s, (_, _), _, Some(e)) => s.err(e), - (_, _, _, None) => unreachable!(), + (s, (_, _), _, Some(e)) => s.err(e), + (_, _, _, None) => unreachable!(), } } } @@ -150,7 +150,7 @@ impl BoundedRange for Range { assert!(self.start <= self.end); // Closed on left side, open on right - let (mut min, mut max) = (self.start, max(self.end, 1) - 1); + let (mut min, mut max) = (self.start, max(self.start, self.end.saturating_sub(1))); loop { if max == 0 { @@ -161,7 +161,8 @@ impl BoundedRange for Range { match f(i).into_inner() { (b, Ok(_)) => { - min = if min == 0 { 0 } else { min - 1 }; + min = min.saturating_sub(1); + // Can't overflow unless we do not quit when max == 0 max -= 1; i = b @@ -194,7 +195,7 @@ impl BoundedRange for Range { parser: p, end: end, // Range is closed on left side, open on right, ie. [self.start, self.end) - state: (usize, usize): (self.start, max(self.end, 1) - 1), + state: (usize, usize): (self.start, max(self.start, self.end.saturating_sub(1))), size_hint(self) { (self.data.0, Some(self.data.1)) @@ -230,7 +231,8 @@ impl BoundedRange for Range { } } on { - self.data.0 = if self.data.0 == 0 { 0 } else { self.data.0 - 1 }; + self.data.0 = self.data.0.saturating_sub(1); + // Can't overflow unless we do not quit when self.data.1 == 0 self.data.1 -= 1; } } @@ -267,7 +269,7 @@ impl BoundedRange for RangeFrom { next(self) { pre {} on { - self.data = if self.data == 0 { 0 } else { self.data - 1 }; + self.data = self.data.saturating_sub(1); } } @@ -292,7 +294,7 @@ impl BoundedRange for RangeFrom { match f(i).into_inner() { (b, Ok(_)) => { - min = if min == 0 { 0 } else { min - 1 }; + min = min.saturating_sub(1); i = b }, @@ -335,19 +337,19 @@ impl BoundedRange for RangeFrom { } } on { - self.data = if self.data == 0 { 0 } else { self.data - 1 }; + self.data = self.data.saturating_sub(1); } } => result : T { // Got all occurrences of the parser - (s, 0, EndStateTill::EndSuccess) => s.ret(result), + (s, 0, EndStateTill::EndSuccess) => s.ret(result), // Did not reach minimum or a failure, propagate (s, _, EndStateTill::Error(e)) => s.err(e), (_, _, EndStateTill::Incomplete) => unreachable!(), // We cannot reach this since we only run the end test once we have reached the // minimum number of matches - (_, _, EndStateTill::EndSuccess) => unreachable!() + (_, _, EndStateTill::EndSuccess) => unreachable!() } } } @@ -1055,7 +1057,7 @@ mod test { let r = skip_many(&b""[..], 2..2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); let r = skip_many(&b"a"[..], 2..2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); let r = skip_many(&b"aa"[..], 2..2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); - let r = skip_many(&b"aaa"[..], 2..2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Ok(()))); + let r = skip_many(&b"aaa"[..], 2..2, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b"a"[..], Ok(()))); let r = skip_many(&b""[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); let r = skip_many(&b"a"[..], 2..4, |i| token(i, b'a')); assert_eq!(r.into_inner(), (&b""[..], Err(Error::expected(b'a')))); From f037f09552498c63309a945e5879bdea1bfe2234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wernst=C3=A5l?= Date: Thu, 1 Sep 2016 00:30:49 +0200 Subject: [PATCH 63/63] Changelog: Added bugfixes --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cff972a9..d4e4e109 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,11 @@ This project adheres to [Semantic Versioning](http://semver.org/). - `primitives::InputClone` - `primitives::State` +## Bugfixes + +- `combinators::bounded` now have a defined behavior when a `Range` has `start == end`: They will parse exactly `start` times. This also fixed a few overflows and unreachable code being reachable. +- `combinators::bounded::many_till` got fixed for an overflow happening when `0: usize` was used to limit the number of iterations. + ## [0.2.6] - 2016-07-07 ### Bugfixes