diff --git a/src/fmt/cursor.rs b/src/fmt/cursor.rs index ec1c8c9..387f737 100644 --- a/src/fmt/cursor.rs +++ b/src/fmt/cursor.rs @@ -1,37 +1,30 @@ use crate::fmt; -use crate::lang::Lang; use crate::tokens::{Item, Kind}; /// Trait for peeking items. -pub(super) trait Parse -where - L: Lang, -{ +pub(super) trait Parse { type Output: ?Sized; /// Parse the given item into its output. - fn parse(item: &Item) -> fmt::Result<&Self::Output>; + fn parse(item: &Item) -> fmt::Result<&Self::Output>; /// Test if the peek matches the given item. - fn peek(item: &Item) -> bool; + fn peek(item: &Item) -> bool; } /// Peek for a literal. pub(super) struct Literal(()); -impl Parse for Literal -where - L: Lang, -{ +impl Parse for Literal { type Output = str; #[inline] - fn peek(item: &Item) -> bool { + fn peek(item: &Item) -> bool { matches!(item.kind, Kind::Literal(..)) } #[inline] - fn parse(item: &Item) -> fmt::Result<&Self::Output> { + fn parse(item: &Item) -> fmt::Result<&Self::Output> { match &item.kind { Kind::Literal(s) => Ok(s), _ => Err(core::fmt::Error), @@ -42,19 +35,16 @@ where /// Peek for an eval marker. pub(super) struct CloseEval(()); -impl Parse for CloseEval -where - L: Lang, -{ +impl Parse for CloseEval { type Output = (); #[inline] - fn peek(item: &Item) -> bool { + fn peek(item: &Item) -> bool { matches!(item.kind, Kind::CloseEval) } #[inline] - fn parse(item: &Item) -> fmt::Result<&Self::Output> { + fn parse(item: &Item) -> fmt::Result<&Self::Output> { match &item.kind { Kind::CloseEval => Ok(&()), _ => Err(core::fmt::Error), @@ -63,34 +53,35 @@ where } /// Parser helper. -pub(super) struct Cursor<'a, L> -where - L: Lang, -{ - items: &'a [(usize, Item)], +pub(super) struct Cursor<'a, T> { + lang: &'a [T], + items: &'a [Item], } -impl<'a, L> Cursor<'a, L> -where - L: Lang, -{ - pub(super) fn new(items: &'a [(usize, Item)]) -> Self { - Self { items } +impl<'a, T> Cursor<'a, T> { + /// Construct a new cursor. + pub(super) fn new(lang: &'a [T], items: &'a [Item]) -> Self { + Self { lang, items } + } + + /// Get a language item by index. + pub(super) fn lang(&self, index: usize) -> fmt::Result<&'a T> { + self.lang.get(index).ok_or(core::fmt::Error) } /// Get the next item. - pub(super) fn next(&mut self) -> Option<&Item> { + pub(super) fn next(&mut self) -> Option<&Item> { let (first, rest) = self.items.split_first()?; self.items = rest; - Some(&first.1) + Some(first) } #[inline] pub(super) fn peek

(&self) -> bool where - P: Parse, + P: Parse, { - if let Some((_, item)) = self.items.first() { + if let Some(item) = self.items.first() { P::peek(item) } else { false @@ -100,9 +91,9 @@ where #[inline] pub(super) fn peek1

(&self) -> bool where - P: Parse, + P: Parse, { - if let Some((_, item)) = self.items.get(1) { + if let Some(item) = self.items.get(1) { P::peek(item) } else { false @@ -112,7 +103,7 @@ where #[inline] pub(super) fn parse

(&mut self) -> fmt::Result<&P::Output> where - P: Parse, + P: Parse, { let item = self.next().ok_or(core::fmt::Error)?; P::parse(item) diff --git a/src/fmt/formatter.rs b/src/fmt/formatter.rs index 46ae4da..0b702e6 100644 --- a/src/fmt/formatter.rs +++ b/src/fmt/formatter.rs @@ -4,7 +4,7 @@ use alloc::string::String; use crate::fmt; use crate::fmt::config::{Config, Indentation}; -use crate::fmt::cursor; +use crate::fmt::cursor::{self, Cursor}; use crate::lang::Lang; use crate::tokens::{Item, Kind}; @@ -77,15 +77,16 @@ impl<'a> Formatter<'a> { /// Format the given stream of tokens. pub(crate) fn format_items( &mut self, - items: &[(usize, Item)], + lang: &[L::Item], + items: &[Item], config: &L::Config, format: &L::Format, ) -> fmt::Result<()> where L: Lang, { - let mut cursor = cursor::Cursor::new(items); - self.format_cursor(&mut cursor, config, format, false) + let mut cursor = Cursor::new(lang, items); + self.format_cursor::(&mut cursor, config, format, false) } /// Forcibly write a line ending, at the end of a file. @@ -142,7 +143,7 @@ impl<'a> Formatter<'a> { /// Internal function for formatting. fn format_cursor( &mut self, - cursor: &mut cursor::Cursor<'_, L>, + cursor: &mut Cursor<'_, L::Item>, config: &L::Config, format: &L::Format, end_on_close_quote: bool, @@ -164,10 +165,9 @@ impl<'a> Formatter<'a> { end_on_eval, } = head; - match &item.kind { - Kind::Register(..) => (), + match item.kind { Kind::Indentation(0) => (), - Kind::Literal(literal) => { + Kind::Literal(ref literal) => { if *in_quote { L::write_quoted(self, literal)?; } else { @@ -175,7 +175,7 @@ impl<'a> Formatter<'a> { } } Kind::OpenQuote(e) if !*in_quote => { - *has_eval = *e; + *has_eval = e; *in_quote = true; L::open_quote(self, config, format, *has_eval)?; } @@ -184,7 +184,7 @@ impl<'a> Formatter<'a> { // // Evaluating quotes are not supported. Kind::OpenQuote(false) if *in_quote => { - self.quoted_quote(cursor, &mut buf, config, format)?; + self.quoted_quote::(cursor, &mut buf, config, format)?; L::write_quoted(self, &buf)?; buf.clear(); } @@ -196,7 +196,7 @@ impl<'a> Formatter<'a> { L::close_quote(self, config, format, mem::take(has_eval))?; } Kind::Lang(lang) => { - lang.format(self, config, format)?; + cursor.lang(lang)?.format(self, config, format)?; } // whitespace below Kind::Push => { @@ -209,7 +209,7 @@ impl<'a> Formatter<'a> { self.space(); } Kind::Indentation(n) => { - self.indentation(*n); + self.indentation(n); } Kind::OpenEval if *in_quote => { if cursor.peek::() && cursor.peek1::() { @@ -251,7 +251,7 @@ impl<'a> Formatter<'a> { /// Support for evaluating an interior quote and returning it as a string. fn quoted_quote( &mut self, - cursor: &mut cursor::Cursor<'_, L>, + cursor: &mut Cursor<'_, L::Item>, buf: &mut String, config: &L::Config, format: &L::Format, @@ -264,7 +264,7 @@ impl<'a> Formatter<'a> { let mut w = FmtWriter::new(buf); let out = &mut Formatter::new(&mut w, self.config); L::open_quote(out, config, format, false)?; - out.format_cursor(cursor, config, format, true)?; + out.format_cursor::(cursor, config, format, true)?; L::close_quote(out, config, format, false)?; Ok(()) } diff --git a/src/lib.rs b/src/lib.rs index 0085fda..5a6b41c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -992,89 +992,57 @@ pub use self::tokens::Tokens; /// Private module used for macros. #[doc(hidden)] pub mod __priv { - use alloc::boxed::Box; - use crate::lang::Lang; use crate::tokens::{from_fn, FormatInto}; use crate::tokens::{Item, ItemStr}; #[inline] - pub const fn static_(string: &'static str) -> Item - where - L: Lang, - { + pub const fn static_(string: &'static str) -> Item { Item::static_(string) } #[inline] - pub const fn literal(string: ItemStr) -> Item - where - L: Lang, - { + pub const fn literal(string: ItemStr) -> Item { Item::literal(string) } #[inline] - pub const fn indentation(level: i16) -> Item - where - L: Lang, - { + pub const fn indentation(level: i16) -> Item { Item::indentation(level) } #[inline] - pub const fn push() -> Item - where - L: Lang, - { + pub const fn push() -> Item { Item::push() } #[inline] - pub const fn line() -> Item - where - L: Lang, - { + pub const fn line() -> Item { Item::line() } #[inline] - pub const fn space() -> Item - where - L: Lang, - { + pub const fn space() -> Item { Item::space() } #[inline] - pub const fn open_quote(is_interpolation: bool) -> Item - where - L: Lang, - { + pub const fn open_quote(is_interpolation: bool) -> Item { Item::open_quote(is_interpolation) } #[inline] - pub const fn close_quote() -> Item - where - L: Lang, - { + pub const fn close_quote() -> Item { Item::close_quote() } #[inline] - pub const fn open_eval() -> Item - where - L: Lang, - { + pub const fn open_eval() -> Item { Item::open_eval() } #[inline] - pub const fn close_eval() -> Item - where - L: Lang, - { + pub const fn close_eval() -> Item { Item::close_eval() } @@ -1090,7 +1058,7 @@ pub mod __priv { L: Lang, { from_fn(|t| { - t.lang_item(Box::new(item)); + t.lang_item(item); }) } @@ -1106,7 +1074,7 @@ pub mod __priv { L: Lang, { from_fn(|t| { - t.lang_item_register(Box::new(item)); + t.lang_item_register(item); }) } } diff --git a/src/macros.rs b/src/macros.rs index 9ccbccf..acc9245 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -96,8 +96,8 @@ /// /// assert_eq! { /// vec![ -/// "import default second", /// "import first", +/// "import default second", /// "", /// "first", /// "default:second" diff --git a/src/tokens/display.rs b/src/tokens/display.rs index e022d98..f3fa3eb 100644 --- a/src/tokens/display.rs +++ b/src/tokens/display.rs @@ -3,7 +3,7 @@ use core::fmt; use alloc::string::ToString; use crate::lang::Lang; -use crate::tokens::{FormatInto, Item}; +use crate::tokens::FormatInto; use crate::Tokens; /// Function to build a string literal. @@ -77,8 +77,6 @@ where { #[inline] fn format_into(self, tokens: &mut Tokens) { - tokens.item(Item::literal( - self.inner.to_string().into_boxed_str().into(), - )); + tokens.literal(self.inner.to_string()); } } diff --git a/src/tokens/format_into.rs b/src/tokens/format_into.rs index 3877180..822507d 100644 --- a/src/tokens/format_into.rs +++ b/src/tokens/format_into.rs @@ -6,7 +6,7 @@ use alloc::string::{String, ToString}; use alloc::vec::Vec; use crate::lang::Lang; -use crate::tokens::{Item, ItemStr, Tokens}; +use crate::tokens::{ItemStr, Tokens}; /// Trait for types that can be formatted in-place into a token stream. /// @@ -48,13 +48,28 @@ where fn format_into(self, tokens: &mut Tokens); } +/// Formatting a reference to a token stream is exactly the same as extending +/// the token stream with a copy of the stream being formatted. +/// +/// # Examples +/// +/// ``` +/// use genco::prelude::*; +/// +/// let a: Tokens = quote!(foo bar); +/// +/// let result = quote!($a baz); +/// +/// assert_eq!("foo bar baz", result.to_string()?); +/// # Ok::<_, genco::fmt::Error>(()) +/// ``` impl FormatInto for Tokens where L: Lang, { #[inline] fn format_into(self, tokens: &mut Self) { - tokens.extend(self); + tokens.extend_by_owned(self); } } @@ -80,7 +95,7 @@ where { #[inline] fn format_into(self, tokens: &mut Tokens) { - tokens.extend(self.iter().cloned()); + tokens.extend_by_ref(self); } } @@ -196,7 +211,7 @@ where { #[inline] fn format_into(self, tokens: &mut Tokens) { - tokens.item(Item::literal(ItemStr::from(self))); + tokens.literal(self); } } @@ -221,7 +236,7 @@ where { #[inline] fn format_into(self, tokens: &mut Tokens) { - tokens.item(Item::literal(ItemStr::from(self))); + tokens.literal(self); } } @@ -247,7 +262,7 @@ where { #[inline] fn format_into(self, tokens: &mut Tokens) { - tokens.item(Item::literal(ItemStr::from(self))); + tokens.literal(self); } } @@ -273,7 +288,7 @@ where { #[inline] fn format_into(self, tokens: &mut Tokens) { - tokens.item(Item::literal(ItemStr::from(self))); + tokens.literal(self); } } @@ -300,7 +315,7 @@ where { #[inline] fn format_into(self, tokens: &mut Tokens) { - tokens.item(Item::literal(ItemStr::from(self.clone()))); + tokens.literal(self.as_ref()); } } @@ -325,9 +340,9 @@ where #[inline] fn format_into(self, tokens: &mut Tokens) { if let Some(s) = self.as_str() { - tokens.item(Item::literal(ItemStr::static_(s))); + tokens.literal(ItemStr::static_(s)); } else { - tokens.item(Item::literal(ItemStr::from(self.to_string()))); + tokens.literal(self.to_string()); } } } @@ -385,7 +400,7 @@ where #[inline] fn format_into(self, tokens: &mut Tokens) { match self { - Cow::Borrowed(b) => tokens.item(Item::literal(ItemStr::from(b))), + Cow::Borrowed(b) => tokens.literal(b), Cow::Owned(o) => o.format_into(tokens), } } @@ -414,7 +429,7 @@ where #[inline] fn format_into(self, tokens: &mut Tokens) { match self { - Cow::Borrowed(b) => tokens.item(Item::literal(ItemStr::from(b))), + Cow::Borrowed(b) => tokens.literal(b), Cow::Owned(o) => o.format_into(tokens), } } diff --git a/src/tokens/item.rs b/src/tokens/item.rs index 0ad1545..ebaf742 100644 --- a/src/tokens/item.rs +++ b/src/tokens/item.rs @@ -1,26 +1,17 @@ //! A single element -use core::cmp::Ordering; use core::fmt; -use core::hash; -use core::mem; - -use alloc::boxed::Box; use crate::lang::Lang; use crate::tokens::{FormatInto, ItemStr, Tokens}; -pub(crate) enum Kind -where - L: Lang, -{ +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) enum Kind { /// A literal item. /// Is added as a raw string to the stream of tokens. Literal(ItemStr), /// A language-specific item. - Lang(Box), - /// A language-specific item that is not rendered. - Register(Box), + Lang(usize), /// Push a new line unless the current line is empty. Will be flushed on /// indentation changes. Push, @@ -46,21 +37,16 @@ where } /// A single item in a stream of tokens. +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] #[non_exhaustive] -pub struct Item -where - L: Lang, -{ - pub(crate) kind: Kind, +pub struct Item { + pub(crate) kind: Kind, } -impl Item -where - L: Lang, -{ +impl Item { /// Construct a new item based on a kind. #[inline] - pub(crate) const fn new(kind: Kind) -> Self { + pub(crate) const fn new(kind: Kind) -> Self { Self { kind } } @@ -72,8 +58,8 @@ where /// use genco::tokens::{Item, ItemStr}; /// use genco::lang::Rust; /// - /// let a = Item::::static_("hello"); - /// let b = Item::::literal("hello".into()); + /// let a = Item::static_("hello"); + /// let b = Item::literal("hello".into()); /// /// assert_eq!(a, b); /// ``` @@ -90,8 +76,8 @@ where /// use genco::tokens::Item; /// use genco::lang::Rust; /// - /// let a = Item::::push(); - /// let b = Item::::push(); + /// let a = Item::push(); + /// let b = Item::push(); /// /// assert_eq!(a, b); /// ``` @@ -108,8 +94,8 @@ where /// use genco::tokens::Item; /// use genco::lang::Rust; /// - /// let a = Item::::line(); - /// let b = Item::::line(); + /// let a = Item::line(); + /// let b = Item::line(); /// /// assert_eq!(a, b); /// ``` @@ -126,8 +112,8 @@ where /// use genco::tokens::Item; /// use genco::lang::Rust; /// - /// let a = Item::::space(); - /// let b = Item::::space(); + /// let a = Item::space(); + /// let b = Item::space(); /// /// assert_eq!(a, b); /// ``` @@ -156,8 +142,8 @@ where /// use genco::tokens::Item; /// use genco::lang::Rust; /// - /// let a = Item::::open_quote(true); - /// let b = Item::::open_quote(false); + /// let a = Item::open_quote(true); + /// let b = Item::open_quote(false); /// /// assert_eq!(a, a); /// assert_ne!(a, b); @@ -175,8 +161,8 @@ where /// use genco::tokens::Item; /// use genco::lang::Rust; /// - /// let a = Item::::close_quote(); - /// let b = Item::::close_quote(); + /// let a = Item::close_quote(); + /// let b = Item::close_quote(); /// /// assert_eq!(a, b); /// ``` @@ -205,8 +191,8 @@ where /// use genco::tokens::{Item, ItemStr}; /// use genco::lang::Rust; /// - /// let a = Item::::static_("hello"); - /// let b = Item::::literal("hello".into()); + /// let a = Item::static_("hello"); + /// let b = Item::literal("hello".into()); /// /// assert_eq!(a, b); /// ``` @@ -215,204 +201,17 @@ where Self::new(Kind::Literal(lit)) } - /// Construct a new language-specific item. - #[inline] - pub(crate) const fn lang(item: Box) -> Self { - Self::new(Kind::Lang(item)) - } - - /// Construct a new language-specific register item. - #[inline] - pub(crate) const fn register(item: Box) -> Self { - Self::new(Kind::Register(item)) - } -} - -impl fmt::Debug for Item -where - L: Lang, - L::Item: core::fmt::Debug, -{ - #[inline] - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match &self.kind { - Kind::Literal(lit) => f.debug_tuple("Literal").field(lit).finish(), - Kind::Lang(item) => f.debug_tuple("Lang").field(item).finish(), - Kind::Register(item) => f.debug_tuple("Register").field(item).finish(), - Kind::Push => write!(f, "Push"), - Kind::Line => write!(f, "Line"), - Kind::Space => write!(f, "Space"), - Kind::Indentation(n) => f.debug_tuple("Indentation").field(n).finish(), - Kind::OpenQuote(b) => f.debug_tuple("OpenQuote").field(b).finish(), - Kind::CloseQuote => write!(f, "CloseQuote"), - Kind::OpenEval => write!(f, "OpenEval"), - Kind::CloseEval => write!(f, "CloseEval"), - } - } -} - -impl Clone for Item -where - L: Lang, - L::Item: Clone, -{ - #[inline] - fn clone(&self) -> Self { - let kind = match &self.kind { - Kind::Literal(lit) => Kind::Literal(lit.clone()), - Kind::Lang(item) => Kind::Lang(item.clone()), - Kind::Register(item) => Kind::Register(item.clone()), - Kind::Push => Kind::Push, - Kind::Line => Kind::Line, - Kind::Space => Kind::Space, - Kind::Indentation(n) => Kind::Indentation(*n), - Kind::OpenQuote(b) => Kind::OpenQuote(*b), - Kind::CloseQuote => Kind::CloseQuote, - Kind::OpenEval => Kind::OpenEval, - Kind::CloseEval => Kind::CloseEval, - }; - - Self { kind } - } -} - -impl PartialEq> for Item -where - L: Lang, - U: Lang, - L::Item: PartialEq, -{ - #[inline] - fn eq(&self, other: &Item) -> bool { - match (&self.kind, &other.kind) { - (Kind::Literal(a), Kind::Literal(b)) => a == b, - (Kind::Lang(a), Kind::Lang(b)) => **a == **b, - (Kind::Register(a), Kind::Register(b)) => **a == **b, - (Kind::Push, Kind::Push) => true, - (Kind::Line, Kind::Line) => true, - (Kind::Space, Kind::Space) => true, - (Kind::Indentation(na), Kind::Indentation(nb)) => na == nb, - (Kind::OpenQuote(ba), Kind::OpenQuote(bb)) => ba == bb, - (Kind::CloseQuote, Kind::CloseQuote) => true, - (Kind::OpenEval, Kind::OpenEval) => true, - (Kind::CloseEval, Kind::CloseEval) => true, - _ => false, - } - } -} - -impl Eq for Item -where - L: Lang, - L::Item: Eq, -{ -} - -impl PartialOrd> for Item -where - L: Lang, - U: Lang, - L::Item: PartialOrd, -{ - #[inline] - fn partial_cmp(&self, other: &Item) -> Option { - match (&self.kind, &other.kind) { - (Kind::Literal(a), Kind::Literal(b)) => a.partial_cmp(b), - (Kind::Lang(a), Kind::Lang(b)) => a.as_ref().partial_cmp(b.as_ref()), - (Kind::Register(a), Kind::Register(b)) => a.as_ref().partial_cmp(b.as_ref()), - (Kind::Push, Kind::Push) => Some(Ordering::Equal), - (Kind::Line, Kind::Line) => Some(Ordering::Equal), - (Kind::Space, Kind::Space) => Some(Ordering::Equal), - (Kind::Indentation(na), Kind::Indentation(nb)) => na.partial_cmp(nb), - (Kind::OpenQuote(ba), Kind::OpenQuote(bb)) => ba.partial_cmp(bb), - (Kind::CloseQuote, Kind::CloseQuote) => Some(Ordering::Equal), - (Kind::OpenEval, Kind::OpenEval) => Some(Ordering::Equal), - (Kind::CloseEval, Kind::CloseEval) => Some(Ordering::Equal), - _ => None, - } - } -} - -impl Ord for Item -where - L: Lang, - L::Item: Ord, -{ + /// Construct a lang item with the given index. #[inline] - fn cmp(&self, other: &Item) -> Ordering { - // NB: This is here because it can't be derived due to the generic - // parameter `L` not implementing `Ord`. - match (&self.kind, &other.kind) { - (Kind::Literal(a), Kind::Literal(b)) => a.cmp(b), - (Kind::Lang(a), Kind::Lang(b)) => a.as_ref().cmp(b), - (Kind::Register(a), Kind::Register(b)) => a.as_ref().cmp(b.as_ref()), - (Kind::Push, Kind::Push) => Ordering::Equal, - (Kind::Line, Kind::Line) => Ordering::Equal, - (Kind::Space, Kind::Space) => Ordering::Equal, - (Kind::Indentation(na), Kind::Indentation(nb)) => na.cmp(nb), - (Kind::OpenQuote(ba), Kind::OpenQuote(bb)) => ba.cmp(bb), - (Kind::CloseQuote, Kind::CloseQuote) => Ordering::Equal, - (Kind::OpenEval, Kind::OpenEval) => Ordering::Equal, - (Kind::CloseEval, Kind::CloseEval) => Ordering::Equal, - (Kind::Literal(_), _) => Ordering::Less, - (_, Kind::Literal(_)) => Ordering::Greater, - (Kind::Lang(_), _) => Ordering::Less, - (_, Kind::Lang(_)) => Ordering::Greater, - (Kind::Register(_), _) => Ordering::Less, - (_, Kind::Register(_)) => Ordering::Greater, - (Kind::Push, _) => Ordering::Less, - (_, Kind::Push) => Ordering::Greater, - (Kind::Line, _) => Ordering::Less, - (_, Kind::Line) => Ordering::Greater, - (Kind::Space, _) => Ordering::Less, - (_, Kind::Space) => Ordering::Greater, - (Kind::Indentation(_), _) => Ordering::Less, - (_, Kind::Indentation(_)) => Ordering::Greater, - (Kind::OpenQuote(_), _) => Ordering::Less, - (_, Kind::OpenQuote(_)) => Ordering::Greater, - (Kind::CloseQuote, _) => Ordering::Less, - (_, Kind::CloseQuote) => Ordering::Greater, - (Kind::OpenEval, _) => Ordering::Less, - (_, Kind::OpenEval) => Ordering::Greater, - } + pub(crate) const fn lang(index: usize) -> Item { + Item::new(Kind::Lang(index)) } } -impl hash::Hash for Item -where - L: Lang, - L::Item: hash::Hash, -{ +impl fmt::Debug for Item { #[inline] - fn hash(&self, state: &mut H) - where - H: hash::Hasher, - { - mem::discriminant(self).hash(state); - - match &self.kind { - Kind::Literal(item_str) => { - item_str.hash(state); - } - Kind::Lang(item) => { - item.hash(state); - } - Kind::Register(item) => { - item.hash(state); - } - Kind::Push => {} - Kind::Line => {} - Kind::Space => {} - Kind::Indentation(n) => { - n.hash(state); - } - Kind::OpenQuote(n) => { - n.hash(state); - } - Kind::CloseQuote => {} - Kind::OpenEval => {} - Kind::CloseEval => {} - } + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.kind.fmt(f) } } @@ -439,15 +238,16 @@ where /// Item::literal("bar".into()), /// Item::space(), /// Item::literal(ItemStr::static_("baz")), -/// ] as Vec>, +/// ], /// result, /// }; /// # Ok::<_, genco::fmt::Error>(()) /// ``` -impl FormatInto for Item +impl FormatInto for Item where L: Lang, { + #[inline] fn format_into(self, tokens: &mut Tokens) { tokens.item(self); } diff --git a/src/tokens/quoted.rs b/src/tokens/quoted.rs index be7422c..5395d79 100644 --- a/src/tokens/quoted.rs +++ b/src/tokens/quoted.rs @@ -1,5 +1,5 @@ use crate::lang::Lang; -use crate::tokens::{FormatInto, Item, Tokens}; +use crate::tokens::{FormatInto, Tokens}; /// Function to provide string quoting. /// @@ -70,8 +70,8 @@ where { #[inline] fn format_into(self, t: &mut Tokens) { - t.item(Item::open_quote(false)); + t.open_quote(false); self.inner.format_into(t); - t.item(Item::close_quote()); + t.close_quote(); } } diff --git a/src/tokens/static_literal.rs b/src/tokens/static_literal.rs index 046f13d..2bd5fbb 100644 --- a/src/tokens/static_literal.rs +++ b/src/tokens/static_literal.rs @@ -1,5 +1,5 @@ use crate::lang::Lang; -use crate::tokens::{FormatInto, Item, ItemStr}; +use crate::tokens::{FormatInto, ItemStr}; /// A formatter from a static literal. /// @@ -15,7 +15,7 @@ where { #[inline] fn format_into(self, tokens: &mut crate::Tokens) { - tokens.item(Item::literal(ItemStr::static_(self.literal))); + tokens.literal(ItemStr::static_(self.literal)); } } diff --git a/src/tokens/tokens.rs b/src/tokens/tokens.rs index bd382e0..e0c3b26 100644 --- a/src/tokens/tokens.rs +++ b/src/tokens/tokens.rs @@ -12,17 +12,14 @@ use core::cmp::Ordering; use core::hash; -use core::iter::FromIterator; -use core::mem; use core::slice; -use alloc::boxed::Box; use alloc::string::String; -use alloc::vec::{self, Vec}; +use alloc::vec::Vec; use crate::fmt; use crate::lang::{Lang, LangSupportsEval}; -use crate::tokens::{FormatInto, Item, Kind, Register}; +use crate::tokens::{FormatInto, Item, ItemStr, Kind, Register}; /// A stream of tokens. /// @@ -74,15 +71,8 @@ pub struct Tokens where L: Lang, { - items: Vec<(usize, Item)>, - /// The last position at which we observed a language item. - /// - /// This references the `position + 1` in the items vector. A position of 0 - /// means that there are no more items. - /// - /// This makes up a singly-linked list over all language items that you can - /// follow. - last_lang_item: usize, + items: Vec, + lang: Vec, } impl Tokens @@ -103,7 +93,7 @@ where pub fn new() -> Self { Tokens { items: Vec::new(), - last_lang_item: 0, + lang: Vec::new(), } } @@ -121,7 +111,7 @@ where pub fn with_capacity(cap: usize) -> Self { Tokens { items: Vec::with_capacity(cap), - last_lang_item: 0, + lang: Vec::new(), } } @@ -136,16 +126,17 @@ where /// let tokens: Tokens<()> = quote!(foo bar baz); /// let mut it = tokens.iter(); /// - /// assert_eq!(Some(&Item::literal(ItemStr::static_("foo"))), it.next()); - /// assert_eq!(Some(&Item::space()), it.next()); - /// assert_eq!(Some(&Item::literal(ItemStr::static_("bar"))), it.next()); - /// assert_eq!(Some(&Item::space()), it.next()); - /// assert_eq!(Some(&Item::literal(ItemStr::static_("baz"))), it.next()); + /// assert_eq!(Item::literal(ItemStr::static_("foo")), it.next().unwrap()); + /// assert_eq!(Item::space(), it.next().unwrap()); + /// assert_eq!(Item::literal(ItemStr::static_("bar")), it.next().unwrap()); + /// assert_eq!(Item::space(), it.next().unwrap()); + /// assert_eq!(Item::literal(ItemStr::static_("baz")), it.next().unwrap()); /// assert_eq!(None, it.next()); /// ``` #[inline] - pub fn iter(&self) -> Iter<'_, L> { + pub fn iter(&self) -> Iter<'_, L::Item> { Iter { + lang: &self.lang[..], iter: self.items.iter(), } } @@ -179,36 +170,66 @@ where tokens.format_into(self) } + #[inline] + pub(crate) fn item(&mut self, item: Item) { + match item.kind { + Kind::Push => self.push(), + Kind::Line => self.line(), + Kind::Space => self.space(), + Kind::Indentation(n) => self.indentation(n), + Kind::Lang(..) => { /* ignored */ } + kind => self.items.push(Item::new(kind)), + } + } + /// Extend with another stream of tokens. /// /// This respects the structural requirements of adding one element at a - /// time, like you would get by calling [`space`], [`push`], or [`line`]. - /// - /// # Examples - /// - /// ``` - /// use genco::prelude::*; - /// use genco::tokens::{Item, ItemStr}; - /// - /// let mut tokens: Tokens<()> = quote!(foo bar); - /// tokens.extend::>(quote!($[' ']baz)); - /// - /// assert_eq!(tokens, quote!(foo bar baz)); - /// ``` - /// - /// [`space`]: Self::space - /// [`push`]: Self::push - /// [`line`]: Self::line - pub fn extend(&mut self, it: I) + /// time, like you would get by calling `space`, `push`, or `line`. + #[inline] + pub(crate) fn extend_by_ref(&mut self, other: &Tokens) where - I: IntoIterator>, + L::Item: Clone, { - let it = it.into_iter(); - let (low, high) = it.size_hint(); - self.items.reserve(high.unwrap_or(low)); + self.items.reserve(other.items.len()); + let base = self.lang.len(); + self.lang.extend(other.lang.iter().cloned()); + + for item in &other.items { + match &item.kind { + Kind::Push => self.push(), + Kind::Line => self.line(), + Kind::Space => self.space(), + Kind::Indentation(n) => self.indentation(*n), + Kind::Lang(lang) => { + self.items.push(Item::lang(base.saturating_add(*lang))); + } + kind => self.items.push(Item::new(kind.clone())), + } + } + } - for item in it { - self.item(item); + /// Extend with another stream of tokens. + /// + /// This respects the structural requirements of adding one element at a + /// time, like you would get by calling `space`, `push`, or `line`. + #[inline] + pub(crate) fn extend_by_owned(&mut self, mut other: Tokens) { + self.items.reserve(other.items.len()); + let base = self.lang.len(); + self.lang.append(&mut other.lang); + + for item in other.items { + match item.kind { + Kind::Push => self.push(), + Kind::Line => self.line(), + Kind::Space => self.space(), + Kind::Indentation(n) => self.indentation(n), + Kind::Lang(lang) => { + self.items.push(Item::lang(base.saturating_add(lang))); + } + kind => self.items.push(Item::new(kind)), + } } } @@ -231,10 +252,10 @@ where /// println!("{:?}", import); /// } /// ``` + #[inline] pub fn iter_lang(&self) -> IterLang<'_, L> { IterLang { - items: &self.items, - pos: self.last_lang_item, + lang: self.lang.iter(), } } @@ -306,11 +327,11 @@ where /// # Ok::<_, genco::fmt::Error>(()) /// ``` pub fn space(&mut self) { - if let Some((_, Item { kind: Kind::Space })) = self.items.last() { + if let Some(Item { kind: Kind::Space }) = self.items.last() { return; } - self.items.push((0, Item::space())); + self.items.push(Item::space()); } /// Add a single push operation. @@ -345,23 +366,23 @@ where /// ``` pub fn push(&mut self) { let item = loop { - let Some((o, item)) = self.items.pop() else { + let Some(item) = self.items.pop() else { break None; }; match &item.kind { // NB: never reconfigure a line into a push. Kind::Line => { - self.items.push((o, item)); + self.items.push(item); return; } Kind::Space | Kind::Push => continue, - _ => break Some((o, item)), + _ => break Some(item), } }; self.items.extend(item); - self.items.push((0, Item::push())); + self.items.push(Item::push()); } /// Add a single line operation. @@ -397,7 +418,7 @@ where /// ``` pub fn line(&mut self) { let item = loop { - let Some((o, item)) = self.items.pop() else { + let Some(item) = self.items.pop() else { break None; }; @@ -405,11 +426,11 @@ where continue; } - break Some((o, item)); + break Some(item); }; self.items.extend(item); - self.items.push((0, Item::line())); + self.items.push(Item::line()); } /// Increase the indentation of the token stream. @@ -556,51 +577,36 @@ where config: &L::Config, format: &L::Format, ) -> fmt::Result { - out.format_items(&self.items, config, format) + out.format_items::(&self.lang, &self.items, config, format) } - /// Push a single item to the stream while checking for structural - /// guarantees. - /// - /// # Examples - /// - /// ``` - /// use genco::prelude::*; - /// use genco::tokens::{Item, ItemStr}; - /// - /// let mut tokens = Tokens::<()>::new(); - /// - /// tokens.append(ItemStr::static_("foo")); - /// tokens.space(); - /// tokens.space(); // Note: second space ignored - /// tokens.append(ItemStr::static_("bar")); - /// - /// assert_eq!(tokens, quote!(foo bar)); - /// ``` - pub(crate) fn item(&mut self, item: Item) { - match item.kind { - Kind::Push => self.push(), - Kind::Line => self.line(), - Kind::Space => self.space(), - Kind::Indentation(n) => self.indentation(n), - Kind::Lang(item) => self.lang_item(item), - Kind::Register(item) => self.lang_item_register(item), - other => self.items.push((0, Item::new(other))), - } + /// Push a literal item to the stream. + #[inline] + pub(crate) fn literal(&mut self, lit: impl Into) { + self.items.push(Item::literal(lit.into())); + } + + /// Push an open quote item to the stream. + #[inline] + pub(crate) fn open_quote(&mut self, is_interpolated: bool) { + self.items.push(Item::open_quote(is_interpolated)); + } + + /// Push a close quote item to the stream. + #[inline] + pub(crate) fn close_quote(&mut self) { + self.items.push(Item::close_quote()); } /// Add a language item directly. - pub(crate) fn lang_item(&mut self, item: Box) { - // NB: recorded position needs to be adjusted. - self.items.push((self.last_lang_item, Item::lang(item))); - self.last_lang_item = self.items.len(); + pub(crate) fn lang_item(&mut self, item: L::Item) { + self.items.push(Item::lang(self.lang.len())); + self.lang.push(item); } /// Register a language item directly. - pub(crate) fn lang_item_register(&mut self, item: Box) { - // NB: recorded position needs to be adjusted. - self.items.push((self.last_lang_item, Item::register(item))); - self.last_lang_item = self.items.len(); + pub(crate) fn lang_item_register(&mut self, item: L::Item) { + self.lang.push(item); } /// File formatting function for token streams that gives full control over the @@ -651,7 +657,7 @@ where fn indentation(&mut self, mut n: i16) { let item = loop { // flush all whitespace preceeding the indentation change. - let Some((o, item)) = self.items.pop() else { + let Some(item) = self.items.pop() else { break None; }; @@ -660,14 +666,14 @@ where Kind::Space => continue, Kind::Line => continue, Kind::Indentation(u) => n += u, - _ => break Some((o, item)), + _ => break Some(item), } }; self.items.extend(item); if n != 0 { - self.items.push((0, Item::new(Kind::Indentation(n)))); + self.items.push(Item::new(Kind::Indentation(n))); } } } @@ -885,54 +891,49 @@ where } } -impl PartialEq>> for Tokens +impl PartialEq> for Tokens where L: Lang, - L::Item: PartialEq, { #[inline] - fn eq(&self, other: &Vec>) -> bool { - self == &other[..] + fn eq(&self, other: &Vec) -> bool { + self.iter().eq(other.iter()) } } -impl PartialEq> for Vec> +impl PartialEq> for Vec where L: Lang, - L::Item: PartialEq, { #[inline] fn eq(&self, other: &Tokens) -> bool { - other == &self[..] + self.iter().eq(other.iter()) } } -impl PartialEq<[Item]> for Tokens +impl PartialEq<[Item]> for Tokens where L: Lang, - L::Item: PartialEq, { #[inline] - fn eq(&self, other: &[Item]) -> bool { + fn eq(&self, other: &[Item]) -> bool { self.iter().eq(other) } } -impl PartialEq<[Item; N]> for Tokens +impl PartialEq<[Item; N]> for Tokens where L: Lang, - L::Item: PartialEq, { #[inline] - fn eq(&self, other: &[Item; N]) -> bool { + fn eq(&self, other: &[Item; N]) -> bool { self == &other[..] } } -impl PartialEq> for [Item] +impl PartialEq> for [Item] where L: Lang, - L::Item: PartialEq, { #[inline] fn eq(&self, other: &Tokens) -> bool { @@ -969,85 +970,85 @@ where } } -/// Iterator over [Tokens]. -/// -/// This is created using [Tokens::into_iter()]. -pub struct IntoIter -where - L: Lang, -{ - iter: vec::IntoIter<(usize, Item)>, +/// An item reference, where language items are translated into a reference to +/// them. +#[derive(PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ItemRef<'a, T> { + kind: ItemRefKind<'a, T>, } -impl Iterator for IntoIter +impl core::fmt::Debug for ItemRef<'_, T> where - L: Lang, + T: core::fmt::Debug, { - type Item = Item; + #[inline] + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + self.kind.fmt(f) + } +} +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +enum ItemRefKind<'a, T> { + Lang(&'a T), + Item(&'a Item), +} + +impl PartialEq for ItemRef<'_, T> { #[inline] - fn next(&mut self) -> Option { - Some(self.iter.next()?.1) + fn eq(&self, other: &Item) -> bool { + match self.kind { + ItemRefKind::Lang(..) => false, + ItemRefKind::Item(item) => *item == *other, + } } +} +impl PartialEq<&Item> for ItemRef<'_, T> { #[inline] - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() + fn eq(&self, other: &&Item) -> bool { + match self.kind { + ItemRefKind::Lang(..) => false, + ItemRefKind::Item(item) => *item == **other, + } } } -/// Construct an owned iterator over the token stream. -/// -/// # Examples -/// -/// ``` -/// use genco::prelude::*; -/// use genco::tokens::{ItemStr, Item}; -/// -/// let tokens: Tokens<()> = quote!(foo bar baz); -/// let mut it = tokens.into_iter(); -/// -/// assert_eq!(Some(Item::literal(ItemStr::static_("foo"))), it.next()); -/// assert_eq!(Some(Item::space()), it.next()); -/// assert_eq!(Some(Item::literal(ItemStr::static_("bar"))), it.next()); -/// assert_eq!(Some(Item::space()), it.next()); -/// assert_eq!(Some(Item::literal(ItemStr::static_("baz"))), it.next()); -/// assert_eq!(None, it.next()); -/// ``` -impl IntoIterator for Tokens -where - L: Lang, -{ - type Item = Item; - type IntoIter = IntoIter; +impl PartialEq> for Item { + #[inline] + fn eq(&self, other: &ItemRef<'_, T>) -> bool { + *other == *self + } +} +impl PartialEq> for &Item { #[inline] - fn into_iter(self) -> Self::IntoIter { - IntoIter { - iter: self.items.into_iter(), - } + fn eq(&self, other: &ItemRef<'_, T>) -> bool { + *other == **self } } -/// Iterator over [Tokens]. +/// Iterator over [`Tokens`]. /// -/// This is created using [Tokens::iter()]. -pub struct Iter<'a, L> -where - L: Lang, -{ - iter: slice::Iter<'a, (usize, Item)>, +/// This is created using [`Tokens::iter()`]. +pub struct Iter<'a, T> { + lang: &'a [T], + iter: slice::Iter<'a, Item>, } -impl<'a, L> Iterator for Iter<'a, L> -where - L: Lang, -{ - type Item = &'a Item; +impl<'a, T> Iterator for Iter<'a, T> { + type Item = ItemRef<'a, T>; #[inline] fn next(&mut self) -> Option { - Some(&self.iter.next()?.1) + let item = self.iter.next()?; + + let kind = if let Kind::Lang(n) = item.kind { + ItemRefKind::Lang(self.lang.get(n)?) + } else { + ItemRefKind::Item(item) + }; + + Some(ItemRef { kind }) } #[inline] @@ -1060,41 +1061,15 @@ impl<'a, L> IntoIterator for &'a Tokens where L: Lang, { - type Item = &'a Item; - type IntoIter = Iter<'a, L>; + type Item = ItemRef<'a, L::Item>; + type IntoIter = Iter<'a, L::Item>; + #[inline] fn into_iter(self) -> Self::IntoIter { self.iter() } } -impl<'a, L> FromIterator<&'a Item> for Tokens -where - L: Lang, - L::Item: Clone, -{ - fn from_iter>>(iter: I) -> Self { - let it = iter.into_iter(); - let (low, high) = it.size_hint(); - let mut tokens = Self::with_capacity(high.unwrap_or(low)); - tokens.extend(it.cloned()); - tokens - } -} - -impl FromIterator> for Tokens -where - L: Lang, -{ - fn from_iter>>(iter: I) -> Self { - let it = iter.into_iter(); - let (low, high) = it.size_hint(); - let mut tokens = Self::with_capacity(high.unwrap_or(low)); - tokens.extend(it); - tokens - } -} - impl core::fmt::Debug for Tokens where L: Lang, @@ -1115,7 +1090,7 @@ where fn clone(&self) -> Self { Self { items: self.items.clone(), - last_lang_item: self.last_lang_item, + lang: self.lang.clone(), } } } @@ -1131,7 +1106,7 @@ where H: hash::Hasher, { self.items.hash(state); - self.last_lang_item.hash(state); + self.lang.hash(state); } } @@ -1142,8 +1117,7 @@ pub struct IterLang<'a, L> where L: Lang, { - items: &'a [(usize, Item)], - pos: usize, + lang: slice::Iter<'a, L::Item>, } impl<'a, L> Iterator for IterLang<'a, L> @@ -1152,26 +1126,9 @@ where { type Item = &'a L::Item; + #[inline] fn next(&mut self) -> Option { - let pos = mem::take(&mut self.pos); - - if pos == 0 { - return None; - } - - // NB: recorded position needs to be adjusted. - match self.items.get(pos - 1)? { - ( - prev, - Item { - kind: Kind::Lang(item) | Kind::Register(item), - }, - ) => { - self.pos = *prev; - Some(item) - } - _ => None, - } + self.lang.next() } }