Rework tokens

src/fmt/cursor.rs

@@ -1,37 +1,30 @@
 use crate::fmt;
-use crate::lang::Lang;
 use crate::tokens::{Item, Kind};
 
 /// Trait for peeking items.
-pub(super) trait Parse<L>
-where
-    L: Lang,
-{
+pub(super) trait Parse {
     type Output: ?Sized;
 
     /// Parse the given item into its output.
-    fn parse(item: &Item<L>) -> fmt::Result<&Self::Output>;
+    fn parse(item: &Item) -> fmt::Result<&Self::Output>;
 
     /// Test if the peek matches the given item.
-    fn peek(item: &Item<L>) -> bool;
+    fn peek(item: &Item) -> bool;
 }
 
 /// Peek for a literal.
 pub(super) struct Literal(());
 
-impl<L> Parse<L> for Literal
-where
-    L: Lang,
-{
+impl Parse for Literal {
     type Output = str;
 
     #[inline]
-    fn peek(item: &Item<L>) -> bool {
+    fn peek(item: &Item) -> bool {
         matches!(item.kind, Kind::Literal(..))
     }
 
     #[inline]
-    fn parse(item: &Item<L>) -> fmt::Result<&Self::Output> {
+    fn parse(item: &Item) -> fmt::Result<&Self::Output> {
         match &item.kind {
             Kind::Literal(s) => Ok(s),
             _ => Err(core::fmt::Error),
@@ -42,19 +35,16 @@ where
 /// Peek for an eval marker.
 pub(super) struct CloseEval(());
 
-impl<L> Parse<L> for CloseEval
-where
-    L: Lang,
-{
+impl Parse for CloseEval {
     type Output = ();
 
     #[inline]
-    fn peek(item: &Item<L>) -> bool {
+    fn peek(item: &Item) -> bool {
         matches!(item.kind, Kind::CloseEval)
     }
 
     #[inline]
-    fn parse(item: &Item<L>) -> fmt::Result<&Self::Output> {
+    fn parse(item: &Item) -> fmt::Result<&Self::Output> {
         match &item.kind {
             Kind::CloseEval => Ok(&()),
             _ => Err(core::fmt::Error),
@@ -63,34 +53,35 @@ where
 }
 
 /// Parser helper.
-pub(super) struct Cursor<'a, L>
-where
-    L: Lang,
-{
-    items: &'a [(usize, Item<L>)],
+pub(super) struct Cursor<'a, T> {
+    lang: &'a [T],
+    items: &'a [Item],
 }
 
-impl<'a, L> Cursor<'a, L>
-where
-    L: Lang,
-{
-    pub(super) fn new(items: &'a [(usize, Item<L>)]) -> Self {
-        Self { items }
+impl<'a, T> Cursor<'a, T> {
+    /// Construct a new cursor.
+    pub(super) fn new(lang: &'a [T], items: &'a [Item]) -> Self {
+        Self { lang, items }
+    }
+
+    /// Get a language item by index.
+    pub(super) fn lang(&self, index: usize) -> fmt::Result<&'a T> {
+        self.lang.get(index).ok_or(core::fmt::Error)
     }
 
     /// Get the next item.
-    pub(super) fn next(&mut self) -> Option<&Item<L>> {
+    pub(super) fn next(&mut self) -> Option<&Item> {
         let (first, rest) = self.items.split_first()?;
         self.items = rest;
-        Some(&first.1)
+        Some(first)
     }
 
     #[inline]
     pub(super) fn peek<P>(&self) -> bool
     where
-        P: Parse<L>,
+        P: Parse,
     {
-        if let Some((_, item)) = self.items.first() {
+        if let Some(item) = self.items.first() {
             P::peek(item)
         } else {
             false
@@ -100,9 +91,9 @@ where
     #[inline]
     pub(super) fn peek1<P>(&self) -> bool
     where
-        P: Parse<L>,
+        P: Parse,
     {
-        if let Some((_, item)) = self.items.get(1) {
+        if let Some(item) = self.items.get(1) {
             P::peek(item)
         } else {
             false
@@ -112,7 +103,7 @@ where
     #[inline]
     pub(super) fn parse<P>(&mut self) -> fmt::Result<&P::Output>
     where
-        P: Parse<L>,
+        P: Parse,
     {
         let item = self.next().ok_or(core::fmt::Error)?;
         P::parse(item)

src/fmt/formatter.rs

@@ -4,7 +4,7 @@ use alloc::string::String;
 
 use crate::fmt;
 use crate::fmt::config::{Config, Indentation};
-use crate::fmt::cursor;
+use crate::fmt::cursor::{self, Cursor};
 use crate::lang::Lang;
 use crate::tokens::{Item, Kind};
 
@@ -77,15 +77,16 @@ impl<'a> Formatter<'a> {
     /// Format the given stream of tokens.
     pub(crate) fn format_items<L>(
         &mut self,
-        items: &[(usize, Item<L>)],
+        lang: &[L::Item],
+        items: &[Item],
         config: &L::Config,
         format: &L::Format,
     ) -> fmt::Result<()>
     where
         L: Lang,
     {
-        let mut cursor = cursor::Cursor::new(items);
-        self.format_cursor(&mut cursor, config, format, false)
+        let mut cursor = Cursor::new(lang, items);
+        self.format_cursor::<L>(&mut cursor, config, format, false)
     }
 
     /// Forcibly write a line ending, at the end of a file.
@@ -142,7 +143,7 @@ impl<'a> Formatter<'a> {
     /// Internal function for formatting.
     fn format_cursor<L>(
         &mut self,
-        cursor: &mut cursor::Cursor<'_, L>,
+        cursor: &mut Cursor<'_, L::Item>,
         config: &L::Config,
         format: &L::Format,
         end_on_close_quote: bool,
@@ -164,18 +165,17 @@ impl<'a> Formatter<'a> {
                 end_on_eval,
             } = head;
 
-            match &item.kind {
-                Kind::Register(..) => (),
+            match item.kind {
                 Kind::Indentation(0) => (),
-                Kind::Literal(literal) => {
+                Kind::Literal(ref literal) => {
                     if *in_quote {
                         L::write_quoted(self, literal)?;
                     } else {
                         self.write_str(literal)?;
                     }
                 }
                 Kind::OpenQuote(e) if !*in_quote => {
-                    *has_eval = *e;
+                    *has_eval = e;
                     *in_quote = true;
                     L::open_quote(self, config, format, *has_eval)?;
                 }
@@ -184,7 +184,7 @@ impl<'a> Formatter<'a> {
                 //
                 // Evaluating quotes are not supported.
                 Kind::OpenQuote(false) if *in_quote => {
-                    self.quoted_quote(cursor, &mut buf, config, format)?;
+                    self.quoted_quote::<L>(cursor, &mut buf, config, format)?;
                     L::write_quoted(self, &buf)?;
                     buf.clear();
                 }
@@ -196,7 +196,7 @@ impl<'a> Formatter<'a> {
                     L::close_quote(self, config, format, mem::take(has_eval))?;
                 }
                 Kind::Lang(lang) => {
-                    lang.format(self, config, format)?;
+                    cursor.lang(lang)?.format(self, config, format)?;
                 }
                 // whitespace below
                 Kind::Push => {
@@ -209,7 +209,7 @@ impl<'a> Formatter<'a> {
                     self.space();
                 }
                 Kind::Indentation(n) => {
-                    self.indentation(*n);
+                    self.indentation(n);
                 }
                 Kind::OpenEval if *in_quote => {
                     if cursor.peek::<cursor::Literal>() && cursor.peek1::<cursor::CloseEval>() {
@@ -251,7 +251,7 @@ impl<'a> Formatter<'a> {
     /// Support for evaluating an interior quote and returning it as a string.
     fn quoted_quote<L>(
         &mut self,
-        cursor: &mut cursor::Cursor<'_, L>,
+        cursor: &mut Cursor<'_, L::Item>,
         buf: &mut String,
         config: &L::Config,
         format: &L::Format,
@@ -264,7 +264,7 @@ impl<'a> Formatter<'a> {
         let mut w = FmtWriter::new(buf);
         let out = &mut Formatter::new(&mut w, self.config);
         L::open_quote(out, config, format, false)?;
-        out.format_cursor(cursor, config, format, true)?;
+        out.format_cursor::<L>(cursor, config, format, true)?;
         L::close_quote(out, config, format, false)?;
         Ok(())
     }

src/lib.rs

@@ -992,89 +992,57 @@ pub use self::tokens::Tokens;
 /// Private module used for macros.
 #[doc(hidden)]
 pub mod __priv {
-    use alloc::boxed::Box;
-
     use crate::lang::Lang;
     use crate::tokens::{from_fn, FormatInto};
     use crate::tokens::{Item, ItemStr};
 
     #[inline]
-    pub const fn static_<L>(string: &'static str) -> Item<L>
-    where
-        L: Lang,
-    {
+    pub const fn static_(string: &'static str) -> Item {
         Item::static_(string)
     }
 
     #[inline]
-    pub const fn literal<L>(string: ItemStr) -> Item<L>
-    where
-        L: Lang,
-    {
+    pub const fn literal(string: ItemStr) -> Item {
         Item::literal(string)
     }
 
     #[inline]
-    pub const fn indentation<L>(level: i16) -> Item<L>
-    where
-        L: Lang,
-    {
+    pub const fn indentation(level: i16) -> Item {
         Item::indentation(level)
     }
 
     #[inline]
-    pub const fn push<L>() -> Item<L>
-    where
-        L: Lang,
-    {
+    pub const fn push() -> Item {
         Item::push()
     }
 
     #[inline]
-    pub const fn line<L>() -> Item<L>
-    where
-        L: Lang,
-    {
+    pub const fn line() -> Item {
         Item::line()
     }
 
     #[inline]
-    pub const fn space<L>() -> Item<L>
-    where
-        L: Lang,
-    {
+    pub const fn space() -> Item {
         Item::space()
     }
 
     #[inline]
-    pub const fn open_quote<L>(is_interpolation: bool) -> Item<L>
-    where
-        L: Lang,
-    {
+    pub const fn open_quote(is_interpolation: bool) -> Item {
         Item::open_quote(is_interpolation)
     }
 
     #[inline]
-    pub const fn close_quote<L>() -> Item<L>
-    where
-        L: Lang,
-    {
+    pub const fn close_quote() -> Item {
         Item::close_quote()
     }
 
     #[inline]
-    pub const fn open_eval<L>() -> Item<L>
-    where
-        L: Lang,
-    {
+    pub const fn open_eval() -> Item {
         Item::open_eval()
     }
 
     #[inline]
-    pub const fn close_eval<L>() -> Item<L>
-    where
-        L: Lang,
-    {
+    pub const fn close_eval() -> Item {
         Item::close_eval()
     }
 
@@ -1090,7 +1058,7 @@ pub mod __priv {
         L: Lang,
     {
         from_fn(|t| {
-            t.lang_item(Box::new(item));
+            t.lang_item(item);
         })
     }
 
@@ -1106,7 +1074,7 @@ pub mod __priv {
         L: Lang,
     {
         from_fn(|t| {
-            t.lang_item_register(Box::new(item));
+            t.lang_item_register(item);
         })
     }
 }

src/macros.rs

@@ -96,8 +96,8 @@
 ///
 /// assert_eq! {
 ///     vec![
-///         "import default second",
 ///         "import first",
+///         "import default second",
 ///         "",
 ///         "first",
 ///         "default:second"

src/tokens/display.rs

@@ -3,7 +3,7 @@ use core::fmt;
 use alloc::string::ToString;
 
 use crate::lang::Lang;
-use crate::tokens::{FormatInto, Item};
+use crate::tokens::FormatInto;
 use crate::Tokens;
 
 /// Function to build a string literal.
@@ -77,8 +77,6 @@ where
 {
     #[inline]
     fn format_into(self, tokens: &mut Tokens<L>) {
-        tokens.item(Item::literal(
-            self.inner.to_string().into_boxed_str().into(),
-        ));
+        tokens.literal(self.inner.to_string());
     }
 }