Let the has_class() check also return the class

crates/math-core/src/character_class.rs

@@ -17,4 +17,6 @@ pub enum Class {
     Punctuation,
     /// `mathinner`
     Inner,
+    /// A class indicating the end of the current formula.
+    End,
 }

crates/math-core/src/parser.rs

@@ -223,10 +223,12 @@ where
     ) -> ParseResult<(Class, &'arena Node<'arena>)> {
         let (cur_token, span) = cur_tokloc?.into_parts();
         let mut class = Class::default();
-        let next_class = self
-            .tokens
-            .peek_class_token()?
-            .class(parse_as.in_sequence(), self.state.right_boundary_hack);
+        let next_class = self.tokens.peek_class_token(parse_as.in_sequence())?;
+        let next_class = if self.state.right_boundary_hack && matches!(next_class, Class::End) {
+            Class::Default
+        } else {
+            next_class
+        };
         let node: Result<Node, LatexError> = match cur_token {
             Token::Digit(number) => 'digit: {
                 if let Some(MathVariant::Transform(tf)) = self.state.transform {
@@ -334,7 +336,7 @@ where
             }
             Token::Punctuation(punc) => {
                 class = Class::Punctuation;
-                let right = if matches!(next_class, Class::Close) || self.state.script_style {
+                let right = if matches!(next_class, Class::End) || self.state.script_style {
                     Some(MathSpacing::Zero)
                 } else {
                     None
@@ -346,6 +348,20 @@ where
                     right,
                 })
             }
+            Token::ForcePunctuation(op) => {
+                class = Class::Punctuation;
+                let right = if matches!(next_class, Class::End) || self.state.script_style {
+                    Some(MathSpacing::Zero)
+                } else {
+                    Some(MathSpacing::ThreeMu)
+                };
+                Ok(Node::Operator {
+                    op,
+                    attrs: OpAttrs::empty(),
+                    left: Some(MathSpacing::Zero),
+                    right,
+                })
+            }
             Token::Ord(ord) => {
                 let attr = if matches!(ord.category(), OrdCategory::FGandForceDefault) {
                     // Category F+G operators will stretch in pre- and postfix positions,
@@ -430,10 +446,7 @@ where
                 };
                 class = Class::BinaryOp;
                 // Recompute the next class:
-                let next_class = self
-                    .tokens
-                    .peek_class_token()?
-                    .class(parse_as.in_sequence(), self.state.right_boundary_hack);
+                let next_class = self.tokens.peek_class_token(parse_as.in_sequence())?;
                 let spacing =
                     self.state
                         .bin_op_spacing(parse_as.in_sequence(), prev_class, next_class, true);
@@ -459,14 +472,16 @@ where
                 } else {
                     None
                 };
-                let right =
-                    if matches!(next_class, Class::Relation | Class::BinaryOp | Class::Close)
-                        || (self.state.script_style && !matches!(next_class, Class::Operator))
-                    {
-                        Some(MathSpacing::Zero)
-                    } else {
-                        None
-                    };
+                let right = if matches!(
+                    next_class,
+                    Class::Relation | Class::BinaryOp | Class::Close | Class::End
+                ) || (self.state.script_style
+                    && !matches!(next_class, Class::Operator))
+                {
+                    Some(MathSpacing::Zero)
+                } else {
+                    None
+                };
                 Ok(Node::Operator {
                     op: op.as_op(),
                     attrs: OpAttrs::empty(),
@@ -476,6 +491,7 @@ where
             }
             Token::OpGreaterThan => {
                 let (left, right) = self.state.relation_spacing(prev_class, next_class);
+                class = Class::Relation;
                 Ok(Node::PseudoOp {
                     name: "&gt;",
                     attrs: OpAttrs::empty(),
@@ -485,6 +501,7 @@ where
             }
             Token::OpLessThan => {
                 let (left, right) = self.state.relation_spacing(prev_class, next_class);
+                class = Class::Relation;
                 Ok(Node::PseudoOp {
                     name: "&lt;",
                     attrs: OpAttrs::empty(),
@@ -802,10 +819,7 @@ where
                 // `\not` has to be followed by something:
                 let (tok, new_span) = self.next_token()?.into_parts();
                 // Recompute the next class:
-                let next_class = self
-                    .tokens
-                    .peek_class_token()?
-                    .class(parse_as.in_sequence(), self.state.right_boundary_hack);
+                let next_class = self.tokens.peek_class_token(parse_as.in_sequence())?;
                 match tok {
                     Token::Relation(op) => {
                         let (left, right) = self.state.relation_spacing(prev_class, next_class);
@@ -1509,11 +1523,8 @@ where
         explicit: bool,
     ) -> ParseResult<(Option<MathSpacing>, Option<MathSpacing>)> {
         // We re-determine the next class here, because the next token may have changed
-        // because we discarded bounds or limits tokens.
-        let next_class = self
-            .tokens
-            .peek_class_token()?
-            .class(parse_as.in_sequence(), self.state.right_boundary_hack);
+        // because we discarded bounds tokens.
+        let next_class = self.tokens.peek_class_token(parse_as.in_sequence())?;
         Ok((
             if matches!(
                 prev_class,
@@ -1532,8 +1543,9 @@ where
             },
             if matches!(
                 next_class,
-                Class::Relation | Class::Punctuation | Class::Open | Class::Close
-            ) {
+                Class::Relation | Class::Punctuation | Class::Open | Class::Close | Class::End
+            ) || (self.state.script_style && matches!(next_class, Class::Inner))
+            {
                 Some(MathSpacing::Zero)
             } else if explicit {
                 Some(MathSpacing::ThreeMu)
@@ -1690,7 +1702,7 @@ impl ParserState<'_> {
             },
             if matches!(
                 next_class,
-                Class::Relation | Class::Punctuation | Class::Close
+                Class::Relation | Class::Punctuation | Class::Close | Class::End
             ) || self.script_style
             {
                 Some(MathSpacing::Zero)
@@ -1715,7 +1727,7 @@ impl ParserState<'_> {
             Class::Relation | Class::Punctuation | Class::BinaryOp | Class::Operator | Class::Open
         ) || matches!(
             next_class,
-            Class::Relation | Class::Punctuation | Class::Close
+            Class::Relation | Class::Punctuation | Class::Close | Class::End
         ) || self.script_style
         {
             Some(MathSpacing::Zero)

crates/math-core/src/predefined.rs

@@ -50,13 +50,13 @@ pub static XLEFTARROW: [Token<'static>; 7] = [
 
 pub static DOTS: [Token<'static>; 3] = [
     Inner(symbol::FULL_STOP),
-    Op(symbol::FULL_STOP),
+    ForcePunctuation(symbol::FULL_STOP.as_op()),
     Inner(symbol::FULL_STOP),
 ];
 
 pub static CDOTS: [Token<'static>; 3] = [
     Inner(symbol::MIDDLE_DOT),
-    Op(symbol::MIDDLE_DOT),
+    ForcePunctuation(symbol::MIDDLE_DOT.as_op()),
     Inner(symbol::MIDDLE_DOT),
 ];
 

crates/math-core/src/token.rs

@@ -110,6 +110,10 @@ pub enum Token<'source> {
     /// The character `&`.
     /// It has its own token because we need to escape it for the HTML output.
     OpAmpersand,
+    /// A token to force an operator to behave like a binary operator (mathbin).
+    /// This is, for example, needed for `×`, which in LaTeX is a binary operator,
+    /// but in MathML Core is a "big operator" (mathop).
+    ForceBinaryOp(MathMLOperator),
     /// A token to force an operator to behave like a relation (mathrel).
     /// This is, for example, needed for `:`, which in LaTeX is a relation,
     /// but in MathML Core is a separator (punctuation).
@@ -118,10 +122,8 @@ pub enum Token<'source> {
     /// This is, for example, needed for `!`, which in LaTeX is a closing symbol,
     /// but in MathML Core is an ordinary operator.
     ForceClose(MathMLOperator),
-    /// A token to force an operator to behave like a binary operator (mathbin).
-    /// This is, for example, needed for `×`, which in LaTeX is a binary operator,
-    /// but in MathML Core is a "big operator" (mathop).
-    ForceBinaryOp(MathMLOperator),
+    /// A token to force an operator to behave like punctuation (mathpunct).
+    ForcePunctuation(MathMLOperator),
     /// `\mathbin`
     Mathbin,
     /// A token to allow a relation to stretch.
@@ -192,35 +194,61 @@ static_assertions::assert_eq_size!(Result<Token<'_>, &'static i32>, [usize; 3]);
 
 impl Token<'_> {
     /// Returns the character class of this token.
-    pub(super) fn class(&self, in_sequence: bool, ignore_end_tokens: bool) -> Class {
-        if !in_sequence {
-            return Class::Default;
-        }
+    pub(super) fn class(&self) -> Option<Class> {
+        use Token::*;
         match self.unwrap_math_ref() {
-            Token::Relation(_) | Token::ForceRelation(_) => Class::Relation,
-            Token::Punctuation(_) => Class::Punctuation,
-            Token::Open(_) | Token::Left | Token::SquareBracketOpen => Class::Open,
-            Token::Close(_)
-            | Token::SquareBracketClose
-            | Token::NewColumn
-            | Token::ForceClose(_) => Class::Close,
-            Token::BinaryOp(_) | Token::ForceBinaryOp(_) | Token::Mathbin => Class::BinaryOp,
-            Token::Op(_) => Class::Operator,
-            Token::End(_) | Token::Right | Token::GroupEnd | Token::Eoi if !ignore_end_tokens => {
-                Class::Close
+            Relation(_) | ForceRelation(_) | OpGreaterThan | OpLessThan | StretchyRel(_) => {
+                Some(Class::Relation)
             }
-            Token::Inner(_) => Class::Inner,
-            // `\big` commands without the "l" or "r" really produce `Class::Default`.
-            Token::Big(_, Some(paren_type)) => {
-                if matches!(paren_type, ParenType::Open) {
-                    Class::Open
-                } else {
-                    Class::Close
-                }
+            Punctuation(_) | ForcePunctuation(_) => Some(Class::Punctuation),
+            Open(_) | Left | SquareBracketOpen | Begin(_) | GroupBegin => Some(Class::Open),
+            Close(_) | SquareBracketClose | ForceClose(_) | Right => Some(Class::Close),
+            BinaryOp(_) | ForceBinaryOp(_) | Mathbin => Some(Class::BinaryOp),
+            Op(_) | PseudoOperator(_) | PseudoOperatorLimits(_) | OperatorName(_) => {
+                Some(Class::Operator)
             }
-            // TODO: This needs to skip spaces and other non-class tokens in the token sequence.
-            Token::CustomCmd(_, [head, ..]) => head.class(in_sequence, ignore_end_tokens),
-            _ => Class::Default,
+            End(_)| NewColumn  | GroupEnd | Eoi => Some(Class::End),
+            Inner(_) => Some(Class::Inner),
+            Big(_, Some(paren_type)) => Some(if matches!(paren_type, ParenType::Open) {
+                Class::Open
+            } else {
+                Class::Close
+            }),
+            CustomCmd(_, toks) => toks.iter().find_map(|tok| tok.class()),
+            Whitespace | Space(_) | Not | TransformSwitch(_) | NoNumber | Tag | CustomSpace
+            | Limits | NonBreakingSpace => None,
+            Letter(_, _)
+            | UprightLetter(_)
+            | Digit(_)
+            // `\big` commands without the "l" or "r" really produce `Class::Default`.
+            | Big(_, None)
+            | NewLine
+            | Middle
+            | Frac(_)
+            | Genfrac
+            | Underscore
+            | Circumflex
+            | Binom(_)
+            | Overset
+            | Underset
+            | OverUnderBrace(_, _)
+            | Sqrt
+            | Transform(_)
+            | Ord(_)
+            | Prime
+            | Enclose(_)
+            | OpAmpersand
+            | Slashed
+            | Text(_)
+            | Style(_)
+            | Color
+            | CustomCmdArg(_)
+            | HardcodedMathML(_)
+            | TextMode(_)
+            | MathOrTextMode(_, _)
+            | UnknownCommand(_)
+            | InternalStringLiteral(_)
+            | Accent(_, _, _) => Some(Class::Default),
         }
     }
 
@@ -332,11 +360,6 @@ impl<'config> TokSpan<'config> {
     pub fn span(&self) -> Span {
         self.1
     }
-
-    #[inline]
-    pub(super) fn class(&self, in_sequence: bool, ignore_end_tokens: bool) -> Class {
-        self.0.class(in_sequence, ignore_end_tokens)
-    }
 }
 
 impl<'config> From<Token<'config>> for TokSpan<'config> {