Add Information About Found Token in UnexpectedToken Error Type (#3299)

gleam-lang · Jun 30, 2024 · a2b12fa · a2b12fa
1 parent 77fe0de
commit a2b12fa
Show file tree

Hide file tree

Showing 22 changed files with 206 additions and 120 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -112,6 +112,24 @@
 - Import cycles now show the location where the import occur.
   ([Ameen Radwan](https://github.com/Acepie))
 
+- Error messages resulting from unexpected tokens now include information on
+  the found token's type. This updated message explains how the lexer handled
+  the token, so as to guide the user towards providing correct syntax.
+
+  Following is an example, where the error message indicates that the name of
+  the provided field conflicts with a keyword:
+
+  ```
+  3 │     A(type: String)
+    │       ^^^^ I was not expecting this
+
+  Found the keyword `type`, expected one of:
+  `)`
+  a constructor argument name
+  ```
+
+  ([Rahul D. Ghosal](https://github.com/rdghosal))
+
 ### Formatter
 
 ### Language Server

diff --git a/compiler-core/src/parse.rs b/compiler-core/src/parse.rs
@@ -241,11 +241,12 @@ where
         parse_result: Result<A, ParseError>,
     ) -> Result<A, ParseError> {
         let parse_result = self.ensure_no_errors(parse_result)?;
-        if let Some((start, _, end)) = self.next_tok() {
+        if let Some((start, token, end)) = self.next_tok() {
             // there are still more tokens
             let expected = vec!["An import, const, type, or function.".into()];
             return parse_error(
                 ParseErrorType::UnexpectedToken {
+                    token,
                     expected,
                     hint: None,
                 },
@@ -2476,8 +2477,9 @@ where
                             })),
                         }
                     }
-                    Some((start, _, end)) => parse_error(
+                    Some((start, token, end)) => parse_error(
                         ParseErrorType::UnexpectedToken {
+                            token,
                             expected: vec!["UpName".into(), "Name".into()],
                             hint: None,
                         },
@@ -2811,7 +2813,7 @@ where
                 Token::UpName { .. } => {
                     parse_error(ParseErrorType::IncorrectName, SrcSpan { start, end })
                 }
-                _ if is_reserved_word(tok) => parse_error(
+                _ if tok.is_reserved_word() => parse_error(
                     ParseErrorType::UnexpectedReservedWord,
                     SrcSpan { start, end },
                 ),
@@ -2968,8 +2970,9 @@ where
     fn next_tok_unexpected<A>(&mut self, expected: Vec<EcoString>) -> Result<A, ParseError> {
         match self.next_tok() {
             None => parse_error(ParseErrorType::UnexpectedEof, SrcSpan { start: 0, end: 0 }),
-            Some((start, _, end)) => parse_error(
+            Some((start, token, end)) => parse_error(
                 ParseErrorType::UnexpectedToken {
+                    token,
                     expected,
                     hint: None,
                 },
@@ -3541,90 +3544,6 @@ fn parse_error<T>(error: ParseErrorType, location: SrcSpan) -> Result<T, ParseEr
 // Misc Helpers
 //
 
-/// Returns whether the given token is a reserved word.
-///
-/// Useful for checking if a user tried to enter a reserved word as a name.
-fn is_reserved_word(tok: Token) -> bool {
-    match tok {
-        Token::As
-        | Token::Assert
-        | Token::Case
-        | Token::Const
-        | Token::Fn
-        | Token::If
-        | Token::Import
-        | Token::Let
-        | Token::Opaque
-        | Token::Pub
-        | Token::Todo
-        | Token::Type
-        | Token::Use
-        | Token::Auto
-        | Token::Delegate
-        | Token::Derive
-        | Token::Echo
-        | Token::Else
-        | Token::Implement
-        | Token::Macro
-        | Token::Panic
-        | Token::Test => true,
-
-        Token::Name { .. }
-        | Token::UpName { .. }
-        | Token::DiscardName { .. }
-        | Token::Int { .. }
-        | Token::Float { .. }
-        | Token::String { .. }
-        | Token::CommentDoc { .. }
-        | Token::LeftParen
-        | Token::RightParen
-        | Token::LeftSquare
-        | Token::RightSquare
-        | Token::LeftBrace
-        | Token::RightBrace
-        | Token::Plus
-        | Token::Minus
-        | Token::Star
-        | Token::Slash
-        | Token::Less
-        | Token::Greater
-        | Token::LessEqual
-        | Token::GreaterEqual
-        | Token::Percent
-        | Token::PlusDot
-        | Token::MinusDot
-        | Token::StarDot
-        | Token::SlashDot
-        | Token::LessDot
-        | Token::GreaterDot
-        | Token::LessEqualDot
-        | Token::GreaterEqualDot
-        | Token::LtGt
-        | Token::Colon
-        | Token::Comma
-        | Token::Hash
-        | Token::Bang
-        | Token::Equal
-        | Token::EqualEqual
-        | Token::NotEqual
-        | Token::Vbar
-        | Token::VbarVbar
-        | Token::AmperAmper
-        | Token::LtLt
-        | Token::GtGt
-        | Token::Pipe
-        | Token::Dot
-        | Token::RArrow
-        | Token::LArrow
-        | Token::DotDot
-        | Token::At
-        | Token::EndOfFile
-        | Token::CommentNormal
-        | Token::CommentModule
-        | Token::NewLine => false,
-    }
-}
-
 // Parsing a function call into the appropriate structure
 #[derive(Debug)]
 pub enum ParserArg {
@@ -3660,6 +3579,7 @@ pub fn make_call(
                 if name != "_" {
                     return parse_error(
                         ParseErrorType::UnexpectedToken {
+                            token: Token::Name { name },
                             expected: vec!["An expression".into(), "An underscore".into()],
                             hint: None,
                         },

diff --git a/compiler-core/src/parse/error.rs b/compiler-core/src/parse/error.rs
@@ -1,5 +1,6 @@
 use crate::ast::SrcSpan;
 use crate::error::wrap;
+use crate::parse::Token;
 use ecow::EcoString;
 use heck::{ToSnakeCase, ToUpperCamelCase};
 
@@ -187,8 +188,23 @@ utf16_codepoint, utf32_codepoint, signed, unsigned, big, little, native, size, u
                 "Argument labels are not allowed for anonymous functions",
                 vec!["Please remove the argument label.".into()],
             ),
-            ParseErrorType::UnexpectedToken { expected, hint } => {
-                let messages = std::iter::once("Expected one of: ".to_string())
+            ParseErrorType::UnexpectedToken {
+                token,
+                expected,
+                hint,
+            } => {
+                let found = match token {
+                    Token::Int { .. } => "an Int".to_string(),
+                    Token::Float { .. } => "a Float".to_string(),
+                    Token::String { .. } => "a String".to_string(),
+                    Token::CommentDoc { .. } => "a comment".to_string(),
+                    Token::DiscardName { .. } => "a discard name".to_string(),
+                    Token::Name { .. } | Token::UpName { .. } => "a name".to_string(),
+                    _ if token.is_reserved_word() => format!("the keyword {}", token),
+                    _ => token.to_string(),
+                };
+
+                let messages = std::iter::once(format!("Found {found}, expected one of: "))
                     .chain(expected.iter().map(|s| s.to_string()));
 
                 let messages = match hint {
@@ -290,6 +306,7 @@ pub enum ParseErrorType {
     UnexpectedEof,
     UnexpectedReservedWord, // reserved word used when a name was expected
     UnexpectedToken {
+        token: Token,
         expected: Vec<EcoString>,
         hint: Option<EcoString>,
     },

diff --git a/...src/parse/snapshots/gleam_core__parse__tests__assignment_pattern_invalid_bit_segment.snap b/...src/parse/snapshots/gleam_core__parse__tests__assignment_pattern_invalid_bit_segment.snap
@@ -8,6 +8,6 @@ error: Syntax error
 3 │     let <<b1, pub>> = <<24, 3>>
   │               ^^^ I was not expecting this
 
-Expected one of: 
-">>"
+Found the keyword `pub`, expected one of: 
+`>>`
 a bit array segment pattern
diff --git a/...-core/src/parse/snapshots/gleam_core__parse__tests__assignment_pattern_invalid_tuple.snap b/...-core/src/parse/snapshots/gleam_core__parse__tests__assignment_pattern_invalid_tuple.snap
@@ -8,6 +8,6 @@ error: Syntax error
 3 │     let #(a, case, c) = #(1, 2, 3)
   │              ^^^^ I was not expecting this
 
-Expected one of: 
-")"
+Found the keyword `case`, expected one of: 
+`)`
 a pattern
diff --git a/compiler-core/src/parse/snapshots/gleam_core__parse__tests__bit_array_invalid_segment.snap b/compiler-core/src/parse/snapshots/gleam_core__parse__tests__bit_array_invalid_segment.snap
@@ -8,6 +8,6 @@ error: Syntax error
 3 │     <<72, 101, 108, 108, 111, 44, 32, 74, 111, 101, const>>
   │                                                     ^^^^^ I was not expecting this
 
-Expected one of: 
-">>"
+Found the keyword `const`, expected one of: 
+`>>`
 a bit array segment
diff --git a/compiler-core/src/parse/snapshots/gleam_core__parse__tests__capture_with_name.snap b/compiler-core/src/parse/snapshots/gleam_core__parse__tests__capture_with_name.snap
@@ -8,6 +8,6 @@ error: Syntax error
 3 │   add(_name, 1)
   │       ^^^^^ I was not expecting this
 
-Expected one of: 
+Found a name, expected one of: 
 An expression
 An underscore
diff --git a/compiler-core/src/parse/snapshots/gleam_core__parse__tests__case_invalid_case_pattern.snap b/compiler-core/src/parse/snapshots/gleam_core__parse__tests__case_invalid_case_pattern.snap
@@ -8,6 +8,6 @@ error: Syntax error
 4 │         -> -> 0
   │         ^^ I was not expecting this
 
-Expected one of: 
-"}"
+Found `->`, expected one of: 
+`}`
 a case clause
diff --git a/compiler-core/src/parse/snapshots/gleam_core__parse__tests__case_invalid_expression.snap b/compiler-core/src/parse/snapshots/gleam_core__parse__tests__case_invalid_expression.snap
@@ -8,6 +8,6 @@ error: Syntax error
 3 │     case 1, type {
   │             ^^^^ I was not expecting this
 
-Expected one of: 
-"{"
+Found the keyword `type`, expected one of: 
+`{`
 an expression
diff --git a/...r-core/src/parse/snapshots/gleam_core__parse__tests__const_invalid_bit_array_segment.snap b/...r-core/src/parse/snapshots/gleam_core__parse__tests__const_invalid_bit_array_segment.snap
@@ -8,6 +8,6 @@ error: Syntax error
 2 │ const a = <<1, 2, <->>
   │                   ^^ I was not expecting this
 
-Expected one of: 
-">>"
+Found `<-`, expected one of: 
+`>>`
 a bit array segment
diff --git a/compiler-core/src/parse/snapshots/gleam_core__parse__tests__const_invalid_list.snap b/compiler-core/src/parse/snapshots/gleam_core__parse__tests__const_invalid_list.snap
@@ -8,6 +8,6 @@ error: Syntax error
 2 │ const a = [1, 2, <-]
   │                  ^^ I was not expecting this
 
-Expected one of: 
-"]"
+Found `<-`, expected one of: 
+`]`
 a constant value
diff --git a/...-core/src/parse/snapshots/gleam_core__parse__tests__const_invalid_record_constructor.snap b/...-core/src/parse/snapshots/gleam_core__parse__tests__const_invalid_record_constructor.snap
@@ -8,6 +8,6 @@ error: Syntax error
 5 │ const a = A("a", let)
   │                  ^^^ I was not expecting this
 
-Expected one of: 
-")"
+Found the keyword `let`, expected one of: 
+`)`
 a constant record argument
diff --git a/compiler-core/src/parse/snapshots/gleam_core__parse__tests__const_invalid_tuple.snap b/compiler-core/src/parse/snapshots/gleam_core__parse__tests__const_invalid_tuple.snap
@@ -8,6 +8,6 @@ error: Syntax error
 2 │ const a = #(1, 2, <-)
   │                   ^^ I was not expecting this
 
-Expected one of: 
-")"
+Found `<-`, expected one of: 
+`)`
 a constant value
diff --git a/compiler-core/src/parse/snapshots/gleam_core__parse__tests__function_invalid_signature.snap b/compiler-core/src/parse/snapshots/gleam_core__parse__tests__function_invalid_signature.snap
@@ -0,0 +1,13 @@
+---
+source: compiler-core/src/parse/tests.rs
+expression: "\nfn f(a, \"b\") -> String {\n    a <> b\n}\n"
+---
+error: Syntax error
+  ┌─ /src/parse/error.gleam:2:9
+  │
+2 │ fn f(a, "b") -> String {
+  │         ^^^ I was not expecting this
+
+Found a String, expected one of: 
+`)`
+a function parameter
diff --git a/...-core/src/parse/snapshots/gleam_core__parse__tests__function_type_invalid_param_type.snap b/...-core/src/parse/snapshots/gleam_core__parse__tests__function_type_invalid_param_type.snap
@@ -8,6 +8,6 @@ error: Syntax error
 2 │ fn f(g: fn(Int, 1) -> Int) -> Int {
   │                 ^ I was not expecting this
 
-Expected one of: 
-")"
+Found an Int, expected one of: 
+`)`
 a type
diff --git a/compiler-core/src/parse/snapshots/gleam_core__parse__tests__tuple_invalid_expr.snap b/compiler-core/src/parse/snapshots/gleam_core__parse__tests__tuple_invalid_expr.snap
@@ -8,6 +8,6 @@ error: Syntax error
 3 │     #(1, 2, const)
   │             ^^^^^ I was not expecting this
 
-Expected one of: 
-")"
+Found the keyword `const`, expected one of: 
+`)`
 an expression
diff --git a/compiler-core/src/parse/snapshots/gleam_core__parse__tests__type_invalid_constructor.snap b/compiler-core/src/parse/snapshots/gleam_core__parse__tests__type_invalid_constructor.snap
@@ -8,6 +8,6 @@ error: Syntax error
 4 │     type
   │     ^^^^ I was not expecting this
 
-Expected one of: 
-"}"
+Found the keyword `type`, expected one of: 
+`}`
 a record constructor
diff --git a/...iler-core/src/parse/snapshots/gleam_core__parse__tests__type_invalid_constructor_arg.snap b/...iler-core/src/parse/snapshots/gleam_core__parse__tests__type_invalid_constructor_arg.snap
@@ -8,6 +8,6 @@ error: Syntax error
 3 │     A(type: String)
   │       ^^^^ I was not expecting this
 
-Expected one of: 
-")"
+Found the keyword `type`, expected one of: 
+`)`
 a constructor argument name
diff --git a/compiler-core/src/parse/snapshots/gleam_core__parse__tests__type_invalid_record.snap b/compiler-core/src/parse/snapshots/gleam_core__parse__tests__type_invalid_record.snap
@@ -0,0 +1,13 @@
+---
+source: compiler-core/src/parse/tests.rs
+expression: "\ntype A {\n    One\n    Two\n    3\n}\n"
+---
+error: Syntax error
+  ┌─ /src/parse/error.gleam:5:5
+  │
+5 │     3
+  │     ^ I was not expecting this
+
+Found an Int, expected one of: 
+`}`
+a record constructor
diff --git a/compiler-core/src/parse/snapshots/gleam_core__parse__tests__type_invalid_type_name.snap b/compiler-core/src/parse/snapshots/gleam_core__parse__tests__type_invalid_type_name.snap
@@ -8,6 +8,6 @@ error: Syntax error
 2 │ type A(a, type) {
   │           ^^^^ I was not expecting this
 
-Expected one of: 
-")"
+Found the keyword `type`, expected one of: 
+`)`
 a name