fix #716: lex numbers with thousands separators

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+### Formatting Changes and Bug Fixes
+
+- sqlfmt now correctly lexes number literals that contain a thousands separator (`_`), like `1_000_000` ([#716](https://github.com/tconbeer/sqlfmt/issues/716) - thank you [@hai-ld](https://github.com/hai-ld)!).
+
 ## [0.28.0] - 2025-10-17
 
 ### Formatting Changes and Bug Fixes

src/sqlfmt/rules/core.py

@@ -103,9 +103,11 @@
         name="number",
         priority=401,
         pattern=group(
-            r"0[xX][0-9a-fA-F]+",
-            r"(\+|-)?\d+(\.\d*)?(e(\+|-)?\d+)?(bd|d|f)?",
-            r"(\+|-)?\.\d+(e(\+|-)?\d+)?(bd|d|f)?",
+            r"0x[0-9a-f]+",
+            r"0o[0-7]+",
+            r"0b[0-1]+",
+            r"(\+|-)?\d+(_\d+)*(\.\d*(_\d+)*)?(e(\+|-)?\d+)?(bd|d|f)?",
+            r"(\+|-)?\.\d+(_\d+)*(e(\+|-)?\d+)?(bd|d|f)?",
         ),
         action=actions.handle_number,
     ),

tests/data/unformatted/125_numeric_constants.sql → tests/data/unformatted/125_numeric_literals.sql

@@ -1,4 +1,4 @@
-select 1, 1.0, +1, +1.0, -1, -1.0, 1e9, 1e+9, 1e-9, -1e9, -1e+9, +1e-9, -1e-9, 1.6e9, 1.6e-9, 1., 1.e9, 1.e-9, -1., -1.e-9, 0xFF
+select 1, 1.0, +1, +1.0, -1, -1.0, 1e9, 1e+9, 1e-9, -1e9, -1e+9, +1e-9, -1e-9, 1.6e9, 1.6e-9, 1., 1.e9, 1.e-9, -1., -1.e-9, 0xFF, 0b000111, 0O2112, 1_000_000, 3.141_592_6
 )))))__SQLFMT_OUTPUT__(((((
 select
     1,
@@ -21,4 +21,8 @@ select
     1.e-9,
     -1.,
     -1.e-9,
-    0xff
+    0xff,
+    0b000111,
+    0o2112,
+    1_000_000,
+    3.141_592_6

tests/functional_tests/test_general_formatting.py

@@ -48,7 +48,7 @@
         "unformatted/122_values.sql",
         "unformatted/123_spark_keywords.sql",
         "unformatted/124_bq_compound_types.sql",
-        "unformatted/125_numeric_constants.sql",
+        "unformatted/125_numeric_literals.sql",
         "unformatted/126_blank_lines.sql",
         "unformatted/127_more_comments.sql",
         "unformatted/128_double_slash_comments.sql",

tests/unit_tests/test_rule.py

@@ -76,6 +76,11 @@ def get_rule(ruleset: List[Rule], rule_name: str) -> Rule:
         (CORE, "number", "1e-9f"),
         (CORE, "number", "1.55e-9bd"),
         (CORE, "number", "0xFF"),
+        (CORE, "number", "0Xff"),
+        (CORE, "number", "0o1234"),
+        (CORE, "number", "0b010101010"),
+        (CORE, "number", "1_000_000"),
+        (CORE, "number", "3.1415_9265"),
         (CORE, "bracket_open", "["),
         (CORE, "bracket_close", ")"),
         (CORE, "double_colon", "::"),