Merge pull request #17 from iaguirre88/support_for_comments_and_semic…

…olon Support for comments and semicolon
san650 · Mar 29, 2019 · 823a192 · 823a192
2 parents 42432db + a244c16
commit 823a192
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 2 deletions.
diff --git a/src/sql_lexer.xrl b/src/sql_lexer.xrl
@@ -5,12 +5,13 @@ KEYWORDS    = [A-Z]+
 INT         = [0-9]+
 NAME        = [a-zA-Z0-9"_.]+
 WHITESPACE  = [\s\t\n\r]
-SEPARATOR   = ,
+SEPARATOR   = [,;]
 OPERATORS   = [*+=<>!']+
 %% STRING   = ".*"
 VARIABLE    = [$?][0-9]+
 PAREN_OPEN  = [([]
 PAREN_CLOSE = [)\]]
+COMMENT     = (-)(-)[\s].*
 
 Rules.
 
@@ -24,5 +25,6 @@ Rules.
 {OPERATORS}   : {token, {operator, TokenLine, TokenChars}}.
 %% {STRING}   : {token, {string, TokenLine, TokenChars}}.
 {VARIABLE}    : {token, {variable, TokenLine, TokenChars}}.
+{COMMENT}     : {token, {comment, TokenLine, TokenChars}}.
 
 Erlang code.
diff --git a/test/pretty_print_formatter/ecto/sql_tokenizer_test.exs b/test/pretty_print_formatter/ecto/sql_tokenizer_test.exs
@@ -96,7 +96,7 @@ defmodule PrettyPrintFormatter.Ecto.SqlTokenizerTest do
 
     test "sqlite parameters" do
       assert SqlTokenizer.tokenize(
-        "SELECT \"value\" FROM \"settings\" WHERE \"name\" = ?1 [\"hCard_note\"]"
+        "SELECT \"value\" FROM \"settings\" WHERE \"name\" = ?1 [\"hCard_note\"];"
       ) ==
         {:ok,
           [
@@ -111,7 +111,30 @@ defmodule PrettyPrintFormatter.Ecto.SqlTokenizerTest do
             {:paren_open},
             {:name, '"hCard_note"'},
             {:paren_close},
+            {:separator}
           ]}
     end
   end
+
+  test "statement with comments" do
+    assert SqlTokenizer.tokenize(
+      "SELECT \"value\" FROM \"settings\"; -- some comment
+      -- another comment; SELECT
+      SELECT \"value\" FROM \"settings\""
+    ) ==
+      {:ok,
+        [
+          {:keyword, 'SELECT'},
+          {:name, '"value"'},
+          {:keyword, 'FROM'},
+          {:name, '"settings"'},
+          {:separator},
+          {:comment, '-- some comment'},
+          {:comment, '-- another comment; SELECT'},
+          {:keyword, 'SELECT'},
+          {:name, '"value"'},
+          {:keyword, 'FROM'},
+          {:name, '"settings"'}
+        ]}
+  end
 end