Refer Antlr file using specific branch (#2893)

* Refer Antlr file using specific branch Signed-off-by: Tomoyuki Morita <[email protected]> * Fix version to current one Signed-off-by: Tomoyuki Morita <[email protected]> * Disable auto download Signed-off-by: Tomoyuki Morita <[email protected]> * Disable auto download Signed-off-by: Tomoyuki Morita <[email protected]> --------- Signed-off-by: Tomoyuki Morita <[email protected]>
opensearch-project · Oct 4, 2024 · d7710d0 · d7710d0
1 parent c1e623d
commit d7710d0
Show file tree

Hide file tree

Showing 4 changed files with 87 additions and 399 deletions.
diff --git a/async-query-core/README.md b/async-query-core/README.md
@@ -32,3 +32,11 @@ Following is the list of extension points where the consumer of the library need
   - [DataSourceSparkParameterComposer](src/main/java/org/opensearch/sql/spark/parameter/DataSourceSparkParameterComposer.java)
   - [GeneralSparkParameterComposer](src/main/java/org/opensearch/sql/spark/parameter/GeneralSparkParameterComposer.java)
   - [SparkSubmitParameterModifier](src/main/java/org/opensearch/sql/spark/config/SparkSubmitParameterModifier.java) To be deprecated in favor of GeneralSparkParameterComposer
+
+## Update Grammar files
+This package uses ANTLR grammar files from `opensearch-spark` and `Spark` repositories.
+To update the grammar files, update `build.gradle` file (in `downloadG4Files` task) as needed and run:
+```
+./gradlew async-query-core:downloadG4Files
+```
+This will overwrite the files under `src/main/antlr`.
diff --git a/async-query-core/build.gradle b/async-query-core/build.gradle
@@ -21,10 +21,12 @@ tasks.register('downloadG4Files', Exec) {
 
     executable 'curl'
 
-    args '-o', 'src/main/antlr/FlintSparkSqlExtensions.g4', 'https://raw.githubusercontent.com/opensearch-project/opensearch-spark/main/flint-spark-integration/src/main/antlr4/FlintSparkSqlExtensions.g4'
-    args '-o', 'src/main/antlr/SparkSqlBase.g4', 'https://raw.githubusercontent.com/opensearch-project/opensearch-spark/main/flint-spark-integration/src/main/antlr4/SparkSqlBase.g4'
-    args '-o', 'src/main/antlr/SqlBaseParser.g4', 'https://raw.githubusercontent.com/apache/spark/master/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4'
-    args '-o', 'src/main/antlr/SqlBaseLexer.g4', 'https://raw.githubusercontent.com/apache/spark/master/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4'
+    def opensearchSparkBranch = "0.5"
+    def apacheSparkVersionTag = "v3.5.1"
+    args '-o', 'src/main/antlr/FlintSparkSqlExtensions.g4', "https://raw.githubusercontent.com/opensearch-project/opensearch-spark/${opensearchSparkBranch}/flint-spark-integration/src/main/antlr4/FlintSparkSqlExtensions.g4"
+    args '-o', 'src/main/antlr/SparkSqlBase.g4', "https://raw.githubusercontent.com/opensearch-project/opensearch-spark/${opensearchSparkBranch}/flint-spark-integration/src/main/antlr4/SparkSqlBase.g4"
+    args '-o', 'src/main/antlr/SqlBaseParser.g4', "https://raw.githubusercontent.com/apache/spark/${apacheSparkVersionTag}/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4"
+    args '-o', 'src/main/antlr/SqlBaseLexer.g4', "https://raw.githubusercontent.com/apache/spark/${apacheSparkVersionTag}/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4"
 }
 
 generateGrammarSource {
@@ -38,12 +40,6 @@ configurations {
     }
 }
 
-// skip download in case of offline build
-if (!gradle.startParameter.offline) {
-    // Make sure the downloadG4File task runs before the generateGrammarSource task
-    generateGrammarSource.dependsOn downloadG4Files
-}
-
 dependencies {
     antlr "org.antlr:antlr4:4.7.1"
 

diff --git a/async-query-core/src/main/antlr/SqlBaseLexer.g4 b/async-query-core/src/main/antlr/SqlBaseLexer.g4
@@ -69,35 +69,6 @@ lexer grammar SqlBaseLexer;
   public void markUnclosedComment() {
     has_unclosed_bracketed_comment = true;
   }
-
-  /**
-   * When greater than zero, it's in the middle of parsing ARRAY/MAP/STRUCT type.
-   */
-  public int complex_type_level_counter = 0;
-
-  /**
-   * Increase the counter by one when hits KEYWORD 'ARRAY', 'MAP', 'STRUCT'.
-   */
-  public void incComplexTypeLevelCounter() {
-    complex_type_level_counter++;
-  }
-
-  /**
-   * Decrease the counter by one when hits close tag '>' && the counter greater than zero
-   * which means we are in the middle of complex type parsing. Otherwise, it's a dangling
-   * GT token and we do nothing.
-   */
-  public void decComplexTypeLevelCounter() {
-    if (complex_type_level_counter > 0) complex_type_level_counter--;
-  }
-
-  /**
-   * If the counter is zero, it's a shift right operator. It can be closing tags of an complex
-   * type definition, such as MAP<INT, ARRAY<INT>>.
-   */
-  public boolean isShiftRightOperator() {
-    return complex_type_level_counter == 0 ? true : false;
-  }
 }
 
 SEMICOLON: ';';
@@ -108,7 +79,6 @@ COMMA: ',';
 DOT: '.';
 LEFT_BRACKET: '[';
 RIGHT_BRACKET: ']';
-BANG: '!';
 
 // NOTE: If you add a new token in the list below, you should update the list of keywords
 // and reserved tag in `docs/sql-ref-ansi-compliance.md#sql-keywords`, and
@@ -129,24 +99,21 @@ ANTI: 'ANTI';
 ANY: 'ANY';
 ANY_VALUE: 'ANY_VALUE';
 ARCHIVE: 'ARCHIVE';
-ARRAY: 'ARRAY' {incComplexTypeLevelCounter();};
+ARRAY: 'ARRAY';
 AS: 'AS';
 ASC: 'ASC';
 AT: 'AT';
 AUTHORIZATION: 'AUTHORIZATION';
-BEGIN: 'BEGIN';
 BETWEEN: 'BETWEEN';
 BIGINT: 'BIGINT';
 BINARY: 'BINARY';
-BINDING: 'BINDING';
 BOOLEAN: 'BOOLEAN';
 BOTH: 'BOTH';
 BUCKET: 'BUCKET';
 BUCKETS: 'BUCKETS';
 BY: 'BY';
 BYTE: 'BYTE';
 CACHE: 'CACHE';
-CALLED: 'CALLED';
 CASCADE: 'CASCADE';
 CASE: 'CASE';
 CAST: 'CAST';
@@ -161,19 +128,16 @@ CLUSTER: 'CLUSTER';
 CLUSTERED: 'CLUSTERED';
 CODEGEN: 'CODEGEN';
 COLLATE: 'COLLATE';
-COLLATION: 'COLLATION';
 COLLECTION: 'COLLECTION';
 COLUMN: 'COLUMN';
 COLUMNS: 'COLUMNS';
 COMMENT: 'COMMENT';
 COMMIT: 'COMMIT';
 COMPACT: 'COMPACT';
 COMPACTIONS: 'COMPACTIONS';
-COMPENSATION: 'COMPENSATION';
 COMPUTE: 'COMPUTE';
 CONCATENATE: 'CONCATENATE';
 CONSTRAINT: 'CONSTRAINT';
-CONTAINS: 'CONTAINS';
 COST: 'COST';
 CREATE: 'CREATE';
 CROSS: 'CROSS';
@@ -197,29 +161,24 @@ DATE_DIFF: 'DATE_DIFF';
 DBPROPERTIES: 'DBPROPERTIES';
 DEC: 'DEC';
 DECIMAL: 'DECIMAL';
-DECLARE: 'DECLARE';
 DEFAULT: 'DEFAULT';
 DEFINED: 'DEFINED';
-DEFINER: 'DEFINER';
 DELETE: 'DELETE';
 DELIMITED: 'DELIMITED';
 DESC: 'DESC';
 DESCRIBE: 'DESCRIBE';
-DETERMINISTIC: 'DETERMINISTIC';
 DFS: 'DFS';
 DIRECTORIES: 'DIRECTORIES';
 DIRECTORY: 'DIRECTORY';
 DISTINCT: 'DISTINCT';
 DISTRIBUTE: 'DISTRIBUTE';
 DIV: 'DIV';
-DO: 'DO';
 DOUBLE: 'DOUBLE';
 DROP: 'DROP';
 ELSE: 'ELSE';
 END: 'END';
 ESCAPE: 'ESCAPE';
 ESCAPED: 'ESCAPED';
-EVOLUTION: 'EVOLUTION';
 EXCEPT: 'EXCEPT';
 EXCHANGE: 'EXCHANGE';
 EXCLUDE: 'EXCLUDE';
@@ -257,28 +216,24 @@ HOURS: 'HOURS';
 IDENTIFIER_KW: 'IDENTIFIER';
 IF: 'IF';
 IGNORE: 'IGNORE';
-IMMEDIATE: 'IMMEDIATE';
 IMPORT: 'IMPORT';
 IN: 'IN';
 INCLUDE: 'INCLUDE';
 INDEX: 'INDEX';
 INDEXES: 'INDEXES';
 INNER: 'INNER';
 INPATH: 'INPATH';
-INPUT: 'INPUT';
 INPUTFORMAT: 'INPUTFORMAT';
 INSERT: 'INSERT';
 INTERSECT: 'INTERSECT';
 INTERVAL: 'INTERVAL';
 INT: 'INT';
 INTEGER: 'INTEGER';
 INTO: 'INTO';
-INVOKER: 'INVOKER';
 IS: 'IS';
 ITEMS: 'ITEMS';
 JOIN: 'JOIN';
 KEYS: 'KEYS';
-LANGUAGE: 'LANGUAGE';
 LAST: 'LAST';
 LATERAL: 'LATERAL';
 LAZY: 'LAZY';
@@ -297,7 +252,7 @@ LOCKS: 'LOCKS';
 LOGICAL: 'LOGICAL';
 LONG: 'LONG';
 MACRO: 'MACRO';
-MAP: 'MAP' {incComplexTypeLevelCounter();};
+MAP: 'MAP';
 MATCHED: 'MATCHED';
 MERGE: 'MERGE';
 MICROSECOND: 'MICROSECOND';
@@ -306,7 +261,6 @@ MILLISECOND: 'MILLISECOND';
 MILLISECONDS: 'MILLISECONDS';
 MINUTE: 'MINUTE';
 MINUTES: 'MINUTES';
-MODIFIES: 'MODIFIES';
 MONTH: 'MONTH';
 MONTHS: 'MONTHS';
 MSCK: 'MSCK';
@@ -317,8 +271,7 @@ NANOSECOND: 'NANOSECOND';
 NANOSECONDS: 'NANOSECONDS';
 NATURAL: 'NATURAL';
 NO: 'NO';
-NONE: 'NONE';
-NOT: 'NOT';
+NOT: 'NOT' | '!';
 NULL: 'NULL';
 NULLS: 'NULLS';
 NUMERIC: 'NUMERIC';
@@ -340,6 +293,8 @@ OVERWRITE: 'OVERWRITE';
 PARTITION: 'PARTITION';
 PARTITIONED: 'PARTITIONED';
 PARTITIONS: 'PARTITIONS';
+PERCENTILE_CONT: 'PERCENTILE_CONT';
+PERCENTILE_DISC: 'PERCENTILE_DISC';
 PERCENTLIT: 'PERCENT';
 PIVOT: 'PIVOT';
 PLACING: 'PLACING';
@@ -352,7 +307,6 @@ PURGE: 'PURGE';
 QUARTER: 'QUARTER';
 QUERY: 'QUERY';
 RANGE: 'RANGE';
-READS: 'READS';
 REAL: 'REAL';
 RECORDREADER: 'RECORDREADER';
 RECORDWRITER: 'RECORDWRITER';
@@ -367,8 +321,6 @@ REPLACE: 'REPLACE';
 RESET: 'RESET';
 RESPECT: 'RESPECT';
 RESTRICT: 'RESTRICT';
-RETURN: 'RETURN';
-RETURNS: 'RETURNS';
 REVOKE: 'REVOKE';
 RIGHT: 'RIGHT';
 RLIKE: 'RLIKE' | 'REGEXP';
@@ -382,7 +334,6 @@ SECOND: 'SECOND';
 SECONDS: 'SECONDS';
 SCHEMA: 'SCHEMA';
 SCHEMAS: 'SCHEMAS';
-SECURITY: 'SECURITY';
 SELECT: 'SELECT';
 SEMI: 'SEMI';
 SEPARATED: 'SEPARATED';
@@ -394,21 +345,18 @@ SETMINUS: 'MINUS';
 SETS: 'SETS';
 SHORT: 'SHORT';
 SHOW: 'SHOW';
-SINGLE: 'SINGLE';
 SKEWED: 'SKEWED';
 SMALLINT: 'SMALLINT';
 SOME: 'SOME';
 SORT: 'SORT';
 SORTED: 'SORTED';
 SOURCE: 'SOURCE';
-SPECIFIC: 'SPECIFIC';
-SQL: 'SQL';
 START: 'START';
 STATISTICS: 'STATISTICS';
 STORED: 'STORED';
 STRATIFY: 'STRATIFY';
 STRING: 'STRING';
-STRUCT: 'STRUCT' {incComplexTypeLevelCounter();};
+STRUCT: 'STRUCT';
 SUBSTR: 'SUBSTR';
 SUBSTRING: 'SUBSTRING';
 SYNC: 'SYNC';
@@ -423,15 +371,13 @@ TEMPORARY: 'TEMPORARY' | 'TEMP';
 TERMINATED: 'TERMINATED';
 THEN: 'THEN';
 TIME: 'TIME';
-TIMEDIFF: 'TIMEDIFF';
 TIMESTAMP: 'TIMESTAMP';
 TIMESTAMP_LTZ: 'TIMESTAMP_LTZ';
 TIMESTAMP_NTZ: 'TIMESTAMP_NTZ';
 TIMESTAMPADD: 'TIMESTAMPADD';
 TIMESTAMPDIFF: 'TIMESTAMPDIFF';
 TINYINT: 'TINYINT';
 TO: 'TO';
-EXECUTE: 'EXECUTE';
 TOUCH: 'TOUCH';
 TRAILING: 'TRAILING';
 TRANSACTION: 'TRANSACTION';
@@ -457,9 +403,6 @@ USER: 'USER';
 USING: 'USING';
 VALUES: 'VALUES';
 VARCHAR: 'VARCHAR';
-VAR: 'VAR';
-VARIABLE: 'VARIABLE';
-VARIANT: 'VARIANT';
 VERSION: 'VERSION';
 VIEW: 'VIEW';
 VIEWS: 'VIEWS';
@@ -468,7 +411,6 @@ WEEK: 'WEEK';
 WEEKS: 'WEEKS';
 WHEN: 'WHEN';
 WHERE: 'WHERE';
-WHILE: 'WHILE';
 WINDOW: 'WINDOW';
 WITH: 'WITH';
 WITHIN: 'WITHIN';
@@ -486,11 +428,8 @@ NEQ : '<>';
 NEQJ: '!=';
 LT  : '<';
 LTE : '<=' | '!>';
-GT  : '>' {decComplexTypeLevelCounter();};
+GT  : '>';
 GTE : '>=' | '!<';
-SHIFT_LEFT: '<<';
-SHIFT_RIGHT: '>>' {isShiftRightOperator()}?;
-SHIFT_RIGHT_UNSIGNED: '>>>' {isShiftRightOperator()}?;
 
 PLUS: '+';
 MINUS: '-';
@@ -503,7 +442,6 @@ PIPE: '|';
 CONCAT_PIPE: '||';
 HAT: '^';
 COLON: ':';
-DOUBLE_COLON: '::';
 ARROW: '->';
 FAT_ARROW : '=>';
 HENT_START: '/*+';
@@ -563,13 +501,8 @@ BIGDECIMAL_LITERAL
     | DECIMAL_DIGITS EXPONENT? 'BD' {isValidDecimal()}?
     ;
 
-// Generalize the identifier to give a sensible INVALID_IDENTIFIER error message:
-// * Unicode letters rather than a-z and A-Z only
-// * URI paths for table references using paths
-// We then narrow down to ANSI rules in exitUnquotedIdentifier() in the parser.
 IDENTIFIER
-    : (UNICODE_LETTER | DIGIT | '_')+
-    | UNICODE_LETTER+ '://' (UNICODE_LETTER | DIGIT | '_' | '/' | '-' | '.' | '?' | '=' | '&' | '#' | '%')+
+    : (LETTER | DIGIT | '_')+
     ;
 
 BACKQUOTED_IDENTIFIER
@@ -593,10 +526,6 @@ fragment LETTER
     : [A-Z]
     ;
 
-fragment UNICODE_LETTER
-    : [\p{L}]
-    ;
-
 SIMPLE_COMMENT
     : '--' ('\\\n' | ~[\r\n])* '\r'? '\n'? -> channel(HIDDEN)
     ;
@@ -606,7 +535,7 @@ BRACKETED_COMMENT
     ;
 
 WS
-    : [ \t\n\f\r\u000B\u00A0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u2028\u202F\u205F\u3000]+ -> channel(HIDDEN)
+    : [ \r\n\t]+ -> channel(HIDDEN)
     ;
 
 // Catch-all for anything we can't recognize.