[PlSql] Add support for CREATE SCHEMA (#4436)

* Update to version 0.23.15 of Trash Toolkit.

* Update dotnet-tools.json

* Update to 0.23.16, add trapc.

* Remaster due to changed grammar.

* Remove all broken ports for antlr/antlr4.

All ports need to be rewritten in target agnostic format.

* Remove testing of PHP for trapc--PHP is not a reliable runtime. Add antlr-ng generator for Antlr4ng testing.

* Update grammar to be target agnostic.

Antlr4 "@Header" blocks are target specific. But, we need this to work for any target. So, we use a comment in the grammar and transformGrammar.py to fix up the problem. _insideOptionsBlock is removed because it's not used anywhere. Untabify source code--tabs are presentation specific and thus depend on the editor used to open it. We don't use the same editors people.

* Create LexerAdaptor.ts

* Create transformGrammar.py

* Update LexerAdaptor.ts

* Fix missing if-then-else.

* Add Antlr4ng port.

* Add Cpp target.

* Update to latest of Trash for Cpp testing.

* Initial version of Dart port--does not work yet.

* Fix Dart port.

* [PlSql] Add support for CREATE SCHEMA

* Add TypeScript port.

* Add Python3 port.

---------

Co-authored-by: Ken Domino <[email protected]>

Author: EastLord

.config/dotnet-tools.json

@@ -3,189 +3,189 @@
   "isRoot": true,
   "tools": {
     "trcaret": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trcaret"
       ],
       "rollForward": false
     },
     "trcover": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trcover"
       ],
       "rollForward": false
     },
     "trgen": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trgen"
       ],
       "rollForward": false
     },
     "trglob": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trglob"
       ],
       "rollForward": false
     },
     "triconv": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "triconv"
       ],
       "rollForward": false
     },
     "trparse": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trparse"
       ],
       "rollForward": false
     },
     "trquery": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trquery"
       ],
       "rollForward": false
     },
     "trtext": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trtext"
       ],
       "rollForward": false
     },
     "trwdog": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trwdog"
       ],
       "rollForward": false
     },
     "trxgrep": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trxgrep"
       ],
       "rollForward": false
     },
     "trxml": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trxml"
       ],
       "rollForward": false
     },
     "trxml2": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trxml2"
       ],
       "rollForward": false
     },
     "trclonereplace": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trclonereplace"
       ],
       "rollForward": false
     },
     "trcombine": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trcombine"
       ],
       "rollForward": false
     },
     "trconvert": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trconvert"
       ],
       "rollForward": false
     },
     "trfoldlit": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trfoldlit"
       ],
       "rollForward": false
     },
     "trgenvsc": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trgenvsc"
       ],
       "rollForward": false
     },
     "tritext": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "tritext"
       ],
       "rollForward": false
     },
     "trjson": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trjson"
       ],
       "rollForward": false
     },
     "trperf": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trperf"
       ],
       "rollForward": false
     },
     "trrename": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trrename"
       ],
       "rollForward": false
     },
     "trsort": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trsort"
       ],
       "rollForward": false
     },
     "trsplit": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trsplit"
       ],
       "rollForward": false
     },
     "trsponge": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trsponge"
       ],
       "rollForward": false
     },
     "trtokens": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trtokens"
       ],
       "rollForward": false
     },
     "trtree": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trtree"
       ],
       "rollForward": false
     },
     "trunfold": {
-      "version": "0.23.16",
+      "version": "0.23.17",
       "commands": [
         "trunfold"
       ],

antlr/antlr4/ANTLRv4Lexer.g4

@@ -54,6 +54,7 @@ options {
     tokenVocab = predefined;
 }
 
+// Insert here @header for lexer.
 
 // Standard set of fragments
 tokens {

antlr/antlr4/Antlr4ng/LexerAdaptor.ts

@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) Mike Lischke. All rights reserved.
+ * Licensed under the BSD 3-clause License. See License.txt in the project root for license information.
+ */
+
+import { CharStream, Lexer, Token } from "antlr4ng";
+
+import { ANTLRv4Lexer } from "./ANTLRv4Lexer.js";
+
+export abstract class LexerAdaptor extends Lexer {
+    /** Generic type for OPTIONS, TOKENS and CHANNELS */
+    private static PREQUEL_CONSTRUCT = -10;
+    private static OPTIONS_CONSTRUCT = -11;
+
+    private currentRuleType: number = Token.INVALID_TYPE;
+
+    public constructor(input: CharStream) {
+        super(input);
+
+        /**
+         * Track whether we are inside of a rule and whether it is lexical parser. _currentRuleType==Token.INVALID_TYPE
+         * means that we are outside of a rule. At the first sign of a rule name reference and _currentRuleType
+         * ==invalid, we can assume that we are starting a parser rule. Similarly, seeing a token reference when not
+         * already in rule means starting a token rule. The terminating ';' of a rule, flips this back to invalid type.
+         *
+         * This is not perfect logic but works. For example, "grammar T;" means that we start and stop a lexical rule
+         * for the "T;". Dangerous but works.
+         *
+         * The whole point of this state information is to distinguish between [..arg actions..] and [char sets].
+         * Char sets can only occur in lexical rules and arg actions cannot occur.
+         */
+        this.currentRuleType = Token.INVALID_TYPE;
+    }
+
+    public override reset(): void {
+        this.currentRuleType = Token.INVALID_TYPE;
+        super.reset();
+    }
+
+    public override emit(): Token {
+        if ((this.type === ANTLRv4Lexer.OPTIONS || this.type === ANTLRv4Lexer.TOKENS
+            || this.type === ANTLRv4Lexer.CHANNELS)
+            && this.currentRuleType === Token.INVALID_TYPE) {
+            // Enter prequel construct ending with an RBRACE.
+            this.currentRuleType = LexerAdaptor.PREQUEL_CONSTRUCT;
+        } else if (this.type === ANTLRv4Lexer.OPTIONS && this.currentRuleType === ANTLRv4Lexer.TOKEN_REF) {
+            this.currentRuleType = LexerAdaptor.OPTIONS_CONSTRUCT;
+        } else if (this.type === ANTLRv4Lexer.RBRACE
+            && this.currentRuleType === LexerAdaptor.PREQUEL_CONSTRUCT) {
+            // Exit prequel construct.
+            this.currentRuleType = Token.INVALID_TYPE;
+        } else if (this.type === ANTLRv4Lexer.RBRACE
+            && this.currentRuleType === LexerAdaptor.OPTIONS_CONSTRUCT) {
+            // Exit options.
+            this.currentRuleType = ANTLRv4Lexer.TOKEN_REF;
+        } else if (this.type === ANTLRv4Lexer.AT && this.currentRuleType === Token.INVALID_TYPE) {
+            // Enter action.
+            this.currentRuleType = ANTLRv4Lexer.AT;
+        } else if (this.type === ANTLRv4Lexer.SEMI
+            && this.currentRuleType === LexerAdaptor.OPTIONS_CONSTRUCT) {
+            // ';' in options { .... }. Don't change anything.
+        } else if (this.type == ANTLRv4Lexer.END_ACTION && this.currentRuleType == ANTLRv4Lexer.AT) {
+            // exit action
+            this.currentRuleType = Token.INVALID_TYPE;
+        } else if (this.type === ANTLRv4Lexer.ID) {
+            const firstChar = this.inputStream.getTextFromRange(this.tokenStartCharIndex, this.tokenStartCharIndex);
+            const c = firstChar.charAt(0);
+            if (c === c.toUpperCase()) {
+                this.type = ANTLRv4Lexer.TOKEN_REF;
+            } else {
+                this.type = ANTLRv4Lexer.RULE_REF;
+            }
+
+            // If outside of rule def.
+            if (this.currentRuleType === Token.INVALID_TYPE) {
+                // Set to inside lexer or parser rule.
+                this.currentRuleType = this.type;
+            }
+        } else if (this.type === ANTLRv4Lexer.SEMI) {
+            // Exit rule def.
+            this.currentRuleType = Token.INVALID_TYPE;
+        }
+
+        return super.emit();
+    }
+
+    protected handleBeginArgument(): void {
+        if (this.currentRuleType === ANTLRv4Lexer.TOKEN_REF) {
+            this.pushMode(ANTLRv4Lexer.LexerCharSet);
+            this.more();
+        } else {
+            this.pushMode(ANTLRv4Lexer.Argument);
+        }
+    }
+
+    protected handleEndArgument(): void {
+        this.popMode();
+        if (this.modeStack.length > 0) {
+            this.type = ANTLRv4Lexer.ARGUMENT_CONTENT;
+        }
+    }
+
+    protected handleEndAction(): void
+    {
+        var oldMode = this.mode;
+        var newMode = this.popMode();
+        var isActionWithinAction = this.modeStack.length > 0
+           && newMode == ANTLRv4Lexer.TargetLanguageAction
+           && oldMode == newMode;
+
+        if (isActionWithinAction) {
+            this.type = ANTLRv4Lexer.ACTION_CONTENT;
+        }
+    }
+}
+

antlr/antlr4/Antlr4ng/transformGrammar.py

@@ -0,0 +1,31 @@
+"""The script transforms the grammar to fit for the c++ target """
+import sys
+import re
+import shutil
+from glob import glob
+from pathlib import Path
+
+def main():
+    """Executes the script."""
+    for file in glob("./*.g4"):
+        transform_grammar(file)
+
+def transform_grammar(file_path):
+    """Transforms the grammar to fit for the target"""
+    print("Altering " + file_path)
+    if not Path(file_path).is_file:
+        print(f"Could not find file: {file_path}")
+        sys.exit(1)
+
+    shutil.move(file_path, file_path + ".bak")
+    with open(file_path + ".bak",'r', encoding="utf-8") as input_file:
+        with open(file_path, 'w', encoding="utf-8") as output_file:
+            for line in input_file:
+                line = re.sub(r"(\/\/ Insert here @header for lexer\.)",\
+                    '@header {import { LexerAdaptor } from "./LexerAdaptor.js"}', line)
+                output_file.write(line)
+
+    print("Writing ...")
+
+if __name__ == '__main__':
+    main()