From 51896f30f56cafc6bd1fddf09ae88b1762498c35 Mon Sep 17 00:00:00 2001
From: Reda Al Sulais <u.yokozuna@gmail.com>
Date: Mon, 15 Apr 2024 17:18:08 +0300
Subject: [PATCH] create separate test case for `CommonTokenBuilder`

---
 packages/parser/tests/greedy.test.ts          | 64 +++++++++++++++++++
 .../parser/tests/sankey-tokenBuilder.test.ts  | 50 ---------------
 2 files changed, 64 insertions(+), 50 deletions(-)
 create mode 100644 packages/parser/tests/greedy.test.ts
 delete mode 100644 packages/parser/tests/sankey-tokenBuilder.test.ts

diff --git a/packages/parser/tests/greedy.test.ts b/packages/parser/tests/greedy.test.ts
new file mode 100644
index 0000000000..9b182cea6c
--- /dev/null
+++ b/packages/parser/tests/greedy.test.ts
@@ -0,0 +1,64 @@
+import { createLangiumGrammarServices, createServicesForGrammar } from 'langium/grammar';
+import { describe, expect, it } from 'vitest';
+import { CommonTokenBuilder } from '../src/index.js';
+import type { TokenType } from 'chevrotain';
+import { EmptyFileSystem } from 'langium';
+
+const grammarServices = createLangiumGrammarServices(EmptyFileSystem).grammar;
+
+async function createServicesFromGrammar(grammar: string) {
+  const services = await createServicesForGrammar({
+    grammar,
+    module: {
+      parser: {
+        TokenBuilder: () => new CommonTokenBuilder([], grammarServices),
+      },
+    },
+  });
+
+  return {
+    grammar: services.Grammar,
+    tokenBuilder: services.parser.TokenBuilder,
+  };
+}
+
+describe('CommonTokenBuilder', async () => {
+  it('should handle grammar with one greedy rule', async () => {
+    const grammar = `
+      grammar TestGrammar
+      entry Rule:
+        'rule' value=(LessGreedy | Greedy);
+
+      hidden terminal WS: /\\s+/;
+
+      /** @greedy */
+      terminal Greedy: /[\\w\\d]+/;
+      terminal LessGreedy: /[\\w]+/;
+    `;
+    const services = await createServicesFromGrammar(grammar);
+    const tokens = services.tokenBuilder.buildTokens(services.grammar) as TokenType[];
+
+    expect(tokens[2].name).toBe('LessGreedy');
+    expect(tokens[3].name).toBe('Greedy');
+  });
+
+  it('should handle grammar with more than one greedy rule', async () => {
+    const grammar = `
+      grammar TestGrammar
+      entry Rule:
+        'rule' value=(LessGreedy | Greedy);
+
+      hidden terminal WS: /\\s+/;
+
+      /** @greedy */
+      terminal LessGreedy: /[\\w]+/;
+      /** @greedy */
+      terminal Greedy: /[\\w\\d]+/;
+    `;
+    const services = await createServicesFromGrammar(grammar);
+    const tokens = services.tokenBuilder.buildTokens(services.grammar) as TokenType[];
+
+    expect(tokens[2].name).toBe('LessGreedy');
+    expect(tokens[3].name).toBe('Greedy');
+  });
+});
diff --git a/packages/parser/tests/sankey-tokenBuilder.test.ts b/packages/parser/tests/sankey-tokenBuilder.test.ts
deleted file mode 100644
index f924170da7..0000000000
--- a/packages/parser/tests/sankey-tokenBuilder.test.ts
+++ /dev/null
@@ -1,50 +0,0 @@
-import { beforeAll, describe, expect, it } from 'vitest';
-import type { TokenType, TokenVocabulary } from 'chevrotain';
-
-import { sankeyServices } from './test-util.js';
-
-describe('SankeyTokenBuilder', () => {
-  describe('token order', () => {
-    let tokenVocab: TokenVocabulary;
-    let tokenVocabNames: string[];
-
-    beforeAll(() => {
-      // Get the ordered tokens (the vocabulary) from the grammar
-      tokenVocab = sankeyServices.parser.TokenBuilder.buildTokens(sankeyServices.Grammar, {
-        caseInsensitive: sankeyServices.LanguageMetaData.caseInsensitive,
-      });
-      // coerce the tokenVocab to a type that can use .map
-      tokenVocabNames = (tokenVocab as TokenType[]).map((tokenVocabEntry: TokenType) => {
-        return tokenVocabEntry.name;
-      });
-    });
-
-    it('whitespace is always first', () => {
-      expect(tokenVocabNames[0]).toEqual('WHITESPACE');
-    });
-    it('sankey-beta comes after whitespace', () => {
-      expect(tokenVocabNames[1]).toEqual('sankey-beta');
-    });
-
-    describe('terminal rules with @greedy in comments are put at the end of the ordered list of tokens', () => {
-      const NUM_EXPECTED_GREEDY_RULES = 2;
-
-      let greedyGroupStartIndex = 0;
-      beforeAll(() => {
-        greedyGroupStartIndex = tokenVocabNames.length - NUM_EXPECTED_GREEDY_RULES - 1;
-      });
-
-      it('SANKEY_LINK_NODE rule has @greedy so it is in the last group of all @greedy terminal rules', () => {
-        expect(tokenVocabNames.indexOf('SANKEY_LINK_NODE')).toBeGreaterThanOrEqual(
-          greedyGroupStartIndex
-        );
-      });
-
-      it('SANKEY_LINK_VALUE rule has @greedy so it is in the last group of all @greedy terminal rules', () => {
-        expect(tokenVocabNames.indexOf('SANKEY_LINK_VALUE')).toBeGreaterThanOrEqual(
-          greedyGroupStartIndex
-        );
-      });
-    });
-  });
-});