From c9f68161997b1c787b804cf21a0f89e7444c1c66 Mon Sep 17 00:00:00 2001 From: Reda Al Sulais Date: Wed, 27 Mar 2024 01:10:31 +0300 Subject: [PATCH] apply review suggestions Co-authored-by: Ashley Engelund (weedySeaDragon @ github) Signed-off-by: Reda Al Sulais --- .../src/language/common/tokenBuilder.ts | 30 +++++++---- .../parser/src/language/sankey/sankey.langium | 5 +- .../parser/tests/sankey-tokenBuilder.test.ts | 50 +++++++++++++++++++ packages/parser/tests/test-util.ts | 2 +- 4 files changed, 75 insertions(+), 12 deletions(-) create mode 100644 packages/parser/tests/sankey-tokenBuilder.test.ts diff --git a/packages/parser/src/language/common/tokenBuilder.ts b/packages/parser/src/language/common/tokenBuilder.ts index 0eeff627bd..07b949e48d 100644 --- a/packages/parser/src/language/common/tokenBuilder.ts +++ b/packages/parser/src/language/common/tokenBuilder.ts @@ -18,16 +18,28 @@ export abstract class AbstractMermaidTokenBuilder extends DefaultTokenBuilder { this.commentProvider = services.documentation.CommentProvider; } + // TODO: This responsibility might better belong in CommentProvider (e.g. AbstractMermaidCommentProvider that is a subclass of CommentProvider). + private ruleHasGreedyComment(rule: GrammarAST.AbstractRule): boolean { + const comment = this.commentProvider.getComment(rule); + return !!comment && /@greedy/.test(comment); + } + protected override buildTerminalTokens(rules: Stream): TokenType[] { - // put the greedy annotated terminal rules at the end of the array - const rulesArray = rules.toArray(); - rules.forEach((rule, index) => { - const comment = this.commentProvider.getComment(rule); - if (comment && /@greedy/.test(comment)) { - rulesArray.push(rulesArray.splice(index, 1)[0]); - } - }); - return super.buildTerminalTokens(stream(rulesArray)); + if (rules.some((rule: GrammarAST.AbstractRule) => this.ruleHasGreedyComment(rule))) { + const notGreedyRules: GrammarAST.AbstractRule[] = []; + const lastRules: GrammarAST.AbstractRule[] = []; + // put terminal rules with @greedy in their comment at the end of the array + rules.forEach((rule) => { + if (this.ruleHasGreedyComment(rule)) { + lastRules.push(rule); + } else { + notGreedyRules.push(rule); + } + }); + return super.buildTerminalTokens(stream([...notGreedyRules, ...lastRules])); + } else { + return super.buildTerminalTokens(rules); + } } protected override buildKeywordTokens( diff --git a/packages/parser/src/language/sankey/sankey.langium b/packages/parser/src/language/sankey/sankey.langium index bcf891ce94..cb5398b5a5 100644 --- a/packages/parser/src/language/sankey/sankey.langium +++ b/packages/parser/src/language/sankey/sankey.langium @@ -16,11 +16,12 @@ entry Sankey returns Sankey: ; SankeyLink: - source=SANKEY_LINK_NODE "," target=SANKEY_LINK_NODE "," value=SANKEY_LINK_VALUE EOL + source=SANKEY_LINK_NODE "," target=SANKEY_LINK_NODE "," + value=SANKEY_LINK_VALUE EOL ; terminal SANKEY_LINK_VALUE returns number: /"(0|[1-9][0-9]*)(\.[0-9]+)?"|[\t ]*(0|[1-9][0-9]*)(\.[0-9]+)?/; /** - * @greedy + * @greedy This ensures that this rule is put at the bottom of the list of tokens. */ terminal SANKEY_LINK_NODE: /sankey-link-node/; diff --git a/packages/parser/tests/sankey-tokenBuilder.test.ts b/packages/parser/tests/sankey-tokenBuilder.test.ts new file mode 100644 index 0000000000..f924170da7 --- /dev/null +++ b/packages/parser/tests/sankey-tokenBuilder.test.ts @@ -0,0 +1,50 @@ +import { beforeAll, describe, expect, it } from 'vitest'; +import type { TokenType, TokenVocabulary } from 'chevrotain'; + +import { sankeyServices } from './test-util.js'; + +describe('SankeyTokenBuilder', () => { + describe('token order', () => { + let tokenVocab: TokenVocabulary; + let tokenVocabNames: string[]; + + beforeAll(() => { + // Get the ordered tokens (the vocabulary) from the grammar + tokenVocab = sankeyServices.parser.TokenBuilder.buildTokens(sankeyServices.Grammar, { + caseInsensitive: sankeyServices.LanguageMetaData.caseInsensitive, + }); + // coerce the tokenVocab to a type that can use .map + tokenVocabNames = (tokenVocab as TokenType[]).map((tokenVocabEntry: TokenType) => { + return tokenVocabEntry.name; + }); + }); + + it('whitespace is always first', () => { + expect(tokenVocabNames[0]).toEqual('WHITESPACE'); + }); + it('sankey-beta comes after whitespace', () => { + expect(tokenVocabNames[1]).toEqual('sankey-beta'); + }); + + describe('terminal rules with @greedy in comments are put at the end of the ordered list of tokens', () => { + const NUM_EXPECTED_GREEDY_RULES = 2; + + let greedyGroupStartIndex = 0; + beforeAll(() => { + greedyGroupStartIndex = tokenVocabNames.length - NUM_EXPECTED_GREEDY_RULES - 1; + }); + + it('SANKEY_LINK_NODE rule has @greedy so it is in the last group of all @greedy terminal rules', () => { + expect(tokenVocabNames.indexOf('SANKEY_LINK_NODE')).toBeGreaterThanOrEqual( + greedyGroupStartIndex + ); + }); + + it('SANKEY_LINK_VALUE rule has @greedy so it is in the last group of all @greedy terminal rules', () => { + expect(tokenVocabNames.indexOf('SANKEY_LINK_VALUE')).toBeGreaterThanOrEqual( + greedyGroupStartIndex + ); + }); + }); + }); +}); diff --git a/packages/parser/tests/test-util.ts b/packages/parser/tests/test-util.ts index 1f1a103f78..4dad2ac9cd 100644 --- a/packages/parser/tests/test-util.ts +++ b/packages/parser/tests/test-util.ts @@ -52,7 +52,7 @@ export function createPieTestServices() { } export const pieParse = createPieTestServices().parse; -const sankeyServices: SankeyServices = createSankeyServices().Sankey; +export const sankeyServices: SankeyServices = createSankeyServices().Sankey; const sankeyParser: LangiumParser = sankeyServices.parser.LangiumParser; export function createSankeyTestServices() { const parse = (input: string) => {