From 6c7b0f276c1089b16e2a3f10c41ece845671e993 Mon Sep 17 00:00:00 2001 From: Reda Al Sulais Date: Tue, 26 Mar 2024 16:08:12 +0300 Subject: [PATCH] implement the greedy annotation approach --- .../src/language/common/tokenBuilder.ts | 27 ++++++++++++++++--- packages/parser/src/language/info/module.ts | 2 +- .../parser/src/language/info/tokenBuilder.ts | 5 ++-- packages/parser/src/language/packet/module.ts | 2 +- .../src/language/packet/tokenBuilder.ts | 5 ++-- packages/parser/src/language/pie/module.ts | 2 +- .../parser/src/language/pie/tokenBuilder.ts | 5 ++-- packages/parser/src/language/sankey/module.ts | 2 +- .../parser/src/language/sankey/sankey.langium | 3 +++ .../src/language/sankey/tokenBuilder.ts | 5 ++-- 10 files changed, 42 insertions(+), 16 deletions(-) diff --git a/packages/parser/src/language/common/tokenBuilder.ts b/packages/parser/src/language/common/tokenBuilder.ts index f997634545..0eeff627bd 100644 --- a/packages/parser/src/language/common/tokenBuilder.ts +++ b/packages/parser/src/language/common/tokenBuilder.ts @@ -1,14 +1,33 @@ -import type { GrammarAST, Stream, TokenBuilderOptions } from 'langium'; import type { TokenType } from 'chevrotain'; - -import { DefaultTokenBuilder } from 'langium'; +import type { + CommentProvider, + GrammarAST, + LangiumCoreServices, + Stream, + TokenBuilderOptions, +} from 'langium'; +import { DefaultTokenBuilder, stream } from 'langium'; export abstract class AbstractMermaidTokenBuilder extends DefaultTokenBuilder { private keywords: Set; + private commentProvider: CommentProvider; - public constructor(keywords: string[]) { + public constructor(keywords: string[], services: LangiumCoreServices) { super(); this.keywords = new Set(keywords); + this.commentProvider = services.documentation.CommentProvider; + } + + protected override buildTerminalTokens(rules: Stream): TokenType[] { + // put the greedy annotated terminal rules at the end of the array + const rulesArray = rules.toArray(); + rules.forEach((rule, index) => { + const comment = this.commentProvider.getComment(rule); + if (comment && /@greedy/.test(comment)) { + rulesArray.push(rulesArray.splice(index, 1)[0]); + } + }); + return super.buildTerminalTokens(stream(rulesArray)); } protected override buildKeywordTokens( diff --git a/packages/parser/src/language/info/module.ts b/packages/parser/src/language/info/module.ts index 83933aeef4..ae4e91aef6 100644 --- a/packages/parser/src/language/info/module.ts +++ b/packages/parser/src/language/info/module.ts @@ -37,7 +37,7 @@ export type InfoServices = LangiumCoreServices & InfoAddedServices; */ export const InfoModule: Module = { parser: { - TokenBuilder: () => new InfoTokenBuilder(), + TokenBuilder: (services) => new InfoTokenBuilder(services), ValueConverter: () => new CommonValueConverter(), }, }; diff --git a/packages/parser/src/language/info/tokenBuilder.ts b/packages/parser/src/language/info/tokenBuilder.ts index 69ad0e689d..260d9ac42d 100644 --- a/packages/parser/src/language/info/tokenBuilder.ts +++ b/packages/parser/src/language/info/tokenBuilder.ts @@ -1,7 +1,8 @@ import { AbstractMermaidTokenBuilder } from '../common/index.js'; +import type { InfoServices } from './module.js'; export class InfoTokenBuilder extends AbstractMermaidTokenBuilder { - public constructor() { - super(['info', 'showInfo']); + public constructor(services: InfoServices) { + super(['info', 'showInfo'], services); } } diff --git a/packages/parser/src/language/packet/module.ts b/packages/parser/src/language/packet/module.ts index 40c68916aa..81776bf128 100644 --- a/packages/parser/src/language/packet/module.ts +++ b/packages/parser/src/language/packet/module.ts @@ -40,7 +40,7 @@ export const PacketModule: Module< PartialLangiumCoreServices & PacketAddedServices > = { parser: { - TokenBuilder: () => new PacketTokenBuilder(), + TokenBuilder: (services) => new PacketTokenBuilder(services), ValueConverter: () => new CommonValueConverter(), }, }; diff --git a/packages/parser/src/language/packet/tokenBuilder.ts b/packages/parser/src/language/packet/tokenBuilder.ts index accba5675a..c560ba6874 100644 --- a/packages/parser/src/language/packet/tokenBuilder.ts +++ b/packages/parser/src/language/packet/tokenBuilder.ts @@ -1,7 +1,8 @@ import { AbstractMermaidTokenBuilder } from '../common/index.js'; +import type { PacketServices } from './module.js'; export class PacketTokenBuilder extends AbstractMermaidTokenBuilder { - public constructor() { - super(['packet-beta']); + public constructor(services: PacketServices) { + super(['packet-beta'], services); } } diff --git a/packages/parser/src/language/pie/module.ts b/packages/parser/src/language/pie/module.ts index b85daee680..75368fc4c0 100644 --- a/packages/parser/src/language/pie/module.ts +++ b/packages/parser/src/language/pie/module.ts @@ -37,7 +37,7 @@ export type PieServices = LangiumCoreServices & PieAddedServices; */ export const PieModule: Module = { parser: { - TokenBuilder: () => new PieTokenBuilder(), + TokenBuilder: (services) => new PieTokenBuilder(services), ValueConverter: () => new PieValueConverter(), }, }; diff --git a/packages/parser/src/language/pie/tokenBuilder.ts b/packages/parser/src/language/pie/tokenBuilder.ts index 85aecf96a0..7334d3002d 100644 --- a/packages/parser/src/language/pie/tokenBuilder.ts +++ b/packages/parser/src/language/pie/tokenBuilder.ts @@ -1,7 +1,8 @@ import { AbstractMermaidTokenBuilder } from '../common/index.js'; +import type { PieServices } from './module.js'; export class PieTokenBuilder extends AbstractMermaidTokenBuilder { - public constructor() { - super(['pie', 'showData']); + public constructor(services: PieServices) { + super(['pie', 'showData'], services); } } diff --git a/packages/parser/src/language/sankey/module.ts b/packages/parser/src/language/sankey/module.ts index fbde011774..6ff5d75675 100644 --- a/packages/parser/src/language/sankey/module.ts +++ b/packages/parser/src/language/sankey/module.ts @@ -44,7 +44,7 @@ export const SankeyModule: Module< > = { parser: { LangiumParser: (services) => createSankeyParser(services), - TokenBuilder: () => new SankeyTokenBuilder(), + TokenBuilder: (services) => new SankeyTokenBuilder(services), ValueConverter: () => new SankeyValueConverter(), }, }; diff --git a/packages/parser/src/language/sankey/sankey.langium b/packages/parser/src/language/sankey/sankey.langium index f718fbdc0d..bcf891ce94 100644 --- a/packages/parser/src/language/sankey/sankey.langium +++ b/packages/parser/src/language/sankey/sankey.langium @@ -20,4 +20,7 @@ SankeyLink: ; terminal SANKEY_LINK_VALUE returns number: /"(0|[1-9][0-9]*)(\.[0-9]+)?"|[\t ]*(0|[1-9][0-9]*)(\.[0-9]+)?/; +/** + * @greedy + */ terminal SANKEY_LINK_NODE: /sankey-link-node/; diff --git a/packages/parser/src/language/sankey/tokenBuilder.ts b/packages/parser/src/language/sankey/tokenBuilder.ts index a42a1bb8ec..ef8ccbb27a 100644 --- a/packages/parser/src/language/sankey/tokenBuilder.ts +++ b/packages/parser/src/language/sankey/tokenBuilder.ts @@ -3,10 +3,11 @@ import type { TokenType } from 'chevrotain'; import { AbstractMermaidTokenBuilder } from '../common/index.js'; import { matchSankeyLinkNode } from './matcher.js'; +import type { SankeyServices } from './module.js'; export class SankeyTokenBuilder extends AbstractMermaidTokenBuilder { - public constructor() { - super(['sankey-beta']); + public constructor(services: SankeyServices) { + super(['sankey-beta'], services); } protected override buildTerminalTokens(rules: Stream): TokenType[] {