implement the greedy annotation approach

Yokozuna59 · Mar 26, 2024 · 6c7b0f2 · 6c7b0f2
1 parent 434d154
commit 6c7b0f2
Show file tree

Hide file tree

Showing 10 changed files with 42 additions and 16 deletions.
diff --git a/packages/parser/src/language/common/tokenBuilder.ts b/packages/parser/src/language/common/tokenBuilder.ts
@@ -1,14 +1,33 @@
-import type { GrammarAST, Stream, TokenBuilderOptions } from 'langium';
 import type { TokenType } from 'chevrotain';
-
-import { DefaultTokenBuilder } from 'langium';
+import type {
+  CommentProvider,
+  GrammarAST,
+  LangiumCoreServices,
+  Stream,
+  TokenBuilderOptions,
+} from 'langium';
+import { DefaultTokenBuilder, stream } from 'langium';
 
 export abstract class AbstractMermaidTokenBuilder extends DefaultTokenBuilder {
   private keywords: Set<string>;
+  private commentProvider: CommentProvider;
 
-  public constructor(keywords: string[]) {
+  public constructor(keywords: string[], services: LangiumCoreServices) {
     super();
     this.keywords = new Set<string>(keywords);
+    this.commentProvider = services.documentation.CommentProvider;
+  }
+
+  protected override buildTerminalTokens(rules: Stream<GrammarAST.AbstractRule>): TokenType[] {
+    // put the greedy annotated terminal rules at the end of the array
+    const rulesArray = rules.toArray();
+    rules.forEach((rule, index) => {
+      const comment = this.commentProvider.getComment(rule);
+      if (comment && /@greedy/.test(comment)) {
+        rulesArray.push(rulesArray.splice(index, 1)[0]);
+      }
+    });
+    return super.buildTerminalTokens(stream(rulesArray));
   }
 
   protected override buildKeywordTokens(

diff --git a/packages/parser/src/language/info/module.ts b/packages/parser/src/language/info/module.ts
@@ -37,7 +37,7 @@ export type InfoServices = LangiumCoreServices & InfoAddedServices;
  */
 export const InfoModule: Module<InfoServices, PartialLangiumCoreServices & InfoAddedServices> = {
   parser: {
-    TokenBuilder: () => new InfoTokenBuilder(),
+    TokenBuilder: (services) => new InfoTokenBuilder(services),
     ValueConverter: () => new CommonValueConverter(),
   },
 };

diff --git a/packages/parser/src/language/info/tokenBuilder.ts b/packages/parser/src/language/info/tokenBuilder.ts
@@ -1,7 +1,8 @@
 import { AbstractMermaidTokenBuilder } from '../common/index.js';
+import type { InfoServices } from './module.js';
 
 export class InfoTokenBuilder extends AbstractMermaidTokenBuilder {
-  public constructor() {
-    super(['info', 'showInfo']);
+  public constructor(services: InfoServices) {
+    super(['info', 'showInfo'], services);
   }
 }
diff --git a/packages/parser/src/language/packet/module.ts b/packages/parser/src/language/packet/module.ts
@@ -40,7 +40,7 @@ export const PacketModule: Module<
   PartialLangiumCoreServices & PacketAddedServices
 > = {
   parser: {
-    TokenBuilder: () => new PacketTokenBuilder(),
+    TokenBuilder: (services) => new PacketTokenBuilder(services),
     ValueConverter: () => new CommonValueConverter(),
   },
 };

diff --git a/packages/parser/src/language/packet/tokenBuilder.ts b/packages/parser/src/language/packet/tokenBuilder.ts
@@ -1,7 +1,8 @@
 import { AbstractMermaidTokenBuilder } from '../common/index.js';
+import type { PacketServices } from './module.js';
 
 export class PacketTokenBuilder extends AbstractMermaidTokenBuilder {
-  public constructor() {
-    super(['packet-beta']);
+  public constructor(services: PacketServices) {
+    super(['packet-beta'], services);
   }
 }
diff --git a/packages/parser/src/language/pie/module.ts b/packages/parser/src/language/pie/module.ts
@@ -37,7 +37,7 @@ export type PieServices = LangiumCoreServices & PieAddedServices;
  */
 export const PieModule: Module<PieServices, PartialLangiumCoreServices & PieAddedServices> = {
   parser: {
-    TokenBuilder: () => new PieTokenBuilder(),
+    TokenBuilder: (services) => new PieTokenBuilder(services),
     ValueConverter: () => new PieValueConverter(),
   },
 };

diff --git a/packages/parser/src/language/pie/tokenBuilder.ts b/packages/parser/src/language/pie/tokenBuilder.ts
@@ -1,7 +1,8 @@
 import { AbstractMermaidTokenBuilder } from '../common/index.js';
+import type { PieServices } from './module.js';
 
 export class PieTokenBuilder extends AbstractMermaidTokenBuilder {
-  public constructor() {
-    super(['pie', 'showData']);
+  public constructor(services: PieServices) {
+    super(['pie', 'showData'], services);
   }
 }
diff --git a/packages/parser/src/language/sankey/module.ts b/packages/parser/src/language/sankey/module.ts
@@ -44,7 +44,7 @@ export const SankeyModule: Module<
 > = {
   parser: {
     LangiumParser: (services) => createSankeyParser(services),
-    TokenBuilder: () => new SankeyTokenBuilder(),
+    TokenBuilder: (services) => new SankeyTokenBuilder(services),
     ValueConverter: () => new SankeyValueConverter(),
   },
 };

diff --git a/packages/parser/src/language/sankey/sankey.langium b/packages/parser/src/language/sankey/sankey.langium
@@ -20,4 +20,7 @@ SankeyLink:
 ;
 
 terminal SANKEY_LINK_VALUE returns number: /"(0|[1-9][0-9]*)(\.[0-9]+)?"|[\t ]*(0|[1-9][0-9]*)(\.[0-9]+)?/;
+/**
+ * @greedy
+ */
 terminal SANKEY_LINK_NODE: /sankey-link-node/;
diff --git a/packages/parser/src/language/sankey/tokenBuilder.ts b/packages/parser/src/language/sankey/tokenBuilder.ts
@@ -3,10 +3,11 @@ import type { TokenType } from 'chevrotain';
 
 import { AbstractMermaidTokenBuilder } from '../common/index.js';
 import { matchSankeyLinkNode } from './matcher.js';
+import type { SankeyServices } from './module.js';
 
 export class SankeyTokenBuilder extends AbstractMermaidTokenBuilder {
-  public constructor() {
-    super(['sankey-beta']);
+  public constructor(services: SankeyServices) {
+    super(['sankey-beta'], services);
   }
 
   protected override buildTerminalTokens(rules: Stream<GrammarAST.AbstractRule>): TokenType[] {