Merge pull request #12 from dyne/sf/plugin

feat(fs)!: parse ignored statements individually

pkg/fs/lexer.test.ts

@@ -1,28 +1,62 @@
 import { lex } from './lexer';
-import { ThenI, SaveThe, IntoTheFile } from './tokens';
+import { Then, I, SaveThe, IntoTheFile } from './tokens';
 
 import { Identifier } from '@slangroom/shared/tokens';
+import { getIgnoredStatements } from '@slangroom/ignored';
 
 test('lexing works', async () => {
 	// Given I have a contract with filesystem statements in it
 	const contract = `Rule unknown ignore
-Given I have a 'string' named 'stringToWrite'
-and I have a 'string' named 'nameOfTheFile'
+Given I have a 'string' named 'stringToWrite0'
+and I have a 'string' named 'nameOfTheFile0'
+and I have a 'string' named 'stringToWrite1'
+and I have a 'string' named 'nameOfTheFile1'
 
-Then I save the 'stringToWrite' into the file 'nameOfTheFile'
+Then I save the 'stringToWrite0' into the file 'nameOfTheFile0'
+and I save the 'stringToWrite1' into the file 'nameOfTheFile1'
 `;
-	// When I lex it
-	const lexed = await lex(contract, { data: { stringToWrite: 'foo', nameOfTheFile: 'bar' } });
-	// Then the result must have no errors
-	expect(lexed.errors).toHaveLength(0);
-	// and it must have 5 tokens
-	expect(lexed.tokens).toHaveLength(5);
+	// When I get the ignored statements of it
+	const ignoreds = await getIgnoredStatements(contract, {
+		data: {
+			stringToWrite0: 'foo0', nameOfTheFile0: 'bar0',
+			stringToWrite1: 'foo1', nameOfTheFile1: 'bar1',
+		},
+	});
+	// and I lex each of them
+	const lexeds = ignoreds.map((x) => lex(x));
+	// Then the result must contain 2 items
+	expect(lexeds).toHaveLength(2);
+	// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+
+	// When I get the first one
+	const first = lexeds[0]!;
+	// Then it must have no errors
+	expect(first.errors).toHaveLength(0);
+	// and it must have 6 tokens
+	expect(first.tokens).toHaveLength(6);
+	// and those tokens must be these:
+	/* eslint-disable @typescript-eslint/no-non-null-assertion */
+	expect(first.tokens[0]!.tokenType).toStrictEqual(Then);
+	expect(first.tokens[1]!.tokenType).toStrictEqual(I);
+	expect(first.tokens[2]!.tokenType).toStrictEqual(SaveThe);
+	expect(first.tokens[3]!.tokenType).toStrictEqual(Identifier);
+	expect(first.tokens[4]!.tokenType).toStrictEqual(IntoTheFile);
+	expect(first.tokens[5]!.tokenType).toStrictEqual(Identifier);
+	/* eslint-enable */
+
+	// When I get the second one
+	const second = lexeds[0]!;
+	// Then it must have no errors
+	expect(second.errors).toHaveLength(0);
+	// and it must have 6 tokens
+	expect(second.tokens).toHaveLength(6);
 	// and those tokens must be these:
 	/* eslint-disable @typescript-eslint/no-non-null-assertion */
-	expect(lexed.tokens[0]!.tokenType).toStrictEqual(ThenI);
-	expect(lexed.tokens[1]!.tokenType).toStrictEqual(SaveThe);
-	expect(lexed.tokens[2]!.tokenType).toStrictEqual(Identifier);
-	expect(lexed.tokens[3]!.tokenType).toStrictEqual(IntoTheFile);
-	expect(lexed.tokens[4]!.tokenType).toStrictEqual(Identifier);
+	expect(second.tokens[0]!.tokenType).toStrictEqual(Then);
+	expect(second.tokens[1]!.tokenType).toStrictEqual(I);
+	expect(second.tokens[2]!.tokenType).toStrictEqual(SaveThe);
+	expect(second.tokens[3]!.tokenType).toStrictEqual(Identifier);
+	expect(second.tokens[4]!.tokenType).toStrictEqual(IntoTheFile);
+	expect(second.tokens[5]!.tokenType).toStrictEqual(Identifier);
 	/* eslint-enable */
 });

pkg/fs/lexer.ts

@@ -1,18 +1,13 @@
 import { vocab } from './tokens';
 
-import { type ZenroomParams } from '@slangroom/shared';
-import { getIgnoredStatements } from '@slangroom/ignored';
 import { Lexer } from '@slangroom/deps/chevrotain';
 
 const FsLexer = new Lexer(vocab);
 
 /**
- * Lexes the input text for filesystem statements.
+ * Lexes the given statement for filesystems statements.
  *
- * @param contract the zencode contract to be lexed.
- * @returns the lexed result.
- */
-export const lex = async (contract: string, params?: ZenroomParams) => {
-	const ignored = await getIgnoredStatements(contract, params);
-	return FsLexer.tokenize(ignored.join('\n'));
-};
+ * @param statement is the statement ignored by Zenroom.
+ * @returns the tokens of the lexed result.
+ **/
+export const lex = (statement: string) => FsLexer.tokenize(statement);

pkg/fs/parser.ts

@@ -1,9 +1,8 @@
-import { IntoTheFile, SaveThe, ThenI, vocab } from './tokens';
+import { IntoTheFile, SaveThe, Then, And, I, vocab } from './tokens';
 import { lex } from './lexer';
 
 import { CstParser, type IToken } from '@slangroom/deps/chevrotain';
 import { Identifier } from '@slangroom/shared/tokens';
-import { ZenroomParams } from '@slangroom/shared/zenroom';
 
 export type FileOverrideStatementCtx = {
 	ThenI: [IToken];
@@ -20,7 +19,11 @@ class Parser extends CstParser {
 	}
 
 	fileOverrideStatement = this.RULE('fileOverrideStatement', () => {
-		this.CONSUME(ThenI);
+		this.OR([
+			{ ALT: () => this.CONSUME(Then) },
+			{ ALT: () => this.CONSUME(And) },
+		]);
+		this.CONSUME(I);
 		this.CONSUME(SaveThe);
 		this.CONSUME(Identifier, { LABEL: 'content' });
 		this.CONSUME(IntoTheFile);
@@ -31,8 +34,14 @@ class Parser extends CstParser {
 export const FsParser = new Parser();
 export const BaseFsVisitor = FsParser.getBaseCstVisitorConstructor();
 
-export const parse = async (contract: string, params?: ZenroomParams) => {
-	const lexed = await lex(contract, params);
+/**
+ * Parses the given statement for filesystems statements.
+ *
+ * @param statement is the statement ignored by Zenroom.
+ * @returns the CST of the lexed statement.
+ **/
+export const parse = (statement: string) => {
+	const lexed = lex(statement);
 	FsParser.input = lexed.tokens;
 	return FsParser.fileOverrideStatement();
 };

pkg/fs/tokens.ts

@@ -1,13 +1,28 @@
 import { Whitespace, Comment, Identifier } from '@slangroom/shared';
 import { createToken } from '@slangroom/deps/chevrotain';
 
-// TODO: maybe this should be put in shared and/or the "I" part should be split up.
 /**
- * The "Then I" statement.
+ * The "Then" statement.
  */
-export const ThenI = createToken({
-	name: 'ThenI',
-	pattern: /Then I/,
+export const Then = createToken({
+	name: 'Then',
+	pattern: /Then/,
+});
+
+/**
+ * The "and" statement.
+ */
+export const And = createToken({
+	name: 'and',
+	pattern: /and/,
+});
+
+/**
+ * The selfish "I" statement.
+ */
+export const I = createToken({
+	name: 'I',
+	pattern: /I/,
 });
 
 /**
@@ -30,4 +45,4 @@ export const IntoTheFile = createToken({
 /**
  * Vocabulary to perform filesystems actions.
  */
-export const vocab = [Whitespace, Comment, ThenI, SaveThe, IntoTheFile, Identifier];
+export const vocab = [Whitespace, Comment, Then, And, I, SaveThe, IntoTheFile, Identifier];

pkg/fs/visitor.test.ts

@@ -1,25 +1,100 @@
 import { visit } from './visitor';
 
-test('ast is correct', async () => {
-	// Given I have a contract with filesystem statements in it
+import { getIgnoredStatements } from '@slangroom/ignored';
+
+test('ast is correct with one statement', async () => {
+	// Given I have a contract with one filesystems statement in it
 	const contract = `Rule unknown ignore
 Given I have a 'string' named 'stringToWrite'
 and I have a 'string' named 'nameOfTheFile'
 
 Then I save the 'stringToWrite' into the file 'nameOfTheFile'
 `;
-	// When I generate ast of the contract
 	const data = {
 		stringToWrite: 'hello world',
 		nameOfTheFile: 'hello-world.txt',
 	};
-	const ast = await visit(contract, { data: data });
-	// Then the content must be "stringToWrite"
+	// When I get the ignored statements of it
+	const ignoreds = await getIgnoredStatements(contract, {
+		data: data,
+	});
+	// and I generate AST of each of them
+	const asts = ignoreds.map((x) => visit(x));
+	// Then the result must contain only one item
+	expect(asts).toHaveLength(1);
+	// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+	const ast = asts[0]!;
+	// and its content must be "stringToWrite"
 	expect(ast.content).toStrictEqual('stringToWrite');
-	// and the filename must be "nameOfTheFile"
+	// and its filename must be "nameOfTheFile"
 	expect(ast.filename).toStrictEqual('nameOfTheFile');
-	// and the value indexed by the content in data must be "hello world"
-	expect(data[ast.content as 'stringToWrite']).toStrictEqual('hello world');
-	// and the value indexed by the filename in data must be "hello-world.txt"
-	expect(data[ast.filename as 'nameOfTheFile']).toStrictEqual('hello-world.txt');
+	// and the value indexed by its content in data must be data's stringToWrite
+	expect(data[ast.content as 'stringToWrite']).toStrictEqual(data.stringToWrite);
+	// and the value indexed by its filename in data must be data's nameOfTheFile
+	expect(data[ast.filename as 'nameOfTheFile']).toStrictEqual(data.nameOfTheFile);
+});
+
+test('ast is correct with multiple statements', async () => {
+	// Given I have a contract with multiple filesystems statements in it
+	const contract = `Rule unknown ignore
+Given I have a 'string' named 'stringToWrite0'
+and I have a 'string' named 'nameOfTheFile0'
+and I have a 'string' named 'stringToWrite1'
+and I have a 'string' named 'nameOfTheFile1'
+and I have a 'string' named 'stringToWrite2'
+and I have a 'string' named 'nameOfTheFile2'
+
+Then I save the 'stringToWrite0' into the file 'nameOfTheFile0'
+and I save the 'stringToWrite1' into the file 'nameOfTheFile1'
+and I save the 'stringToWrite2' into the file 'nameOfTheFile2'
+`;
+	const data = {
+		stringToWrite0: 'hello world0',
+		nameOfTheFile0: 'hello-world0.txt',
+		stringToWrite1: 'hello world1',
+		nameOfTheFile1: 'hello-world1.txt',
+		stringToWrite2: 'hello world2',
+		nameOfTheFile2: 'hello-world2.txt',
+	};
+	// When I get the ignored statements of it
+	const ignoreds = await getIgnoredStatements(contract, {
+		data: data,
+	});
+	// and I generate AST of each of them
+	const asts = ignoreds.map((x) => visit(x));
+	// Then the result must contain 3 items
+	expect(asts).toHaveLength(3);
+	// and I get the first one
+	// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+	const first = asts[0]!;
+	// and the its content must be "stringToWrite0"
+	expect(first.content).toStrictEqual('stringToWrite0');
+	// and the its filename must be "nameOfTheFile"
+	expect(first.filename).toStrictEqual('nameOfTheFile0');
+	// and the value indexed by its content in data must be data's stringToWrite0
+	expect(data[first.content as 'stringToWrite0']).toStrictEqual(data.stringToWrite0);
+	// and the value indexed by its filename in data must be data's nameOfTheFile0
+	expect(data[first.filename as 'nameOfTheFile0']).toStrictEqual(data.nameOfTheFile0);
+	// and I get the second one
+	// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+	const second = asts[1]!;
+	// and the its content must be "stringToWrite0"
+	expect(second.content).toStrictEqual('stringToWrite1');
+	// and the its filename must be "nameOfTheFile"
+	expect(second.filename).toStrictEqual('nameOfTheFile1');
+	// and the value indexed by its content in data must be data's stringToWrite1
+	expect(data[second.content as 'stringToWrite1']).toStrictEqual(data.stringToWrite1);
+	// and the value indexed by its filename in data must be data's nameOfTheFile1
+	expect(data[second.filename as 'nameOfTheFile1']).toStrictEqual(data.nameOfTheFile1);
+	// and I get the third one
+	// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+	const third = asts[2]!;
+	// and the its content must be "stringToWrite2"
+	expect(third.content).toStrictEqual('stringToWrite2');
+	// and the its filename must be "nameOfTheFile"
+	expect(third.filename).toStrictEqual('nameOfTheFile2');
+	// and the value indexed by its content in data must be data's stringToWrite2
+	expect(data[third.content as 'stringToWrite2']).toStrictEqual(data.stringToWrite2);
+	// and the value indexed by its filename in data must be data's nameOfTheFile2
+	expect(data[third.filename as 'nameOfTheFile2']).toStrictEqual(data.nameOfTheFile2);
 });

pkg/fs/visitor.ts

@@ -1,6 +1,5 @@
 import { BaseFsVisitor, parse, type FileOverrideStatementCtx } from './parser';
 
-import type { ZenroomParams } from '@slangroom/shared';
 import type { CstNode } from '@slangroom/deps/chevrotain';
 
 export type FileOverrideStatement = {
@@ -31,7 +30,13 @@ class Visitor extends BaseFsVisitor {
 
 export const FsVisitor = new Visitor();
 
-export const visit = async (contract: string, params?: ZenroomParams) => {
-	const parsed = await parse(contract, params);
-	return FsVisitor.visit(parsed);
+/**
+ * Visits the given statement for filesystems statements.
+ *
+ * @param statement is the statement ignored by Zenroom.
+ * @returns the AST of the parsed CST.
+ **/
+export const visit = (statement: string) => {
+	const cst = parse(statement);
+	return FsVisitor.visit(cst);
 };