From 944d7ce51a74be0ff18e6601a6de6cc8ed4508ba Mon Sep 17 00:00:00 2001 From: Bayheck Date: Wed, 11 Oct 2023 20:23:35 +0600 Subject: [PATCH 01/22] feat: added typescript testcaferc configuration file support --- src/configuration/configuration-base.ts | 33 ++++++++++++++++++++++++- src/configuration/formats.ts | 1 + 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/configuration/configuration-base.ts b/src/configuration/configuration-base.ts index 5e2922b4940..847efb1fea0 100644 --- a/src/configuration/configuration-base.ts +++ b/src/configuration/configuration-base.ts @@ -1,5 +1,9 @@ -import { extname, isAbsolute } from 'path'; + +import { + extname, isAbsolute, +} from 'path'; import debug from 'debug'; + import JSON5 from 'json5'; import { @@ -21,6 +25,7 @@ import Extensions from './formats'; import { ReadConfigFileError } from '../errors/runtime'; import { RUNTIME_ERRORS } from '../errors/types'; + const DEBUG_LOGGER = debug('testcafe:configuration'); export default class Configuration { @@ -155,6 +160,7 @@ export default class Configuration { } public async _load (): Promise { + if (!this.defaultPaths?.length) return null; @@ -166,6 +172,8 @@ export default class Configuration { if (this._isJSConfiguration(filePath)) options = this._readJsConfigurationFileContent(filePath); + else if (this._isTSConfiguration(filePath)) + options = await this._readTsConfigurationFileContent(filePath); else { const configurationFileContent = await this._readConfigurationFileContent(filePath); @@ -178,6 +186,7 @@ export default class Configuration { const existedConfigs = configs.filter(config => !!config.options); + if (!existedConfigs.length) return null; @@ -210,6 +219,10 @@ export default class Configuration { return Configuration._hasExtension(filePath, Extensions.js) || Configuration._hasExtension(filePath, Extensions.cjs); } + protected _isTSConfiguration (filePath = this.filePath): boolean { + return Configuration._hasExtension(filePath, Extensions.ts) || Configuration._hasExtension(filePath, Extensions.cjs); + } + protected _isJSONConfiguration (filePath = this.filePath): boolean { return Configuration._hasExtension(filePath, Extensions.json); } @@ -229,6 +242,24 @@ export default class Configuration { return null; } + public async _readTsConfigurationFileContent (filePath = this.filePath): Promise { + if (filePath) { + delete require.cache[filePath]; + const TypeScriptTestFileCompiler = require('../compiler/test-file/formats/typescript/compiler'); + const compiler = new TypeScriptTestFileCompiler(); + + const precompiledCode = await compiler.precompile([{ filename: filePath }]); + + await compiler.compile(precompiledCode?.[0], filePath); + const options = require(filePath); + + return options; + } + + return null; + } + + public async _readConfigurationFileContent (filePath = this.filePath): Promise { try { return await readFile(filePath); diff --git a/src/configuration/formats.ts b/src/configuration/formats.ts index a26d8a45df4..3e3cbf542d9 100644 --- a/src/configuration/formats.ts +++ b/src/configuration/formats.ts @@ -2,6 +2,7 @@ enum Extensions { js = '.js', json = '.json', cjs = '.cjs', + ts = '.ts', } export default Extensions; From 56ae266078c2928c2b087bdd70dc66939bd3c0fb Mon Sep 17 00:00:00 2001 From: Bayheck Date: Fri, 13 Oct 2023 21:53:09 +0600 Subject: [PATCH 02/22] testing: added test for configuration-test with custom ts config --- test/server/configuration-test.js | 208 +++++++++++------- .../data/configuration/module/module.js | 7 + .../configuration/typescript-module/module.ts | 5 + 3 files changed, 135 insertions(+), 85 deletions(-) create mode 100644 test/server/data/configuration/module/module.js create mode 100644 test/server/data/configuration/typescript-module/module.ts diff --git a/test/server/configuration-test.js b/test/server/configuration-test.js index 6f41c35b30e..e2b5e8a3690 100644 --- a/test/server/configuration-test.js +++ b/test/server/configuration-test.js @@ -36,6 +36,19 @@ const createJsConfig = (filePath, options) => { fs.writeFileSync(filePath, `module.exports = ${JSON.stringify(options)}`); }; +const createTsConfig = (filePath, options) => { + options = options || {}; + fs.writeFileSync(filePath, ` + const fs = require('fs'); + const path = require('path'); + const { nanoid } = require('nanoid'); + const jsCustomModule = require('./test/server/data/configuration/module/module.js'); + const tsCustomModule = require('./test/server/data/configuration/typescript-module/module.ts'); + + \n + module.exports = ${JSON.stringify(options)}`); +}; + const jsConfigIndex = TestCafeConfiguration.FILENAMES.findIndex(file=>file.includes(Extensions.js)); const jsonConfigIndex = TestCafeConfiguration.FILENAMES.findIndex(file=>file.includes(Extensions.json)); @@ -729,6 +742,116 @@ describe('TypeScriptConfiguration', function () { ); }); + describe('Custom Testcafe Config Path', () => { + let configuration; + + afterEach(async () => { + await del(configuration.defaultPaths); + }); + + it('Custom config path is used', () => { + const customConfigFile = 'custom11.testcaferc.json'; + + const options = { + 'hostname': '123.456.789', + 'port1': 1234, + 'port2': 5678, + 'src': 'path1/folder', + 'browser': 'edge', + }; + + createJSONConfig(customConfigFile, options); + + configuration = new TestCafeConfiguration(customConfigFile); + + return configuration.init() + .then(() => { + expect(pathUtil.basename(configuration.filePath)).eql(customConfigFile); + expect(configuration.getOption('hostname')).eql(options.hostname); + expect(configuration.getOption('port1')).eql(options.port1); + expect(configuration.getOption('port2')).eql(options.port2); + expect(configuration.getOption('src')).eql([ options.src ]); + expect(configuration.getOption('browser')).eql(options.browser); + }); + }); + + it('Custom js config path is used', async () => { + const customConfigFile = 'custom11.testcaferc.js'; + + const options = { + 'hostname': '123.456.789', + 'port1': 1234, + 'port2': 5678, + 'src': 'path1/folder', + 'browser': 'edge', + }; + + createJsConfig(customConfigFile, options); + + configuration = new TestCafeConfiguration(customConfigFile); + + await configuration.init(); + + expect(pathUtil.basename(configuration.filePath)).eql(customConfigFile); + expect(configuration.getOption('hostname')).eql(options.hostname); + expect(configuration.getOption('port1')).eql(options.port1); + expect(configuration.getOption('port2')).eql(options.port2); + expect(configuration.getOption('src')).eql([ options.src ]); + expect(configuration.getOption('browser')).eql(options.browser); + }); + + it('Custom ts config path is used', async () => { + const customConfigFile = 'custom11.testcaferc.ts'; + + const options = { + 'hostname': '123.456.789', + 'port1': 1234, + 'port2': 5678, + 'src': 'path1/folder', + 'browser': 'edge', + }; + + createTsConfig(customConfigFile, options); + + configuration = new TestCafeConfiguration(customConfigFile); + + await configuration.init(); + + expect(pathUtil.basename(configuration.filePath)).eql(customConfigFile); + expect(configuration.getOption('hostname')).eql(options.hostname); + expect(configuration.getOption('port1')).eql(options.port1); + expect(configuration.getOption('port2')).eql(options.port2); + expect(configuration.getOption('src')).eql([ options.src ]); + expect(configuration.getOption('browser')).eql(options.browser); + }); + + it('Constructor should revert back to default when no custom config', () => { + const defaultFileLocation = '.testcaferc.json'; + + const options = { + 'hostname': '123.456.789', + 'port1': 1234, + 'port2': 5678, + 'src': 'path1/folder', + 'browser': 'edge', + }; + + createJSONConfig(defaultFileLocation, options); + + configuration = new TestCafeConfiguration(); + + return configuration.init() + .then(() => { + expect(pathUtil.basename(configuration.filePath)).eql(defaultFileLocation); + expect(configuration.getOption('hostname')).eql(options.hostname); + expect(configuration.getOption('port1')).eql(options.port1); + expect(configuration.getOption('port2')).eql(options.port2); + expect(configuration.getOption('src')).eql([ options.src ]); + expect(configuration.getOption('browser')).eql(options.browser); + }); + }); + }); + describe('With configuration file', () => { tmp.setGracefulCleanup(); @@ -917,89 +1040,4 @@ describe('TypeScriptConfiguration', function () { }); }); }); - - describe('Custom Testcafe Config Path', () => { - let configuration; - - afterEach(async () => { - await del(configuration.defaultPaths); - }); - - it('Custom config path is used', () => { - const customConfigFile = 'custom11.testcaferc.json'; - - const options = { - 'hostname': '123.456.789', - 'port1': 1234, - 'port2': 5678, - 'src': 'path1/folder', - 'browser': 'edge', - }; - - createJSONConfig(customConfigFile, options); - - configuration = new TestCafeConfiguration(customConfigFile); - - return configuration.init() - .then(() => { - expect(pathUtil.basename(configuration.filePath)).eql(customConfigFile); - expect(configuration.getOption('hostname')).eql(options.hostname); - expect(configuration.getOption('port1')).eql(options.port1); - expect(configuration.getOption('port2')).eql(options.port2); - expect(configuration.getOption('src')).eql([ options.src ]); - expect(configuration.getOption('browser')).eql(options.browser); - }); - }); - - it('Custom js config path is used', async () => { - const customConfigFile = 'custom11.testcaferc.js'; - - const options = { - 'hostname': '123.456.789', - 'port1': 1234, - 'port2': 5678, - 'src': 'path1/folder', - 'browser': 'edge', - }; - - createJsConfig(customConfigFile, options); - - configuration = new TestCafeConfiguration(customConfigFile); - - await configuration.init(); - - expect(pathUtil.basename(configuration.filePath)).eql(customConfigFile); - expect(configuration.getOption('hostname')).eql(options.hostname); - expect(configuration.getOption('port1')).eql(options.port1); - expect(configuration.getOption('port2')).eql(options.port2); - expect(configuration.getOption('src')).eql([ options.src ]); - expect(configuration.getOption('browser')).eql(options.browser); - }); - - it('Constructor should revert back to default when no custom config', () => { - const defaultFileLocation = '.testcaferc.json'; - - const options = { - 'hostname': '123.456.789', - 'port1': 1234, - 'port2': 5678, - 'src': 'path1/folder', - 'browser': 'edge', - }; - - createJSONConfig(defaultFileLocation, options); - - configuration = new TestCafeConfiguration(); - - return configuration.init() - .then(() => { - expect(pathUtil.basename(configuration.filePath)).eql(defaultFileLocation); - expect(configuration.getOption('hostname')).eql(options.hostname); - expect(configuration.getOption('port1')).eql(options.port1); - expect(configuration.getOption('port2')).eql(options.port2); - expect(configuration.getOption('src')).eql([ options.src ]); - expect(configuration.getOption('browser')).eql(options.browser); - }); - }); - }); }); diff --git a/test/server/data/configuration/module/module.js b/test/server/data/configuration/module/module.js new file mode 100644 index 00000000000..a7ddc5cf0e4 --- /dev/null +++ b/test/server/data/configuration/module/module.js @@ -0,0 +1,7 @@ +function module() { + return 'module'; +} + +module.exports = { + module: module, +} diff --git a/test/server/data/configuration/typescript-module/module.ts b/test/server/data/configuration/typescript-module/module.ts new file mode 100644 index 00000000000..26581fb7116 --- /dev/null +++ b/test/server/data/configuration/typescript-module/module.ts @@ -0,0 +1,5 @@ +function greet (): string { + return 'Hello World!'; +} + +module.exports = { greet }; From 82e07668cf8f3ef7f266013198ad6458775c95de Mon Sep 17 00:00:00 2001 From: Bayheck Date: Wed, 18 Oct 2023 23:18:26 +0600 Subject: [PATCH 03/22] refactor: basic decomposition of typescript compiler for configuration --- src/compiler/test-file/api-based.js | 4 + .../formats/typescript/configuration.ts | 331 +++ src/configuration/configuration-base.ts | 13 +- test/server/configuration-test.js | 1862 ++++++++--------- 4 files changed, 1276 insertions(+), 934 deletions(-) create mode 100644 src/compiler/test-file/formats/typescript/configuration.ts diff --git a/src/compiler/test-file/api-based.js b/src/compiler/test-file/api-based.js index f921b0ae2b6..bb44863d6f5 100644 --- a/src/compiler/test-file/api-based.js +++ b/src/compiler/test-file/api-based.js @@ -82,7 +82,11 @@ export default class APIBasedTestFileCompilerBase extends TestFileCompilerBase { mod._compile(code, filename); + Module._cache[filename] = mod; + cacheProxy.stopExternalCaching(); + + Module._cache[filename] = mod; } } diff --git a/src/compiler/test-file/formats/typescript/configuration.ts b/src/compiler/test-file/formats/typescript/configuration.ts new file mode 100644 index 00000000000..61f5f72bdd3 --- /dev/null +++ b/src/compiler/test-file/formats/typescript/configuration.ts @@ -0,0 +1,331 @@ +/* eslint-disable no-debugger */ +import path from 'path'; +import { zipObject } from 'lodash'; +import OS from 'os-family'; +import APIBasedTestFileCompilerBase from '../../api-based'; +import ESNextTestFileCompiler from '../es-next/compiler'; +import { GeneralError } from '../../../../errors/runtime'; +import { RUNTIME_ERRORS } from '../../../../errors/types'; +import debug from 'debug'; +// import cacheProxy from '../../cache-proxy'; +import { isRelative } from '../../../../api/test-page-url'; + +import getExportableLibPath from '../../get-exportable-lib-path'; +import DISABLE_V8_OPTIMIZATION_NOTE from '../../disable-v8-optimization-note'; + +// NOTE: For type definitions only +import TypeScript, { + SyntaxKind, + VisitResult, + Visitor, + Node, + visitEachChild, + visitNode, + TransformerFactory, + SourceFile, + addSyntheticLeadingComment, +} from 'typescript'; + +import { TypeScriptCompilerOptions } from '../../../../configuration/interfaces'; +import { OptionalCompilerArguments } from '../../../interfaces'; +import Extensions from '../extensions'; + +declare type TypeScriptInstance = typeof TypeScript; + +const tsFactory = TypeScript.factory; + +interface TestFileInfo { + filename: string; +} + +declare interface RequireCompilerFunction { + (code: string, filename: string): string; +} + +interface RequireCompilers { + [extension: string]: RequireCompilerFunction; +} + +function testcafeImportPathReplacer (esm?: boolean): TransformerFactory { + return context => { + const visit: Visitor = (node): VisitResult => { + // @ts-ignore + if (node.parent?.kind === SyntaxKind.ImportDeclaration && node.kind === SyntaxKind.StringLiteral && node.text === 'testcafe') { + const libPath = getExportableLibPath(esm); + + return tsFactory.createStringLiteral(libPath); + } + + return visitEachChild(node, child => visit(child), context); + }; + + return node => visitNode(node, visit); + }; +} + +function disableV8OptimizationCodeAppender (): TransformerFactory { + return () => { + const visit: Visitor = (node): VisitResult => { + const evalStatement = tsFactory.createExpressionStatement(tsFactory.createCallExpression( + tsFactory.createIdentifier('eval'), + void 0, + [tsFactory.createStringLiteral('')] + )); + + const evalStatementWithComment = addSyntheticLeadingComment(evalStatement, SyntaxKind.MultiLineCommentTrivia, DISABLE_V8_OPTIMIZATION_NOTE, true); + + // @ts-ignore + return tsFactory.updateSourceFile(node, [...node.statements, evalStatementWithComment]); + }; + + return node => visitNode(node, visit); + }; +} + + +const DEBUG_LOGGER = debug('testcafe:compiler:typescript'); + +const RENAMED_DEPENDENCIES_MAP = new Map([['testcafe', getExportableLibPath()]]); + +const DEFAULT_TYPESCRIPT_COMPILER_PATH = 'typescript'; + + +export default class TypeScriptConfigurationCompiler extends APIBasedTestFileCompilerBase { + private static tsDefsPath = TypeScriptConfigurationCompiler._getTSDefsPath(); + + + private readonly _compilerPath: string; + + public constructor (compilerOptions?: TypeScriptCompilerOptions, { baseUrl, esm }: OptionalCompilerArguments = {}) { + super({ baseUrl, esm }); + + this._compilerPath = TypeScriptConfigurationCompiler._getCompilerPath(compilerOptions); + } + + private static _getCompilerPath (compilerOptions?: TypeScriptCompilerOptions): string { + let compilerPath = compilerOptions && compilerOptions.customCompilerModulePath; + + if (!compilerPath || compilerPath === DEFAULT_TYPESCRIPT_COMPILER_PATH) + return DEFAULT_TYPESCRIPT_COMPILER_PATH; + + // NOTE: if the relative path to custom TypeScript compiler module is specified + // then we will resolve the path from the root of the 'testcafe' module + if (isRelative(compilerPath)) { + const testcafeRootFolder = path.resolve(__dirname, '../../../../../'); + + compilerPath = path.resolve(testcafeRootFolder, compilerPath); + } + + return compilerPath; + } + + private _loadTypeScriptCompiler (): TypeScriptInstance { + try { + return require(this._compilerPath); + } + catch (err: any) { + throw new GeneralError(RUNTIME_ERRORS.typeScriptCompilerLoadingError, err.message); + } + } + + private static _normalizeFilename (filename: string): string { + filename = path.resolve(filename); + + if (OS.win) + filename = filename.toLowerCase(); + + return filename; + } + + private static _getTSDefsPath (): string { + return TypeScriptConfigurationCompiler._normalizeFilename(path.resolve(__dirname, '../../../../../ts-defs/index.d.ts')); + } + + private _reportErrors (diagnostics: Readonly): void { + // NOTE: lazy load the compiler + const ts: TypeScriptInstance = this._loadTypeScriptCompiler(); + let errMsg = 'TypeScript compilation failed.\n'; + + diagnostics.forEach(d => { + const message = ts.flattenDiagnosticMessageText(d.messageText, '\n'); + const file = d.file; + + if (file && d.start !== void 0) { + const { line, character } = file.getLineAndCharacterOfPosition(d.start); + + errMsg += `${file.fileName} (${line + 1}, ${character + 1}): `; + } + + errMsg += `${message}\n`; + }); + + throw new Error(errMsg); + } + + + private _compileFilesToCache (ts: TypeScriptInstance, filenames: string[]): void { + // const opts = this._tsConfig.getOptions() as Dictionary; + // const program = ts.createProgram([TypeScriptConfigurationCompiler.tsDefsPath, ...filenames], opts); + const program = ts.createProgram([TypeScriptConfigurationCompiler.tsDefsPath, ...filenames], { + 'experimentalDecorators': true, + 'emitDecoratorMetadata': true, + 'allowJs': true, + 'pretty': true, + 'inlineSourceMap': true, + 'noImplicitAny': false, + 'module': 1, + 'moduleResolution': 2, + 'target': 3, + 'jsx': 2, + 'suppressOutputPathCheck': true, + 'skipLibCheck': true, + }); + + DEBUG_LOGGER('version: %s', ts.version); + // DEBUG_LOGGER('options: %O', opts); + + program.getSourceFiles().forEach(sourceFile => { + // @ts-ignore A hack to allow import globally installed TestCafe in tests + sourceFile.renamedDependencies = RENAMED_DEPENDENCIES_MAP; + }); + + const diagnostics = ts.getPreEmitDiagnostics(program); + + if (diagnostics.length) + this._reportErrors(diagnostics); + + // NOTE: The first argument of emit() is a source file to be compiled. If it's undefined, all files in + // will be compiled. contains a file specified in createProgram() plus all its dependencies. + // This mode is much faster than compiling files one-by-one, and it is used in the tsc CLI compiler. + program.emit(void 0, (outputName, result, writeBOM, onError, sources) => { + if (!sources) + return; + + const sourcePath = TypeScriptConfigurationCompiler._normalizeFilename(sources[0].fileName); + + this.cache[sourcePath] = result; + }, void 0, void 0, { + before: this._getTypescriptTransformers(), + }); + } + + private _getTypescriptTransformers (): TransformerFactory[] { + const transformers: TransformerFactory[] = [testcafeImportPathReplacer(this.esm)]; + + if (this.esm) + transformers.push(disableV8OptimizationCodeAppender()); + + return transformers; + } + + public _precompileCode (testFilesInfo: TestFileInfo[]): string[] { + DEBUG_LOGGER('path: "%s"', this._compilerPath); + + // NOTE: lazy load the compiler + const ts: TypeScriptInstance = this._loadTypeScriptCompiler(); + const filenames = testFilesInfo.map(({ filename }) => filename); + const normalizedFilenames = filenames.map(filename => TypeScriptConfigurationCompiler._normalizeFilename(filename)); + const normalizedFilenamesMap = zipObject(normalizedFilenames, filenames); + + const uncachedFiles = normalizedFilenames + .filter(filename => filename !== TypeScriptConfigurationCompiler.tsDefsPath && !this.cache[filename]) + .map(filename => normalizedFilenamesMap[filename]); + + if (uncachedFiles.length) + this._compileFilesToCache(ts, uncachedFiles); + + return normalizedFilenames.map(filename => this.cache[filename]); + } + + public _getRequireCompilers (): RequireCompilers { + debugger; + const requireCompilers: RequireCompilers = { + [Extensions.ts]: (code, filename) => this._compileCode(code, filename), + [Extensions.tsx]: (code, filename) => this._compileCode(code, filename), + [Extensions.js]: (code, filename) => ESNextTestFileCompiler.prototype._compileCode.call(this, code, filename), + [Extensions.cjs]: (code, filename) => ESNextTestFileCompiler.prototype._compileCode.call(this, code, filename), + [Extensions.jsx]: (code, filename) => ESNextTestFileCompiler.prototype._compileCode.call(this, code, filename), + }; + + if (this.esm) + requireCompilers[Extensions.mjs] = (code, filename) => ESNextTestFileCompiler.prototype._compileCode.call(this, code, filename); + + return requireCompilers; + } + + // async _execAsModule (code: string, filename: string): Promise { + // const mod = Module(filename, module.parent); + + // mod.filename = filename; + // mod.paths = APIBasedTestFileCompilerBase._getNodeModulesLookupPath(filename); + + // cacheProxy.startExternalCaching(this.cachePrefix); + + // mod._compile(code, filename); + + // Module._cache[filename] = mod; + + // cacheProxy.stopExternalCaching(); + + // this.emit('module-compiled', mod.exports); + + // Module._cache[filename] = mod; + // } + + _setupRequireHook (): void { + const requireCompilers = this._getRequireCompilers(); + + debugger; + this.origRequireExtensions = Object.create(null); + + Object.keys(requireCompilers).forEach(ext => { + const origExt = require.extensions[ext]; + + this.origRequireExtensions[ext] = origExt; + + require.extensions[ext] = (mod, filename) => { + this._compileExternalModule(mod, filename, requireCompilers[ext], origExt); + }; + }); + } + + async _runCompiledCode (compiledCode?: string, filename?:string): Promise { + this._setupRequireHook(); + debugger; + try { + await this._execAsModule(compiledCode || '', filename || ''); + } + + finally { + this._removeRequireHook(); + } + } + + execute (compiledCode: string, filename: string): any { + debugger; + return this._runCompiledCode(compiledCode, filename); + } + + async compile (code: string, filename: string): Promise { + const [compiledCode] = await this.precompile([{ code, filename }]); + + debugger; + if (compiledCode) + return this.execute(compiledCode, filename); + + return Promise.resolve(); + } + + + public get canPrecompile (): boolean { + return true; + } + + public get canCompileInEsm (): boolean { + return true; + } + + public getSupportedExtension (): string[] { + return [Extensions.ts, Extensions.tsx]; + } +} diff --git a/src/configuration/configuration-base.ts b/src/configuration/configuration-base.ts index 847efb1fea0..496ffbc2597 100644 --- a/src/configuration/configuration-base.ts +++ b/src/configuration/configuration-base.ts @@ -1,3 +1,5 @@ +/* eslint-disable no-console */ +/* eslint-disable no-debugger */ import { extname, isAbsolute, @@ -19,6 +21,7 @@ import OptionSource from './option-source'; import resolvePathRelativelyCwd from '../utils/resolve-path-relatively-cwd'; import renderTemplate from '../utils/render-template'; import WARNING_MESSAGES from '../notifications/warning-message'; +import TypeScriptConfigurationCompiler from '../compiler/test-file/formats/typescript/configuration'; import log from '../cli/log'; import { Dictionary } from './interfaces'; import Extensions from './formats'; @@ -42,8 +45,10 @@ export default class Configuration { } protected static _fromObj (obj: object): Dictionary