feat: allow passing playwright config path at a check level (#1154)

* feat: allow passing playwright config path at a check level

* fix: pw-test should load playwright config even if not defined on checkly config

* fix: pw-test

* chore: remove unused import

* chore: log command output in failing test if exit code is not as expected

* fix: must install dependencies in fixture or we can't resolve pw correctly

---------

Co-authored-by: Simo Kinnunen <[email protected]>

Author: ferrandiaz

packages/cli/e2e/__tests__/pw-test.spec.ts

@@ -2,25 +2,39 @@ import path from 'node:path'
 import fs from 'node:fs'
 
 import config from 'config'
-import { describe, it, expect, afterEach } from 'vitest'
+import { describe, it, expect, afterEach, beforeAll } from 'vitest'
 
 import { runChecklyCli } from '../run-checkly'
 import { loadChecklyConfig } from '../../src/services/checkly-config-loader'
 
+const FIXTURE_TEST_PWT_NATIVE = path.join(__dirname, 'fixtures', 'test-pwt-native')
+
 describe('pw-test', { timeout: 45000 }, () => {
   afterEach(() => {
-    const configPath = path.join(__dirname, 'fixtures', 'test-pwt-native')
-    fs.copyFileSync(path.join(configPath, 'checkly.config.original.ts'), path.join(configPath, 'checkly.config.ts'))
+    fs.copyFileSync(
+      path.join(FIXTURE_TEST_PWT_NATIVE, 'checkly.config.original.ts'),
+      path.join(FIXTURE_TEST_PWT_NATIVE, 'checkly.config.ts'),
+    )
+  })
+
+  beforeAll(async () => {
+    // Install fixture dependencies or they will not resolve correctly.
+    const { execa } = await import('execa')
+    await execa('npm', ['install'], { cwd: FIXTURE_TEST_PWT_NATIVE })
   })
 
   it('Playwright test should run successfully', async () => {
     const result = await runChecklyCli({
       args: ['pw-test', '--', `--grep`, '@TAG-B'],
       apiKey: config.get('apiKey'),
       accountId: config.get('accountId'),
-      directory: path.join(__dirname, 'fixtures', 'test-pwt-native'),
+      directory: FIXTURE_TEST_PWT_NATIVE,
       timeout: 120000, // 2 minutes
     })
+    if (result.status !== 0) {
+      // eslint-disable-next-line no-console
+      console.log(result)
+    }
     expect(result.status).toBe(0)
   }, 130000)
 
@@ -29,11 +43,11 @@ describe('pw-test', { timeout: 45000 }, () => {
       args: ['pw-test', '--create-check', '--', `--grep`, '@TAG-B'],
       apiKey: config.get('apiKey'),
       accountId: config.get('accountId'),
-      directory: path.join(__dirname, 'fixtures', 'test-pwt-native'),
+      directory: FIXTURE_TEST_PWT_NATIVE,
       timeout: 120000, // 2 minutes
     })
     expect(result.status).toBe(0)
-    const checklyConfig = await loadChecklyConfig(path.join(__dirname, 'fixtures', 'test-pwt-native'))
+    const checklyConfig = await loadChecklyConfig(FIXTURE_TEST_PWT_NATIVE)
     expect(checklyConfig.config?.checks).toBeDefined()
     expect(checklyConfig.config?.checks?.playwrightConfigPath).toBe('./playwright.config.ts')
     expect(checklyConfig.config?.checks?.playwrightChecks).toBeDefined()

packages/cli/src/commands/pw-test.ts

@@ -1,5 +1,6 @@
 import { AuthCommand } from './authCommand'
 import {
+  findPlaywrightConfigPath,
   getCiInformation,
   getDefaultChecklyConfig,
   getEnvs,
@@ -115,8 +116,20 @@ export default class PwTestCommand extends AuthCommand {
       config: checklyConfig,
       constructs: checklyConfigConstructs,
     } = await loadChecklyConfig(configDirectory, configFilenames, false, pwPathFlag)
-    const playwrightConfigPath = checklyConfig.checks?.playwrightConfigPath
-    const dir = path.dirname(playwrightConfigPath || '.')
+    let playwrightConfigPath = pwPathFlag ?? checklyConfig.checks?.playwrightConfigPath
+
+    if (!playwrightConfigPath) {
+      const foundPath = findPlaywrightConfigPath(configDirectory)
+      if (!foundPath) {
+        this.style.actionFailure()
+        this.style.shortError('No Playwright config found. You can specify a custom path using the playwright --config flag.')
+        this.exit(1)
+      }
+      playwrightConfigPath = `./${path.relative(configDirectory, foundPath)}`
+    }
+
+    const absoluteConfigPath = path.resolve(configDirectory, playwrightConfigPath)
+    const dir = path.dirname(absoluteConfigPath)
     const playwrightCheck = await PwTestCommand.createPlaywrightCheck(
       playwrightFlags,
       runLocation as keyof Region,

packages/cli/src/services/checkly-config-loader.ts

@@ -1,7 +1,6 @@
 import * as path from 'path'
 import fs from 'node:fs/promises'
-import { existsSync } from 'fs'
-import { getDefaultChecklyConfig, writeChecklyConfigFile } from './util'
+import { findPlaywrightConfigPath, getDefaultChecklyConfig, writeChecklyConfigFile } from './util'
 import { CheckProps, RuntimeCheckProps } from '../constructs/check'
 import { PlaywrightCheckProps } from '../constructs/playwright-check'
 import { Session } from '../constructs'
@@ -35,7 +34,7 @@ export type CheckConfigDefaults =
 export type PlaywrightSlimmedProp = Pick<PlaywrightCheckProps, 'name' | 'activated'
   | 'muted' | 'shouldFail' | 'locations' | 'tags' | 'frequency' | 'environmentVariables'
   | 'alertChannels' | 'privateLocations' | 'retryStrategy' | 'alertEscalationPolicy'
-  | 'pwProjects' | 'pwTags' | 'installCommand' | 'testCommand' | 'group' | 'groupName' | 'runParallel'> & { logicalId: string }
+  | 'pwProjects' | 'pwTags' | 'installCommand' | 'testCommand' | 'group' | 'groupName' | 'runParallel'> & { logicalId: string, playwrightConfigPath?: string }
 
 export type ChecklyConfig = {
   /**
@@ -192,12 +191,6 @@ async function handleMissingConfig (
   throw new ConfigNotFoundError(`Unable to locate a config at ${dir} with ${filenames.join(', ')}.`)
 }
 
-function findPlaywrightConfigPath (dir: string): string | undefined {
-  return ['playwright.config.ts', 'playwright.config.js']
-    .map(file => path.resolve(dir, file))
-    .find(filePath => existsSync(filePath))
-}
-
 function validateConfigFields (config: ChecklyConfig, fields: (keyof ChecklyConfig)[]): void {
   for (const field of fields) {
     if (!config?.[field] || !isString(config[field])) {

packages/cli/src/services/project-parser.ts

@@ -1,7 +1,6 @@
 import * as path from 'path'
 import {
-  findFilesWithPattern,
-
+  findFilesWithPattern, getPlaywrightConfigPath,
   pathToPosix,
 } from './util'
 import {
@@ -121,21 +120,15 @@ async function loadPlaywrightChecks (
   playwrightConfigPath?: string,
   include?: string | string[],
 ) {
-  if (!playwrightConfigPath) {
-    return
-  }
-
-  // Resolve the playwrightConfigPath relative to the project directory
-  const resolvedPlaywrightConfigPath = path.resolve(directory, playwrightConfigPath)
-
   if (playwrightChecks?.length) {
     try {
-      setCheckFilePaths(playwrightConfigPath, directory)
       for (const playwrightCheckProps of playwrightChecks) {
+        const configPath = getPlaywrightConfigPath(playwrightCheckProps, playwrightConfigPath, directory)
+        setCheckFilePaths(configPath, directory)
         // eslint-disable-next-line @typescript-eslint/no-unused-vars
         const playwrightCheck = new PlaywrightCheck(playwrightCheckProps.logicalId, {
           ...playwrightCheckProps,
-          playwrightConfigPath: resolvedPlaywrightConfigPath,
+          playwrightConfigPath: configPath,
           include,
         })
       }
@@ -144,7 +137,11 @@ async function loadPlaywrightChecks (
     }
   } else {
     try {
+      if (!playwrightConfigPath) {
+        return
+      }
       setCheckFilePaths(playwrightConfigPath, directory)
+      const resolvedPlaywrightConfigPath = path.resolve(directory, playwrightConfigPath)
       const basePath = path.basename(resolvedPlaywrightConfigPath)
       // eslint-disable-next-line @typescript-eslint/no-unused-vars
       const playwrightCheck = new PlaywrightCheck(basePath, {

packages/cli/src/services/util.ts

@@ -26,6 +26,7 @@ import {
   PNpmDetector,
   YarnDetector,
 } from './check-parser/package-files/package-manager'
+import { existsSync } from 'fs'
 
 export interface GitInformation {
   commitId: string
@@ -358,3 +359,23 @@ export async function writeChecklyConfigFile (dir: string, config: ChecklyConfig
 
   await fs.writeFile(configFile, configContent, { encoding: 'utf-8' })
 }
+
+export function getPlaywrightConfigPath (
+  playwrightCheckProps: PlaywrightSlimmedProp,
+  playwrightConfigPath: string | undefined,
+  dir: string,
+): string {
+  if (playwrightCheckProps.playwrightConfigPath) {
+    return path.resolve(dir, playwrightCheckProps.playwrightConfigPath)
+  } else if (playwrightConfigPath) {
+    return path.resolve(dir, playwrightConfigPath)
+  } else {
+    throw new Error('No Playwright config path provided.')
+  }
+}
+
+export function findPlaywrightConfigPath (dir: string): string | undefined {
+  return ['playwright.config.ts', 'playwright.config.js']
+    .map(file => path.resolve(dir, file))
+    .find(filePath => existsSync(filePath))
+}