From 7fb63eab338f762b550b77dd2aff9c51b2be8174 Mon Sep 17 00:00:00 2001 From: serbangeorge-m Date: Fri, 19 Sep 2025 17:08:05 +0200 Subject: [PATCH 1/5] chore(test): enable test and skip for ci and windows Signed-off-by: serbangeorge-m --- .github/workflows/ai-lab-e2e-nightly-windows.yaml | 1 + tests/playwright/src/ai-lab-extension.spec.ts | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ai-lab-e2e-nightly-windows.yaml b/.github/workflows/ai-lab-e2e-nightly-windows.yaml index 088f73e52..75a3a1aa1 100644 --- a/.github/workflows/ai-lab-e2e-nightly-windows.yaml +++ b/.github/workflows/ai-lab-e2e-nightly-windows.yaml @@ -98,6 +98,7 @@ jobs: - name: Set the default env. variables env: + CI: true DEFAULT_PODMAN_DESKTOP_REPO_ARGS: 'REPO=podman-desktop,FORK=podman-desktop,BRANCH=main' DEFAULT_NPM_TARGET: 'test:e2e' DEFAULT_ENV_VARS: 'TEST_PODMAN_MACHINE=true,ELECTRON_ENABLE_INSPECT=true' diff --git a/tests/playwright/src/ai-lab-extension.spec.ts b/tests/playwright/src/ai-lab-extension.spec.ts index f43977b2f..e3c67f99f 100644 --- a/tests/playwright/src/ai-lab-extension.spec.ts +++ b/tests/playwright/src/ai-lab-extension.spec.ts @@ -217,8 +217,8 @@ test.describe.serial(`AI Lab extension installation and verification`, () => { ); }); - //The test is currently failing on ci, relates to https://github.com/redhat-developer/mapt/issues/583 - test.describe.skip(`Start Llama Stack from sidebar and verify containers`, { tag: '@smoke' }, () => { + test.describe.serial(`Start Llama Stack from sidebar and verify containers`, { tag: '@smoke' }, () => { + test.skip(!!isCI && !!isWindows, 'Skipping Llama Stack tests on GitHub Actions with Windows platform'); let llamaStackPage: AiLlamaStackPage; const llamaStackContainerNames: string[] = []; From f686dd13e97941bbf6f09736b943914b2317a6b5 Mon Sep 17 00:00:00 2001 From: serbangeorge-m Date: Fri, 19 Sep 2025 17:27:06 +0200 Subject: [PATCH 2/5] chore(test): update locator Signed-off-by: serbangeorge-m --- tests/playwright/src/model/ai-lab-model-llamastack-page.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/playwright/src/model/ai-lab-model-llamastack-page.ts b/tests/playwright/src/model/ai-lab-model-llamastack-page.ts index 12bb5552d..ac9332acb 100644 --- a/tests/playwright/src/model/ai-lab-model-llamastack-page.ts +++ b/tests/playwright/src/model/ai-lab-model-llamastack-page.ts @@ -26,7 +26,9 @@ export class AiLlamaStackPage extends AILabBasePage { constructor(page: Page, webview: Page) { super(page, webview, 'Llama Stack'); this.startLlamaStackContainerButton = this.webview.getByRole('button', { name: 'Start Llama Stack container' }); - this.openLlamaStackContainerButton = this.webview.getByRole('button', { name: 'Open Llama Stack container' }); + this.openLlamaStackContainerButton = this.webview.getByRole('button', { + name: 'Open Llama Stack Server container', + }); this.exploreLlamaStackEnvironmentButton = this.webview.getByRole('button', { name: 'Explore Llama-Stack environment', }); From 56f9ff6cb4819de63df4601a61a5b5639632a373 Mon Sep 17 00:00:00 2001 From: serbangeorge-m Date: Mon, 22 Sep 2025 11:40:17 +0200 Subject: [PATCH 3/5] chore(tes): skip tests Signed-off-by: serbangeorge-m --- tests/playwright/src/ai-lab-extension.spec.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/playwright/src/ai-lab-extension.spec.ts b/tests/playwright/src/ai-lab-extension.spec.ts index e3c67f99f..850ba3926 100644 --- a/tests/playwright/src/ai-lab-extension.spec.ts +++ b/tests/playwright/src/ai-lab-extension.spec.ts @@ -291,7 +291,7 @@ test.describe.serial(`AI Lab extension installation and verification`, () => { }); }); - test.describe.serial('AI Lab API endpoint e2e test', { tag: '@smoke' }, () => { + test.describe.skip('AI Lab API endpoint e2e test', { tag: '@smoke' }, () => { let localServerPort: string; let extensionVersion: string | undefined; const model: string = 'facebook/detr-resnet-101'; @@ -390,7 +390,7 @@ test.describe.serial(`AI Lab extension installation and verification`, () => { // Do not use non-instruct models in playground tests. // They break out of guilderails and fail the tests. ['ibm-granite/granite-3.3-8b-instruct-GGUF'].forEach(modelName => { - test.describe.serial(`AI Lab playground creation and deletion for ${modelName}`, { tag: '@smoke' }, () => { + test.describe.skip(`AI Lab playground creation and deletion for ${modelName}`, { tag: '@smoke' }, () => { let catalogPage: AILabCatalogPage; let playgroundsPage: AILabPlaygroundsPage; let playgroundDetailsPage: AILabPlaygroundDetailsPage; @@ -478,7 +478,7 @@ test.describe.serial(`AI Lab extension installation and verification`, () => { AI_APP_MODEL_AND_NAMES.forEach((appNames, appModel) => { /* eslint-disable sonarjs/no-nested-functions */ - test.describe.serial(`AI Recipe installation for ${appModel}`, { tag: '@smoke' }, () => { + test.describe.skip(`AI Recipe installation for ${appModel}`, { tag: '@smoke' }, () => { appNames.forEach(appName => { test.describe.serial(`AI Recipe installation ${appName}`, () => { test.skip( @@ -630,7 +630,7 @@ test.describe.serial(`AI Lab extension installation and verification`, () => { }); }); - test.describe.serial('InstructLab container startup', { tag: '@instructlab' }, () => { + test.describe.skip('InstructLab container startup', { tag: '@instructlab' }, () => { let instructLabPage: AILabTryInstructLabPage; const instructLabContainerName = /^instructlab-\d+$/; let exactInstructLabContainerName = ''; From 1c634c3e3c657e8267e576dc77b559f3f72fb510 Mon Sep 17 00:00:00 2001 From: serbangeorge-m Date: Mon, 22 Sep 2025 11:53:40 +0200 Subject: [PATCH 4/5] chore(test): test cleanup Signed-off-by: serbangeorge-m --- tests/playwright/src/ai-lab-extension.spec.ts | 156 +++++++++--------- 1 file changed, 78 insertions(+), 78 deletions(-) diff --git a/tests/playwright/src/ai-lab-extension.spec.ts b/tests/playwright/src/ai-lab-extension.spec.ts index 850ba3926..1a9d4a339 100644 --- a/tests/playwright/src/ai-lab-extension.spec.ts +++ b/tests/playwright/src/ai-lab-extension.spec.ts @@ -217,81 +217,7 @@ test.describe.serial(`AI Lab extension installation and verification`, () => { ); }); - test.describe.serial(`Start Llama Stack from sidebar and verify containers`, { tag: '@smoke' }, () => { - test.skip(!!isCI && !!isWindows, 'Skipping Llama Stack tests on GitHub Actions with Windows platform'); - let llamaStackPage: AiLlamaStackPage; - const llamaStackContainerNames: string[] = []; - - test.beforeAll(`Open Llama Stack`, async ({ runner, page, navigationBar }) => { - aiLabPage = await reopenAILabDashboard(runner, page, navigationBar); - await aiLabPage.navigationBar.waitForLoad(); - llamaStackPage = await aiLabPage.navigationBar.openLlamaStack(); - await llamaStackPage.waitForLoad(); - }); - - test(`Start Llama Stack containers`, async () => { - test.setTimeout(300_000); - await llamaStackPage.waitForLoad(); - await llamaStackPage.runLlamaStackContainer(); - await playExpect(llamaStackPage.openLlamaStackContainerButton).toBeVisible({ timeout: 120_000 }); - await playExpect(llamaStackPage.exploreLlamaStackEnvironmentButton).toBeVisible({ timeout: 120_000 }); - await playExpect(llamaStackPage.openLlamaStackContainerButton).toBeEnabled({ timeout: 30_000 }); - await playExpect(llamaStackPage.exploreLlamaStackEnvironmentButton).toBeEnabled({ timeout: 30_000 }); - }); - - test(`Verify Llama Stack containers are running`, async ({ navigationBar }) => { - let containersPage = await navigationBar.openContainers(); - await playExpect(containersPage.heading).toBeVisible(); - - await playExpect - .poll( - async () => { - const allRows = await containersPage.getAllTableRows(); - llamaStackContainerNames.length = 0; - for (const row of allRows) { - const text = await row.textContent(); - if (text?.includes('llama-stack')) { - const containerNameMatch = RegExp(/\b(llama-stack[^\s]*)/).exec(text); - if (containerNameMatch) { - llamaStackContainerNames.push(containerNameMatch[1]); - } - } - } - return llamaStackContainerNames.length; - }, - { - timeout: 30_000, - intervals: [5_000], - }, - ) - .toBe(2); - - console.log(`Found containers: ${llamaStackContainerNames.join(', ')}`); - - for (const container of llamaStackContainerNames) { - containersPage = await navigationBar.openContainers(); - await playExpect(containersPage.heading).toBeVisible(); - const containersDetailsPage = await containersPage.openContainersDetails(container); - await playExpect(containersDetailsPage.heading).toBeVisible(); - await playExpect - .poll(async () => containersDetailsPage.getState(), { timeout: 30_000 }) - .toContain(ContainerState.Running); - } - }); - - test.afterAll(`Stop Llama Stack containers`, async ({ navigationBar }) => { - for (const container of llamaStackContainerNames) { - const containersPage = await navigationBar.openContainers(); - await playExpect(containersPage.heading).toBeVisible(); - await containersPage.deleteContainer(container); - await playExpect - .poll(async () => await containersPage.containerExists(container), { timeout: 30_000 }) - .toBeFalsy(); - } - }); - }); - - test.describe.skip('AI Lab API endpoint e2e test', { tag: '@smoke' }, () => { + test.describe.serial('AI Lab API endpoint e2e test', { tag: '@smoke' }, () => { let localServerPort: string; let extensionVersion: string | undefined; const model: string = 'facebook/detr-resnet-101'; @@ -390,7 +316,7 @@ test.describe.serial(`AI Lab extension installation and verification`, () => { // Do not use non-instruct models in playground tests. // They break out of guilderails and fail the tests. ['ibm-granite/granite-3.3-8b-instruct-GGUF'].forEach(modelName => { - test.describe.skip(`AI Lab playground creation and deletion for ${modelName}`, { tag: '@smoke' }, () => { + test.describe.serial(`AI Lab playground creation and deletion for ${modelName}`, { tag: '@smoke' }, () => { let catalogPage: AILabCatalogPage; let playgroundsPage: AILabPlaygroundsPage; let playgroundDetailsPage: AILabPlaygroundDetailsPage; @@ -478,7 +404,7 @@ test.describe.serial(`AI Lab extension installation and verification`, () => { AI_APP_MODEL_AND_NAMES.forEach((appNames, appModel) => { /* eslint-disable sonarjs/no-nested-functions */ - test.describe.skip(`AI Recipe installation for ${appModel}`, { tag: '@smoke' }, () => { + test.describe.serial(`AI Recipe installation for ${appModel}`, { tag: '@smoke' }, () => { appNames.forEach(appName => { test.describe.serial(`AI Recipe installation ${appName}`, () => { test.skip( @@ -630,7 +556,7 @@ test.describe.serial(`AI Lab extension installation and verification`, () => { }); }); - test.describe.skip('InstructLab container startup', { tag: '@instructlab' }, () => { + test.describe.serial('InstructLab container startup', { tag: '@instructlab' }, () => { let instructLabPage: AILabTryInstructLabPage; const instructLabContainerName = /^instructlab-\d+$/; let exactInstructLabContainerName = ''; @@ -693,6 +619,80 @@ test.describe.serial(`AI Lab extension installation and verification`, () => { }); }); +test.describe.serial(`Start Llama Stack from sidebar and verify containers`, { tag: '@smoke' }, () => { + test.skip(!!isCI && !!isWindows, 'Skipping Llama Stack tests on GitHub Actions with Windows platform'); + let llamaStackPage: AiLlamaStackPage; + const llamaStackContainerNames: string[] = []; + + test.beforeAll(`Open Llama Stack`, async ({ runner, page, navigationBar }) => { + aiLabPage = await reopenAILabDashboard(runner, page, navigationBar); + await aiLabPage.navigationBar.waitForLoad(); + llamaStackPage = await aiLabPage.navigationBar.openLlamaStack(); + await llamaStackPage.waitForLoad(); + }); + + test(`Start Llama Stack containers`, async () => { + test.setTimeout(300_000); + await llamaStackPage.waitForLoad(); + await llamaStackPage.runLlamaStackContainer(); + await playExpect(llamaStackPage.openLlamaStackContainerButton).toBeVisible({ timeout: 120_000 }); + await playExpect(llamaStackPage.exploreLlamaStackEnvironmentButton).toBeVisible({ timeout: 120_000 }); + await playExpect(llamaStackPage.openLlamaStackContainerButton).toBeEnabled({ timeout: 30_000 }); + await playExpect(llamaStackPage.exploreLlamaStackEnvironmentButton).toBeEnabled({ timeout: 30_000 }); + }); + + test(`Verify Llama Stack containers are running`, async ({ navigationBar }) => { + let containersPage = await navigationBar.openContainers(); + await playExpect(containersPage.heading).toBeVisible(); + + await playExpect + .poll( + async () => { + const allRows = await containersPage.getAllTableRows(); + llamaStackContainerNames.length = 0; + for (const row of allRows) { + const text = await row.textContent(); + if (text?.includes('llama-stack')) { + const containerNameMatch = RegExp(/\b(llama-stack[^\s]*)/).exec(text); + if (containerNameMatch) { + llamaStackContainerNames.push(containerNameMatch[1]); + } + } + } + return llamaStackContainerNames.length; + }, + { + timeout: 30_000, + intervals: [5_000], + }, + ) + .toBe(2); + + console.log(`Found containers: ${llamaStackContainerNames.join(', ')}`); + + for (const container of llamaStackContainerNames) { + containersPage = await navigationBar.openContainers(); + await playExpect(containersPage.heading).toBeVisible(); + const containersDetailsPage = await containersPage.openContainersDetails(container); + await playExpect(containersDetailsPage.heading).toBeVisible(); + await playExpect + .poll(async () => containersDetailsPage.getState(), { timeout: 30_000 }) + .toContain(ContainerState.Running); + } + }); + + test.afterAll(`Stop Llama Stack containers`, async ({ navigationBar }) => { + for (const container of llamaStackContainerNames) { + const containersPage = await navigationBar.openContainers(); + await playExpect(containersPage.heading).toBeVisible(); + await containersPage.deleteContainer(container); + await playExpect + .poll(async () => await containersPage.containerExists(container), { timeout: 30_000 }) + .toBeFalsy(); + } + }); +}); + async function cleanupServices(): Promise { try { const modelServicePage = await aiLabPage.navigationBar.openServices(); From 1e3468c1ffdc985075123637149b9a57edc7e6b1 Mon Sep 17 00:00:00 2001 From: serbangeorge-m Date: Mon, 22 Sep 2025 13:36:54 +0200 Subject: [PATCH 5/5] chore(test): update test Signed-off-by: serbangeorge-m --- tests/playwright/src/ai-lab-extension.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/playwright/src/ai-lab-extension.spec.ts b/tests/playwright/src/ai-lab-extension.spec.ts index 1a9d4a339..08d4ca970 100644 --- a/tests/playwright/src/ai-lab-extension.spec.ts +++ b/tests/playwright/src/ai-lab-extension.spec.ts @@ -690,6 +690,7 @@ test.describe.serial(`Start Llama Stack from sidebar and verify containers`, { t .poll(async () => await containersPage.containerExists(container), { timeout: 30_000 }) .toBeFalsy(); } + await deleteUnusedImages(navigationBar); }); });