diff --git a/app-backend/Dockerfile b/app-backend/Dockerfile index 2beaac3..6169150 100644 --- a/app-backend/Dockerfile +++ b/app-backend/Dockerfile @@ -1,6 +1,7 @@ FROM python:3.11-slim RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ + libsqlite3-0=3.40.1-2+deb12u1 \ libgl1-mesa-glx=22.3.6-1+deb12u1 \ libjemalloc-dev=5.3.0-1 \ git=1:2.39.5-0+deb12u1 && \ diff --git a/app-frontend/Dockerfile b/app-frontend/Dockerfile index 7800ffb..2018069 100644 --- a/app-frontend/Dockerfile +++ b/app-frontend/Dockerfile @@ -7,7 +7,7 @@ FROM node:20.11.1 AS vite-app COPY react /usr/app/react WORKDIR /usr/app/react -RUN npm install && npm run build +RUN npm install --legacy-peer-deps && npm run build FROM nginx:alpine diff --git a/setup-scripts/setup-genai-studio/manifests/studio-manifest.yaml b/setup-scripts/setup-genai-studio/manifests/studio-manifest.yaml index 03d1ddb..dd82589 100644 --- a/setup-scripts/setup-genai-studio/manifests/studio-manifest.yaml +++ b/setup-scripts/setup-genai-studio/manifests/studio-manifest.yaml @@ -285,6 +285,10 @@ spec: value: ${REGISTRY}/app-frontend:${TAG} - name: APP_BACKEND_IMAGE value: ${REGISTRY}/app-backend:${TAG} + - name: REGISTRY + value: opea + - name: TAG + value: latest ports: - containerPort: 5000 resources: diff --git a/setup-scripts/setup-genai-studio/readme.md b/setup-scripts/setup-genai-studio/readme.md index 16b3e04..181e82e 100644 --- a/setup-scripts/setup-genai-studio/readme.md +++ b/setup-scripts/setup-genai-studio/readme.md @@ -18,7 +18,7 @@ Run below commands: ```sh sudo apt install ansible ansible-galaxy collection install kubernetes.core -ansible-playbook genai-studio-playbook.yml +ansible-playbook genai-studio.yml ``` ### Quick health test diff --git a/studio-backend/Dockerfile b/studio-backend/Dockerfile index c3de1db..bb55723 100644 --- a/studio-backend/Dockerfile +++ b/studio-backend/Dockerfile @@ -7,6 +7,11 @@ WORKDIR /usr/src/ # Copy the current directory contents into the container at /usr/src/app COPY app /usr/src/app +# Upgrade libsqlite3 to a safe version +RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ + libsqlite3-0=3.40.1-2+deb12u1 && \ + rm -rf /var/lib/apt/lists/* + # Upgrade setuptools to a safe version and install any needed packages specified in requirements.txt RUN pip install --no-cache-dir --upgrade pip==24.3.1 setuptools==75.3.0 && \ pip install --no-cache-dir -r /usr/src/app/requirements.txt diff --git a/studio-backend/app/requirements.txt b/studio-backend/app/requirements.txt index 6e69d6e..4da7b91 100644 --- a/studio-backend/app/requirements.txt +++ b/studio-backend/app/requirements.txt @@ -1,7 +1,7 @@ -fastapi==0.115.0 +fastapi==0.115.4 uvicorn==0.30.6 kubernetes==30.1.0 requests==2.32.3 pydantic==1.10.18 -starlette==0.38.6 +starlette==0.41.2 websockets==10.3 \ No newline at end of file diff --git a/studio-backend/app/services/exporter_service.py b/studio-backend/app/services/exporter_service.py index 5d0ab11..ab909b8 100644 --- a/studio-backend/app/services/exporter_service.py +++ b/studio-backend/app/services/exporter_service.py @@ -24,7 +24,7 @@ def convert_proj_info_to_manifest(proj_info_json, output_file=None): output_manifest.extend((doc, service_name) for doc in service_manifest) manifest_string = "" - for index, (doc, service_name) in enumerate(output_manifest): + for _, (doc, service_name) in enumerate(output_manifest): # Skip if the document is None if doc is None: continue diff --git a/studio-backend/app/templates/microsvc-composes/data-prep.yaml b/studio-backend/app/templates/microsvc-composes/data-prep.yaml index 3025333..69a86da 100644 --- a/studio-backend/app/templates/microsvc-composes/data-prep.yaml +++ b/studio-backend/app/templates/microsvc-composes/data-prep.yaml @@ -1,5 +1,5 @@ "{{endpoint}}": - image: opea/dataprep-redis:latest + image: ${REGISTRY}/dataprep-redis:${TAG} container_name: "{{endpoint}}" depends_on: - "{{redis_vector_store_endpoint}}" diff --git a/studio-backend/app/templates/microsvc-composes/embedding-usvc.yaml b/studio-backend/app/templates/microsvc-composes/embedding-usvc.yaml index 961e84e..4720dce 100644 --- a/studio-backend/app/templates/microsvc-composes/embedding-usvc.yaml +++ b/studio-backend/app/templates/microsvc-composes/embedding-usvc.yaml @@ -1,5 +1,5 @@ "{{endpoint}}": - image: opea/embedding-tei:latest + image: ${REGISTRY}/embedding-tei:${TAG} container_name: "{{endpoint}}" depends_on: - "{{tei_endpoint}}" diff --git a/studio-backend/app/templates/microsvc-composes/llm-uservice.yaml b/studio-backend/app/templates/microsvc-composes/llm-uservice.yaml index 08c97b2..17b426f 100644 --- a/studio-backend/app/templates/microsvc-composes/llm-uservice.yaml +++ b/studio-backend/app/templates/microsvc-composes/llm-uservice.yaml @@ -1,5 +1,5 @@ "{{endpoint}}": - image: opea/llm-tgi:latest + image: ${REGISTRY}/llm-tgi:${TAG} container_name: "{{endpoint}}" depends_on: - "{{tgi_endpoint}}" diff --git a/studio-backend/app/templates/microsvc-composes/reranking-usvc.yaml b/studio-backend/app/templates/microsvc-composes/reranking-usvc.yaml index c49ec65..04c0b4c 100644 --- a/studio-backend/app/templates/microsvc-composes/reranking-usvc.yaml +++ b/studio-backend/app/templates/microsvc-composes/reranking-usvc.yaml @@ -1,5 +1,5 @@ "{{endpoint}}": - image: opea/reranking-tei:latest + image: ${REGISTRY}/reranking-tei:${TAG} container_name: "{{endpoint}}" depends_on: - "{{tei_endpoint}}" diff --git a/studio-backend/app/templates/microsvc-composes/retriever-usvc.yaml b/studio-backend/app/templates/microsvc-composes/retriever-usvc.yaml index b8c7460..648d402 100644 --- a/studio-backend/app/templates/microsvc-composes/retriever-usvc.yaml +++ b/studio-backend/app/templates/microsvc-composes/retriever-usvc.yaml @@ -1,5 +1,5 @@ "{{endpoint}}": - image: opea/retriever-redis:latest + image: ${REGISTRY}/retriever-redis:${TAG} container_name: "{{endpoint}}" depends_on: - "{{redis_vector_store_endpoint}}" diff --git a/studio-backend/app/templates/microsvc-manifests/data-prep.yaml b/studio-backend/app/templates/microsvc-manifests/data-prep.yaml index 88875e6..41d50b0 100644 --- a/studio-backend/app/templates/microsvc-manifests/data-prep.yaml +++ b/studio-backend/app/templates/microsvc-manifests/data-prep.yaml @@ -76,7 +76,7 @@ spec: runAsUser: 1000 seccompProfile: type: RuntimeDefault - image: "opea/dataprep-redis:latest" + image: "${REGISTRY}/dataprep-redis:${TAG}" imagePullPolicy: IfNotPresent ports: - name: data-prep diff --git a/studio-backend/app/templates/microsvc-manifests/embedding-usvc.yaml b/studio-backend/app/templates/microsvc-manifests/embedding-usvc.yaml index 5f10cce..efbe8b6 100644 --- a/studio-backend/app/templates/microsvc-manifests/embedding-usvc.yaml +++ b/studio-backend/app/templates/microsvc-manifests/embedding-usvc.yaml @@ -69,7 +69,7 @@ spec: runAsUser: 1000 seccompProfile: type: RuntimeDefault - image: "opea/embedding-tei:latest" + image: "${REGISTRY}/embedding-tei:${TAG}" imagePullPolicy: IfNotPresent ports: - name: embedding-usvc diff --git a/studio-backend/app/templates/microsvc-manifests/llm-uservice.yaml b/studio-backend/app/templates/microsvc-manifests/llm-uservice.yaml index 8231e53..0614344 100644 --- a/studio-backend/app/templates/microsvc-manifests/llm-uservice.yaml +++ b/studio-backend/app/templates/microsvc-manifests/llm-uservice.yaml @@ -71,7 +71,7 @@ spec: runAsUser: 1000 seccompProfile: type: RuntimeDefault - image: "opea/llm-tgi:latest" + image: "${REGISTRY}/llm-tgi:${TAG}" imagePullPolicy: IfNotPresent ports: - name: llm-uservice diff --git a/studio-backend/app/templates/microsvc-manifests/reranking-usvc.yaml b/studio-backend/app/templates/microsvc-manifests/reranking-usvc.yaml index ab17631..fa2871a 100644 --- a/studio-backend/app/templates/microsvc-manifests/reranking-usvc.yaml +++ b/studio-backend/app/templates/microsvc-manifests/reranking-usvc.yaml @@ -69,7 +69,7 @@ spec: runAsUser: 1000 seccompProfile: type: RuntimeDefault - image: "opea/reranking-tei:latest" + image: "${REGISTRY}/reranking-tei:${TAG}" imagePullPolicy: IfNotPresent ports: - name: reranking-usvc diff --git a/studio-backend/app/templates/microsvc-manifests/retriever-usvc.yaml b/studio-backend/app/templates/microsvc-manifests/retriever-usvc.yaml index 65658f9..7b89d73 100644 --- a/studio-backend/app/templates/microsvc-manifests/retriever-usvc.yaml +++ b/studio-backend/app/templates/microsvc-manifests/retriever-usvc.yaml @@ -75,7 +75,7 @@ spec: runAsUser: 1000 seccompProfile: type: RuntimeDefault - image: "opea/retriever-redis:latest" + image: "${REGISTRY}/retriever-redis:${TAG}" imagePullPolicy: IfNotPresent ports: - name: retriever-usvc diff --git a/studio-backend/app/utils/placeholders_utils.py b/studio-backend/app/utils/placeholders_utils.py index ffdc394..d1e70ca 100644 --- a/studio-backend/app/utils/placeholders_utils.py +++ b/studio-backend/app/utils/placeholders_utils.py @@ -41,6 +41,9 @@ def replace_manifest_placeholders(obj, variables): if key == "default.conf" or key == "project-info.json": continue if isinstance(value, str): + # Replace ${REGISTRY} and ${TAG} with the value from environment variables + value = value.replace("${REGISTRY}", os.getenv("REGISTRY", "opea")) + value = value.replace("${TAG}", os.getenv("TAG", "latest")) # Attempt to replace placeholders in the string formatted_value = value.format(**variables) # If the key is a port-related field and the formatted value is a digit, convert to int @@ -118,7 +121,10 @@ def replace_compose_placeholders(obj, variables): return [replace_compose_placeholders(value, variables) for value in obj] elif isinstance(obj, str): # Replace {{}} placeholders in strings - return re.sub(r'\{\{(.*?)\}\}', lambda m: str(variables.get(m.group(1), m.group(0))), obj) + value = re.sub(r'\{\{(.*?)\}\}', lambda m: str(variables.get(m.group(1), m.group(0))), obj) + value = value.replace("${REGISTRY}", os.getenv("REGISTRY", "opea")) + value = value.replace("${TAG}", os.getenv("TAG", "latest")) + return value return obj def replace_dynamic_compose_placeholder(value_str, service_info): diff --git a/studio-frontend/Dockerfile b/studio-frontend/Dockerfile index 14cf024..986ede4 100644 --- a/studio-frontend/Dockerfile +++ b/studio-frontend/Dockerfile @@ -19,7 +19,10 @@ WORKDIR /usr/src COPY . . # Install dependencies and build the app -RUN pnpm install && pnpm build +RUN pnpm config set store-dir .pnpm-store && \ + pnpm install && \ + rm -rf .pnpm-store && \ + pnpm build EXPOSE 3000 diff --git a/studio-frontend/package.json b/studio-frontend/package.json index e482fad..1081d31 100644 --- a/studio-frontend/package.json +++ b/studio-frontend/package.json @@ -68,7 +68,15 @@ "@qdrant/openapi-typescript-fetch": "1.2.6", "@google/generative-ai": "^0.15.0", "openai": "4.51.0", - "@langchain/core": "0.2.18" + "@langchain/core": "0.2.18", + "axios": "1.7.4", + "lunary": "0.7.13", + "nth-check": "2.0.1", + "pdfjs-dist": "4.2.67", + "prismjs": "1.27.0", + "semver": "7.5.2", + "ws": "8.17.1", + "@esbuild/linux-x64": "0.21.5" }, "eslintIgnore": [ "**/dist", diff --git a/tests/playwright/playwright.config.js b/tests/playwright/playwright.config.js index dd963d3..0de2ff0 100644 --- a/tests/playwright/playwright.config.js +++ b/tests/playwright/playwright.config.js @@ -14,15 +14,13 @@ module.exports = defineConfig({ testDir: './', fullyParallel: false, // Disable fully parallel tests forbidOnly: !!process.env.CI, - retries: process.env.CI ? 2 : 0, + retries: 0, workers: 1, // Set the number of workers to 1 - reporter: 'html', + reporter: [['html', { outputFolder: 'playwright-report' }]], use: { /* Update Base URL to use in actions like `await page.goto('/')`. */ baseURL: 'http://', - /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ - trace: 'on-first-retry', video: 'retain-on-failure', screenshot: 'only-on-failure' }, diff --git a/tests/playwright/studio-e2e/001_test_sandbox_deployment.spec.ts b/tests/playwright/studio-e2e/001_test_sandbox_deployment.spec.ts index 327df93..c9bcd3d 100644 --- a/tests/playwright/studio-e2e/001_test_sandbox_deployment.spec.ts +++ b/tests/playwright/studio-e2e/001_test_sandbox_deployment.spec.ts @@ -1,4 +1,5 @@ import { test, expect } from '@playwright/test'; +import { waitForStatusText } from '../utils'; import fs from 'fs'; import path from 'path'; import os from 'os'; @@ -7,7 +8,6 @@ test('001_test_sandbox_deployment', async ({ page, baseURL }) => { test.setTimeout(600000); const IDC_URL = baseURL || "" - const statusChangeTimeout = 300000; // 5 minutes await page.goto(IDC_URL); await page.getByRole('button', { name: 'Create New Workflow' }).click(); await page.getByRole('button', { name: 'Settings' }).click(); @@ -18,21 +18,20 @@ test('001_test_sandbox_deployment', async ({ page, baseURL }) => { await fileChooser.setFiles(filePath); await page.getByRole('button', { name: 'Save Workflow' }).click(); await page.getByPlaceholder('My New Chatflow').click(); - await page.getByPlaceholder('My New Chatflow').fill('Wf1'); + await page.getByPlaceholder('My New Chatflow').fill('test_001'); await page.getByRole('button', { name: 'Save' }).click(); await page.goto(IDC_URL); await expect(page.locator('td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root').first()).toHaveText('Not Running', { timeout: 60000 }); await page.getByLabel('a dense table').locator('button').first().click(); - // Verify that the status has changed to "Ready" - for (let i = 0; i < 2; i++) { - await page.reload(); - try { - await expect(page.locator('td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root').first()).toHaveText('Ready', { timeout: statusChangeTimeout }); - break; - } catch (error) { - console.log(`Attempt ${i + 1} failed: ${error}`); - } - } + // for (let i = 0; i < 5; i++) { + // try { + // await expect(page.locator('td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root').first()).toHaveText('Ready', { timeout: 60000 }); + // break; + // } catch (error) { + // console.log(`Attempt ${i + 1} failed: ${error}`); + // } + // } + await waitForStatusText(page, 'td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root', 'Ready', 5, 60000); await page.waitForTimeout(8000); // Open APP-UI @@ -61,7 +60,8 @@ test('001_test_sandbox_deployment', async ({ page, baseURL }) => { // Stop & Delete Sandbox await page.locator('button:has([data-testid="StopCircleOutlinedIcon"])').first().click(); - await expect(page.locator('td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root').first()).toHaveText('Not Running', { timeout: statusChangeTimeout }); + // await expect(page.locator('td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root').first()).toHaveText('Not Running', { timeout: statusChangeTimeout }); + await waitForStatusText(page, 'td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root', 'Not Running', 5, 60000); await page.locator('#demo-customized-button').first().click(); await page.getByRole('menuitem', { name: 'Delete' }).click(); await page.getByRole('button', { name: 'Delete' }).click(); diff --git a/tests/playwright/studio-e2e/002_test_sandbox_chatqna.spec.ts b/tests/playwright/studio-e2e/002_test_sandbox_chatqna.spec.ts index 716e755..04186a6 100644 --- a/tests/playwright/studio-e2e/002_test_sandbox_chatqna.spec.ts +++ b/tests/playwright/studio-e2e/002_test_sandbox_chatqna.spec.ts @@ -1,4 +1,5 @@ import { test, expect } from '@playwright/test'; +import { waitForStatusText } from '../utils'; import path from 'path'; const sampleWorkflow = path.resolve(__dirname, '../../../sample-workflows/sample_workflow_chatqna.json'); @@ -37,7 +38,6 @@ test('002_test_sandbox_chatqna', async ({ page, baseURL }) => { let apiResponse = { value: '' }; const IDC_URL = baseURL || "" - const statusChangeTimeout = 300000; // 5 minutes await page.goto(IDC_URL); await page.getByRole('button', { name: 'Create New Workflow' }).click(); await page.getByRole('button', { name: 'Settings' }).click(); @@ -47,21 +47,12 @@ test('002_test_sandbox_chatqna', async ({ page, baseURL }) => { await fileChooser.setFiles(sampleWorkflow); await page.getByRole('button', { name: 'Save Workflow' }).click(); await page.getByPlaceholder('My New Chatflow').click(); - await page.getByPlaceholder('My New Chatflow').fill('Wf1'); + await page.getByPlaceholder('My New Chatflow').fill('test_002'); await page.getByRole('button', { name: 'Save' }).click(); await page.goto(IDC_URL); await expect(page.locator('td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root').first()).toHaveText('Not Running', { timeout: 60000 }); await page.getByLabel('a dense table').locator('button').first().click(); - // Verify that the status has changed to "Ready" - for (let i = 0; i < 2; i++) { - await page.reload(); - try { - await expect(page.locator('td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root').first()).toHaveText('Ready', { timeout: statusChangeTimeout }); - break; - } catch (error) { - console.log(`Attempt ${i + 1} failed: ${error}`); - } - } + await waitForStatusText(page, 'td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root', 'Ready', 5, 60000); await page.waitForTimeout(8000); // Open APP-UI @@ -160,14 +151,15 @@ test('002_test_sandbox_chatqna', async ({ page, baseURL }) => { } // Delete 1 document + Check data sources successfully deduct 1 or not + await page2.waitForTimeout(5000); await page2.getByRole('button').nth(3).click(); await page2.getByRole('row', { name: 'tennis_tutorial.pdf' }).getByRole('button').click(); - await expect(page2.getByRole('cell', { name: 'tennis_tutorial.pdf' })).toBeHidden(); + await expect(page2.getByRole('cell', { name: 'tennis_tutorial.pdf' })).toBeHidden( { timeout: 60000 } ); // Stop & Delete Sandbox await page.bringToFront(); await page.locator('button:has([data-testid="StopCircleOutlinedIcon"])').first().click(); - await expect(page.locator('td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root').first()).toHaveText('Not Running', { timeout: statusChangeTimeout }); + await waitForStatusText(page, 'td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root', 'Not Running', 5, 60000); await page.locator('#demo-customized-button').first().click(); await page.getByRole('menuitem', { name: 'Delete' }).click(); await page.getByRole('button', { name: 'Delete' }).click(); diff --git a/tests/playwright/utils.ts b/tests/playwright/utils.ts new file mode 100644 index 0000000..a32dea1 --- /dev/null +++ b/tests/playwright/utils.ts @@ -0,0 +1,17 @@ +import { expect } from '@playwright/test'; + +export async function waitForStatusText(page: any, selector: string, statusText: string, maxAttempts: number = 5, intervalTimeout: number = 60000) { + for (let i = 0; i < maxAttempts; i++) { + try { + const text = await page.locator(selector).first().innerText(); + if (text === 'Error') { + throw new Error(`Encountered unwanted status text "Error" in element "${selector}"`); + } + await expect(page.locator(selector).first()).toHaveText(statusText, { timeout: intervalTimeout }); + return; + } catch (error) { + console.log(`Attempt ${i + 1} failed: ${error}`); + } + } + throw new Error(`Failed to find text "${statusText}" in element "${selector}" after ${maxAttempts} attempts`); +} \ No newline at end of file