fix most trivy issues, update playwright tests, update studio-backend

app-backend/Dockerfile

@@ -1,6 +1,7 @@
 FROM python:3.11-slim
 
 RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
+    libsqlite3-0=3.40.1-2+deb12u1 \
     libgl1-mesa-glx=22.3.6-1+deb12u1 \
     libjemalloc-dev=5.3.0-1 \
     git=1:2.39.5-0+deb12u1 && \

app-frontend/Dockerfile

@@ -7,7 +7,7 @@ FROM node:20.11.1 AS vite-app
 COPY react /usr/app/react
 WORKDIR /usr/app/react
 
-RUN npm install && npm run build
+RUN npm install --legacy-peer-deps && npm run build
 
 FROM nginx:alpine
 

setup-scripts/setup-genai-studio/manifests/studio-manifest.yaml

@@ -285,6 +285,10 @@ spec:
           value: ${REGISTRY}/app-frontend:${TAG}
         - name: APP_BACKEND_IMAGE
           value: ${REGISTRY}/app-backend:${TAG}
+        - name: REGISTRY
+          value: opea
+        - name: TAG
+          value: latest
         ports:
         - containerPort: 5000
         resources:

setup-scripts/setup-genai-studio/readme.md

@@ -18,7 +18,7 @@ Run below commands:
 ```sh
 sudo apt install ansible
 ansible-galaxy collection install kubernetes.core
-ansible-playbook genai-studio-playbook.yml
+ansible-playbook genai-studio.yml
 ```
 
 ### Quick health test

studio-backend/Dockerfile

@@ -7,6 +7,11 @@ WORKDIR /usr/src/
 # Copy the current directory contents into the container at /usr/src/app
 COPY app /usr/src/app
 
+# Upgrade libsqlite3 to a safe version
+RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
+    libsqlite3-0=3.40.1-2+deb12u1 && \
+    rm -rf /var/lib/apt/lists/*
+
 # Upgrade setuptools to a safe version and install any needed packages specified in requirements.txt
 RUN pip install --no-cache-dir --upgrade pip==24.3.1 setuptools==75.3.0 && \
     pip install --no-cache-dir -r /usr/src/app/requirements.txt

studio-backend/app/requirements.txt

@@ -1,7 +1,7 @@
-fastapi==0.115.0
+fastapi==0.115.4
 uvicorn==0.30.6
 kubernetes==30.1.0
 requests==2.32.3
 pydantic==1.10.18
-starlette==0.38.6
+starlette==0.41.2
 websockets==10.3

studio-backend/app/services/exporter_service.py

@@ -24,7 +24,7 @@ def convert_proj_info_to_manifest(proj_info_json, output_file=None):
         output_manifest.extend((doc, service_name) for doc in service_manifest)
 
     manifest_string = ""
-    for index, (doc, service_name) in enumerate(output_manifest):
+    for _, (doc, service_name) in enumerate(output_manifest):
         # Skip if the document is None
         if doc is None:
             continue

studio-backend/app/templates/microsvc-composes/data-prep.yaml

@@ -1,5 +1,5 @@
 "{{endpoint}}":
-  image: opea/dataprep-redis:latest
+  image: ${REGISTRY}/dataprep-redis:${TAG}
   container_name: "{{endpoint}}"
   depends_on:
     - "{{redis_vector_store_endpoint}}"

studio-backend/app/templates/microsvc-composes/embedding-usvc.yaml

@@ -1,5 +1,5 @@
 "{{endpoint}}":
-  image: opea/embedding-tei:latest
+  image: ${REGISTRY}/embedding-tei:${TAG}
   container_name: "{{endpoint}}"
   depends_on:
     - "{{tei_endpoint}}"

studio-backend/app/templates/microsvc-composes/llm-uservice.yaml

@@ -1,5 +1,5 @@
 "{{endpoint}}":
-  image: opea/llm-tgi:latest
+  image: ${REGISTRY}/llm-tgi:${TAG}
   container_name: "{{endpoint}}"
   depends_on:
     - "{{tgi_endpoint}}"

studio-backend/app/templates/microsvc-composes/reranking-usvc.yaml

@@ -1,5 +1,5 @@
 "{{endpoint}}":
-  image: opea/reranking-tei:latest
+  image: ${REGISTRY}/reranking-tei:${TAG}
   container_name: "{{endpoint}}"
   depends_on:
     - "{{tei_endpoint}}"

studio-backend/app/templates/microsvc-composes/retriever-usvc.yaml

@@ -1,5 +1,5 @@
 "{{endpoint}}":
-  image: opea/retriever-redis:latest
+  image: ${REGISTRY}/retriever-redis:${TAG}
   container_name: "{{endpoint}}"
   depends_on:
     - "{{redis_vector_store_endpoint}}"

studio-backend/app/templates/microsvc-manifests/data-prep.yaml

@@ -76,7 +76,7 @@ spec:
             runAsUser: 1000
             seccompProfile:
               type: RuntimeDefault
-          image: "opea/dataprep-redis:latest"
+          image: "${REGISTRY}/dataprep-redis:${TAG}"
           imagePullPolicy: IfNotPresent
           ports:
             - name: data-prep

studio-backend/app/templates/microsvc-manifests/embedding-usvc.yaml

@@ -69,7 +69,7 @@ spec:
             runAsUser: 1000
             seccompProfile:
               type: RuntimeDefault
-          image: "opea/embedding-tei:latest"
+          image: "${REGISTRY}/embedding-tei:${TAG}"
           imagePullPolicy: IfNotPresent
           ports:
             - name: embedding-usvc

studio-backend/app/templates/microsvc-manifests/llm-uservice.yaml

@@ -71,7 +71,7 @@ spec:
             runAsUser: 1000
             seccompProfile:
               type: RuntimeDefault
-          image: "opea/llm-tgi:latest"
+          image: "${REGISTRY}/llm-tgi:${TAG}"
           imagePullPolicy: IfNotPresent
           ports:
             - name: llm-uservice

studio-backend/app/templates/microsvc-manifests/reranking-usvc.yaml

@@ -69,7 +69,7 @@ spec:
             runAsUser: 1000
             seccompProfile:
               type: RuntimeDefault
-          image: "opea/reranking-tei:latest"
+          image: "${REGISTRY}/reranking-tei:${TAG}"
           imagePullPolicy: IfNotPresent
           ports:
             - name: reranking-usvc

studio-backend/app/templates/microsvc-manifests/retriever-usvc.yaml

@@ -75,7 +75,7 @@ spec:
             runAsUser: 1000
             seccompProfile:
               type: RuntimeDefault
-          image: "opea/retriever-redis:latest"
+          image: "${REGISTRY}/retriever-redis:${TAG}"
           imagePullPolicy: IfNotPresent
           ports:
             - name: retriever-usvc

studio-backend/app/utils/placeholders_utils.py

@@ -41,6 +41,9 @@ def replace_manifest_placeholders(obj, variables):
             if key == "default.conf" or key == "project-info.json":
                 continue
             if isinstance(value, str):
+                # Replace ${REGISTRY} and ${TAG} with the value from environment variables
+                value = value.replace("${REGISTRY}", os.getenv("REGISTRY", "opea"))
+                value = value.replace("${TAG}", os.getenv("TAG", "latest"))
                 # Attempt to replace placeholders in the string
                 formatted_value = value.format(**variables)
                 # If the key is a port-related field and the formatted value is a digit, convert to int
@@ -118,7 +121,10 @@ def replace_compose_placeholders(obj, variables):
         return [replace_compose_placeholders(value, variables) for value in obj]
     elif isinstance(obj, str):
         # Replace {{}} placeholders in strings
-        return re.sub(r'\{\{(.*?)\}\}', lambda m: str(variables.get(m.group(1), m.group(0))), obj)
+        value = re.sub(r'\{\{(.*?)\}\}', lambda m: str(variables.get(m.group(1), m.group(0))), obj)
+        value = value.replace("${REGISTRY}", os.getenv("REGISTRY", "opea"))
+        value = value.replace("${TAG}", os.getenv("TAG", "latest"))
+        return value
     return obj
 
 def replace_dynamic_compose_placeholder(value_str, service_info):

studio-frontend/Dockerfile

@@ -19,7 +19,10 @@ WORKDIR /usr/src
 COPY . .
 
 # Install dependencies and build the app
-RUN pnpm install && pnpm build
+RUN pnpm config set store-dir .pnpm-store && \
+    pnpm install && \
+    rm -rf .pnpm-store && \
+    pnpm build
 
 EXPOSE 3000
 

studio-frontend/package.json

@@ -68,7 +68,15 @@
         "@qdrant/openapi-typescript-fetch": "1.2.6",
         "@google/generative-ai": "^0.15.0",
         "openai": "4.51.0",
-        "@langchain/core": "0.2.18"
+        "@langchain/core": "0.2.18",
+        "axios": "1.7.4",
+        "lunary": "0.7.13",
+        "nth-check": "2.0.1",
+        "pdfjs-dist": "4.2.67",
+        "prismjs": "1.27.0",
+        "semver": "7.5.2",
+        "ws": "8.17.1",
+        "@esbuild/linux-x64": "0.21.5"
     },
     "eslintIgnore": [
         "**/dist",

tests/playwright/playwright.config.js

@@ -14,15 +14,13 @@ module.exports = defineConfig({
   testDir: './',
   fullyParallel: false, // Disable fully parallel tests
   forbidOnly: !!process.env.CI,
-  retries: process.env.CI ? 2 : 0,
+  retries: 0,
   workers: 1, // Set the number of workers to 1
-  reporter: 'html',
+  reporter: [['html', { outputFolder: 'playwright-report' }]],
   use: {
     /* Update Base URL to use in actions like `await page.goto('/')`. */
     baseURL: 'http://<baseURL>',
 
-    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
-    trace: 'on-first-retry',
     video: 'retain-on-failure',
     screenshot: 'only-on-failure'
   },

tests/playwright/studio-e2e/001_test_sandbox_deployment.spec.ts

@@ -1,4 +1,5 @@
 import { test, expect } from '@playwright/test';
+import { waitForStatusText } from '../utils';
 import fs from 'fs';
 import path from 'path';
 import os from 'os';
@@ -7,7 +8,6 @@ test('001_test_sandbox_deployment', async ({ page, baseURL }) => {
     test.setTimeout(600000);
 
     const IDC_URL = baseURL || ""
-    const statusChangeTimeout = 300000; // 5 minutes
     await page.goto(IDC_URL);
     await page.getByRole('button', { name: 'Create New Workflow' }).click();
     await page.getByRole('button', { name: 'Settings' }).click();
@@ -18,21 +18,20 @@ test('001_test_sandbox_deployment', async ({ page, baseURL }) => {
     await fileChooser.setFiles(filePath);
     await page.getByRole('button', { name: 'Save Workflow' }).click();
     await page.getByPlaceholder('My New Chatflow').click();
-    await page.getByPlaceholder('My New Chatflow').fill('Wf1');
+    await page.getByPlaceholder('My New Chatflow').fill('test_001');
     await page.getByRole('button', { name: 'Save' }).click();
     await page.goto(IDC_URL);
     await expect(page.locator('td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root').first()).toHaveText('Not Running', { timeout: 60000 });
     await page.getByLabel('a dense table').locator('button').first().click();
-    // Verify that the status has changed to "Ready"
-    for (let i = 0; i < 2; i++) {
-        await page.reload();
-        try {
-            await expect(page.locator('td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root').first()).toHaveText('Ready', { timeout: statusChangeTimeout });
-            break;
-        } catch (error) {
-            console.log(`Attempt ${i + 1} failed: ${error}`);
-        }
-    }
+    // for (let i = 0; i < 5; i++) {
+    //     try {
+    //         await expect(page.locator('td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root').first()).toHaveText('Ready', { timeout: 60000 });
+    //         break;
+    //     } catch (error) {
+    //         console.log(`Attempt ${i + 1} failed: ${error}`);
+    //     }
+    // }
+    await waitForStatusText(page, 'td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root', 'Ready', 5, 60000);
     await page.waitForTimeout(8000);
 
     // Open APP-UI
@@ -61,7 +60,8 @@ test('001_test_sandbox_deployment', async ({ page, baseURL }) => {
 
     // Stop & Delete Sandbox
     await page.locator('button:has([data-testid="StopCircleOutlinedIcon"])').first().click();
-    await expect(page.locator('td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root').first()).toHaveText('Not Running', { timeout: statusChangeTimeout });
+    // await expect(page.locator('td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root').first()).toHaveText('Not Running', { timeout: statusChangeTimeout });
+    await waitForStatusText(page, 'td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root', 'Not Running', 5, 60000);
     await page.locator('#demo-customized-button').first().click();
     await page.getByRole('menuitem', { name: 'Delete' }).click();
     await page.getByRole('button', { name: 'Delete' }).click();

tests/playwright/studio-e2e/002_test_sandbox_chatqna.spec.ts

@@ -1,4 +1,5 @@
 import { test, expect } from '@playwright/test';
+import { waitForStatusText } from '../utils';
 import path from 'path';
 
 const sampleWorkflow = path.resolve(__dirname, '../../../sample-workflows/sample_workflow_chatqna.json');
@@ -37,7 +38,6 @@ test('002_test_sandbox_chatqna', async ({ page, baseURL }) => {
     let apiResponse = { value: '' };
 
     const IDC_URL = baseURL || ""
-    const statusChangeTimeout = 300000; // 5 minutes
     await page.goto(IDC_URL);
     await page.getByRole('button', { name: 'Create New Workflow' }).click();
     await page.getByRole('button', { name: 'Settings' }).click();
@@ -47,21 +47,12 @@ test('002_test_sandbox_chatqna', async ({ page, baseURL }) => {
     await fileChooser.setFiles(sampleWorkflow);
     await page.getByRole('button', { name: 'Save Workflow' }).click();
     await page.getByPlaceholder('My New Chatflow').click();
-    await page.getByPlaceholder('My New Chatflow').fill('Wf1');
+    await page.getByPlaceholder('My New Chatflow').fill('test_002');
     await page.getByRole('button', { name: 'Save' }).click();
     await page.goto(IDC_URL);
     await expect(page.locator('td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root').first()).toHaveText('Not Running', { timeout: 60000 });
     await page.getByLabel('a dense table').locator('button').first().click();
-    // Verify that the status has changed to "Ready"
-    for (let i = 0; i < 2; i++) {
-        await page.reload();
-        try {
-            await expect(page.locator('td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root').first()).toHaveText('Ready', { timeout: statusChangeTimeout });
-            break;
-        } catch (error) {
-            console.log(`Attempt ${i + 1} failed: ${error}`);
-        }
-    }
+    await waitForStatusText(page, 'td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root', 'Ready', 5, 60000);
     await page.waitForTimeout(8000);
 
     // Open APP-UI
@@ -160,14 +151,15 @@ test('002_test_sandbox_chatqna', async ({ page, baseURL }) => {
     }
 
     // Delete 1 document + Check data sources successfully deduct 1 or not
+    await page2.waitForTimeout(5000);
     await page2.getByRole('button').nth(3).click();
     await page2.getByRole('row', { name: 'tennis_tutorial.pdf' }).getByRole('button').click();
-    await expect(page2.getByRole('cell', { name: 'tennis_tutorial.pdf' })).toBeHidden();
+    await expect(page2.getByRole('cell', { name: 'tennis_tutorial.pdf' })).toBeHidden( { timeout: 60000 } );
 
     // Stop & Delete Sandbox
     await page.bringToFront();
     await page.locator('button:has([data-testid="StopCircleOutlinedIcon"])').first().click();
-    await expect(page.locator('td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root').first()).toHaveText('Not Running', { timeout: statusChangeTimeout });
+    await waitForStatusText(page, 'td.MuiTableCell-root div.MuiStack-root p.MuiTypography-root', 'Not Running', 5, 60000);
     await page.locator('#demo-customized-button').first().click();
     await page.getByRole('menuitem', { name: 'Delete' }).click();
     await page.getByRole('button', { name: 'Delete' }).click();

tests/playwright/utils.ts

@@ -0,0 +1,17 @@
+import { expect } from '@playwright/test';
+
+export async function waitForStatusText(page: any, selector: string, statusText: string, maxAttempts: number = 5, intervalTimeout: number = 60000) {
+    for (let i = 0; i < maxAttempts; i++) {
+        try {
+            const text = await page.locator(selector).first().innerText();
+            if (text === 'Error') {
+                throw new Error(`Encountered unwanted status text "Error" in element "${selector}"`);
+            }
+            await expect(page.locator(selector).first()).toHaveText(statusText, { timeout: intervalTimeout });
+            return;
+        } catch (error) {
+            console.log(`Attempt ${i + 1} failed: ${error}`);
+        }
+    }
+    throw new Error(`Failed to find text "${statusText}" in element "${selector}" after ${maxAttempts} attempts`);
+}