diff --git a/.github/workflows/_e2e-test.yml b/.github/workflows/_e2e-test.yml index 664ed33..23b213d 100644 --- a/.github/workflows/_e2e-test.yml +++ b/.github/workflows/_e2e-test.yml @@ -53,6 +53,7 @@ jobs: fi sleep 5 sudo apt install ansible -y + sed -i 's/value: ${TAG}/value: latest/' manifests/studio-manifest.yaml ansible-playbook genai-studio.yml -e "container_registry=${OPEA_IMAGE_REPO}opea" -e "container_tag=${{ inputs.tag }}" -e "mysql_host=$(hostname -I | awk '{print $1}')" sleep 5 kubectl wait --for=condition=ready pod --all --namespace=studio --timeout=300s --field-selector=status.phase!=Succeeded diff --git a/app-frontend/react/src/components/Conversation/DataSource.tsx b/app-frontend/react/src/components/Conversation/DataSource.tsx index cc2f5c9..f2187c4 100644 --- a/app-frontend/react/src/components/Conversation/DataSource.tsx +++ b/app-frontend/react/src/components/Conversation/DataSource.tsx @@ -36,7 +36,7 @@ export default function DataSource({ opened, onClose }: Props) { try { setTimeout(async () => { const response = await client.post( - `${DATA_PREP_URL}/get_file`, + `${DATA_PREP_URL}/get`, {}, // Request body (if needed, replace the empty object with actual data) { headers: { @@ -55,7 +55,7 @@ export default function DataSource({ opened, onClose }: Props) { const deleteFile = async (id: string) => { try { await client.post( - `${DATA_PREP_URL}/delete_file`, + `${DATA_PREP_URL}/delete`, { file_path: id }, // Request body (if needed, replace the empty object with actual data) { headers: { diff --git a/app-frontend/react/src/redux/Conversation/ConversationSlice.ts b/app-frontend/react/src/redux/Conversation/ConversationSlice.ts index 0776e7a..5445a35 100644 --- a/app-frontend/react/src/redux/Conversation/ConversationSlice.ts +++ b/app-frontend/react/src/redux/Conversation/ConversationSlice.ts @@ -114,7 +114,7 @@ export const submitDataSourceURL = createAsyncThunkWrapper( try { const body = new FormData(); body.append("link_list", JSON.stringify(link_list)); - const response = await client.post(DATA_PREP_URL, body); + const response = await client.post(`${DATA_PREP_URL}/ingest`, body); return response.data; } catch (error) { console.log ("error", error); @@ -132,7 +132,7 @@ export const uploadFile = createAsyncThunkWrapper("conversation/uploadFile", asy message: "uploading File", loading: true, }); - const response = await client.post(DATA_PREP_URL, body); + const response = await client.post(`${DATA_PREP_URL}/ingest`, body); return response.data; } catch (error) { throw error; diff --git a/sample-workflows/import_workflow.png b/assets/screenshots/import_workflow.png similarity index 100% rename from sample-workflows/import_workflow.png rename to assets/screenshots/import_workflow.png diff --git a/setup-scripts/setup-genai-studio/manifests/studio-manifest.yaml b/setup-scripts/setup-genai-studio/manifests/studio-manifest.yaml index ea9945e..9dc3ae0 100644 --- a/setup-scripts/setup-genai-studio/manifests/studio-manifest.yaml +++ b/setup-scripts/setup-genai-studio/manifests/studio-manifest.yaml @@ -326,9 +326,9 @@ spec: - name: APP_BACKEND_IMAGE value: ${REGISTRY}/app-backend:${TAG} - name: REGISTRY - value: opea + value: ${REGISTRY} - name: TAG - value: latest + value: ${TAG} - name: SBX_HTTP_PROXY value: ${HTTP_PROXY} - name: SBX_NO_PROXY diff --git a/setup-scripts/setup-genai-studio/playbooks/deploy-studio.yml b/setup-scripts/setup-genai-studio/playbooks/deploy-studio.yml index e1f422a..52163f3 100644 --- a/setup-scripts/setup-genai-studio/playbooks/deploy-studio.yml +++ b/setup-scripts/setup-genai-studio/playbooks/deploy-studio.yml @@ -77,7 +77,7 @@ MYSQL_HOST: "{{ mysql_host }}" - name: Wait for all pods to be ready in studio namespace - shell: kubectl wait --for=condition=ready pod --all --namespace=studio --timeout=180s + shell: kubectl wait --for=condition=ready pod --all --namespace=studio --timeout=300s register: pod_ready_check failed_when: pod_ready_check.rc != 0 changed_when: false \ No newline at end of file diff --git a/studio-backend/app/templates/microsvc-composes/data-prep.yaml b/studio-backend/app/templates/microsvc-composes/data-prep.yaml index b1698e2..3ce6afe 100644 --- a/studio-backend/app/templates/microsvc-composes/data-prep.yaml +++ b/studio-backend/app/templates/microsvc-composes/data-prep.yaml @@ -1,5 +1,5 @@ "{{endpoint}}": - image: ${REGISTRY}/dataprep-redis:${TAG} + image: ${REGISTRY}/dataprep:${TAG} container_name: "{{endpoint}}" depends_on: "{{redis_vector_store_endpoint}}": @@ -7,7 +7,7 @@ "{{tei_endpoint}}": condition: service_healthy ports: - - 6007:6007 + - 6007:5000 environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} @@ -15,5 +15,6 @@ REDIS_URL: "redis://${public_host_ip}:{{redis_vector_store_port}}" REDIS_HOST: "${public_host_ip}" INDEX_NAME: "rag-redis" - TEI_ENDPOINT: "http://${public_host_ip}:{{tei_port}}" + TEI_EMBEDDING_ENDPOINT: "http://${public_host_ip}:{{tei_port}}" + DATAPREP_COMPONENT_NAME: "OPEA_DATAPREP_REDIS" HUGGINGFACEHUB_API_TOKEN: "{{tei_huggingFaceToken}}" \ No newline at end of file diff --git a/studio-backend/app/templates/microsvc-manifests/data-prep.yaml b/studio-backend/app/templates/microsvc-manifests/data-prep.yaml index 92bbde6..b88246d 100644 --- a/studio-backend/app/templates/microsvc-manifests/data-prep.yaml +++ b/studio-backend/app/templates/microsvc-manifests/data-prep.yaml @@ -9,8 +9,8 @@ metadata: name: config-{endpoint} data: HEALTHCHECK_ENDPOINT: "{tei_endpoint}" - TEI_ENDPOINT: "http://{tei_endpoint}" - EMBED_MODEL: "" + TEI_EMBEDDING_ENDPOINT: "http://{tei_endpoint}" + DATAPREP_COMPONENT_NAME: "OPEA_DATAPREP_REDIS" REDIS_URL: "redis://{redis_vector_store_endpoint}:{redis_vector_store_port}" INDEX_NAME: "rag-redis" KEY_INDEX_NAME: "file-keys" @@ -34,7 +34,7 @@ spec: type: ClusterIP ports: - port: 6007 - targetPort: 6007 + targetPort: 5000 protocol: TCP name: "{endpoint}" selector: @@ -84,11 +84,11 @@ spec: runAsUser: 1000 seccompProfile: type: RuntimeDefault - image: "${REGISTRY}/dataprep-redis:${TAG}" + image: "${REGISTRY}/dataprep:${TAG}" imagePullPolicy: Always ports: - name: data-prep - containerPort: 6007 + containerPort: 5000 protocol: TCP volumeMounts: - mountPath: /tmp diff --git a/studio-backend/tests/exporter-groundtruth/gt_app-compose.yaml b/studio-backend/tests/exporter-groundtruth/gt_app-compose.yaml index 0cc47fb..f74ef89 100644 --- a/studio-backend/tests/exporter-groundtruth/gt_app-compose.yaml +++ b/studio-backend/tests/exporter-groundtruth/gt_app-compose.yaml @@ -95,7 +95,7 @@ services: HF_HUB_ENABLE_HF_TRANSFER: 0 restart: unless-stopped prepare-doc-redis-prep-0: - image: opea/dataprep-redis:latest + image: opea/dataprep:latest container_name: prepare-doc-redis-prep-0 depends_on: - redis-vector-store-0 diff --git a/studio-backend/tests/exporter-groundtruth/gt_app-manifest-with-nginx.yaml b/studio-backend/tests/exporter-groundtruth/gt_app-manifest-with-nginx.yaml index a3d64d6..1a2b814 100644 --- a/studio-backend/tests/exporter-groundtruth/gt_app-manifest-with-nginx.yaml +++ b/studio-backend/tests/exporter-groundtruth/gt_app-manifest-with-nginx.yaml @@ -778,7 +778,7 @@ spec: runAsUser: 1000 seccompProfile: type: RuntimeDefault - image: opea/dataprep-redis:latest + image: opea/dataprep:latest imagePullPolicy: Always ports: - name: data-prep diff --git a/studio-backend/tests/exporter-groundtruth/gt_app-manifest.yaml b/studio-backend/tests/exporter-groundtruth/gt_app-manifest.yaml index 5f8d992..6ba7b1d 100644 --- a/studio-backend/tests/exporter-groundtruth/gt_app-manifest.yaml +++ b/studio-backend/tests/exporter-groundtruth/gt_app-manifest.yaml @@ -782,7 +782,7 @@ spec: runAsUser: 1000 seccompProfile: type: RuntimeDefault - image: opea/dataprep-redis:latest + image: opea/dataprep:latest imagePullPolicy: Always ports: - name: data-prep diff --git a/studio-frontend/Dockerfile b/studio-frontend/Dockerfile index 24e7119..f504441 100644 --- a/studio-frontend/Dockerfile +++ b/studio-frontend/Dockerfile @@ -1,7 +1,7 @@ FROM node:20-alpine # Install necessary packages -RUN apk add --no-cache gcompat python3 make g++ \ +RUN apk add --no-cache gcompat python3 make g++ git \ # Needed for pdfjs-dist build-base cairo-dev pango-dev \ # Install Chromium @@ -9,9 +9,6 @@ RUN apk add --no-cache gcompat python3 make g++ \ # Install PNPM globally npm install -g pnpm@9.12.3 -# Install git -RUN apk add --no-cache git - # Debug step to verify git installation RUN git --version diff --git a/studio-frontend/packages/ui/src/ui-component/table/FlowListTable.jsx b/studio-frontend/packages/ui/src/ui-component/table/FlowListTable.jsx index 107ef78..8fa17f6 100644 --- a/studio-frontend/packages/ui/src/ui-component/table/FlowListTable.jsx +++ b/studio-frontend/packages/ui/src/ui-component/table/FlowListTable.jsx @@ -31,7 +31,7 @@ import { Analytics, PlayCircleOutline, UnarchiveOutlined -} from '@mui/icons-material' +} from '@mui/icons-material' import BuildDeploymentPackageDialog from '../dialog/BuildDeploymentPackageDialog' import chatflowsApi from '@/api/chatflows' @@ -75,7 +75,7 @@ export const FlowListTable = ({ data, images, isLoading, filterFunction, updateF const [order, setOrder] = useState(localStorage.getItem(localStorageKeyOrder) || 'desc') const [orderBy, setOrderBy] = useState(localStorage.getItem(localStorageKeyOrderBy) || 'updatedDate') - + const handleRequestSort = (property) => { const isAsc = orderBy === property && order === 'asc' @@ -115,7 +115,7 @@ export const FlowListTable = ({ data, images, isLoading, filterFunction, updateF const openWebSocketConnection = (id, status) => { const ws = new WebSocket(`${studio_server_url}/${statusCheckEndpoint}`); ws.onopen = () => { - const payload = JSON.stringify({id: id, status: status}); + const payload = JSON.stringify({ id: id, status: status }); ws.send(payload); console.log('Connected to WebSocket server', id); }; @@ -126,7 +126,7 @@ export const FlowListTable = ({ data, images, isLoading, filterFunction, updateF ws.close(); openConnections.splice(openConnections.indexOf(ws), 1); updateSandboxStatus(id, data.status, data.sandbox_app_url, data.sandbox_grafana_url); - updateFlowToServerApi(id, {sandboxStatus: data.status, sandboxAppUrl: data.sandbox_app_url, sandboxGrafanaUrl: data.sandbox_grafana_url}); + updateFlowToServerApi(id, { sandboxStatus: data.status, sandboxAppUrl: data.sandbox_app_url, sandboxGrafanaUrl: data.sandbox_grafana_url }); } }; ws.onclose = () => { @@ -144,7 +144,7 @@ export const FlowListTable = ({ data, images, isLoading, filterFunction, updateF openConnections.forEach((ws) => { ws.close(); }); - }; + }; }, [sortedData]); const updateSandboxStatus = (id, newStatus, sandboxAppUrl = null, sandboxGrafanaUrl = null) => { @@ -179,7 +179,7 @@ export const FlowListTable = ({ data, images, isLoading, filterFunction, updateF const [buildDeploymentPackageDialogOpen, setBuildDeploymentPackageDialogOpen] = useState(false) const [buildDeploymentPackageDialogProps, setBuildDeploymentPackageDialogProps] = useState({}) - const downloadDeploymentPackage = async(id, deploymentConfig) => { + const downloadDeploymentPackage = async (id, deploymentConfig) => { console.log('downloadDeploymentPackage', id, deploymentConfig); try { const response = await chatflowsApi.buildDeploymentPackage(id, deploymentConfig, { @@ -200,7 +200,7 @@ export const FlowListTable = ({ data, images, isLoading, filterFunction, updateF } setBuildDeploymentPackageDialogOpen(false) } - + const handleBuildDeploymentPackage = (id) => { setBuildDeploymentPackageDialogProps({ id: id @@ -208,7 +208,7 @@ export const FlowListTable = ({ data, images, isLoading, filterFunction, updateF setBuildDeploymentPackageDialogOpen(true) } - + useEffect(() => { setSortedData(handleSortData()); @@ -253,25 +253,52 @@ export const FlowListTable = ({ data, images, isLoading, filterFunction, updateF Workflow Name - + + + Sandbox Control + + + + + Sandbox Status + + + - Sandbox + Launch App - + - Deployment + Launch Monitoring Dashboard - + + + Deployment Package Generation + + + - + - {userRole === 'admin' && - + {userRole === 'admin' && + + + + + + + @@ -343,6 +376,12 @@ export const FlowListTable = ({ data, images, isLoading, filterFunction, updateF + + + + + + ) : ( @@ -363,7 +402,7 @@ export const FlowListTable = ({ data, images, isLoading, filterFunction, updateF }} > {row.templateName || row.name} @@ -371,100 +410,126 @@ export const FlowListTable = ({ data, images, isLoading, filterFunction, updateF - - - - - {row.sandboxStatus === "Getting Ready" || row.sandboxStatus === "Stopping" ? ( - - ): null - } - {row.sandboxStatus} - - - - - - {row.sandboxStatus === "Ready" || row.sandboxStatus === "Getting Ready"? ( - - - - ) : ( - - - - )} - - - - - - - - - - - - - - - - - + + + {row.sandboxStatus === "Ready" || row.sandboxStatus === "Getting Ready" ? ( + + + + ) : ( + + + + )} + + + + + {row.sandboxStatus === "Getting Ready" || row.sandboxStatus === "Stopping" ? ( + + ) : null + } + {row.sandboxStatus} - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + - {moment(row.updatedDate).format('MMMM Do, YYYY')} - {userRole=='admin' && {row.userid}} + {moment(row.updatedDate).format('MMMM Do, YYYY')} + {userRole == 'admin' && {row.userid}} ))}