From e919c094ec30fc336b648669a644c88ae4f1f045 Mon Sep 17 00:00:00 2001 From: wwanarif Date: Fri, 3 Jan 2025 09:24:54 +0000 Subject: [PATCH 01/15] refactor backend from 'projects' to 'workflows' Signed-off-by: wwanarif --- app-backend/app_gateway.py | 10 +- app-backend/megaservice.py | 12 +- studio-backend/app/models/pipeline_model.py | 2 +- ...t_info_model.py => workflow_info_model.py} | 2 +- studio-backend/app/routers/download_router.py | 20 +- studio-backend/app/routers/sandbox_router.py | 10 +- .../app/services/exporter_service.py | 4 +- ...fo_service.py => workflow_info_service.py} | 2 +- .../app/templates/app/app.compose.yaml | 2 +- .../templates/app/app.manifest.aws.ecr.yaml | 10 +- .../app/templates/app/app.manifest.yaml | 10 +- .../grafana-dashboards/sandbox-dashboard.json | 4231 +++++++++++++++-- .../app/utils/placeholders_utils.py | 8 +- .../exporter-groundtruth/gt_app-compose.yaml | 2 +- .../gt_app-manifest-with-nginx.yaml | 75 +- .../exporter-groundtruth/gt_app-manifest.yaml | 24 +- .../{project-info.json => workflow-info.json} | 0 studio-backend/tests/test_compose-exporter.py | 2 +- studio-backend/tests/test_download-zip.py | 2 +- .../tests/test_flowise-pipeline-translator.py | 8 +- .../tests/test_manifest-exporter.py | 9 +- 21 files changed, 4093 insertions(+), 352 deletions(-) rename studio-backend/app/models/{project_info_model.py => workflow_info_model.py} (95%) rename studio-backend/app/services/{project_info_service.py => workflow_info_service.py} (99%) rename studio-backend/tests/flowise-pipeline-translator/{project-info.json => workflow-info.json} (100%) diff --git a/app-backend/app_gateway.py b/app-backend/app_gateway.py index 9323344..27576ca 100644 --- a/app-backend/app_gateway.py +++ b/app-backend/app_gateway.py @@ -31,10 +31,10 @@ class AppGateway(Gateway): def __init__(self, megaservice, host='0.0.0.0', port=8888): try: - with open('config/project-info.json', 'r') as f: - self.project_info = json.load(f) + with open('config/workflow-info.json', 'r') as f: + self.workflow_info = json.load(f) except: - logging.error('Failed to load project-info.json') + logging.error('Failed to load workflow-info.json') super().__init__( megaservice, host, port, '/v1/app-backend', ChatCompletionRequest, ChatCompletionResponse ) @@ -42,11 +42,11 @@ def __init__(self, megaservice, host='0.0.0.0', port=8888): async def handle_request(self, request: Request): data = await request.json() print('\n'*5, '====== handle_request ======\n', data) - if 'chat_completion_ids' in self.project_info: + if 'chat_completion_ids' in self.workflow_info: prompt = self._handle_message(data['messages']) params = {} llm_parameters = None - for id, node in self.project_info['nodes'].items(): + for id, node in self.workflow_info['nodes'].items(): if node['category'] in category_params_map: param_class = category_params_map[node['category']]() param_keys = [key for key in dir(param_class) if not key.startswith('__') and not callable(getattr(param_class, key))] diff --git a/app-backend/megaservice.py b/app-backend/megaservice.py index 0b4bd44..130be60 100644 --- a/app-backend/megaservice.py +++ b/app-backend/megaservice.py @@ -19,8 +19,8 @@ def __init__(self, host="0.0.0.0", port=8000): self.host = host self.port = port self.megaservice = ServiceOrchestrator() - with open('config/project-info.json', 'r') as f: - self.project_info = json.load(f) + with open('config/workflow-info.json', 'r') as f: + self.workflow_info = json.load(f) def import_all_microservices_from_template(self): template_dir = os.path.join(os.path.dirname(__file__), 'templates', 'microservices') @@ -35,13 +35,13 @@ def import_all_microservices_from_template(self): def add_remote_service(self): print("add_remote_service") templates = self.import_all_microservices_from_template() - if 'chat_input_ids' not in self.project_info: - raise Exception('chat_input_ids not found in project_info') - nodes = self.project_info['chat_input_ids'] + if 'chat_input_ids' not in self.workflow_info: + raise Exception('chat_input_ids not found in workflow_info') + nodes = self.workflow_info['chat_input_ids'] services = {} while nodes: node_id = nodes.pop(0) - node = self.project_info['nodes'][node_id] + node = self.workflow_info['nodes'][node_id] print('node', node) if node['inMegaservice']: print('adding Node', node_id) diff --git a/studio-backend/app/models/pipeline_model.py b/studio-backend/app/models/pipeline_model.py index c94798b..7563b42 100644 --- a/studio-backend/app/models/pipeline_model.py +++ b/studio-backend/app/models/pipeline_model.py @@ -65,5 +65,5 @@ class PipelineFlow(BaseModel): createdDate: datetime updatedDate: datetime -class ProjectId(BaseModel): +class WorkflowId(BaseModel): id: str \ No newline at end of file diff --git a/studio-backend/app/models/project_info_model.py b/studio-backend/app/models/workflow_info_model.py similarity index 95% rename from studio-backend/app/models/project_info_model.py rename to studio-backend/app/models/workflow_info_model.py index 14a03b5..c507059 100644 --- a/studio-backend/app/models/project_info_model.py +++ b/studio-backend/app/models/workflow_info_model.py @@ -19,7 +19,7 @@ class UIConfigModel(BaseModel): chat_input: bool doc_input: bool -class ProjectInfoModel(BaseModel): +class WorkflowInfoModel(BaseModel): chat_completion_ids: List[str] chat_input_ids: List[str] doc_input_ids: List[str] diff --git a/studio-backend/app/routers/download_router.py b/studio-backend/app/routers/download_router.py index 243a080..a1d0b7c 100644 --- a/studio-backend/app/routers/download_router.py +++ b/studio-backend/app/routers/download_router.py @@ -10,7 +10,7 @@ from app.services.exporter_service import convert_proj_info_to_compose from app.models.pipeline_model import PipelineFlow -from app.services.project_info_service import ProjectInfo +from app.services.workflow_info_service import WorkflowInfo router = APIRouter() @@ -26,12 +26,12 @@ def create_and_download_zip(request: PipelineFlow, background_tasks: BackgroundT env_file_path = os.path.join(temp_dir, ".env") readme_file_path = os.path.join(temp_dir, "readme.MD") compose_file_path = os.path.join(temp_dir, "compose.yaml") - project_info_file_path = os.path.join(temp_dir, "project-info.json") + workflow_info_file_path = os.path.join(temp_dir, "workflow-info.json") zip_path = os.path.join(temp_dir, "docker-compose.zip") - # Covnert request to project info json - project_info_raw = ProjectInfo(request.dict()) - project_info = json.loads(project_info_raw.export_to_json()) + # Covnert request to workflow info json + workflow_info_raw = WorkflowInfo(request.dict()) + workflow_info = json.loads(workflow_info_raw.export_to_json()) # Write the strings to files try: @@ -46,20 +46,20 @@ def create_and_download_zip(request: PipelineFlow, background_tasks: BackgroundT f.write(readme_content) # compose.yaml contents - compose_content = convert_proj_info_to_compose(project_info) + compose_content = convert_proj_info_to_compose(workflow_info) with open(compose_file_path, 'w') as f: f.write(compose_content) - # project-info.json contents - with open(project_info_file_path, 'w') as f: - f.write(json.dumps(project_info, indent=4)) + # workflow-info.json contents + with open(workflow_info_file_path, 'w') as f: + f.write(json.dumps(workflow_info, indent=4)) with zipfile.ZipFile(zip_path, 'w') as zipf: # Specify the folder structure in the arcname zipf.write(env_file_path, arcname=os.path.join("docker-compose", ".env")) zipf.write(readme_file_path, arcname=os.path.join("docker-compose", "readme.MD")) zipf.write(compose_file_path, arcname=os.path.join("docker-compose", "compose.yaml")) - zipf.write(project_info_file_path, arcname=os.path.join("docker-compose", "project-info.json")) + zipf.write(workflow_info_file_path, arcname=os.path.join("docker-compose", "workflow-info.json")) # Schedule the cleanup task to run after the response has been sent background_tasks.add_task(clean_up_temp_dir, temp_dir) diff --git a/studio-backend/app/routers/sandbox_router.py b/studio-backend/app/routers/sandbox_router.py index e04af4f..ab8c101 100644 --- a/studio-backend/app/routers/sandbox_router.py +++ b/studio-backend/app/routers/sandbox_router.py @@ -4,8 +4,8 @@ import json -from app.models.pipeline_model import PipelineFlow, ProjectId -from app.services.project_info_service import ProjectInfo +from app.models.pipeline_model import PipelineFlow, WorkflowId +from app.services.workflow_info_service import WorkflowInfo from app.services.namespace_service import deploy_manifest_in_namespace, delete_namespace, check_ns_status router = APIRouter() @@ -13,18 +13,18 @@ @router.post("/deploy-sandbox") async def deploy_sandbox(request: PipelineFlow): print('deploy-sandbox') - project_info = ProjectInfo(request.dict()) + workflow_info = WorkflowInfo(request.dict()) core_v1_api = client.CoreV1Api() apps_v1_api = client.AppsV1Api() try: - response = deploy_manifest_in_namespace(core_v1_api, apps_v1_api, json.loads(project_info.export_to_json())) + response = deploy_manifest_in_namespace(core_v1_api, apps_v1_api, json.loads(workflow_info.export_to_json())) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) return response @router.post("/delete-sandbox") -async def delete_sandbox(request: ProjectId): +async def delete_sandbox(request: WorkflowId): print('deploy-sandbox') core_v1_api = client.CoreV1Api() try: diff --git a/studio-backend/app/services/exporter_service.py b/studio-backend/app/services/exporter_service.py index ab909b8..5d6bf44 100644 --- a/studio-backend/app/services/exporter_service.py +++ b/studio-backend/app/services/exporter_service.py @@ -7,7 +7,7 @@ def convert_proj_info_to_manifest(proj_info_json, output_file=None): - print("Converting project info json to manifest.") + print("Converting workflow info json to manifest.") opea_services = process_opea_services(proj_info_json) # print(json.dumps(opea_services, indent=4)) @@ -56,7 +56,7 @@ def convert_proj_info_to_manifest(proj_info_json, output_file=None): def convert_proj_info_to_compose(proj_info_json, output_file=None): - print("Converting project info json to compose.") + print("Converting workflow info json to compose.") opea_services = process_opea_services(proj_info_json) # print(json.dumps(opea_services, indent=4)) diff --git a/studio-backend/app/services/project_info_service.py b/studio-backend/app/services/workflow_info_service.py similarity index 99% rename from studio-backend/app/services/project_info_service.py rename to studio-backend/app/services/workflow_info_service.py index b664467..1b7c43b 100644 --- a/studio-backend/app/services/project_info_service.py +++ b/studio-backend/app/services/workflow_info_service.py @@ -1,7 +1,7 @@ import json import re -class ProjectInfo: +class WorkflowInfo: def __init__(self, pipeline): data = pipeline self.id = data['id'] diff --git a/studio-backend/app/templates/app/app.compose.yaml b/studio-backend/app/templates/app/app.compose.yaml index 368b843..9903ff5 100644 --- a/studio-backend/app/templates/app/app.compose.yaml +++ b/studio-backend/app/templates/app/app.compose.yaml @@ -2,7 +2,7 @@ app-backend: image: __APP_BACKEND_IMAGE__ container_name: app-backend volumes: - - ./project-info.json:/home/user/config/project-info.json + - ./workflow-info.json:/home/user/config/workflow-info.json depends_on: __BACKEND_ENDPOINTS_LIST_PLACEHOLDER__ ports: diff --git a/studio-backend/app/templates/app/app.manifest.aws.ecr.yaml b/studio-backend/app/templates/app/app.manifest.aws.ecr.yaml index a795156..51ff5ce 100644 --- a/studio-backend/app/templates/app/app.manifest.aws.ecr.yaml +++ b/studio-backend/app/templates/app/app.manifest.aws.ecr.yaml @@ -13,7 +13,7 @@ kind: ConfigMap metadata: name: app-backend-config data: - project-info.json: |+ + workflow-info.json: |+ __BACKEND_PROJECT_INFO_JSON_PLACEHOLDER__ --- apiVersion: v1 @@ -67,9 +67,9 @@ spec: volumeMounts: - mountPath: /tmp name: tmp - - mountPath: /home/user/config/project-info.json - name: project-info-volume - subPath: project-info.json + - mountPath: /home/user/config/workflow-info.json + name: workflow-info-volume + subPath: workflow-info.json ports: - name: app-backend containerPort: 8888 @@ -80,7 +80,7 @@ spec: volumes: - name: tmp emptyDir: {} - - name: project-info-volume + - name: workflow-info-volume configMap: name: app-backend-config --- diff --git a/studio-backend/app/templates/app/app.manifest.yaml b/studio-backend/app/templates/app/app.manifest.yaml index 931de63..8944f93 100644 --- a/studio-backend/app/templates/app/app.manifest.yaml +++ b/studio-backend/app/templates/app/app.manifest.yaml @@ -4,7 +4,7 @@ kind: ConfigMap metadata: name: app-backend-config data: - project-info.json: |+ + workflow-info.json: |+ __BACKEND_PROJECT_INFO_JSON_PLACEHOLDER__ --- apiVersion: v1 @@ -58,9 +58,9 @@ spec: volumeMounts: - mountPath: /tmp name: tmp - - mountPath: /home/user/config/project-info.json - name: project-info-volume - subPath: project-info.json + - mountPath: /home/user/config/workflow-info.json + name: workflow-info-volume + subPath: workflow-info.json ports: - name: app-backend containerPort: 8888 @@ -69,7 +69,7 @@ spec: volumes: - name: tmp emptyDir: {} - - name: project-info-volume + - name: workflow-info-volume configMap: name: app-backend-config --- diff --git a/studio-backend/app/templates/grafana-dashboards/sandbox-dashboard.json b/studio-backend/app/templates/grafana-dashboards/sandbox-dashboard.json index 36eced1..e84fc3f 100644 --- a/studio-backend/app/templates/grafana-dashboards/sandbox-dashboard.json +++ b/studio-backend/app/templates/grafana-dashboards/sandbox-dashboard.json @@ -22,104 +22,138 @@ "id": null, "links": [], "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "title": "Cluster CPU / MEM / DISK", + "type": "row" + }, { "datasource": { "default": true, "type": "prometheus", "uid": "prometheus" }, + "description": "Resource pressure via PSI", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, + "decimals": 1, + "links": [], "mappings": [], + "max": 1, + "min": 0, "thresholds": { - "mode": "absolute", + "mode": "percentage", "steps": [ { "color": "green", "value": null }, { - "color": "red", - "value": 80 + "color": "dark-yellow", + "value": 70 + }, + { + "color": "dark-red", + "value": 90 } ] }, - "unit": "short" + "unit": "percentunit" }, "overrides": [] }, "gridPos": { - "h": 9, - "w": 12, + "h": 4, + "w": 3, "x": 0, - "y": 0 + "y": 1 }, - "id": 2, + "id": 3, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "displayMode": "basic", + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "showUnfilled": true, + "sizing": "auto", + "text": {}, + "valueMode": "color" }, - "pluginVersion": "8.0.0", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"___EXPR_NAMESPACE___\", container!=\"\"}[5m])) by (pod)", - "interval": "", - "legendFormat": "{{pod}}", - "range": true, - "refId": "A" + "exemplar": false, + "expr": "irate(node_pressure_cpu_waiting_seconds_total{job=\"node-exporter\"}[$__rate_interval])", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "CPU", + "range": false, + "refId": "CPU some", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "irate(node_pressure_memory_waiting_seconds_total{job=\"node-exporter\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "Mem", + "range": false, + "refId": "Memory some", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "irate(node_pressure_io_waiting_seconds_total{job=\"node-exporter\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "I/O", + "range": false, + "refId": "I/O some", + "step": 240 } ], - "title": "CPU Usage (container)", - "type": "timeseries" + "title": "Pressure", + "type": "bargauge" }, { "datasource": { @@ -127,98 +161,90 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Busy state of all CPU cores together", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "mode": "thresholds" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" } - }, - "mappings": [], + ], + "max": 100, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "rgba(50, 172, 45, 0.97)", "value": null }, { - "color": "red", - "value": 80 + "color": "rgba(237, 129, 40, 0.89)", + "value": 85 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 95 } ] }, - "unit": "bytes" + "unit": "percent" }, "overrides": [] }, "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 0 + "h": 4, + "w": 3, + "x": 3, + "y": 1 }, "id": 4, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" }, - "pluginVersion": "8.0.0", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(container_memory_usage_bytes{namespace=\"___EXPR_NAMESPACE___\", container!=\"\"}[5m])) by (pod)", - "interval": "", - "legendFormat": "{{pod}}", - "range": true, - "refId": "B" + "exemplar": false, + "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])))", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "range": false, + "refId": "A", + "step": 240 } ], - "title": "Memory Usage (container)", - "type": "timeseries" + "title": "CPU Busy", + "type": "gauge" }, { "datasource": { @@ -226,98 +252,90 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "System load over all CPU cores together", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "mode": "thresholds" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" } - }, - "mappings": [], + ], + "max": 100, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "rgba(50, 172, 45, 0.97)", "value": null }, { - "color": "red", - "value": 80 + "color": "rgba(237, 129, 40, 0.89)", + "value": 85 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 95 } ] }, - "unit": "short" + "unit": "percent" }, "overrides": [] }, "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 9 + "h": 4, + "w": 3, + "x": 6, + "y": 1 }, "id": 5, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" }, - "pluginVersion": "8.0.0", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(process_cpu_seconds_total{namespace=\"___EXPR_NAMESPACE___\"}[5m])", - "interval": "", - "legendFormat": "{{pod}}", - "range": true, - "refId": "A" + "exemplar": false, + "expr": "scalar(node_load1{job=\"node-exporter\"}) * 100 / count(count(node_cpu_seconds_total{job=\"node-exporter\"}) by (cpu))", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 } ], - "title": "CPU Usage (process)", - "type": "timeseries" + "title": "Sys Load", + "type": "gauge" }, { "datasource": { @@ -325,100 +343,3804 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Non available RAM memory", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, + "decimals": 1, "mappings": [], + "max": 100, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "rgba(50, 172, 45, 0.97)", "value": null }, { - "color": "red", + "color": "rgba(237, 129, 40, 0.89)", "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 } ] }, - "unit": "short" + "unit": "percent" }, "overrides": [] }, "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 9 + "h": 4, + "w": 3, + "x": 9, + "y": 1 }, + "hideTimeOverride": false, "id": 6, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" }, - "pluginVersion": "8.0.0", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(process_resident_memory_bytes{namespace=\"___EXPR_NAMESPACE___\"}[5m])", + "exemplar": false, + "expr": "((node_memory_MemTotal_bytes{ job=\"node-exporter\"} - node_memory_MemFree_bytes{ job=\"node-exporter\"}) / node_memory_MemTotal_bytes{ job=\"node-exporter\"}) * 100", + "format": "time_series", + "hide": true, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(1 - (node_memory_MemAvailable_bytes{ job=\"node-exporter\"} / node_memory_MemTotal_bytes{ job=\"node-exporter\"})) * 100", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "B", + "step": 240 + } + ], + "title": "RAM Used", + "type": "gauge" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Used Swap", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 10 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 25 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 7, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "((node_memory_SwapTotal_bytes{job=\"node-exporter\"} - node_memory_SwapFree_bytes{job=\"node-exporter\"}) / (node_memory_SwapTotal_bytes{job=\"node-exporter\"})) * 100", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "SWAP Used", + "type": "gauge" + }, + { + "datasource": {}, + "description": "Used Root FS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 8, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "100 - ((node_filesystem_avail_bytes{job=\"node-exporter\",mountpoint=\"/\",fstype!=\"rootfs\"} * 100) / node_filesystem_size_bytes{job=\"node-exporter\",mountpoint=\"/\",fstype!=\"rootfs\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "Root FS Used", + "type": "gauge" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of CPU cores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 18, + "y": 1 + }, + "id": 9, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(count(node_cpu_seconds_total{job=\"node-exporter\"}) by (cpu))", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "description": "System uptime", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 20, + "y": 1 + }, + "hideTimeOverride": true, + "id": 10, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_time_seconds{job=\"node-exporter\"} - node_boot_time_seconds{job=\"node-exporter\"}", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total RootFS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 70 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 18, + "y": 3 + }, + "id": 11, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_filesystem_size_bytes{job=\"node-exporter\",mountpoint=\"/\",fstype!=\"rootfs\"}", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "RootFS Total", + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total RAM", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 20, + "y": 3 + }, + "id": 12, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_memory_MemTotal_bytes{job=\"node-exporter\"}", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "RAM Total", + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total SWAP", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 22, + "y": 3 + }, + "id": 13, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_memory_SwapTotal_bytes{job=\"node-exporter\"}", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "SWAP Total", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 14, + "panels": [], + "title": "Sandbox CPU / MEM / REQ", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"___EXPR_NAMESPACE___\", container!=\"\"}[5m])) by (pod)", + "interval": "", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage (container)", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(process_cpu_seconds_total{namespace=\"___EXPR_NAMESPACE___\"}[5m])", + "interval": "", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage (process)", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(container_memory_usage_bytes{namespace=\"___EXPR_NAMESPACE___\", container!=\"\"}[5m])) by (pod)", + "interval": "", + "legendFormat": "{{pod}}", + "range": true, + "refId": "B" + } + ], + "title": "Memory Usage (container)", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(process_resident_memory_bytes{namespace=\"___EXPR_NAMESPACE___\"}[5m])", + "interval": "", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Memory Usage (process)", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 15, + "x": 0, + "y": 22 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "$$hashKey": "object:214", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(handler) (rate(http_requests_total{namespace=\"___EXPR_NAMESPACE___\", handler!~\"/metrics|/v1/health_check|none\"}[1m]))", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ method }} {{ handler }}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total requests per second", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "HTTP 500" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#bf1b00", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 22 + }, + "id": 26, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "$$hashKey": "object:140", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(status) (rate(http_requests_total{namespace=\"___EXPR_NAMESPACE___\"}[1m]))", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, "interval": "", - "legendFormat": "{{pod}}", + "intervalFactor": 1, + "legendFormat": "{{ status }}", "range": true, - "refId": "A" + "refId": "A", + "useBackend": false + } + ], + "title": "Request per second", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 19, + "panels": [ + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1000 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 31 + }, + "id": 20, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "(histogram_quantile(0.5, sum by (le) (rate(tgi_request_queue_duration_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m]))) * 1000) > 0", + "hide": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "(histogram_quantile(0.5, sum by (le) (rate(tgi_batch_inference_duration_bucket{method=\"prefill\", namespace=\"___EXPR_NAMESPACE___\"}[10m]))) * 1000) > 0", + "hide": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "C" + }, + { + "datasource": { + "name": "Expression", + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "$B + $C", + "hide": false, + "refId": "D", + "type": "math" + } + ], + "title": "Time to first token", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 31 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "(histogram_quantile(0.5, sum by (le) (rate(tgi_batch_forward_duration_bucket{method=\"decode\", namespace=\"___EXPR_NAMESPACE___\"}[10m]))) * 1000)>0", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Decode per-token latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 31 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum((rate(tgi_request_generated_tokens_sum{namespace=\"___EXPR_NAMESPACE___\"}[10m]) / rate(tgi_request_generated_tokens_count{namespace=\"___EXPR_NAMESPACE___\"}[10m]))>0)", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Throughput (generated tok/s)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le) (rate(tgi_request_input_length_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le) (rate(tgi_request_input_length_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "hide": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(tgi_request_input_length_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "hide": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + } + ], + "title": "Number of tokens per prompt", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le) (rate(tgi_request_generated_tokens_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le) (rate(tgi_request_generated_tokens_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "hide": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(tgi_request_generated_tokens_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "hide": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + } + ], + "title": "Number of generated tokens per request", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 46 + }, + "id": 27, + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(tgi_request_success{namespace=\"___EXPR_NAMESPACE___\"}[1m]))", + "legendFormat": "Success", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(tgi_request_failure{namespace=\"___EXPR_NAMESPACE___\"}[1m])) by (err)", + "hide": false, + "legendFormat": "Error: {{err}}", + "range": true, + "refId": "B" + } + ], + "title": "Requests", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 9, + "x": 6, + "y": 46 + }, + "id": 30, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le) (rate(tgi_request_mean_time_per_token_duration_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le) (rate(tgi_request_mean_time_per_token_duration_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "hide": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(tgi_request_mean_time_per_token_duration_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "hide": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + } + ], + "title": "Mean Time Per Token quantiles", + "type": "timeseries" + }, + { + "cards": {}, + "color": { + "cardColor": "#5794F2", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "opacity" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 9, + "x": 15, + "y": 46 + }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 31, + "legend": { + "show": false + }, + "maxDataPoints": 25, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#5794F2", + "min": 0, + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 1, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.2.0", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(tgi_request_mean_time_per_token_duration_bucket{namespace=\"___EXPR_NAMESPACE___\"}[5m])) by (le)", + "format": "heatmap", + "interval": "", + "legendFormat": "{{ le }}", + "range": true, + "refId": "A" + } + ], + "title": "Mean Time Per Token", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 54 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(tgi_request_count{namespace=\"___EXPR_NAMESPACE___\"})", + "legendFormat": "Replicas", + "range": true, + "refId": "A" + } + ], + "title": "Number of replicas", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 3, + "y": 54 + }, + "id": 29, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(tgi_queue_size{namespace=\"___EXPR_NAMESPACE___\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Queue Size", + "type": "gauge" } ], - "title": "Memory Usage (process)", - "type": "timeseries" + "title": "Sandbox TGI", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 32, + "panels": [ + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 60 + }, + "id": 33, + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(te_embed_success{namespace=\"___EXPR_NAMESPACE___\"}[1m]))", + "legendFormat": "Success", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(te_embed_count{namespace=\"___EXPR_NAMESPACE___\"}[1m]))", + "hide": false, + "legendFormat": "Total Count", + "range": true, + "refId": "B" + } + ], + "title": "Requests", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 9, + "x": 6, + "y": 60 + }, + "id": 36, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le) (rate(te_embed_queue_duration_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le) (rate(te_embed_queue_duration_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "hide": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(te_embed_queue_duration_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "hide": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + } + ], + "title": "Queue Duration quantiles", + "type": "timeseries" + }, + { + "cards": {}, + "color": { + "cardColor": "#5794F2", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "opacity" + }, + "dataFormat": "tsbuckets", + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 9, + "x": 15, + "y": 60 + }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 37, + "legend": { + "show": false + }, + "maxDataPoints": 25, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#5794F2", + "min": 0, + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 1, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.2.0", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(te_request_duration_bucket{namespace=\"___EXPR_NAMESPACE___\"}[5m])) by (le)", + "format": "heatmap", + "interval": "", + "legendFormat": "{{ le }}", + "range": true, + "refId": "A" + } + ], + "title": "E2E Latency", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 68 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(te_request_count{namespace=\"___EXPR_NAMESPACE___\"})", + "legendFormat": "Replicas", + "range": true, + "refId": "A" + } + ], + "title": "Number of replicas", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 3, + "y": 68 + }, + "id": 35, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(te_queue_size{namespace=\"___EXPR_NAMESPACE___\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Queue Size", + "type": "gauge" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 14, + "w": 6, + "x": 0, + "y": 73 + }, + "id": 38, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le) (rate(te_embed_duration_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le) (rate(te_embed_duration_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "hide": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(te_embed_duration_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "hide": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + } + ], + "title": "Latency quantiles", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 14, + "w": 9, + "x": 6, + "y": 73 + }, + "id": 39, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le) (rate(te_request_tokenization_duration_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le) (rate(te_request_tokenization_duration_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "hide": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(te_request_tokenization_duration_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "hide": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + } + ], + "title": "Tokenization Duration quantiles", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 9, + "x": 15, + "y": 73 + }, + "id": 40, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le) (rate(te_request_inference_duration_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le) (rate(te_request_inference_duration_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "hide": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(te_request_inference_duration_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "hide": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + } + ], + "title": "Inference quantiles", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 9, + "x": 15, + "y": 80 + }, + "id": 41, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le) (rate(te_embed_tokenization_duration_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le) (rate(te_embed_tokenization_duration_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "hide": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(te_embed_tokenization_duration_bucket{namespace=\"___EXPR_NAMESPACE___\"}[10m])))", + "hide": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + } + ], + "title": "Tokenization Latency quantiles", + "type": "timeseries" + } + ], + "title": "Sandbox TEI", + "type": "row" } ], + "refresh": "30s", "schemaVersion": 39, "tags": [], "templating": { @@ -428,38 +4150,11 @@ "from": "now-30m", "to": "now" }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", + "timepicker": {}, + "timezone": "browser", "title": "___DASHBOARD_TITLE___", "uid": "___DASHBOARD_UID___", - "version": 2, - "weekStart": "", - "overwrite": false, - "folderId": 0, - "message": "Imported by API" + "version": 1, + "weekStart": "" } } \ No newline at end of file diff --git a/studio-backend/app/utils/placeholders_utils.py b/studio-backend/app/utils/placeholders_utils.py index c4afa29..0b7bebb 100644 --- a/studio-backend/app/utils/placeholders_utils.py +++ b/studio-backend/app/utils/placeholders_utils.py @@ -38,7 +38,7 @@ def replace_manifest_placeholders(obj, variables): if isinstance(obj, dict): for key, value in obj.items(): # Skip nginx.conf as it contains {} that will clashe with .format() - if key == "default.conf" or key == "project-info.json": + if key == "default.conf" or key == "workflow-info.json": continue if isinstance(value, str): # Replace ${REGISTRY} and ${TAG} with the value from environment variables @@ -67,7 +67,7 @@ def replace_dynamic_manifest_placeholder(value_str, service_info, proj_info_json if service_info['service_type'] == 'app': ui_env_config_info_str = "" ui_nginx_config_info_str = "" - backend_project_info_str = "" + backend_workflow_info_str = "" for key, value in service_info['ui_config_info'].items(): # For __UI_CONFIG_INFO_ENV_PLACEHOLDER__ url_name = value['url_name'] @@ -91,7 +91,7 @@ def replace_dynamic_manifest_placeholder(value_str, service_info, proj_info_json ui_nginx_config_info_str += indented_location_block # For __BACKEND_PROJECT_INFO_JSON_PLACEHOLDER__ - backend_project_info_str = json.dumps(proj_info_json, indent=4) + backend_workflow_info_str = json.dumps(proj_info_json, indent=4) # Get app images from environment variables app_frontend_image = os.getenv("APP_FRONTEND_IMAGE", "opea/app-frontend:latest") @@ -100,7 +100,7 @@ def replace_dynamic_manifest_placeholder(value_str, service_info, proj_info_json # Replace the unique placeholders with the actual strings final_config = value_str.replace("__UI_CONFIG_INFO_ENV_PLACEHOLDER__", ui_env_config_info_str.strip()).replace( "__UI_CONFIG_INFO_NGINX_PLACEHOLDER__", ui_nginx_config_info_str.strip()).replace( - "__BACKEND_PROJECT_INFO_JSON_PLACEHOLDER__", backend_project_info_str.replace(f"\n", f"\n{indent_str}")).replace( + "__BACKEND_PROJECT_INFO_JSON_PLACEHOLDER__", backend_workflow_info_str.replace(f"\n", f"\n{indent_str}")).replace( "__APP_FRONTEND_IMAGE__", app_frontend_image).replace( "__APP_BACKEND_IMAGE__", app_backend_image) diff --git a/studio-backend/tests/exporter-groundtruth/gt_app-compose.yaml b/studio-backend/tests/exporter-groundtruth/gt_app-compose.yaml index c9bcbaa..ba03bb3 100644 --- a/studio-backend/tests/exporter-groundtruth/gt_app-compose.yaml +++ b/studio-backend/tests/exporter-groundtruth/gt_app-compose.yaml @@ -150,7 +150,7 @@ services: image: opea/app-backend:latest container_name: app-backend volumes: - - ./project-info.json:/home/user/config/project-info.json + - ./workflow-info.json:/home/user/config/workflow-info.json depends_on: - redis-vector-store-0 - tei-0 diff --git a/studio-backend/tests/exporter-groundtruth/gt_app-manifest-with-nginx.yaml b/studio-backend/tests/exporter-groundtruth/gt_app-manifest-with-nginx.yaml index 4ab3214..8da9daf 100644 --- a/studio-backend/tests/exporter-groundtruth/gt_app-manifest-with-nginx.yaml +++ b/studio-backend/tests/exporter-groundtruth/gt_app-manifest-with-nginx.yaml @@ -180,9 +180,8 @@ spec: resources: {} volumes: - name: model-volume - hostPath: - path: /mnt/opea-models - type: Directory + persistentVolumeClaim: + claimName: model-pvc - name: shm emptyDir: medium: Memory @@ -285,9 +284,8 @@ spec: resources: {} volumes: - name: model-volume - hostPath: - path: /mnt/opea-models - type: Directory + persistentVolumeClaim: + claimName: model-pvc - name: shm emptyDir: medium: Memory @@ -395,9 +393,8 @@ spec: resources: {} volumes: - name: model-volume - hostPath: - path: /mnt/opea-models - type: Directory + persistentVolumeClaim: + claimName: model-pvc - name: shm emptyDir: medium: Memory @@ -505,9 +502,8 @@ spec: resources: {} volumes: - name: model-volume - hostPath: - path: /mnt/opea-models - type: Directory + persistentVolumeClaim: + claimName: model-pvc - name: shm emptyDir: medium: Memory @@ -1023,7 +1019,7 @@ kind: ConfigMap metadata: name: app-backend-config data: - project-info.json: | + workflow-info.json: | { "chat_completion_ids": [ "chat_completion_0" @@ -1043,6 +1039,7 @@ data: "opea_service@llm_tgi_0" ], "connected_to": [], + "dependent_services": null, "hideOutput": true, "id": "chat_completion_0", "inMegaservice": false, @@ -1057,6 +1054,8 @@ data: "connected_to": [ "opea_service@embedding_tei_langchain_0" ], + "dependent_services": null, + "hideOutput": null, "id": "chat_input_0", "inMegaservice": false, "inference_params": {}, @@ -1070,6 +1069,8 @@ data: "connected_to": [ "opea_service@prepare_doc_redis_prep_0" ], + "dependent_services": null, + "hideOutput": null, "id": "doc_input_0", "inMegaservice": false, "inference_params": {}, @@ -1093,6 +1094,7 @@ data: "modelName": "BAAI/bge-large-en-v1.5" } }, + "hideOutput": null, "id": "opea_service@embedding_tei_langchain_0", "inMegaservice": true, "inference_params": {}, @@ -1116,16 +1118,17 @@ data: "modelName": "Intel/neural-chat-7b-v3-3" } }, + "hideOutput": null, "id": "opea_service@llm_tgi_0", "inMegaservice": true, "inference_params": { "chat_template": "### You are a helpful, respectful and honest assistant to help the user with questions.\n### Context: {context}\n### Question: {question}\n### Answer:", - "frequency_penalty": "", - "max_tokens": 17, + "frequency_penalty": 0, + "max_tokens": 17.0, "presence_penalty": 1.03, - "streaming": true, + "streaming": "True", "temperature": 0.01, - "top_k": 10, + "top_k": 10.0, "top_p": 0.95, "typical_p": 0.95 }, @@ -1149,6 +1152,7 @@ data: "modelName": "BAAI/bge-large-en-v1.5" } }, + "hideOutput": null, "id": "opea_service@prepare_doc_redis_prep_0", "inMegaservice": false, "inference_params": {}, @@ -1172,10 +1176,11 @@ data: "modelName": "BAAI/bge-reranker-base" } }, + "hideOutput": null, "id": "opea_service@reranking_tei_0", "inMegaservice": true, "inference_params": { - "top_n": 1 + "top_n": 1.0 }, "name": "opea_service@reranking_tei", "params": {}, @@ -1198,6 +1203,7 @@ data: "modelName": "BAAI/bge-base-en-v1.5" } }, + "hideOutput": null, "id": "opea_service@retriever_redis_0", "inMegaservice": true, "inference_params": { @@ -1215,6 +1221,8 @@ data: "connected_to": [ "opea_service@retriever_redis_0" ], + "dependent_services": null, + "hideOutput": null, "id": "redis_vector_store_0", "inMegaservice": true, "inference_params": {}, @@ -1281,13 +1289,13 @@ spec: seccompProfile: type: RuntimeDefault image: opea/app-backend:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: Always volumeMounts: - mountPath: /tmp name: tmp - - mountPath: /home/user/config/project-info.json - name: project-info-volume - subPath: project-info.json + - mountPath: /home/user/config/workflow-info.json + name: workflow-info-volume + subPath: workflow-info.json ports: - name: app-backend containerPort: 8888 @@ -1296,7 +1304,7 @@ spec: volumes: - name: tmp emptyDir: {} - - name: project-info-volume + - name: workflow-info-volume configMap: name: app-backend-config --- @@ -1405,6 +1413,13 @@ data: proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + + # Disable buffering for SSE + proxy_buffering off; + proxy_cache off; + proxy_http_version 1.1; + proxy_set_header Connection ''; + chunked_transfer_encoding off; } location /v1/embeddings { @@ -1493,3 +1508,17 @@ spec: defaultMode: 420 name: app-nginx-config name: nginx-config-volume +--- +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: model-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi + storageClassName: local-path \ No newline at end of file diff --git a/studio-backend/tests/exporter-groundtruth/gt_app-manifest.yaml b/studio-backend/tests/exporter-groundtruth/gt_app-manifest.yaml index 2116558..c52eeb3 100644 --- a/studio-backend/tests/exporter-groundtruth/gt_app-manifest.yaml +++ b/studio-backend/tests/exporter-groundtruth/gt_app-manifest.yaml @@ -1023,7 +1023,7 @@ kind: ConfigMap metadata: name: app-backend-config data: - project-info.json: | + workflow-info.json: | { "chat_completion_ids": [ "chat_completion_0" @@ -1285,9 +1285,9 @@ spec: volumeMounts: - mountPath: /tmp name: tmp - - mountPath: /home/user/config/project-info.json - name: project-info-volume - subPath: project-info.json + - mountPath: /home/user/config/workflow-info.json + name: workflow-info-volume + subPath: workflow-info.json ports: - name: app-backend containerPort: 8888 @@ -1296,7 +1296,7 @@ spec: volumes: - name: tmp emptyDir: {} - - name: project-info-volume + - name: workflow-info-volume configMap: name: app-backend-config --- @@ -1364,3 +1364,17 @@ spec: volumes: - name: tmp emptyDir: {} +--- +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: model-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi + storageClassName: local-path \ No newline at end of file diff --git a/studio-backend/tests/flowise-pipeline-translator/project-info.json b/studio-backend/tests/flowise-pipeline-translator/workflow-info.json similarity index 100% rename from studio-backend/tests/flowise-pipeline-translator/project-info.json rename to studio-backend/tests/flowise-pipeline-translator/workflow-info.json diff --git a/studio-backend/tests/test_compose-exporter.py b/studio-backend/tests/test_compose-exporter.py index 9ee5f71..6bf8c76 100644 --- a/studio-backend/tests/test_compose-exporter.py +++ b/studio-backend/tests/test_compose-exporter.py @@ -11,7 +11,7 @@ def setup_and_teardown(): test_dir = os.path.dirname(os.path.abspath(__file__)) # Paths for the `mega.yaml` and output file - proj_info_file = os.path.join(test_dir, "flowise-pipeline-translator", "project-info.json") + proj_info_file = os.path.join(test_dir, "flowise-pipeline-translator", "workflow-info.json") output_file = os.path.join(test_dir, "exporter-groundtruth", "app-compose.yaml") gt_file = os.path.join(test_dir, "exporter-groundtruth", "gt_app-compose.yaml") diff --git a/studio-backend/tests/test_download-zip.py b/studio-backend/tests/test_download-zip.py index 7ee632a..765e096 100644 --- a/studio-backend/tests/test_download-zip.py +++ b/studio-backend/tests/test_download-zip.py @@ -55,4 +55,4 @@ def test_create_and_download_zip(setup_and_teardown): assert 'docker-compose/.env' in zipf.namelist() assert 'docker-compose/readme.MD' in zipf.namelist() assert 'docker-compose/compose.yaml' in zipf.namelist() - assert 'docker-compose/project-info.json' in zipf.namelist() \ No newline at end of file + assert 'docker-compose/workflow-info.json' in zipf.namelist() \ No newline at end of file diff --git a/studio-backend/tests/test_flowise-pipeline-translator.py b/studio-backend/tests/test_flowise-pipeline-translator.py index bad3499..7c9a9d4 100644 --- a/studio-backend/tests/test_flowise-pipeline-translator.py +++ b/studio-backend/tests/test_flowise-pipeline-translator.py @@ -7,7 +7,7 @@ import json -from app.services.project_info_service import ProjectInfo +from app.services.workflow_info_service import WorkflowInfo class TestFlowisePipelineTranslator(unittest.TestCase): @@ -21,9 +21,9 @@ def setUp(self): def test_flowise_pipeline_translator(self): # Call the function directly - print("converting flowise_pipeline to project_info") - project_info = ProjectInfo(json.loads(self.pipeline_json)) - print('project_info', project_info.export_to_json()) + print("converting flowise_pipeline to workflow_info") + workflow_info = WorkflowInfo(json.loads(self.pipeline_json)) + print('workflow_info', workflow_info.export_to_json()) self.assertTrue = True diff --git a/studio-backend/tests/test_manifest-exporter.py b/studio-backend/tests/test_manifest-exporter.py index 0808f94..7de37af 100644 --- a/studio-backend/tests/test_manifest-exporter.py +++ b/studio-backend/tests/test_manifest-exporter.py @@ -12,7 +12,7 @@ def setup_and_teardown(): test_dir = os.path.dirname(os.path.abspath(__file__)) # Paths for the `mega.yaml` and output file - proj_info_file = os.path.join(test_dir, "flowise-pipeline-translator", "project-info.json") + proj_info_file = os.path.join(test_dir, "flowise-pipeline-translator", "workflow-info.json") output_file = os.path.join(test_dir, "exporter-groundtruth", "app-manifest.yaml") gt_file = os.path.join(test_dir, "exporter-groundtruth", "gt_app-manifest.yaml") gt_nginx_file = os.path.join(test_dir, "exporter-groundtruth", "gt_app-manifest-with-nginx.yaml") @@ -23,7 +23,7 @@ def setup_and_teardown(): os.unlink(output_file) def test_convert_chatqna_proj_info_to_manifest_obj(setup_and_teardown): - proj_info_file, _, gt_file, _ = setup_and_teardown + proj_info_file, output_file, gt_file, _ = setup_and_teardown with open(proj_info_file, "r") as file: proj_info = json.load(file) @@ -39,9 +39,12 @@ def create_identifiers_set(documents): identifiers.add((doc['kind'], doc['metadata']['name'])) return identifiers - manifest_dict = yaml.safe_load_all(output_manifest) + manifest_dict = list(yaml.safe_load_all(output_manifest)) output_identifiers = create_identifiers_set(manifest_dict) + with open(output_file, "w") as f: + yaml.dump_all(manifest_dict, f) + # Load the documents from the gt manifest with open(gt_file, 'r') as f: gt_manifest_docs = list(yaml.safe_load_all(f)) From 162a5f9f224dfcf4b1b78a13f254a9b5e3df2eb5 Mon Sep 17 00:00:00 2001 From: wwanarif Date: Fri, 3 Jan 2025 09:26:30 +0000 Subject: [PATCH 02/15] enabled keycloak and mysql integration Signed-off-by: wwanarif --- .../manifests/studio-manifest-aws-ecr.yaml | 161 +++++++++++++- .../manifests/studio-manifest.yaml | 199 +++++++++++++++++- .../playbooks/deploy-studio.yml | 63 +++++- .../playbooks/setup-mysqldb.yml | 92 ++++++++ .../setup-genai-studio/studio-config.yaml | 14 ++ studio-frontend/packages/server/.gitignore | 6 + studio-frontend/packages/server/bin/run | 0 .../packages/server/src/DataSource.ts | 14 +- .../packages/server/src/Interface.ts | 1 + .../server/src/controllers/chatflows/index.ts | 11 + .../server/src/database/entities/ChatFlow.ts | 3 + .../migrations/mysql/1693840429259-Init.ts | 4 + .../1732778337650-OPEAAddUserIdtoChatFlow.ts | 20 ++ .../src/database/migrations/sqlite/index.ts | 5 +- .../server/src/routes/chatflows/index.ts | 2 +- .../server/src/services/chatflows/index.ts | 61 ++++++ studio-frontend/packages/ui/package.json | 2 + studio-frontend/packages/ui/src/App.jsx | 44 ++-- .../packages/ui/src/KeycloakContext.jsx | 72 +++++++ .../packages/ui/src/api/chatflows.js | 3 + .../ui/src/layout/MainLayout/Header/index.jsx | 80 ++++--- .../ui/src/layout/MainLayout/index.jsx | 88 +++++--- .../src/ui-component/table/FlowListTable.jsx | 23 +- .../packages/ui/src/views/canvas/index.jsx | 8 + .../packages/ui/src/views/opeaflows/index.jsx | 41 +++- studio-frontend/packages/ui/vite.config.js | 4 +- .../001_test_sandbox_deployment.spec.ts | 11 +- .../002_test_sandbox_chatqna.spec.ts | 11 +- 28 files changed, 906 insertions(+), 137 deletions(-) create mode 100644 setup-scripts/setup-genai-studio/playbooks/setup-mysqldb.yml create mode 100644 setup-scripts/setup-genai-studio/studio-config.yaml create mode 100644 studio-frontend/packages/server/.gitignore mode change 100644 => 100755 studio-frontend/packages/server/bin/run create mode 100644 studio-frontend/packages/server/src/database/migrations/sqlite/1732778337650-OPEAAddUserIdtoChatFlow.ts create mode 100644 studio-frontend/packages/ui/src/KeycloakContext.jsx diff --git a/setup-scripts/setup-genai-studio/manifests/studio-manifest-aws-ecr.yaml b/setup-scripts/setup-genai-studio/manifests/studio-manifest-aws-ecr.yaml index d3bc2f5..988d03b 100644 --- a/setup-scripts/setup-genai-studio/manifests/studio-manifest-aws-ecr.yaml +++ b/setup-scripts/setup-genai-studio/manifests/studio-manifest-aws-ecr.yaml @@ -20,8 +20,13 @@ data: # SPDX-License-Identifier: Apache-2.0 server { - listen 80; - listen [::]:80; + # listen 80; + # listen [::]:80; + listen 443 ssl; + listen [::]:443 ssl; + + ssl_certificate /etc/ssl/tls.crt; + ssl_certificate_key /etc/ssl/tls.key; proxy_connect_timeout 600; proxy_send_timeout 600; @@ -34,7 +39,7 @@ data: resolver_timeout 5s; location /home { - root /usr/share/nginx/html; # Use root to serve files from a directory + root /usr/share/nginx/html; index index.html; } @@ -71,6 +76,15 @@ data: proxy_set_header X-Forwarded-Proto $scheme; } + # Location block for keycloak + location /auth { + proxy_pass https://${KEYCLOAK_DNS}/auth/; + proxy_set_header Host $host:30007; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + # Location block for app-backend location /v1/app-backend { # Initialize the variable for namespace @@ -170,10 +184,11 @@ spec: selector: app: studio-nginx ports: - - protocol: TCP - port: 80 - targetPort: 80 - nodePort: 30007 + - name: https + protocol: TCP + port: 443 + targetPort: 443 + nodePort: 30007 type: NodePort --- apiVersion: apps/v1 @@ -201,7 +216,7 @@ spec: envsubst "$(env | grep _DNS= | awk -F= '{print "${"$1"}"}' | tr '\n' ' ')" < /tmp/default.conf > /etc/nginx/conf.d/default.conf envFrom: - configMapRef: - name: internal-dns-config + name: studio-config volumeMounts: - name: tmp-volume mountPath: /tmp @@ -214,6 +229,8 @@ spec: volumeMounts: - name: nginx-conf-volume mountPath: /etc/nginx/conf.d + - name: tls + mountPath: /etc/ssl securityContext: {} volumes: - name: tmp-volume @@ -222,6 +239,9 @@ spec: name: studio-nginx-config - name: nginx-conf-volume emptyDir: {} + - name: tls + secret: + secretName: tls-secret --- apiVersion: v1 kind: Service @@ -272,6 +292,9 @@ rules: - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "create", "list", "watch"] +- apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "create", "list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -322,7 +345,7 @@ spec: value: ${NO_PROXY} envFrom: - configMapRef: - name: internal-dns-config + name: studio-config ports: - containerPort: 5000 resources: @@ -394,4 +417,122 @@ spec: - name: ecr-registry-secret volumes: - name: tmp - emptyDir: {} \ No newline at end of file + emptyDir: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: keycloak + namespace: studio + labels: + app: keycloak +spec: + replicas: 1 + selector: + matchLabels: + app: keycloak + template: + metadata: + labels: + app: keycloak + spec: + initContainers: + - name: keycloak-assets + image: curlimages/curl:latest + command: ["/bin/sh", "-c"] + args: + - | + OWNER=$(echo ${KC_ASSETS_GIT_URL} | sed -E 's|https://github.com/([^/]+)/([^/]+)/tree/([^/]+)/.*|\1|') + REPO=$(echo ${KC_ASSETS_GIT_URL} | sed -E 's|https://github.com/([^/]+)/([^/]+)/tree/([^/]+)/.*|\2|') + BRANCH=$(echo ${KC_ASSETS_GIT_URL} | sed -E 's|https://github.com/[^/]+/[^/]+/tree/([^/]+)/.*|\1|') + KC_ASSETS_DIR=$(echo ${KC_ASSETS_GIT_URL} | sed -E 's|https://github.com/[^/]+/[^/]+/tree/[^/]+/(.*?)/?$|\1|') + if [[ "${KC_ASSETS_DIR: -1}" == "/" ]]; then KC_ASSETS_DIR="${KC_ASSETS_DIR%/}"; fi + DOWNLOAD_URL="https://codeload.github.com/${OWNER}/${REPO}/tar.gz/${BRANCH}" + curl "${DOWNLOAD_URL}" | tar -xz --strip-components=4 -C /opt/keycloak/themes "${REPO}-${BRANCH}/${KC_ASSETS_DIR}/themes" + curl "${DOWNLOAD_URL}" | tar -xz --strip-components=4 -C /opt/keycloak/data "${REPO}-${BRANCH}/${KC_ASSETS_DIR}/data" + envFrom: + - configMapRef: + name: studio-config + volumeMounts: + - name: keycloak-themes-volume + mountPath: /opt/keycloak/themes + - name: keycloak-dataimport-volume + mountPath: /opt/keycloak/data/import + securityContext: + runAsUser: 0 + runAsGroup: 0 + containers: + - name: keycloak + image: quay.io/keycloak/keycloak:latest + volumeMounts: + - name: tls + mountPath: /etc/ssl + readOnly: true + - name: keycloak-themes-volume + mountPath: /opt/keycloak/themes + - name: keycloak-dataimport-volume + mountPath: /opt/keycloak/data/import + args: + - start + - --import-realm + ports: + - containerPort: 8080 + - containerPort: 8443 + env: + - name: KC_BOOTSTRAP_ADMIN_USERNAME + value: "admin" + - name: KC_BOOTSTRAP_ADMIN_PASSWORD + value: "admin" + - name: KC_PROXY_HEADERS + value: "forwarded" + - name: KC_HTTP_RELATIVE_PATH + value: "/auth" + - name: KC_PROXY + value: edge + - name: KC_HTTPS_CERTIFICATE_FILE + value: /etc/ssl/tls.crt + - name: KC_HTTPS_CERTIFICATE_KEY_FILE + value: /etc/ssl/tls.key + - name: KC_HOSTNAME_STRICT + value: "false" + - name: KC_HOSTNAME_STRICT_HTTPS + value: "true" + readinessProbe: + failureThreshold: 3 + httpGet: + path: auth/realms/master + port: 8443 + scheme: HTTPS + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "1Gi" + cpu: "1" + volumes: + - name: tls + secret: + secretName: tls-secret + - name: keycloak-themes-volume + emptyDir: {} + - name: keycloak-dataimport-volume + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: keycloak + namespace: studio +spec: + type: ClusterIP + ports: + - name: https + protocol: TCP + port: 8443 + targetPort: 8443 + selector: + app: keycloak \ No newline at end of file diff --git a/setup-scripts/setup-genai-studio/manifests/studio-manifest.yaml b/setup-scripts/setup-genai-studio/manifests/studio-manifest.yaml index 5885f11..ea9945e 100644 --- a/setup-scripts/setup-genai-studio/manifests/studio-manifest.yaml +++ b/setup-scripts/setup-genai-studio/manifests/studio-manifest.yaml @@ -10,8 +10,13 @@ data: # SPDX-License-Identifier: Apache-2.0 server { - listen 80; - listen [::]:80; + # listen 80; + # listen [::]:80; + listen 443 ssl; + listen [::]:443 ssl; + + ssl_certificate /etc/ssl/app-tls.crt; + ssl_certificate_key /etc/ssl/app-tls.key; proxy_connect_timeout 600; proxy_send_timeout 600; @@ -24,7 +29,7 @@ data: resolver_timeout 5s; location /home { - root /usr/share/nginx/html; # Use root to serve files from a directory + root /usr/share/nginx/html; index index.html; } @@ -61,6 +66,15 @@ data: proxy_set_header X-Forwarded-Proto $scheme; } + # Location block for keycloak + location /auth { + proxy_pass https://${KEYCLOAK_DNS}/auth/; + proxy_set_header Host $host:30007; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + # Location block for app-backend location /v1/app-backend { # Initialize the variable for namespace @@ -160,10 +174,11 @@ spec: selector: app: studio-nginx ports: - - protocol: TCP - port: 80 - targetPort: 80 - nodePort: 30007 + - name: https + protocol: TCP + port: 443 + targetPort: 443 + nodePort: 30007 type: NodePort --- apiVersion: apps/v1 @@ -191,7 +206,7 @@ spec: envsubst "$(env | grep _DNS= | awk -F= '{print "${"$1"}"}' | tr '\n' ' ')" < /tmp/default.conf > /etc/nginx/conf.d/default.conf envFrom: - configMapRef: - name: internal-dns-config + name: studio-config volumeMounts: - name: tmp-volume mountPath: /tmp @@ -204,6 +219,8 @@ spec: volumeMounts: - name: nginx-conf-volume mountPath: /etc/nginx/conf.d + - name: app-tls + mountPath: /etc/ssl securityContext: {} volumes: - name: tmp-volume @@ -212,6 +229,9 @@ spec: name: studio-nginx-config - name: nginx-conf-volume emptyDir: {} + - name: app-tls + secret: + secretName: app-tls --- apiVersion: v1 kind: Service @@ -262,6 +282,9 @@ rules: - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "create", "list", "watch"] +- apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "create", "list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -312,7 +335,7 @@ spec: value: ${NO_PROXY} envFrom: - configMapRef: - name: internal-dns-config + name: studio-config ports: - containerPort: 5000 resources: @@ -370,6 +393,21 @@ spec: securityContext: {} image: ${REGISTRY}/studio-frontend:${TAG} imagePullPolicy: Always + env: + - name: DATABASE_TYPE + value: mysql + - name: DATABASE_HOST + value: ${MYSQL_HOST} + - name: DATABASE_PORT + value: "3306" + - name: DATABASE_USER + value: studio + - name: DATABASE_PASSWORD + value: studio + - name: DATABASE_NAME + value: studio + - name: DATABASE_SSL + value: "true" ports: - name: studio-frontend containerPort: 8080 @@ -378,6 +416,147 @@ spec: volumeMounts: - mountPath: /tmp name: tmp + - name: mysql-tls + mountPath: /etc/mysql/ssl + readOnly: true volumes: - name: tmp - emptyDir: {} \ No newline at end of file + emptyDir: {} + - name: mysql-tls + secret: + secretName: mysql-tls +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: keycloak + namespace: studio + labels: + app: keycloak +spec: + replicas: 1 + selector: + matchLabels: + app: keycloak + template: + metadata: + labels: + app: keycloak + spec: + initContainers: + - name: keycloak-assets + image: curlimages/curl:latest + command: ["/bin/sh", "-c"] + args: + - | + OWNER=$(echo ${KC_ASSETS_GIT_URL} | sed -E 's|https://github.com/([^/]+)/([^/]+)/tree/([^/]+)/.*|\1|') + REPO=$(echo ${KC_ASSETS_GIT_URL} | sed -E 's|https://github.com/([^/]+)/([^/]+)/tree/([^/]+)/.*|\2|') + BRANCH=$(echo ${KC_ASSETS_GIT_URL} | sed -E 's|https://github.com/[^/]+/[^/]+/tree/([^/]+)/.*|\1|') + KC_ASSETS_DIR=$(echo ${KC_ASSETS_GIT_URL} | sed -E 's|https://github.com/[^/]+/[^/]+/tree/[^/]+/(.*?)/?$|\1|') + if [[ "${KC_ASSETS_DIR: -1}" == "/" ]]; then KC_ASSETS_DIR="${KC_ASSETS_DIR%/}"; fi + DOWNLOAD_URL="https://codeload.github.com/${OWNER}/${REPO}/tar.gz/${BRANCH}" + curl "${DOWNLOAD_URL}" | tar -xz --strip-components=4 -C /opt/keycloak/themes "${REPO}-${BRANCH}/${KC_ASSETS_DIR}/themes" + curl "${DOWNLOAD_URL}" | tar -xz --strip-components=4 -C /opt/keycloak/data "${REPO}-${BRANCH}/${KC_ASSETS_DIR}/data" + envFrom: + - configMapRef: + name: studio-config + volumeMounts: + - name: keycloak-themes-volume + mountPath: /opt/keycloak/themes + - name: keycloak-dataimport-volume + mountPath: /opt/keycloak/data/import + securityContext: + runAsUser: 0 + runAsGroup: 0 + containers: + - name: keycloak + image: quay.io/keycloak/keycloak:latest + volumeMounts: + - name: mysql-tls + mountPath: /etc/mysql/ssl + readOnly: true + - name: app-tls + mountPath: /etc/ssl + readOnly: true + - name: keycloak-themes-volume + mountPath: /opt/keycloak/themes + - name: keycloak-dataimport-volume + mountPath: /opt/keycloak/data/import + args: + - start + - --import-realm + ports: + - containerPort: 8080 + - containerPort: 8443 + env: + - name: KC_BOOTSTRAP_ADMIN_USERNAME + value: "admin" + - name: KC_BOOTSTRAP_ADMIN_PASSWORD + value: "admin" + - name: KC_PROXY_HEADERS + value: "forwarded" + - name: KC_HTTP_RELATIVE_PATH + value: "/auth" + - name: KC_PROXY + value: edge + - name: KC_HTTPS_CERTIFICATE_FILE + value: /etc/ssl/app-tls.crt + - name: KC_HTTPS_CERTIFICATE_KEY_FILE + value: /etc/ssl/app-tls.key + - name: KC_HOSTNAME_STRICT + value: "false" + - name: KC_HOSTNAME_STRICT_HTTPS + value: "true" + # Database Configuration for MySQL + - name: KC_DB + value: "mysql" + - name: KC_DB_URL + value: "jdbc:mysql://${MYSQL_HOST}:3306/keycloak?useSSL=true&requireSSL=true&clientCertificateKeyStoreUrl=file:/etc/mysql/ssl/client-keystore.p12&trustCertificateKeyStoreUrl=file:/etc/mysql/ssl/ca.pem" + - name: KC_DB_USERNAME + value: "studio" + - name: KC_DB_PASSWORD + value: "studio" + - name: KC_DB_DATABASE + value: "keycloak" + readinessProbe: + failureThreshold: 3 + httpGet: + path: auth/realms/master + port: 8443 + scheme: HTTPS + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "1Gi" + cpu: "1" + volumes: + - name: app-tls + secret: + secretName: app-tls + - name: mysql-tls + secret: + secretName: mysql-tls + - name: keycloak-themes-volume + emptyDir: {} + - name: keycloak-dataimport-volume + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: keycloak + namespace: studio +spec: + type: ClusterIP + ports: + - name: https + protocol: TCP + port: 8443 + targetPort: 8443 + selector: + app: keycloak \ No newline at end of file diff --git a/setup-scripts/setup-genai-studio/playbooks/deploy-studio.yml b/setup-scripts/setup-genai-studio/playbooks/deploy-studio.yml index 70d2f3a..e1f422a 100644 --- a/setup-scripts/setup-genai-studio/playbooks/deploy-studio.yml +++ b/setup-scripts/setup-genai-studio/playbooks/deploy-studio.yml @@ -4,9 +4,14 @@ - ../vars.yml tasks: + - name: Check if studio namespace exists + command: kubectl get namespace studio + register: studio_namespace + ignore_errors: yes + - name: Create studio namespace command: kubectl create namespace studio - ignore_errors: yes + when: studio_namespace.rc != 0 - name: Check for coredns service shell: kubectl get svc coredns -n kube-system --ignore-not-found @@ -18,13 +23,61 @@ shell: sed -i 's/kube-dns/coredns/g' ../manifests/studio-manifest.yaml when: coredns_check.stdout != '' - - name: Apply internal DNS configuration - command: kubectl apply -f ../internal-dns-config.yaml + - name: Check if app-tls exists in studio namespace + command: kubectl get secret app-tls -n studio + register: app_tls_secret_check + ignore_errors: yes + + - name: Generate TLS certificate and create app-tls + shell: | + openssl req -x509 -nodes -days 365 -newkey rsa:4096 -keyout app-tls.key -out app-tls.crt -subj "/CN=studio/O=studio" + kubectl create secret generic app-tls --from-file=app-tls.crt --from-file=app-tls.key -n studio + rm app-tls.key app-tls.crt + when: app_tls_secret_check.rc != 0 + + - name: Check if mysql-tls exists in studio namespace + command: kubectl get secret mysql-tls -n studio + register: mysql_tls_secret_check + ignore_errors: yes + + - name: Copy mysql ssl to current user + become: yes + become_user: root + shell: | + cp /var/lib/mysql/ca.pem . + cp /var/lib/mysql/client-cert.pem . + cp /var/lib/mysql/client-key.pem . + chown -R {{ ansible_env.USER }}:{{ ansible_env.USER }} ca.pem + chown -R {{ ansible_env.USER }}:{{ ansible_env.USER }} client-key.pem + chown -R {{ ansible_env.USER }}:{{ ansible_env.USER }} client-cert.pem + when: mysql_tls_secret_check.rc != 0 + + - name: Create mysql-tls from mysql ssl + shell: | + openssl pkcs12 -export -in client-cert.pem -inkey client-key.pem -out client-keystore.p12 -name keycloak -CAfile ca.pem -caname root -password pass: + kubectl create secret generic mysql-tls \ + --from-file=ca.pem \ + --from-file=client-cert.pem \ + --from-file=client-key.pem \ + --from-file=client-keystore.p12 \ + -n studio + rm ca.pem client-key.pem client-cert.pem client-keystore.p12 + when: mysql_tls_secret_check.rc != 0 + + - name: Apply studio configuration + command: kubectl apply -f ../studio-config.yaml - name: Apply customized studio manifest - shell: "envsubst '${REGISTRY} ${TAG} ${HTTP_PROXY} ${NO_PROXY}' < ../manifests/studio-manifest.yaml | kubectl apply -f -" + shell: "envsubst '${REGISTRY} ${TAG} ${HTTP_PROXY} ${NO_PROXY} ${MYSQL_HOST}' < ../manifests/studio-manifest.yaml | kubectl apply -f -" environment: REGISTRY: "{{ container_registry }}" TAG: "{{ container_tag }}" HTTP_PROXY: "{{ http_proxy }}" - NO_PROXY: "{{ no_proxy }}" \ No newline at end of file + NO_PROXY: "{{ no_proxy }}" + MYSQL_HOST: "{{ mysql_host }}" + + - name: Wait for all pods to be ready in studio namespace + shell: kubectl wait --for=condition=ready pod --all --namespace=studio --timeout=180s + register: pod_ready_check + failed_when: pod_ready_check.rc != 0 + changed_when: false \ No newline at end of file diff --git a/setup-scripts/setup-genai-studio/playbooks/setup-mysqldb.yml b/setup-scripts/setup-genai-studio/playbooks/setup-mysqldb.yml new file mode 100644 index 0000000..8b5261a --- /dev/null +++ b/setup-scripts/setup-genai-studio/playbooks/setup-mysqldb.yml @@ -0,0 +1,92 @@ +- name: Setup MySQL Server + hosts: localhost + become: yes + tasks: + + - name: Install PyMySQL module + pip: + name: PyMySQL + state: present + + - name: Check if MySQL user 'studio' exists + mysql_user: + login_user: root + login_password: root + name: studio + check_implicit_admin: yes + state: present + register: studio_user_exists + ignore_errors: yes + + - name: End playbook if MySQL user 'studio' exists + meta: end_play + when: studio_user_exists is succeeded + + - name: Install MySQL server + apt: + name: mysql-server + state: present + update_cache: yes + + - name: Configure MySQL to listen on all interfaces + lineinfile: + path: /etc/mysql/mysql.conf.d/mysqld.cnf + regexp: '^bind-address' + line: 'bind-address = 0.0.0.0' + state: present + + - name: Restart MySQL service + service: + name: mysql + state: restarted + + - name: Secure MySQL installation updating root password + mysql_user: + name: root + host: localhost + password: root + login_unix_socket: /var/run/mysqld/mysqld.sock + priv: '*.*:ALL,GRANT' + state: present + plugin: mysql_native_password + ignore_errors: yes + + - name: Create MySQL user 'studio' for all hosts + mysql_user: + login_user: root + login_password: root + name: studio + host: '%' + password: studio + priv: '*.*:ALL,GRANT' + state: present + + - name: Enforce SSL for MySQL user 'studio' for all hosts + mysql_query: + login_user: root + login_password: root + query: "ALTER USER 'studio'@'%' REQUIRE X509;" + + - name: Create MySQL user 'studio' for localhost without X509 + mysql_user: + login_user: root + login_password: root + name: studio + host: localhost + password: studio + priv: '*.*:ALL,GRANT' + state: present + + - name: Create database 'keycloak' + mysql_db: + login_user: studio + login_password: studio + name: keycloak + state: present + + - name: Create database 'studio' + mysql_db: + login_user: studio + login_password: studio + name: studio + state: present \ No newline at end of file diff --git a/setup-scripts/setup-genai-studio/studio-config.yaml b/setup-scripts/setup-genai-studio/studio-config.yaml new file mode 100644 index 0000000..002efdf --- /dev/null +++ b/setup-scripts/setup-genai-studio/studio-config.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: studio-config + namespace: studio +data: + KC_ASSETS_GIT_URL: "https://github.com/opea-project/GenAIStudio/tree/main/assets/keycloak" + KEYCLOAK_DNS: "keycloak.studio.svc.cluster.local:8443" + GRAFANA_DNS: "kube-prometheus-stack-grafana.monitoring.svc.cluster.local" + STUDIO_FRONTEND_DNS: "studio-frontend.studio.svc.cluster.local:3000" + APP_FRONTEND_DNS: "app-frontend.$namespace.svc.cluster.local:5175" + APP_BACKEND_DNS: "app-backend.$namespace.svc.cluster.local:8888" + PREPARE_DOC_REDIS_PREP_DNS: "prepare-doc-redis-prep-0.$namespace.svc.cluster.local:6007" + STUDIO_BACKEND_DNS: "studio-backend.studio.svc.cluster.local:5000" \ No newline at end of file diff --git a/studio-frontend/packages/server/.gitignore b/studio-frontend/packages/server/.gitignore new file mode 100644 index 0000000..b3ab1ae --- /dev/null +++ b/studio-frontend/packages/server/.gitignore @@ -0,0 +1,6 @@ +.idea/ +.vscode/ +node_modules/ +build/ +tmp/ +temp/ \ No newline at end of file diff --git a/studio-frontend/packages/server/bin/run b/studio-frontend/packages/server/bin/run old mode 100644 new mode 100755 diff --git a/studio-frontend/packages/server/src/DataSource.ts b/studio-frontend/packages/server/src/DataSource.ts index 811f62b..5ac39bc 100644 --- a/studio-frontend/packages/server/src/DataSource.ts +++ b/studio-frontend/packages/server/src/DataSource.ts @@ -78,6 +78,7 @@ export const init = async (): Promise => { break default: homePath = process.env.DATABASE_PATH ?? flowisePath + console.log ("***", path.resolve(homePath, 'database.sqlite')) appDataSource = new DataSource({ type: 'sqlite', database: path.resolve(homePath, 'database.sqlite'), @@ -104,7 +105,18 @@ const getDatabaseSSLFromEnv = () => { ca: Buffer.from(process.env.DATABASE_SSL_KEY_BASE64, 'base64') } } else if (process.env.DATABASE_SSL === 'true') { - return true + // return true + try { + return { + rejectUnauthorized: false, + ca: fs.readFileSync('/etc/mysql/ssl/ca.pem'), + cert: fs.readFileSync('/etc/mysql/ssl/client-cert.pem'), + key: fs.readFileSync('/etc/mysql/ssl/client-key.pem') + }; + } catch (error) { + console.error('Error reading certificates from mounted path:', error); + return undefined; + } } return undefined } diff --git a/studio-frontend/packages/server/src/Interface.ts b/studio-frontend/packages/server/src/Interface.ts index 2da6787..228ddc3 100644 --- a/studio-frontend/packages/server/src/Interface.ts +++ b/studio-frontend/packages/server/src/Interface.ts @@ -21,6 +21,7 @@ export enum ChatMessageRatingType { export interface IChatFlow { id: string name: string + userid: string flowData: string updatedDate: Date createdDate: Date diff --git a/studio-frontend/packages/server/src/controllers/chatflows/index.ts b/studio-frontend/packages/server/src/controllers/chatflows/index.ts index 3c91449..c09cfd3 100644 --- a/studio-frontend/packages/server/src/controllers/chatflows/index.ts +++ b/studio-frontend/packages/server/src/controllers/chatflows/index.ts @@ -58,6 +58,15 @@ const getAllChatflows = async (req: Request, res: Response, next: NextFunction) } } +const getAllChatflowsbyUserId = async (req: Request, res: Response, next: NextFunction) => { + try { + const apiResponse = await chatflowsService.getAllChatflowsbyUserId(req.query.userid as string, req.query.type as ChatflowType) + return res.json(apiResponse) + } catch (error) { + next(error) + } +} + // Get specific chatflow via api key const getChatflowByApiKey = async (req: Request, res: Response, next: NextFunction) => { try { @@ -98,6 +107,7 @@ const saveChatflow = async (req: Request, res: Response, next: NextFunction) => const body = req.body const newChatFlow = new ChatFlow() Object.assign(newChatFlow, body) + console.log ('newChatFlow', newChatFlow) const apiResponse = await chatflowsService.saveChatflow(newChatFlow) return res.json(apiResponse) } catch (error) { @@ -257,6 +267,7 @@ export default { checkIfChatflowIsValidForUploads, deleteChatflow, getAllChatflows, + getAllChatflowsbyUserId, getChatflowByApiKey, getChatflowById, saveChatflow, diff --git a/studio-frontend/packages/server/src/database/entities/ChatFlow.ts b/studio-frontend/packages/server/src/database/entities/ChatFlow.ts index f86caf2..122d097 100644 --- a/studio-frontend/packages/server/src/database/entities/ChatFlow.ts +++ b/studio-frontend/packages/server/src/database/entities/ChatFlow.ts @@ -10,6 +10,9 @@ export class ChatFlow implements IChatFlow { @Column() name: string + @Column({ nullable: true, type: 'text' }) + userid: string + @Column({ type: 'text' }) flowData: string diff --git a/studio-frontend/packages/server/src/database/migrations/mysql/1693840429259-Init.ts b/studio-frontend/packages/server/src/database/migrations/mysql/1693840429259-Init.ts index 9d07206..4e7e8f8 100644 --- a/studio-frontend/packages/server/src/database/migrations/mysql/1693840429259-Init.ts +++ b/studio-frontend/packages/server/src/database/migrations/mysql/1693840429259-Init.ts @@ -6,11 +6,15 @@ export class Init1693840429259 implements MigrationInterface { `CREATE TABLE IF NOT EXISTS \`chat_flow\` ( \`id\` varchar(36) NOT NULL, \`name\` varchar(255) NOT NULL, + \`userid\` varchar(255) DEFAULT NULL, \`flowData\` text NOT NULL, \`deployed\` tinyint DEFAULT NULL, \`isPublic\` tinyint DEFAULT NULL, \`apikeyid\` varchar(255) DEFAULT NULL, \`chatbotConfig\` varchar(255) DEFAULT NULL, + \`sandboxStatus\` varchar(255) DEFAULT NULL, + \`sandboxAppUrl\` varchar(255) DEFAULT NULL, + \`sandboxGrafanaUrl\` varchar(255) DEFAULT NULL, \`createdDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), \`updatedDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), PRIMARY KEY (\`id\`) diff --git a/studio-frontend/packages/server/src/database/migrations/sqlite/1732778337650-OPEAAddUserIdtoChatFlow.ts b/studio-frontend/packages/server/src/database/migrations/sqlite/1732778337650-OPEAAddUserIdtoChatFlow.ts new file mode 100644 index 0000000..b802061 --- /dev/null +++ b/studio-frontend/packages/server/src/database/migrations/sqlite/1732778337650-OPEAAddUserIdtoChatFlow.ts @@ -0,0 +1,20 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class OPEAAddUserIdtoChatFlow1732778337650 implements MigrationInterface { + name = 'OPEAAddUserIdtoChatFlow1732778337650' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE TABLE "temporary_chat_flow" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "flowData" text NOT NULL, "deployed" boolean, "isPublic" boolean, "apikeyid" varchar, "chatbotConfig" text, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')), "sandboxStatus" text, "sandboxAppUrl" text, "sandboxGrafanaUrl" text, "apiConfig" text, "analytic" text, "category" text, "speechToText" text, "type" text, "userid" text)`); + await queryRunner.query(`INSERT INTO "temporary_chat_flow"("id", "name", "flowData", "deployed", "isPublic", "apikeyid", "chatbotConfig", "createdDate", "updatedDate", "sandboxStatus", "sandboxAppUrl", "sandboxGrafanaUrl", "apiConfig", "analytic", "category", "speechToText", "type", "userid") SELECT "id", "name", "flowData", "deployed", "isPublic", "apikeyid", "chatbotConfig", "createdDate", "updatedDate", "sandboxStatus", "sandboxAppUrl", "sandboxGrafanaUrl", "apiConfig", "analytic", "category", "speechToText", "type", "userid" FROM "chat_flow"`); + await queryRunner.query(`DROP TABLE "chat_flow"`); + await queryRunner.query(`ALTER TABLE "temporary_chat_flow" RENAME TO "chat_flow"`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "chat_flow" RENAME TO "temporary_chat_flow"`); + await queryRunner.query(`CREATE TABLE "chat_flow" ("id" varchar PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "flowData" text NOT NULL, "deployed" text, "isPublic" boolean, "apikeyid" varchar, "chatbotConfig" text, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "updatedDate" datetime NOT NULL DEFAULT (datetime('now')), "sandboxStatus" text, "sandboxAppUrl" text, "sandboxGrafanaUrl" text, "apiConfig" text, "analytic" text, "category" text, "speechToText" text, "type" text, "userid" varchar NOT NULL)`); + await queryRunner.query(`INSERT INTO "chat_flow"("id", "name", "flowData", "deployed", "isPublic", "apikeyid", "chatbotConfig", "createdDate", "updatedDate", "sandboxStatus", "sandboxAppUrl", "sandboxGrafanaUrl", "apiConfig", "analytic", "category", "speechToText", "type", "userid") SELECT "id", "name", "flowData", "deployed", "isPublic", "apikeyid", "chatbotConfig", "createdDate", "updatedDate", "sandboxStatus", "sandboxAppUrl", "sandboxGrafanaUrl", "apiConfig", "analytic", "category", "speechToText", "type", "userid" FROM "temporary_chat_flow"`); + await queryRunner.query(`DROP TABLE "temporary_chat_flow"`); + } + +} diff --git a/studio-frontend/packages/server/src/database/migrations/sqlite/index.ts b/studio-frontend/packages/server/src/database/migrations/sqlite/index.ts index 207dc0c..67401e6 100644 --- a/studio-frontend/packages/server/src/database/migrations/sqlite/index.ts +++ b/studio-frontend/packages/server/src/database/migrations/sqlite/index.ts @@ -26,6 +26,8 @@ import { AddActionToChatMessage1721078251523 } from './1721078251523-AddActionTo import { AddArtifactsToChatMessage1726156258465 } from './1726156258465-AddArtifactsToChatMessage' import { AddCustomTemplate1725629836652 } from './1725629836652-AddCustomTemplate' import { OPEAAddSandboxStatustoChatFlow1727419719000 } from './1727419719000-OPEAAddSandboxStatustoChatFlow' +import { OPEAAddUserIdtoChatFlow1732778337650 } from './1732778337650-OPEAAddUserIdtoChatFlow' + export const sqliteMigrations = [ Init1693835579790, ModifyChatFlow1693920824108, @@ -54,5 +56,6 @@ export const sqliteMigrations = [ AddActionToChatMessage1721078251523, AddArtifactsToChatMessage1726156258465, AddCustomTemplate1725629836652, - OPEAAddSandboxStatustoChatFlow1727419719000 + OPEAAddSandboxStatustoChatFlow1727419719000, + OPEAAddUserIdtoChatFlow1732778337650, ] diff --git a/studio-frontend/packages/server/src/routes/chatflows/index.ts b/studio-frontend/packages/server/src/routes/chatflows/index.ts index b0c5350..ab79258 100644 --- a/studio-frontend/packages/server/src/routes/chatflows/index.ts +++ b/studio-frontend/packages/server/src/routes/chatflows/index.ts @@ -7,7 +7,7 @@ router.post('/', chatflowsController.saveChatflow) router.post('/importchatflows', chatflowsController.importChatflows) // READ -router.get('/', chatflowsController.getAllChatflows) +router.get('/', chatflowsController.getAllChatflowsbyUserId) router.get(['/', '/:id'], chatflowsController.getChatflowById) router.get(['/apikey/', '/apikey/:apikey'], chatflowsController.getChatflowByApiKey) diff --git a/studio-frontend/packages/server/src/services/chatflows/index.ts b/studio-frontend/packages/server/src/services/chatflows/index.ts index 00b674c..d6b9337 100644 --- a/studio-frontend/packages/server/src/services/chatflows/index.ts +++ b/studio-frontend/packages/server/src/services/chatflows/index.ts @@ -134,6 +134,66 @@ const getAllChatflows = async (type?: ChatflowType): Promise => { } } +const getAllChatflowsbyUserId = async (userid: string, type?: ChatflowType): Promise => { + try { + const appServer = getRunningExpressApp() + + // Use find with a where condition to filter by userid + let dbResponse = await appServer.AppDataSource.getRepository(ChatFlow).find({ + where: { + userid: userid, // Filter by the specific userid + }, + }) + + // If no chatflows are found for the user, create a new one based from sample workflow + if (dbResponse.length === 0) { + // URL to fetch the sample workflow + const url = 'https://raw.githubusercontent.com/opea-project/GenAIStudio/refs/heads/main/sample-workflows/sample_workflow_chatqna.json'; + + try { + // Fetch and parse the JSON data from the URL + const response = await axios.get(url); + const parsedFlowData = response.data; + + // Create a new chatflow with the flowData from the URL + const newChatflow: Partial = { + userid: userid, + name: 'sample-chatqna', + flowData: JSON.stringify(parsedFlowData), + type: 'OPEA', + deployed: false, + isPublic: false + }; + + // Call the importChatflows function to insert the new chatflow + await importChatflows([newChatflow]); + } catch (error) { + throw new Error('Failed to import sample chatflow'); + } + + // Rerun the find query to fetch the latest state of chatflows after insertion + dbResponse = await appServer.AppDataSource.getRepository(ChatFlow).find({ + where: { + userid: userid, + }, + }); + } + + // Filter further by type if needed + if (type) { + return dbResponse.filter((chatflow) => chatflow.type === type) + } + + return dbResponse + } catch (error) { + throw new InternalFlowiseError( + StatusCodes.INTERNAL_SERVER_ERROR, + `Error: chatflowsService.getAllChatflows - ${getErrorMessage(error)}` + ) + } +} + + const getChatflowByApiKey = async (apiKeyId: string, keyonly?: unknown): Promise => { try { // Here we only get chatflows that are bounded by the apikeyid and chatflows that are not bounded by any apikey @@ -455,6 +515,7 @@ export default { checkIfChatflowIsValidForUploads, deleteChatflow, getAllChatflows, + getAllChatflowsbyUserId, getChatflowByApiKey, getChatflowById, saveChatflow, diff --git a/studio-frontend/packages/ui/package.json b/studio-frontend/packages/ui/package.json index d8ed6fc..55a60ef 100644 --- a/studio-frontend/packages/ui/package.json +++ b/studio-frontend/packages/ui/package.json @@ -20,6 +20,7 @@ "@mui/lab": "5.0.0-alpha.156", "@mui/material": "5.15.0", "@mui/x-data-grid": "6.8.0", + "@react-keycloak/web": "^3.4.0", "@tabler/icons-react": "^3.3.0", "@uiw/codemirror-theme-sublime": "^4.21.21", "@uiw/codemirror-theme-vscode": "^4.21.21", @@ -34,6 +35,7 @@ "framer-motion": "^4.1.13", "history": "^5.0.0", "html-react-parser": "^3.0.4", + "keycloak-js": "^26.0.5", "lodash": "^4.17.21", "moment": "^2.29.3", "notistack": "^2.0.4", diff --git a/studio-frontend/packages/ui/src/App.jsx b/studio-frontend/packages/ui/src/App.jsx index 857d4ea..f547d3b 100644 --- a/studio-frontend/packages/ui/src/App.jsx +++ b/studio-frontend/packages/ui/src/App.jsx @@ -1,32 +1,26 @@ -import { useSelector } from 'react-redux' - -import { ThemeProvider } from '@mui/material/styles' -import { CssBaseline, StyledEngineProvider } from '@mui/material' - -// routing -import Routes from '@/routes' - -// defaultTheme -import themes from '@/themes' - -// project imports -import NavigationScroll from '@/layout/NavigationScroll' - -// ==============================|| APP ||============================== // +import { useSelector } from 'react-redux'; +import { ThemeProvider } from '@mui/material/styles'; +import { CssBaseline, StyledEngineProvider } from '@mui/material'; +import Routes from '@/routes'; +import themes from '@/themes'; +import NavigationScroll from '@/layout/NavigationScroll'; +import KeycloakProvider from './KeycloakContext'; // Import the updated KeycloakProvider const App = () => { - const customization = useSelector((state) => state.customization) + const customization = useSelector((state) => state.customization); return ( - - - - - - + + + + + + + + - ) -} + ); +}; -export default App +export default App; diff --git a/studio-frontend/packages/ui/src/KeycloakContext.jsx b/studio-frontend/packages/ui/src/KeycloakContext.jsx new file mode 100644 index 0000000..9753ee6 --- /dev/null +++ b/studio-frontend/packages/ui/src/KeycloakContext.jsx @@ -0,0 +1,72 @@ +import React, { createContext, useContext, useEffect, useState } from 'react'; +import Keycloak from 'keycloak-js'; + +// Create the Keycloak context +const KeycloakContext = createContext(null); + +// Provide the Keycloak context to the application +export const KeycloakProvider = ({ children }) => { + const [keycloak, setKeycloak] = useState(null); + const [isInitialized, setIsInitialized] = useState(false); + + useEffect(() => { + if (!window.crypto || !window.crypto.subtle) { + console.error("Web Crypto API is not available. This may cause security issues."); + } + + const initOptions = { + url: '/auth/', + realm: 'genaistudio', + clientId: 'genaistudio', + onLoad: 'login-required', // check-sso | login-required + responseType: 'code', // Corrected from KeycloakResponseType to responseType + silentCheckSsoRedirectUri: window.location.origin + "/silent-check-sso.html", + checkLoginIframe: false, + }; + + const kc = new Keycloak(initOptions); + + kc.init({ + onLoad: initOptions.onLoad, + responseType: 'code', // Corrected from KeycloakResponseType to responseType + }).then((auth) => { + if (!auth) { + window.location.reload(); + } else { + console.info("Authenticated"); + console.log('auth', auth); + console.log('Keycloak', kc); + + kc.onTokenExpired = () => { + console.log('token expired'); + }; + + setKeycloak(kc); // Set the Keycloak instance in state + setIsInitialized(true); // Mark initialization as complete + } + }).catch((error) => { + console.error("Authentication Failed", error); + }); + }, []); + + if (!isInitialized) { + return
Loading...
; // Show a loading state until Keycloak is initialized + } + + return ( + + {children} + + ); +}; + +// Custom hook to use Keycloak context +export const useKeycloak = () => { + const context = useContext(KeycloakContext); + if (!context) { + throw new Error('useKeycloak must be used within a KeycloakProvider'); + } + return context; +}; + +export default KeycloakProvider; \ No newline at end of file diff --git a/studio-frontend/packages/ui/src/api/chatflows.js b/studio-frontend/packages/ui/src/api/chatflows.js index 25ab861..a51798c 100644 --- a/studio-frontend/packages/ui/src/api/chatflows.js +++ b/studio-frontend/packages/ui/src/api/chatflows.js @@ -7,6 +7,8 @@ const getAllAgentflows = () => client.get('/chatflows?type=MULTIAGENT') const getAllOpeaflows = () => client.get('/chatflows?type=OPEA') +const getUserOpeaflows = (userid) => client.get(`/chatflows?userid=${userid}&type=OPEA`) + const getSpecificChatflow = (id) => client.get(`/chatflows/${id}`) const getSpecificChatflowFromPublicEndpoint = (id) => client.get(`/public-chatflows/${id}`) @@ -33,6 +35,7 @@ export default { getAllChatflows, getAllAgentflows, getAllOpeaflows, + getUserOpeaflows, getSpecificChatflow, getSpecificChatflowFromPublicEndpoint, createNewChatflow, diff --git a/studio-frontend/packages/ui/src/layout/MainLayout/Header/index.jsx b/studio-frontend/packages/ui/src/layout/MainLayout/Header/index.jsx index 5d4fba2..8a28478 100644 --- a/studio-frontend/packages/ui/src/layout/MainLayout/Header/index.jsx +++ b/studio-frontend/packages/ui/src/layout/MainLayout/Header/index.jsx @@ -18,6 +18,26 @@ import { IconMenu2 } from '@tabler/icons-react' // store import { SET_DARKMODE } from '@/store/actions' +// keycloak context +import LogoutIcon from '@mui/icons-material/Logout'; +import { useKeycloak } from '../../../KeycloakContext' + +const LogoutButton = () => { + const keycloak = useKeycloak(); // Access the Keycloak instance + + const handleLogout = () => { + keycloak.logout({ + redirectUri: window.location.origin, // Redirect to the home page or desired URL after logout + }); + }; + + return ( + + + + ); +}; + // ==============================|| MAIN NAVBAR / HEADER ||============================== // const MaterialUISwitch = styled(Switch)(({ theme }) => ({ @@ -92,45 +112,43 @@ const Header = ({ handleLeftDrawerToggle }) => { return ( <> - {/* logo & toggler button */} + {/* Container for logo and logout button */} - - + {/* Logo Section */} + + + + + + + {/* Logout Button */} + + - {/* - - - - */} - - {/* */} - - {/* */} + ) } diff --git a/studio-frontend/packages/ui/src/layout/MainLayout/index.jsx b/studio-frontend/packages/ui/src/layout/MainLayout/index.jsx index 5f9acea..83ae6f2 100644 --- a/studio-frontend/packages/ui/src/layout/MainLayout/index.jsx +++ b/studio-frontend/packages/ui/src/layout/MainLayout/index.jsx @@ -12,6 +12,8 @@ import Sidebar from './Sidebar' import { drawerWidth, headerHeight } from '@/store/constant' import { SET_MENU } from '@/store/actions' +import {useKeycloak } from '../../KeycloakContext.jsx' + // styles const Main = styled('main', { shouldForwardProp: (prop) => prop !== 'open' })(({ theme, open }) => ({ ...theme.typography.mainContent, @@ -57,6 +59,16 @@ const Main = styled('main', { shouldForwardProp: (prop) => prop !== 'open' })(({ // ==============================|| MAIN LAYOUT ||============================== // const MainLayout = () => { + const keycloak = useKeycloak() + console.log ("login roles", keycloak?.tokenParsed?.resource_access?.genaistudio?.roles[0]) + let userRole = keycloak?.tokenParsed?.resource_access?.genaistudio?.roles[0] + + const handleLogout = () => { + keycloak.logout({ + redirectUri: window.location.origin, // Redirect to the home page or desired URL after logout + }); + }; + const theme = useTheme() const matchDownMd = useMediaQuery(theme.breakpoints.down('lg')) @@ -73,45 +85,51 @@ const MainLayout = () => { }, [matchDownMd]) return ( - - - {/* header */} - - -
- - - - {/* drawer */} - {/* */} - - {/* main content */} -
- - + + {/* header */} + - - FlowiseAI - - -
+ +
+ + - + {/* drawer */} + {/* */} + + {/* main content */} + (
+ + + + FlowiseAI + + +
) + ): + ( + +

You are unauthorised. Please contact your admin for approval.

+ + + ) ) } diff --git a/studio-frontend/packages/ui/src/ui-component/table/FlowListTable.jsx b/studio-frontend/packages/ui/src/ui-component/table/FlowListTable.jsx index 8ba6d73..107ef78 100644 --- a/studio-frontend/packages/ui/src/ui-component/table/FlowListTable.jsx +++ b/studio-frontend/packages/ui/src/ui-component/table/FlowListTable.jsx @@ -61,11 +61,12 @@ const getLocalStorageKeyName = (name, isAgentCanvas) => { return (isAgentCanvas ? 'agentcanvas' : 'chatflowcanvas') + '_' + name } -export const FlowListTable = ({ data, images, isLoading, filterFunction, updateFlowsApi, setError, isAgentCanvas, isOpeaCanvas, stopSandboxApi, updateFlowToServerApi }) => { +export const FlowListTable = ({ data, images, isLoading, filterFunction, updateFlowsApi, setError, isAgentCanvas, isOpeaCanvas, stopSandboxApi, updateFlowToServerApi, userRole }) => { // overwrite setError setError = (error) => { console.error(error) } + // console.log ("table user", userRole) const theme = useTheme() const customization = useSelector((state) => state.customization) @@ -89,7 +90,7 @@ export const FlowListTable = ({ data, images, isLoading, filterFunction, updateF const handleSortData = () => { if (!data) return []; - console.log('handleSortData', data); + // console.log('handleSortData', data); const sorted = [...data].map((row) => ({ ...row, sandboxStatus: row.sandboxStatus || 'Not Running' // Ensure initial status @@ -288,6 +289,16 @@ export const FlowListTable = ({ data, images, isLoading, filterFunction, updateF Last Modified Date + {userRole === 'admin' && + + + User + + } @@ -309,6 +320,9 @@ export const FlowListTable = ({ data, images, isLoading, filterFunction, updateF + + + @@ -326,6 +340,9 @@ export const FlowListTable = ({ data, images, isLoading, filterFunction, updateF + + + ) : ( @@ -465,6 +482,8 @@ export const FlowListTable = ({ data, images, isLoading, filterFunction, updateF {moment(row.updatedDate).format('MMMM Do, YYYY')} + {userRole=='admin' && {row.userid}} + ))} diff --git a/studio-frontend/packages/ui/src/views/canvas/index.jsx b/studio-frontend/packages/ui/src/views/canvas/index.jsx index cc29da6..6b05709 100644 --- a/studio-frontend/packages/ui/src/views/canvas/index.jsx +++ b/studio-frontend/packages/ui/src/views/canvas/index.jsx @@ -55,12 +55,16 @@ import { usePrompt } from '@/utils/usePrompt' // const import { FLOWISE_CREDENTIAL_ID } from '@/store/constant' +// keycloak context +import { useKeycloak } from '../../KeycloakContext' + const nodeTypes = { customNode: CanvasNode, stickyNote: StickyNote } const edgeTypes = { buttonedge: ButtonEdge } // ==============================|| CANVAS ||============================== // const Canvas = () => { + const keycloak = useKeycloak() const theme = useTheme() const navigate = useNavigate() @@ -223,12 +227,16 @@ const Canvas = () => { rfInstanceObject.nodes = nodes const flowData = JSON.stringify(rfInstanceObject) + console.log (chatflowName) + console.log (keycloak?.tokenParsed) + if (!chatflow.id) { const newChatflowBody = { name: chatflowName, deployed: false, isPublic: false, flowData, + userid: keycloak?.tokenParsed?.email ? keycloak.tokenParsed.email : '', type: isAgentCanvas ? 'MULTIAGENT' : isOpeaCanvas ? 'OPEA' : 'CHATFLOW' } createNewChatflowApi.request(newChatflowBody) diff --git a/studio-frontend/packages/ui/src/views/opeaflows/index.jsx b/studio-frontend/packages/ui/src/views/opeaflows/index.jsx index d2b6423..8286db8 100644 --- a/studio-frontend/packages/ui/src/views/opeaflows/index.jsx +++ b/studio-frontend/packages/ui/src/views/opeaflows/index.jsx @@ -29,9 +29,13 @@ import { baseURL } from '@/store/constant' // icons import { IconPlus, IconLayoutGrid, IconList } from '@tabler/icons-react' +//keycloak +import { useKeycloak } from '../../KeycloakContext' + // ==============================|| OPEAFlows ||============================== // const Opeaflows = () => { + const keycloak = useKeycloak() const navigate = useNavigate() const theme = useTheme() @@ -42,7 +46,22 @@ const Opeaflows = () => { const [loginDialogOpen, setLoginDialogOpen] = useState(false) const [loginDialogProps, setLoginDialogProps] = useState({}) - const getAllOpeaflowsApi = useApi(chatflowsApi.getAllOpeaflows) + console.log ("roles", keycloak?.tokenParsed?.resource_access?.genaistudio?.roles[0]) + let userRole = keycloak?.tokenParsed?.resource_access?.genaistudio?.roles[0] + let getAllOpeaflowsApi = null + if (keycloak.authenticated) { + getAllOpeaflowsApi = useApi(chatflowsApi.getAllOpeaflows) + + if (userRole === 'admin') { + getAllOpeaflowsApi = useApi(chatflowsApi.getAllOpeaflows) + } + else if (userRole === 'user') { + getAllOpeaflowsApi = useApi(() => chatflowsApi.getUserOpeaflows(keycloak.tokenParsed.email)); + console.log("email", keycloak.tokenParsed.email) + console.log ("get user opeaflows", getAllOpeaflowsApi) + } + } + const stopSandboxApi = chatflowsApi.stopSandbox const updateFlowToServerApi = chatflowsApi.updateChatflow const [view, setView] = useState(localStorage.getItem('flowDisplayStyle') || 'list') @@ -86,15 +105,16 @@ const Opeaflows = () => { useEffect(() => { if (getAllOpeaflowsApi.error) { - if (getAllOpeaflowsApi.error?.response?.status === 401) { - setLoginDialogProps({ - title: 'Login', - confirmButtonName: 'Login' - }) - setLoginDialogOpen(true) - } else { - setError(getAllOpeaflowsApi.error) - } + console.log ("error", getAllOpeaflowsApi.error) + // if (getAllOpeaflowsApi.error?.response?.status === 401) { + // setLoginDialogProps({ + // title: 'Login', + // confirmButtonName: 'Login' + // }) + // setLoginDialogOpen(true) + // } else { + // setError(getAllOpeaflowsApi.error) + // } } }, [getAllOpeaflowsApi.error]) @@ -196,6 +216,7 @@ const Opeaflows = () => { setError={setError} stopSandboxApi={stopSandboxApi} isOpeaCanvas={true} + userRole={userRole} /> )} {!isLoading && (!getAllOpeaflowsApi.data || getAllOpeaflowsApi.data.length === 0) && ( diff --git a/studio-frontend/packages/ui/vite.config.js b/studio-frontend/packages/ui/vite.config.js index 1d51668..c987920 100644 --- a/studio-frontend/packages/ui/vite.config.js +++ b/studio-frontend/packages/ui/vite.config.js @@ -37,8 +37,8 @@ export default defineConfig(async ({ mode }) => { server: { open: true, proxy, - port: process.env.VITE_PORT ?? 8080, - host: process.env.VITE_HOST + port: process.env.VITE_PORT ?? 8088, + host: process.env.VITE_HOST ?? '0.0.0.0' } } }) diff --git a/tests/playwright/studio-e2e/001_test_sandbox_deployment.spec.ts b/tests/playwright/studio-e2e/001_test_sandbox_deployment.spec.ts index 4815eda..146ba7a 100644 --- a/tests/playwright/studio-e2e/001_test_sandbox_deployment.spec.ts +++ b/tests/playwright/studio-e2e/001_test_sandbox_deployment.spec.ts @@ -4,11 +4,18 @@ import fs from 'fs'; import path from 'path'; import os from 'os'; -test('001_test_sandbox_deployment', async ({ page, baseURL }) => { +test('001_test_sandbox_deployment', async ({ browser, baseURL }) => { test.setTimeout(600000); - + const context = await browser.newContext({ + ignoreHTTPSErrors: true + }); + const page = await context.newPage(); const IDC_URL = baseURL || "" await page.goto(IDC_URL); + await page.getByLabel('Username or email').fill('test_automation@gmail.com'); + await page.getByLabel('Password', { exact: true }).click(); + await page.getByLabel('Password', { exact: true }).fill('test'); + await page.getByRole('button', { name: 'Sign In' }).click(); await page.getByRole('button', { name: 'Create New Workflow' }).click(); await page.getByRole('button', { name: 'Settings' }).click(); let fileChooserPromise = page.waitForEvent('filechooser'); diff --git a/tests/playwright/studio-e2e/002_test_sandbox_chatqna.spec.ts b/tests/playwright/studio-e2e/002_test_sandbox_chatqna.spec.ts index 3f2583a..056e3e6 100644 --- a/tests/playwright/studio-e2e/002_test_sandbox_chatqna.spec.ts +++ b/tests/playwright/studio-e2e/002_test_sandbox_chatqna.spec.ts @@ -37,12 +37,19 @@ async function setupResponseListener(page, apiResponse) { }); } -test('002_test_sandbox_chatqna', async ({ page, baseURL }) => { +test('002_test_sandbox_chatqna', async ({ browser, baseURL }) => { test.setTimeout(600000); let apiResponse = { value: '' }; - + const context = await browser.newContext({ + ignoreHTTPSErrors: true + }); + const page = await context.newPage(); const IDC_URL = baseURL || "" await page.goto(IDC_URL); + await page.getByLabel('Username or email').fill('test_automation@gmail.com'); + await page.getByLabel('Password', { exact: true }).click(); + await page.getByLabel('Password', { exact: true }).fill('test'); + await page.getByRole('button', { name: 'Sign In' }).click(); await page.getByRole('button', { name: 'Create New Workflow' }).click(); await page.getByRole('button', { name: 'Settings' }).click(); let fileChooserPromise = page.waitForEvent('filechooser'); From 56b92a331403e0189e6f33cd0e7b92cc33f8b341 Mon Sep 17 00:00:00 2001 From: wwanarif Date: Mon, 6 Jan 2025 05:48:56 +0000 Subject: [PATCH 03/15] update e2e workflow Signed-off-by: wwanarif --- .github/workflows/_e2e-test.yml | 4 ++-- .../setup-genai-studio/internal-dns-config.yaml | 12 ------------ 2 files changed, 2 insertions(+), 14 deletions(-) delete mode 100644 setup-scripts/setup-genai-studio/internal-dns-config.yaml diff --git a/.github/workflows/_e2e-test.yml b/.github/workflows/_e2e-test.yml index 8009f58..652f209 100644 --- a/.github/workflows/_e2e-test.yml +++ b/.github/workflows/_e2e-test.yml @@ -53,7 +53,7 @@ jobs: fi sleep 5 sudo apt install ansible -y - ansible-playbook genai-studio.yml -e "container_registry=${OPEA_IMAGE_REPO}opea" -e "container_tag=${{ inputs.tag }}" + ansible-playbook genai-studio.yml -e "container_registry=${OPEA_IMAGE_REPO}opea" -e "container_tag=${{ inputs.tag }}" -e "mysql_host=${OPEA_IMAGE_REPO%%:*}" sleep 5 kubectl wait --for=condition=ready pod --all --namespace=studio --timeout=300s --field-selector=status.phase!=Succeeded kubectl wait --for=condition=ready pod --all --namespace=monitoring --timeout=300s --field-selector=status.phase!=Succeeded @@ -74,7 +74,7 @@ jobs: - name: Update Playwright Config run: | NODE_IP=$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}') - sed -i "s|baseURL:.*|baseURL: \"http://$NODE_IP:30007\",|" playwright.config.js + sed -i "s|baseURL:.*|baseURL: \"https://$NODE_IP:30007\",|" playwright.config.js working-directory: ${{ github.workspace }}/tests/playwright - name: Run Playwright Tests diff --git a/setup-scripts/setup-genai-studio/internal-dns-config.yaml b/setup-scripts/setup-genai-studio/internal-dns-config.yaml deleted file mode 100644 index e87c12e..0000000 --- a/setup-scripts/setup-genai-studio/internal-dns-config.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: internal-dns-config - namespace: studio -data: - GRAFANA_DNS: "kube-prometheus-stack-grafana.monitoring.svc.cluster.local" - STUDIO_FRONTEND_DNS: "studio-frontend.studio.svc.cluster.local:3000" - APP_FRONTEND_DNS: "app-frontend.$namespace.svc.cluster.local:5175" - APP_BACKEND_DNS: "app-backend.$namespace.svc.cluster.local:8888" - PREPARE_DOC_REDIS_PREP_DNS: "prepare-doc-redis-prep-0.$namespace.svc.cluster.local:6007" - STUDIO_BACKEND_DNS: "studio-backend.studio.svc.cluster.local:5000" \ No newline at end of file From 8458f41602f3dbc82727c381580499a519f4e299 Mon Sep 17 00:00:00 2001 From: wwanarif Date: Tue, 7 Jan 2025 05:48:03 +0000 Subject: [PATCH 04/15] removed gateway dependency Signed-off-by: wwanarif --- app-backend/Dockerfile | 4 +- app-backend/app_gateway.py | 89 ---------------------------------- app-backend/megaservice.py | 97 ++++++++++++++++++++++++++++++++++---- 3 files changed, 90 insertions(+), 100 deletions(-) delete mode 100644 app-backend/app_gateway.py diff --git a/app-backend/Dockerfile b/app-backend/Dockerfile index 8f9c4e5..8f857df 100644 --- a/app-backend/Dockerfile +++ b/app-backend/Dockerfile @@ -12,14 +12,12 @@ RUN useradd -m -s /bin/bash user && \ chown -R user /home/user/ WORKDIR /home/user/ -# temporary pointing to v1.1 GenAIComps for Gateway dependency -RUN git clone https://github.com/opea-project/GenAIComps.git -b v1.1rc +RUN git clone https://github.com/opea-project/GenAIComps.git WORKDIR /home/user/GenAIComps RUN pip install --no-cache-dir --upgrade pip==24.3.1 setuptools==75.3.0 && \ pip install --no-cache-dir -r /home/user/GenAIComps/requirements.txt -COPY ./app_gateway.py /home/user/app_gateway.py COPY ./templates/microservices/* /home/user/templates/microservices/ COPY ./megaservice.py /home/user/megaservice.py COPY config/* /home/user/config/ diff --git a/app-backend/app_gateway.py b/app-backend/app_gateway.py deleted file mode 100644 index 27576ca..0000000 --- a/app-backend/app_gateway.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright (C) 2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -import os -import json -import logging - -# library import -from fastapi import Request -from fastapi.responses import StreamingResponse - -# comps import -from comps import Gateway, MicroService, ServiceOrchestrator, ServiceType -from comps.cores.proto.api_protocol import ( - AudioChatCompletionRequest, - ChatCompletionRequest, - ChatCompletionResponse, - ChatCompletionResponseChoice, - ChatMessage, - EmbeddingRequest, - UsageInfo -) -from comps.cores.proto.docarray import LLMParams, LLMParamsDoc, RerankedDoc, RerankerParms, RetrieverParms, TextDoc - -category_params_map = { - 'LLM': LLMParams, - 'Reranking': RerankerParms, - 'Retreiver': RetrieverParms, -} - -class AppGateway(Gateway): - def __init__(self, megaservice, host='0.0.0.0', port=8888): - try: - with open('config/workflow-info.json', 'r') as f: - self.workflow_info = json.load(f) - except: - logging.error('Failed to load workflow-info.json') - super().__init__( - megaservice, host, port, '/v1/app-backend', ChatCompletionRequest, ChatCompletionResponse - ) - - async def handle_request(self, request: Request): - data = await request.json() - print('\n'*5, '====== handle_request ======\n', data) - if 'chat_completion_ids' in self.workflow_info: - prompt = self._handle_message(data['messages']) - params = {} - llm_parameters = None - for id, node in self.workflow_info['nodes'].items(): - if node['category'] in category_params_map: - param_class = category_params_map[node['category']]() - param_keys = [key for key in dir(param_class) if not key.startswith('__') and not callable(getattr(param_class, key))] - print('param_keys', param_keys) - params_dict = {} - for key in param_keys: - if key in data: - params_dict[key] = data[key] - # hadle special case for stream and streaming - if key in ['stream', 'streaming']: - params_dict[key] = data.get('stream', True) and data.get('streaming', True) - elif key in node['inference_params']: - params_dict[key] = node['inference_params'][key] - params[id] = params_dict - if node['category'] == 'LLM': - params[id]['max_new_tokens'] = params[id]['max_tokens'] - llm_parameters = LLMParams(**params[id]) - result_dict, runtime_graph = await self.megaservice.schedule( - initial_inputs={'query':prompt, 'text': prompt}, - llm_parameters=llm_parameters, - params=params, - ) - print('runtime_graph', runtime_graph.graph) - for node, response in result_dict.items(): - if isinstance(response, StreamingResponse): - return response - last_node = runtime_graph.all_leaves()[-1] - print('result_dict:', result_dict) - print('last_node:',last_node) - response = result_dict[last_node]['text'] - choices = [] - usage = UsageInfo() - choices.append( - ChatCompletionResponseChoice( - index=0, - message=ChatMessage(role='assistant', content=response), - finish_reason='stop', - ) - ) - return ChatCompletionResponse(model='custom_app', choices=choices, usage=usage) \ No newline at end of file diff --git a/app-backend/megaservice.py b/app-backend/megaservice.py index 130be60..b942b48 100644 --- a/app-backend/megaservice.py +++ b/app-backend/megaservice.py @@ -6,12 +6,28 @@ import importlib # library import +from fastapi import Request +from fastapi.responses import StreamingResponse # comps import -from comps import MicroService, ServiceOrchestrator, ServiceType -from app_gateway import AppGateway +from comps import MicroService, ServiceOrchestrator, ServiceRoleType, ServiceType +from comps.cores.mega.utils import handle_message +from comps.cores.proto.api_protocol import ( + ChatCompletionRequest, + ChatCompletionResponse, + ChatCompletionResponseChoice, + ChatMessage, + UsageInfo, +) +from comps.cores.proto.docarray import LLMParams, RerankerParms, RetrieverParms -HOST_IP = os.getenv("HOST_IP", "0,0,0,0") +category_params_map = { + 'LLM': LLMParams, + 'Reranking': RerankerParms, + 'Retreiver': RetrieverParms, +} + +HOST_IP = os.getenv("HOST_IP", "0.0.0.0") USE_NODE_ID_AS_IP = os.getenv("USE_NODE_ID_AS_IP","").lower() == 'true' class AppService: @@ -19,6 +35,8 @@ def __init__(self, host="0.0.0.0", port=8000): self.host = host self.port = port self.megaservice = ServiceOrchestrator() + self.megaservice.align_inputs = self.align_inputs + self.endpoint = "/v1/app-backend" with open('config/workflow-info.json', 'r') as f: self.workflow_info = json.load(f) @@ -56,8 +74,7 @@ def add_remote_service(self): self.megaservice.flow_to(services[prev_node], microservice) for next_node in node['connected_to']: nodes.append(next_node) - self.megaservice.align_inputs = self.align_inputs - self.gateway = AppGateway(megaservice=self.megaservice, host="0.0.0.0", port=self.port) + def align_inputs(self, inputs, *args, **kwargs): """Override this method in megaservice definition.""" print('\n'*2,'align_inputs') @@ -73,10 +90,74 @@ def align_inputs(self, inputs, *args, **kwargs): except Exception as e: print('unable to parse input', e) return inputs + + async def handle_request(self, request: Request): + data = await request.json() + print('\n'*5, '====== handle_request ======\n', data) + if 'chat_completion_ids' in self.workflow_info: + prompt = handle_message(data['messages']) + params = {} + llm_parameters = None + for id, node in self.workflow_info['nodes'].items(): + if node['category'] in category_params_map: + param_class = category_params_map[node['category']]() + param_keys = [key for key in dir(param_class) if not key.startswith('__') and not callable(getattr(param_class, key))] + print('param_keys', param_keys) + params_dict = {} + for key in param_keys: + if key in data: + params_dict[key] = data[key] + # hadle special case for stream and streaming + if key in ['stream', 'streaming']: + params_dict[key] = data.get('stream', True) and data.get('streaming', True) + elif key in node['inference_params']: + params_dict[key] = node['inference_params'][key] + params[id] = params_dict + if node['category'] == 'LLM': + params[id]['max_new_tokens'] = params[id]['max_tokens'] + llm_parameters = LLMParams(**params[id]) + result_dict, runtime_graph = await self.megaservice.schedule( + initial_inputs={'query':prompt, 'text': prompt}, + llm_parameters=llm_parameters, + params=params, + ) + print('runtime_graph', runtime_graph.graph) + for node, response in result_dict.items(): + if isinstance(response, StreamingResponse): + return response + last_node = runtime_graph.all_leaves()[-1] + print('result_dict:', result_dict) + print('last_node:',last_node) + response = result_dict[last_node]['text'] + choices = [] + usage = UsageInfo() + choices.append( + ChatCompletionResponseChoice( + index=0, + message=ChatMessage(role='assistant', content=response), + finish_reason='stop', + ) + ) + return ChatCompletionResponse(model='custom_app', choices=choices, usage=usage) + + def start(self): + + self.service = MicroService( + self.__class__.__name__, + service_role=ServiceRoleType.MEGASERVICE, + host=self.host, + port=self.port, + endpoint=self.endpoint, + input_datatype=ChatCompletionRequest, + output_datatype=ChatCompletionResponse, + ) + self.service.add_route(self.endpoint, self.handle_request, methods=["POST"]) + self.service.start() if __name__ == "__main__": - megaservice_host_ip = None if USE_NODE_ID_AS_IP else HOST_IP - chatqna = AppService(host=HOST_IP, port=8888) + print('pre initialize appService') + app = AppService(host=HOST_IP, port=8888) print('after initialize appService') - chatqna.add_remote_service() \ No newline at end of file + app.add_remote_service() + app.start() \ No newline at end of file From 447bc9bc00fd65d4194589493bcf9a77889cff98 Mon Sep 17 00:00:00 2001 From: wwanarif Date: Wed, 8 Jan 2025 06:37:48 +0000 Subject: [PATCH 05/15] fixed @tabler/icons-react version Signed-off-by: wwanarif --- app-frontend/react/package.json | 2 +- studio-frontend/packages/ui/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app-frontend/react/package.json b/app-frontend/react/package.json index c56e0a5..99941f1 100644 --- a/app-frontend/react/package.json +++ b/app-frontend/react/package.json @@ -17,7 +17,7 @@ "@mantine/notifications": "^7.13.4", "@microsoft/fetch-event-source": "^2.0.1", "@reduxjs/toolkit": "^2.2.5", - "@tabler/icons-react": "^3.5.0", + "@tabler/icons-react": "3.7.0", "axios": "^1.7.2", "luxon": "^3.4.4", "react": "^18.2.0", diff --git a/studio-frontend/packages/ui/package.json b/studio-frontend/packages/ui/package.json index 55a60ef..4a31ee1 100644 --- a/studio-frontend/packages/ui/package.json +++ b/studio-frontend/packages/ui/package.json @@ -21,7 +21,7 @@ "@mui/material": "5.15.0", "@mui/x-data-grid": "6.8.0", "@react-keycloak/web": "^3.4.0", - "@tabler/icons-react": "^3.3.0", + "@tabler/icons-react": "3.7.0", "@uiw/codemirror-theme-sublime": "^4.21.21", "@uiw/codemirror-theme-vscode": "^4.21.21", "@uiw/react-codemirror": "^4.21.21", From 07971fe7f29c093d1c515a8f400ba8848b73ea5f Mon Sep 17 00:00:00 2001 From: wwanarif Date: Wed, 8 Jan 2025 08:49:35 +0000 Subject: [PATCH 06/15] update templates to always pull opea images Signed-off-by: wwanarif --- .../app/templates/microsvc-manifests/data-prep.yaml | 2 +- .../templates/microsvc-manifests/embedding-usvc.yaml | 2 +- .../templates/microsvc-manifests/llm-uservice.yaml | 2 +- .../templates/microsvc-manifests/reranking-usvc.yaml | 2 +- .../templates/microsvc-manifests/retriever-usvc.yaml | 2 +- .../gt_app-manifest-with-nginx.yaml | 10 +++++----- .../tests/exporter-groundtruth/gt_app-manifest.yaml | 12 ++++++------ 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/studio-backend/app/templates/microsvc-manifests/data-prep.yaml b/studio-backend/app/templates/microsvc-manifests/data-prep.yaml index 41d50b0..6f03f5e 100644 --- a/studio-backend/app/templates/microsvc-manifests/data-prep.yaml +++ b/studio-backend/app/templates/microsvc-manifests/data-prep.yaml @@ -77,7 +77,7 @@ spec: seccompProfile: type: RuntimeDefault image: "${REGISTRY}/dataprep-redis:${TAG}" - imagePullPolicy: IfNotPresent + imagePullPolicy: Always ports: - name: data-prep containerPort: 6007 diff --git a/studio-backend/app/templates/microsvc-manifests/embedding-usvc.yaml b/studio-backend/app/templates/microsvc-manifests/embedding-usvc.yaml index efbe8b6..e0b2e71 100644 --- a/studio-backend/app/templates/microsvc-manifests/embedding-usvc.yaml +++ b/studio-backend/app/templates/microsvc-manifests/embedding-usvc.yaml @@ -70,7 +70,7 @@ spec: seccompProfile: type: RuntimeDefault image: "${REGISTRY}/embedding-tei:${TAG}" - imagePullPolicy: IfNotPresent + imagePullPolicy: Always ports: - name: embedding-usvc containerPort: 6000 diff --git a/studio-backend/app/templates/microsvc-manifests/llm-uservice.yaml b/studio-backend/app/templates/microsvc-manifests/llm-uservice.yaml index f9017c3..7cf6b1e 100644 --- a/studio-backend/app/templates/microsvc-manifests/llm-uservice.yaml +++ b/studio-backend/app/templates/microsvc-manifests/llm-uservice.yaml @@ -72,7 +72,7 @@ spec: seccompProfile: type: RuntimeDefault image: "${REGISTRY}/llm-tgi:${TAG}" - imagePullPolicy: IfNotPresent + imagePullPolicy: Always ports: - name: llm-uservice containerPort: 9000 diff --git a/studio-backend/app/templates/microsvc-manifests/reranking-usvc.yaml b/studio-backend/app/templates/microsvc-manifests/reranking-usvc.yaml index fa2871a..d91e645 100644 --- a/studio-backend/app/templates/microsvc-manifests/reranking-usvc.yaml +++ b/studio-backend/app/templates/microsvc-manifests/reranking-usvc.yaml @@ -70,7 +70,7 @@ spec: seccompProfile: type: RuntimeDefault image: "${REGISTRY}/reranking-tei:${TAG}" - imagePullPolicy: IfNotPresent + imagePullPolicy: Always ports: - name: reranking-usvc containerPort: 8000 diff --git a/studio-backend/app/templates/microsvc-manifests/retriever-usvc.yaml b/studio-backend/app/templates/microsvc-manifests/retriever-usvc.yaml index 7b89d73..17edc0b 100644 --- a/studio-backend/app/templates/microsvc-manifests/retriever-usvc.yaml +++ b/studio-backend/app/templates/microsvc-manifests/retriever-usvc.yaml @@ -76,7 +76,7 @@ spec: seccompProfile: type: RuntimeDefault image: "${REGISTRY}/retriever-redis:${TAG}" - imagePullPolicy: IfNotPresent + imagePullPolicy: Always ports: - name: retriever-usvc containerPort: 7000 diff --git a/studio-backend/tests/exporter-groundtruth/gt_app-manifest-with-nginx.yaml b/studio-backend/tests/exporter-groundtruth/gt_app-manifest-with-nginx.yaml index 8da9daf..08ff0c4 100644 --- a/studio-backend/tests/exporter-groundtruth/gt_app-manifest-with-nginx.yaml +++ b/studio-backend/tests/exporter-groundtruth/gt_app-manifest-with-nginx.yaml @@ -576,7 +576,7 @@ spec: seccompProfile: type: RuntimeDefault image: opea/embedding-tei:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: Always ports: - name: embedding-usvc containerPort: 6000 @@ -675,7 +675,7 @@ spec: seccompProfile: type: RuntimeDefault image: opea/llm-tgi:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: Always ports: - name: llm-uservice containerPort: 9000 @@ -779,7 +779,7 @@ spec: seccompProfile: type: RuntimeDefault image: opea/dataprep-redis:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: Always ports: - name: data-prep containerPort: 6007 @@ -876,7 +876,7 @@ spec: seccompProfile: type: RuntimeDefault image: opea/reranking-tei:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: Always ports: - name: reranking-usvc containerPort: 8000 @@ -979,7 +979,7 @@ spec: seccompProfile: type: RuntimeDefault image: opea/retriever-redis:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: Always ports: - name: retriever-usvc containerPort: 7000 diff --git a/studio-backend/tests/exporter-groundtruth/gt_app-manifest.yaml b/studio-backend/tests/exporter-groundtruth/gt_app-manifest.yaml index c52eeb3..a227332 100644 --- a/studio-backend/tests/exporter-groundtruth/gt_app-manifest.yaml +++ b/studio-backend/tests/exporter-groundtruth/gt_app-manifest.yaml @@ -580,7 +580,7 @@ spec: seccompProfile: type: RuntimeDefault image: opea/embedding-tei:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: Always ports: - name: embedding-usvc containerPort: 6000 @@ -679,7 +679,7 @@ spec: seccompProfile: type: RuntimeDefault image: opea/llm-tgi:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: Always ports: - name: llm-uservice containerPort: 9000 @@ -783,7 +783,7 @@ spec: seccompProfile: type: RuntimeDefault image: opea/dataprep-redis:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: Always ports: - name: data-prep containerPort: 6007 @@ -880,7 +880,7 @@ spec: seccompProfile: type: RuntimeDefault image: opea/reranking-tei:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: Always ports: - name: reranking-usvc containerPort: 8000 @@ -983,7 +983,7 @@ spec: seccompProfile: type: RuntimeDefault image: opea/retriever-redis:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: Always ports: - name: retriever-usvc containerPort: 7000 @@ -1281,7 +1281,7 @@ spec: seccompProfile: type: RuntimeDefault image: opea/app-backend:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: Always volumeMounts: - mountPath: /tmp name: tmp From 2ec779fc6d54de6e96e53112901b360b9109e7e4 Mon Sep 17 00:00:00 2001 From: wwanarif Date: Thu, 9 Jan 2025 19:24:27 +0000 Subject: [PATCH 07/15] add button to import sample-workflows in studio-fe and fix UI_FEATURES disabling in app-fe Signed-off-by: wwanarif --- app-frontend/react/.env.production | 2 - app-frontend/react/src/config.ts | 14 ++-- .../app/templates/app/app.compose.yaml | 2 +- .../templates/app/app.manifest.aws.ecr.yaml | 2 +- .../app/templates/app/app.manifest.yaml | 2 +- .../app/utils/placeholders_utils.py | 4 +- .../server/src/controllers/chatflows/index.ts | 10 +++ .../server/src/routes/chatflows/index.ts | 1 + .../server/src/services/chatflows/index.ts | 70 +++++++++---------- .../packages/ui/src/api/chatflows.js | 3 + .../packages/ui/src/views/opeaflows/index.jsx | 18 ++++- 11 files changed, 77 insertions(+), 51 deletions(-) diff --git a/app-frontend/react/.env.production b/app-frontend/react/.env.production index 7f151e6..16b02d1 100644 --- a/app-frontend/react/.env.production +++ b/app-frontend/react/.env.production @@ -1,3 +1 @@ -VITE_CHAT_SERVICE_URL=APP_BACKEND_SERVICE_URL -VITE_DATA_PREP_SERVICE_URL=APP_DATA_PREP_SERVICE_URL VITE_APP_UUID=APP_UUID \ No newline at end of file diff --git a/app-frontend/react/src/config.ts b/app-frontend/react/src/config.ts index 2763b33..281cab7 100644 --- a/app-frontend/react/src/config.ts +++ b/app-frontend/react/src/config.ts @@ -3,12 +3,9 @@ // console.log("Environment variables:", import.meta.env); -export const DATA_PREP_URL = import.meta.env.VITE_DATA_PREP_SERVICE_URL; -export const CHAT_QNA_URL = import.meta.env.VITE_CHAT_SERVICE_URL; export const APP_UUID = import.meta.env.VITE_APP_UUID; - -console.log("data prep", DATA_PREP_URL); -console.log("chat qna", CHAT_QNA_URL); +export const CHAT_QNA_URL = "VITE_APP_BACKEND_SERVICE_URL"; +export const DATA_PREP_URL = "VITE_APP_DATA_PREP_SERVICE_URL"; type UiFeatures = { dataprep: boolean; @@ -16,6 +13,9 @@ type UiFeatures = { }; export const UI_FEATURES: UiFeatures = { - dataprep: !!DATA_PREP_URL, - chat: !!CHAT_QNA_URL + chat: CHAT_QNA_URL.startsWith('VITE_') ? false : true, + dataprep: DATA_PREP_URL.startsWith('VITE_') ? false : true }; + +console.log("chat qna", CHAT_QNA_URL, UI_FEATURES.chat); +console.log("data prep", DATA_PREP_URL, UI_FEATURES.dataprep); \ No newline at end of file diff --git a/studio-backend/app/templates/app/app.compose.yaml b/studio-backend/app/templates/app/app.compose.yaml index 9903ff5..4eab4f5 100644 --- a/studio-backend/app/templates/app/app.compose.yaml +++ b/studio-backend/app/templates/app/app.compose.yaml @@ -25,7 +25,7 @@ app-frontend: - no_proxy=${no_proxy} - https_proxy=${https_proxy} - http_proxy=${http_proxy} - - APP_BACKEND_SERVICE_URL=/v1/app-backend + - VITE_APP_BACKEND_SERVICE_URL=/v1/app-backend __UI_CONFIG_INFO_ENV_PLACEHOLDER__ ipc: host restart: always diff --git a/studio-backend/app/templates/app/app.manifest.aws.ecr.yaml b/studio-backend/app/templates/app/app.manifest.aws.ecr.yaml index 51ff5ce..b2c9437 100644 --- a/studio-backend/app/templates/app/app.manifest.aws.ecr.yaml +++ b/studio-backend/app/templates/app/app.manifest.aws.ecr.yaml @@ -118,7 +118,7 @@ spec: containers: - name: app-frontend env: - - name: APP_BACKEND_SERVICE_URL + - name: VITE_APP_BACKEND_SERVICE_URL value: /v1/app-backend __UI_CONFIG_INFO_ENV_PLACEHOLDER__ securityContext: {} diff --git a/studio-backend/app/templates/app/app.manifest.yaml b/studio-backend/app/templates/app/app.manifest.yaml index 8944f93..5b12824 100644 --- a/studio-backend/app/templates/app/app.manifest.yaml +++ b/studio-backend/app/templates/app/app.manifest.yaml @@ -107,7 +107,7 @@ spec: containers: - name: app-frontend env: - - name: APP_BACKEND_SERVICE_URL + - name: VITE_APP_BACKEND_SERVICE_URL value: /v1/app-backend __UI_CONFIG_INFO_ENV_PLACEHOLDER__ securityContext: {} diff --git a/studio-backend/app/utils/placeholders_utils.py b/studio-backend/app/utils/placeholders_utils.py index 0b7bebb..c37e2b8 100644 --- a/studio-backend/app/utils/placeholders_utils.py +++ b/studio-backend/app/utils/placeholders_utils.py @@ -72,7 +72,7 @@ def replace_dynamic_manifest_placeholder(value_str, service_info, proj_info_json # For __UI_CONFIG_INFO_ENV_PLACEHOLDER__ url_name = value['url_name'] endpoint_path = value['endpoint_path'] - env_block = f"{indent_str}- name: {url_name}\n{indent_str} value: {endpoint_path}\n" + env_block = f"{indent_str}- name: VITE_{url_name}\n{indent_str} value: {endpoint_path}\n" ui_env_config_info_str += env_block # For __UI_CONFIG_INFO_ENV_PLACEHOLDER__ @@ -144,7 +144,7 @@ def replace_dynamic_compose_placeholder(value_str, service_info): for _, value in service_info['ui_config_info'].items(): url_name = value['url_name'] endpoint_path = value['endpoint_path'] - endpoint_block = f"{indent_str} - {url_name}={endpoint_path}\n" + endpoint_block = f"{indent_str} - VITE_{url_name}={endpoint_path}\n" ui_env_config_info_str += endpoint_block # Get app images from environment variables diff --git a/studio-frontend/packages/server/src/controllers/chatflows/index.ts b/studio-frontend/packages/server/src/controllers/chatflows/index.ts index c09cfd3..5b33000 100644 --- a/studio-frontend/packages/server/src/controllers/chatflows/index.ts +++ b/studio-frontend/packages/server/src/controllers/chatflows/index.ts @@ -67,6 +67,15 @@ const getAllChatflowsbyUserId = async (req: Request, res: Response, next: NextFu } } +const importSampleChatflowsbyUserId = async (req: Request, res: Response, next: NextFunction) => { + try { + const apiResponse = await chatflowsService.importSampleChatflowsbyUserId(req.query.userid as string, req.query.type as ChatflowType) + return res.json(apiResponse) + } catch (error) { + next(error) + } +} + // Get specific chatflow via api key const getChatflowByApiKey = async (req: Request, res: Response, next: NextFunction) => { try { @@ -268,6 +277,7 @@ export default { deleteChatflow, getAllChatflows, getAllChatflowsbyUserId, + importSampleChatflowsbyUserId, getChatflowByApiKey, getChatflowById, saveChatflow, diff --git a/studio-frontend/packages/server/src/routes/chatflows/index.ts b/studio-frontend/packages/server/src/routes/chatflows/index.ts index ab79258..495548b 100644 --- a/studio-frontend/packages/server/src/routes/chatflows/index.ts +++ b/studio-frontend/packages/server/src/routes/chatflows/index.ts @@ -4,6 +4,7 @@ const router = express.Router() // CREATE router.post('/', chatflowsController.saveChatflow) +router.post('/importsamples', chatflowsController.importSampleChatflowsbyUserId) router.post('/importchatflows', chatflowsController.importChatflows) // READ diff --git a/studio-frontend/packages/server/src/services/chatflows/index.ts b/studio-frontend/packages/server/src/services/chatflows/index.ts index d6b9337..f0d80b4 100644 --- a/studio-frontend/packages/server/src/services/chatflows/index.ts +++ b/studio-frontend/packages/server/src/services/chatflows/index.ts @@ -139,45 +139,11 @@ const getAllChatflowsbyUserId = async (userid: string, type?: ChatflowType): Pro const appServer = getRunningExpressApp() // Use find with a where condition to filter by userid - let dbResponse = await appServer.AppDataSource.getRepository(ChatFlow).find({ + const dbResponse = await appServer.AppDataSource.getRepository(ChatFlow).find({ where: { userid: userid, // Filter by the specific userid }, }) - - // If no chatflows are found for the user, create a new one based from sample workflow - if (dbResponse.length === 0) { - // URL to fetch the sample workflow - const url = 'https://raw.githubusercontent.com/opea-project/GenAIStudio/refs/heads/main/sample-workflows/sample_workflow_chatqna.json'; - - try { - // Fetch and parse the JSON data from the URL - const response = await axios.get(url); - const parsedFlowData = response.data; - - // Create a new chatflow with the flowData from the URL - const newChatflow: Partial = { - userid: userid, - name: 'sample-chatqna', - flowData: JSON.stringify(parsedFlowData), - type: 'OPEA', - deployed: false, - isPublic: false - }; - - // Call the importChatflows function to insert the new chatflow - await importChatflows([newChatflow]); - } catch (error) { - throw new Error('Failed to import sample chatflow'); - } - - // Rerun the find query to fetch the latest state of chatflows after insertion - dbResponse = await appServer.AppDataSource.getRepository(ChatFlow).find({ - where: { - userid: userid, - }, - }); - } // Filter further by type if needed if (type) { @@ -193,6 +159,39 @@ const getAllChatflowsbyUserId = async (userid: string, type?: ChatflowType): Pro } } +const importSampleChatflowsbyUserId = async (userid: string, type?: ChatflowType): Promise => { + try { + const response = await axios.get('https://api.github.com/repos/opea-project/GenAIStudio/contents/sample-workflows'); + const files = response.data.filter((item: any) => item.type === 'file'); + console.log(`Number of files: ${files.length}`); + + const chatflows: Partial[] = []; + + for (const file of files) { + console.log(`Download URL: ${file.download_url}`); + const fileResponse = await axios.get(file.download_url); + const parsedFlowData = fileResponse.data; + + const newChatflow: Partial = { + userid: userid, + name: file.name.replace('.json', ''), + flowData: JSON.stringify(parsedFlowData), + type: 'OPEA', + deployed: false, + isPublic: false + }; + + chatflows.push(newChatflow); + } + const insertResponse = await importChatflows(chatflows); + return insertResponse; + } catch (error) { + throw new InternalFlowiseError( + StatusCodes.INTERNAL_SERVER_ERROR, + `Error: chatflowsService.importSampleChatflowsbyUserId - ${getErrorMessage(error)}` + ); + } +} const getChatflowByApiKey = async (apiKeyId: string, keyonly?: unknown): Promise => { try { @@ -516,6 +515,7 @@ export default { deleteChatflow, getAllChatflows, getAllChatflowsbyUserId, + importSampleChatflowsbyUserId, getChatflowByApiKey, getChatflowById, saveChatflow, diff --git a/studio-frontend/packages/ui/src/api/chatflows.js b/studio-frontend/packages/ui/src/api/chatflows.js index a51798c..bacd1af 100644 --- a/studio-frontend/packages/ui/src/api/chatflows.js +++ b/studio-frontend/packages/ui/src/api/chatflows.js @@ -31,6 +31,8 @@ const stopSandbox = (id) => client.post(`/chatflows-sandbox/stop/${id}`) const buildDeploymentPackage = (id, body) => client.post(`chatflows-sandbox/build-deployment-package/${id}`, body, {responseType: "arraybuffer"}) +const importSampleChatflowsbyUserId = (userid) => client.post(`/chatflows/importsamples?userid=${userid}&type=OPEA`) + export default { getAllChatflows, getAllAgentflows, @@ -39,6 +41,7 @@ export default { getSpecificChatflow, getSpecificChatflowFromPublicEndpoint, createNewChatflow, + importSampleChatflowsbyUserId, importChatflows, updateChatflow, deleteChatflow, diff --git a/studio-frontend/packages/ui/src/views/opeaflows/index.jsx b/studio-frontend/packages/ui/src/views/opeaflows/index.jsx index 8286db8..1beefee 100644 --- a/studio-frontend/packages/ui/src/views/opeaflows/index.jsx +++ b/studio-frontend/packages/ui/src/views/opeaflows/index.jsx @@ -93,6 +93,15 @@ const Opeaflows = () => { navigate('/opeacanvas') } + const importSamples = () => { + setLoading(true); + chatflowsApi.importSampleChatflowsbyUserId(keycloak.tokenParsed.email).then(() => { + getAllOpeaflowsApi.request(); + }).catch(() => { + setLoading(false); + }); + } + const goToCanvas = (selectedChatflow) => { navigate(`/opeacanvas/${selectedChatflow.id}`) } @@ -186,9 +195,14 @@ const Opeaflows = () => { */} - } sx={{ borderRadius: 2, height: 40, width:250}}> + + } sx={{ borderRadius: 2, height: 40, width: 250 }}> Create New Workflow - + + } sx={{ borderRadius: 2, height: 40, width: 250 }}> + Import Sample Workflows + + {!view || view === 'card' ? ( <> {isLoading && !getAllOpeaflowsApi.data ? ( From 90d9ca6157214b51065f1128447eae316c48065b Mon Sep 17 00:00:00 2001 From: "Chin, Yi Xiang" Date: Wed, 15 Jan 2025 07:50:28 +0000 Subject: [PATCH 08/15] Fix Input and output alignment to follow latest images Signed-off-by: Chin, Yi Xiang --- app-backend/megaservice.py | 201 +++++++++++++++++- .../microsvc-composes/llm-uservice.yaml | 2 +- .../microsvc-manifests/llm-uservice.yaml | 2 +- 3 files changed, 197 insertions(+), 8 deletions(-) diff --git a/app-backend/megaservice.py b/app-backend/megaservice.py index b942b48..bdeb3d1 100644 --- a/app-backend/megaservice.py +++ b/app-backend/megaservice.py @@ -4,6 +4,7 @@ import os import json import importlib +import re # library import from fastapi import Request @@ -20,6 +21,7 @@ UsageInfo, ) from comps.cores.proto.docarray import LLMParams, RerankerParms, RetrieverParms +from langchain_core.prompts import PromptTemplate category_params_map = { 'LLM': LLMParams, @@ -30,12 +32,39 @@ HOST_IP = os.getenv("HOST_IP", "0.0.0.0") USE_NODE_ID_AS_IP = os.getenv("USE_NODE_ID_AS_IP","").lower() == 'true' + +class ChatTemplate: + @staticmethod + def generate_rag_prompt(question, documents): + context_str = "\n".join(documents) + if context_str and len(re.findall("[\u4E00-\u9FFF]", context_str)) / len(context_str) >= 0.3: + # chinese context + template = """ +### 你将扮演一个乐于助人、尊重他人并诚实的助手，你的目标是帮助用户解答问题。有效地利用来自本地知识库的搜索结果。确保你的回答中只包含相关信息。如果你不确定问题的答案，请避免分享不准确的信息。 +### 搜索结果：{context} +### 问题：{question} +### 回答： +""" + else: + template = """ +### You are a helpful, respectful and honest assistant to help the user with questions. \ +Please refer to the search results obtained from the local knowledge base. \ +But be careful to not incorporate the information that you think is not relevant to the question. \ +If you don't know the answer to a question, please don't share false information. \n +### Search results: {context} \n +### Question: {question} \n +### Answer: +""" + return template.format(context=context_str, question=question) + class AppService: def __init__(self, host="0.0.0.0", port=8000): self.host = host self.port = port self.megaservice = ServiceOrchestrator() self.megaservice.align_inputs = self.align_inputs + self.megaservice.align_outputs = self.align_outputs + self.megaservice.align_generator = self.align_generator self.endpoint = "/v1/app-backend" with open('config/workflow-info.json', 'r') as f: self.workflow_info = json.load(f) @@ -56,7 +85,7 @@ def add_remote_service(self): if 'chat_input_ids' not in self.workflow_info: raise Exception('chat_input_ids not found in workflow_info') nodes = self.workflow_info['chat_input_ids'] - services = {} + self.services = {} while nodes: node_id = nodes.pop(0) node = self.workflow_info['nodes'][node_id] @@ -68,10 +97,11 @@ def add_remote_service(self): microservice = templates[microservice_name].get_service(host_ip=service_node_ip, node_id_as_ip=USE_NODE_ID_AS_IP) microservice.name = node_id self.megaservice.add(microservice) - services[node_id] = microservice + + self.services[node_id] = microservice for prev_node in node['connected_from']: - if prev_node in services: - self.megaservice.flow_to(services[prev_node], microservice) + if prev_node in self.services: + self.megaservice.flow_to(self.services[prev_node], microservice) for next_node in node['connected_to']: nodes.append(next_node) @@ -79,18 +109,177 @@ def align_inputs(self, inputs, *args, **kwargs): """Override this method in megaservice definition.""" print('\n'*2,'align_inputs') node_id = args[0] - # print('node_id', node_id) + llm_parameters_dict = args[2] + print('node_id', node_id) params = kwargs.get('params', {}) + print('original_inputs', inputs) + print('-'*20) # print('params', params) if node_id in params: try: new_input = params[node_id] inputs.update(new_input) - print('inputs', inputs) except Exception as e: print('unable to parse input', e) + if self.services[node_id].service_type == ServiceType.EMBEDDING: + inputs["input"] = inputs["text"] + inputs["inputs"] = inputs["text"] + del inputs["text"] + elif self.services[node_id].service_type == ServiceType.RETRIEVER: + # prepare the retriever params + retriever_parameters = kwargs.get("retriever_parameters", None) + if retriever_parameters: + inputs.update(retriever_parameters.dict()) + elif self.services[node_id].service_type == ServiceType.LLM: + # convert TGI/vLLM to unified OpenAI /v1/chat/completions format + next_inputs = {} + next_inputs["model"] = inputs.get("model") or "Intel/neural-chat-7b-v3-3" + if inputs.get("inputs"): + next_inputs["messages"] = [{"role": "user", "content": inputs["inputs"]}] + else: + # for rag case + next_inputs["query"] = inputs["query"] + next_inputs["documents"] = inputs.get("documents",[]) + next_inputs["max_tokens"] = llm_parameters_dict["max_tokens"] + next_inputs["top_p"] = llm_parameters_dict["top_p"] + next_inputs["stream"] = inputs["stream"] + next_inputs["frequency_penalty"] = inputs["frequency_penalty"] + # next_inputs["presence_penalty"] = inputs["presence_penalty"] + # next_inputs["repetition_penalty"] = inputs["repetition_penalty"] + next_inputs["temperature"] = inputs["temperature"] + inputs = next_inputs + print('final_inputs', inputs) + print('-'*20) return inputs + def align_outputs(self, data, cur_node, inputs, runtime_graph, llm_parameters_dict, **kwargs): + print('\n'*2,'align_outputs') + print('cur_node', cur_node) + print('data', data) + print('-'*20) + print('inputs', inputs) + print('-'*20) + next_data = {} + if self.services[cur_node].service_type == ServiceType.EMBEDDING: + # assert isinstance(data, dict) + next_data = {"text": inputs["inputs"], "embedding": data['data'][0]['embedding']} + elif self.services[cur_node].service_type == ServiceType.RETRIEVER: + + docs = [doc["text"] for doc in data["retrieved_docs"]] + + with_rerank = runtime_graph.downstream(cur_node)[0].startswith("opea_service@rerank") + if with_rerank and docs: + print("Rerank with docs") + # forward to rerank + # prepare inputs for rerank + next_data["initial_query"] = data["initial_query"] + next_data["texts"] = [doc["text"] for doc in data["retrieved_docs"]] + next_data["retrieved_docs"] = data["retrieved_docs"] + else: + print("No rerank") + # forward to llm + if not docs and with_rerank: + # delete the rerank from retriever -> rerank -> llm + for ds in reversed(runtime_graph.downstream(cur_node)): + for nds in runtime_graph.downstream(ds): + runtime_graph.add_edge(cur_node, nds) + runtime_graph.delete_node_if_exists(ds) + + # handle template + # if user provides template, then format the prompt with it + # otherwise, use the default template + prompt = data["initial_query"] + chat_template = llm_parameters_dict["chat_template"] + if chat_template: + prompt_template = PromptTemplate.from_template(chat_template) + input_variables = prompt_template.input_variables + if sorted(input_variables) == ["context", "question"]: + prompt = prompt_template.format(question=data["initial_query"], context="\n".join(docs)) + elif input_variables == ["question"]: + prompt = prompt_template.format(question=data["initial_query"]) + else: + print(f"{prompt_template} not used, we only support 2 input variables ['question', 'context']") + prompt = ChatTemplate.generate_rag_prompt(data["initial_query"], docs) + else: + prompt = ChatTemplate.generate_rag_prompt(data["initial_query"], docs) + + next_data["inputs"] = prompt + + elif self.services[cur_node].service_type == ServiceType.RERANK: + # rerank the inputs with the scores + # reranker_parameters = kwargs.get("reranker_parameters", None) + # top_n = reranker_parameters.top_n if reranker_parameters else 1 + # docs = inputs["texts"] + # reranked_docs = [] + # for best_response in data['documents'][:top_n]: + # reranked_docs.append(docs[best_response["index"]]) + + # # handle template + # # if user provides template, then format the prompt with it + # # otherwise, use the default template + # prompt = inputs["query"] + # chat_template = llm_parameters_dict["chat_template"] + # if chat_template: + # prompt_template = PromptTemplate.from_template(chat_template) + # input_variables = prompt_template.input_variables + # if sorted(input_variables) == ["context", "question"]: + # prompt = prompt_template.format(question=prompt, context="\n".join(reranked_docs)) + # elif input_variables == ["question"]: + # prompt = prompt_template.format(question=prompt) + # else: + # print(f"{prompt_template} not used, we only support 2 input variables ['question', 'context']") + # prompt = ChatTemplate.generate_rag_prompt(prompt, reranked_docs) + # else: + # prompt = ChatTemplate.generate_rag_prompt(prompt, reranked_docs) + + # next_data["inputs"] = prompt + next_data = data + + elif self.services[cur_node].service_type == ServiceType.LLM and not llm_parameters_dict["stream"]: + next_data["text"] = data["choices"][0]["message"]["content"] + else: + next_data = data + + print('next_data', next_data) + print('-'*20) + return next_data + + def align_generator(self, gen, **kwargs): + print('\n'*2,'align_generator') + + buffer = "" + for line in gen: + line = line.decode("utf-8") + print('line', line) + start = line.find("{") + end = line.rfind("}") + 1 + + json_str = line[start:end] + try: + json_data = json.loads(json_str) + if json_data["choices"][0]["finish_reason"] != "eos_token": + choice = json_data["choices"][0] + if "delta" in choice and "content" in choice["delta"]: + buffer += choice["delta"]["content"] + elif "text" in choice: + buffer += choice["text"] + buffer = buffer.replace("\\n", "\n") + print("buffer", buffer) + + words = buffer.split() + if len(words) > 1: + output_word = words[0] + ' ' + yield f"data: {repr(output_word.encode('utf-8'))}\n\n" + buffer = " ".join(words[1:]) + else: + buffer = words[0] if words else "" + except Exception as e: + yield f"data: {repr(json_str.encode('utf-8'))}\n\n" + if buffer: + yield f"data: {repr(buffer.encode('utf-8'))}\n\n" + yield "data: [DONE]\n\n" + + async def handle_request(self, request: Request): data = await request.json() print('\n'*5, '====== handle_request ======\n', data) diff --git a/studio-backend/app/templates/microsvc-composes/llm-uservice.yaml b/studio-backend/app/templates/microsvc-composes/llm-uservice.yaml index 17b426f..59f4b63 100644 --- a/studio-backend/app/templates/microsvc-composes/llm-uservice.yaml +++ b/studio-backend/app/templates/microsvc-composes/llm-uservice.yaml @@ -10,7 +10,7 @@ no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} - TGI_LLM_ENDPOINT: "http://${public_host_ip}:{{tgi_port}}" + LLM_ENDPOINT: "http://${public_host_ip}:{{tgi_port}}" HUGGINGFACEHUB_API_TOKEN: "{{tgi_huggingFaceToken}}" HF_HUB_DISABLE_PROGRESS_BARS: 1 HF_HUB_ENABLE_HF_TRANSFER: 0 diff --git a/studio-backend/app/templates/microsvc-manifests/llm-uservice.yaml b/studio-backend/app/templates/microsvc-manifests/llm-uservice.yaml index 7cf6b1e..9ae5004 100644 --- a/studio-backend/app/templates/microsvc-manifests/llm-uservice.yaml +++ b/studio-backend/app/templates/microsvc-manifests/llm-uservice.yaml @@ -8,7 +8,7 @@ kind: ConfigMap metadata: name: config-{endpoint} data: - TGI_LLM_ENDPOINT: "http://{tgi_endpoint}" + LLM_ENDPOINT: "http://{tgi_endpoint}" HUGGINGFACEHUB_API_TOKEN: "{tgi_huggingFaceToken}" HF_HOME: "/tmp/.cache/huggingface" http_proxy: "${HTTP_PROXY}" From e5801b5412a567636ceb74b8b1b88bcb415e44db Mon Sep 17 00:00:00 2001 From: wwanarif Date: Wed, 15 Jan 2025 09:45:56 +0000 Subject: [PATCH 09/15] replace harbor with local registry as the default under onpremise k8 setup Signed-off-by: wwanarif --- .../{ => cleanup_registry}/harbor_cleanup.sh | 0 .../onpremise-kubernetes.yml | 4 +- .../playbooks/install-requirements.yml | 6 +- .../playbooks/setup-local-registry.yml | 137 ++++++++++++++++++ .../setup-onpremise-kubernetes/readme.md | 4 +- .../registry.config.yml | 20 +++ 6 files changed, 166 insertions(+), 5 deletions(-) rename setup-scripts/setup-onpremise-kubernetes/{ => cleanup_registry}/harbor_cleanup.sh (100%) create mode 100644 setup-scripts/setup-onpremise-kubernetes/playbooks/setup-local-registry.yml create mode 100644 setup-scripts/setup-onpremise-kubernetes/registry.config.yml diff --git a/setup-scripts/setup-onpremise-kubernetes/harbor_cleanup.sh b/setup-scripts/setup-onpremise-kubernetes/cleanup_registry/harbor_cleanup.sh similarity index 100% rename from setup-scripts/setup-onpremise-kubernetes/harbor_cleanup.sh rename to setup-scripts/setup-onpremise-kubernetes/cleanup_registry/harbor_cleanup.sh diff --git a/setup-scripts/setup-onpremise-kubernetes/onpremise-kubernetes.yml b/setup-scripts/setup-onpremise-kubernetes/onpremise-kubernetes.yml index 740036d..8d74987 100644 --- a/setup-scripts/setup-onpremise-kubernetes/onpremise-kubernetes.yml +++ b/setup-scripts/setup-onpremise-kubernetes/onpremise-kubernetes.yml @@ -4,5 +4,5 @@ - name: Setup K8 cluster import_playbook: playbooks/setup-k8-cluster.yml -- name: Setup harbor container registry - import_playbook: playbooks/setup-harbor.yml \ No newline at end of file +- name: Setup local container registry + import_playbook: playbooks/setup-local-registry.yml \ No newline at end of file diff --git a/setup-scripts/setup-onpremise-kubernetes/playbooks/install-requirements.yml b/setup-scripts/setup-onpremise-kubernetes/playbooks/install-requirements.yml index 4dbc94a..4083ee2 100644 --- a/setup-scripts/setup-onpremise-kubernetes/playbooks/install-requirements.yml +++ b/setup-scripts/setup-onpremise-kubernetes/playbooks/install-requirements.yml @@ -161,7 +161,7 @@ line: "{{ item }}" loop: - 'export http_proxy={{ http_proxy }}' - - 'export http_proxy={{ http_proxy }}' + - 'export https_proxy={{ http_proxy }}' - 'export no_proxy={{ no_proxy }}' - name: Set up proxy environment variables for Docker service block: @@ -219,6 +219,10 @@ name: ['kubelet', 'kubeadm', 'kubectl'] state: present update_cache: yes + environment: + http_proxy: "{{ http_proxy }}" + https_proxy: "{{ http_proxy }}" + no_proxy: "{{ no_proxy }}" - name: Hold Kubernetes packages dpkg_selections: diff --git a/setup-scripts/setup-onpremise-kubernetes/playbooks/setup-local-registry.yml b/setup-scripts/setup-onpremise-kubernetes/playbooks/setup-local-registry.yml new file mode 100644 index 0000000..6e71265 --- /dev/null +++ b/setup-scripts/setup-onpremise-kubernetes/playbooks/setup-local-registry.yml @@ -0,0 +1,137 @@ +- name: Update Docker daemon configuration on all hosts and run Docker registry on localhost + hosts: all + become: yes + become_method: sudo + become_user: root + vars_files: + - ../vars.yml + tasks: + - name: Ensure /etc/docker/daemon.json exists + copy: + dest: /etc/docker/daemon.json + content: | + { + "exec-opts": ["native.cgroupdriver=systemd"], + "log-driver" : "json-file", + "log-opts": { + "max-size" : "100m" + }, + "storage-driver": "overlay2". + "insecure-registries": [ + "{{ groups['k8_master'][0] }}:5000" + ] + } + + handlers: + - name: Reload Docker daemon + command: systemctl daemon-reload + + - name: Restart Docker service + service: + name: docker + state: restarted + +- name: Run Docker registry on localhost + hosts: localhost + become: yes + become_method: sudo + become_user: root + vars_files: + - ../vars.yml + tasks: + - name: Install pip3 if not present + apt: + name: python3-pip + state: present + update_cache: yes + environment: + http_proxy: "{{ http_proxy }}" + https_proxy: "{{ http_proxy }}" + no_proxy: "{{ no_proxy }}" + + - name: Install Docker SDK for Python + pip: + name: docker + executable: pip3 + state: present + environment: + http_proxy: "{{ http_proxy }}" + https_proxy: "{{ http_proxy }}" + no_proxy: "{{ no_proxy }}" + + - name: Ensure /var/lib/registry directory exists + file: + path: /var/lib/registry + state: directory + owner: root + group: root + mode: '0755' + + - name: Run Docker registry container + docker_container: + name: registry + image: registry:2 + state: started + restart_policy: always + ports: + - "5000:5000" + volumes: + - ../registry.config.yml:/etc/docker/registry/config.yml + - /var/lib/registry:/var/lib/registry + container_default_behavior: "compatibility" + + - name: Ensure /etc/systemd/system/containerd.service.d directory exists + file: + path: /etc/systemd/system/containerd.service.d + state: directory + owner: root + group: root + mode: '0755' + + - name: Create /etc/systemd/system/containerd.service.d/http-proxy.conf + copy: + dest: /etc/systemd/system/containerd.service.d/http-proxy.conf + content: | + [Service] + Environment="HTTP_PROXY={{ http_proxy }}" + Environment="HTTPS_PROXY={{ http_proxy }}" + Environment="NO_PROXY={{ no_proxy }}" + notify: + - Reload systemd daemon + + - name: Create /etc/containerd/config.toml + copy: + dest: /etc/containerd/config.toml + content: | + version = 2 + root = "/var/lib/containerd" + state = "/run/containerd" + oom_score = 0 + + [grpc] + max_recv_message_size = 16777216 + max_send_message_size = 16777216 + + [debug] + level = "info" + + [metrics] + address = "" + grpc_histogram = false + + [plugins] + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ groups['k8_master'][0] }}:5000"] + endpoint = ["http://{{ groups['k8_master'][0] }}:5000"] + notify: + - Restart containerd service + + handlers: + - name: Reload systemd daemon + command: systemctl daemon-reload + + - name: Restart containerd service + service: + name: containerd + state: restarted \ No newline at end of file diff --git a/setup-scripts/setup-onpremise-kubernetes/readme.md b/setup-scripts/setup-onpremise-kubernetes/readme.md index a7d52fa..f544924 100644 --- a/setup-scripts/setup-onpremise-kubernetes/readme.md +++ b/setup-scripts/setup-onpremise-kubernetes/readme.md @@ -20,6 +20,6 @@ ansible-playbook -i inventory.ini onpremise-kubernetes.yml To push your local docker images into the harbor container registry, run below: ```sh -docker tag : :8443/k8s/: -docker push :8443/k8s/: +docker tag : :5000/opea/: +docker push :5000/opea/: ``` \ No newline at end of file diff --git a/setup-scripts/setup-onpremise-kubernetes/registry.config.yml b/setup-scripts/setup-onpremise-kubernetes/registry.config.yml new file mode 100644 index 0000000..879d3af --- /dev/null +++ b/setup-scripts/setup-onpremise-kubernetes/registry.config.yml @@ -0,0 +1,20 @@ +version: 0.1 +log: + fields: + service: registry +storage: + cache: + blobdescriptor: inmemory + filesystem: + rootdirectory: /var/lib/registry + delete: + enabled: true +http: + addr: :5000 + headers: + X-Content-Type-Options: [nosniff] +health: + storagedriver: + enabled: true + interval: 10s + threshold: 3 \ No newline at end of file From a0fe7bda219c0b85b52974dec08b4a2922982270 Mon Sep 17 00:00:00 2001 From: wwanarif Date: Thu, 16 Jan 2025 15:31:40 +0000 Subject: [PATCH 10/15] enabled remote svc healthcheck before running microsvc Signed-off-by: wwanarif --- app-backend/Dockerfile | 4 ++-- app-backend/megaservice.py | 2 +- .../microservices/embedding_tei_langchain.py | 3 +-- app-backend/templates/microservices/llm_tgi.py | 3 +-- .../templates/microservices/reranking_tei.py | 3 +-- .../templates/microservices/retriever_redis.py | 3 +-- .../templates/microsvc-composes/data-prep.yaml | 6 ++++-- .../microsvc-composes/embedding-usvc.yaml | 5 +++-- .../microsvc-composes/llm-uservice.yaml | 5 +++-- .../microsvc-composes/reranking-usvc.yaml | 5 +++-- .../microsvc-composes/retriever-usvc.yaml | 8 +++++--- .../app/templates/microsvc-composes/tei.yaml | 7 ++++++- .../app/templates/microsvc-composes/tgi.yaml | 7 ++++++- .../templates/microsvc-manifests/data-prep.yaml | 8 ++++++++ .../microsvc-manifests/embedding-usvc.yaml | 8 ++++++++ .../microsvc-manifests/llm-uservice.yaml | 8 ++++++++ .../app/templates/microsvc-manifests/readme.MD | 17 ----------------- .../microsvc-manifests/reranking-usvc.yaml | 8 ++++++++ .../microsvc-manifests/retriever-usvc.yaml | 8 ++++++++ 19 files changed, 77 insertions(+), 41 deletions(-) delete mode 100644 studio-backend/app/templates/microsvc-manifests/readme.MD diff --git a/app-backend/Dockerfile b/app-backend/Dockerfile index 8f857df..3a47f14 100644 --- a/app-backend/Dockerfile +++ b/app-backend/Dockerfile @@ -12,7 +12,7 @@ RUN useradd -m -s /bin/bash user && \ chown -R user /home/user/ WORKDIR /home/user/ -RUN git clone https://github.com/opea-project/GenAIComps.git +RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git WORKDIR /home/user/GenAIComps RUN pip install --no-cache-dir --upgrade pip==24.3.1 setuptools==75.3.0 && \ @@ -30,4 +30,4 @@ WORKDIR /home/user RUN echo 'ulimit -S -n 999999' >> ~/.bashrc -ENTRYPOINT ["python", "megaservice.py"] \ No newline at end of file +ENTRYPOINT ["python", "megaservice.py"] diff --git a/app-backend/megaservice.py b/app-backend/megaservice.py index bdeb3d1..ef4b033 100644 --- a/app-backend/megaservice.py +++ b/app-backend/megaservice.py @@ -346,7 +346,7 @@ def start(self): if __name__ == "__main__": print('pre initialize appService') - app = AppService(host=HOST_IP, port=8888) + app = AppService(host="0.0.0.0", port=8888) print('after initialize appService') app.add_remote_service() app.start() \ No newline at end of file diff --git a/app-backend/templates/microservices/embedding_tei_langchain.py b/app-backend/templates/microservices/embedding_tei_langchain.py index 54b290a..1bea51d 100644 --- a/app-backend/templates/microservices/embedding_tei_langchain.py +++ b/app-backend/templates/microservices/embedding_tei_langchain.py @@ -4,8 +4,7 @@ def get_service(host_ip = "0.0.0.0", **kwargs): return MicroService( name="embedding", host=host_ip, - # port=None if kwargs.get("node_id_as_ip") else 6000, - port=6000, + port=6000 if kwargs.get("node_id_as_ip") else 6009, endpoint="/v1/embeddings", use_remote_service=True, service_type=ServiceType.EMBEDDING diff --git a/app-backend/templates/microservices/llm_tgi.py b/app-backend/templates/microservices/llm_tgi.py index 3f5c942..9977d0b 100644 --- a/app-backend/templates/microservices/llm_tgi.py +++ b/app-backend/templates/microservices/llm_tgi.py @@ -4,8 +4,7 @@ def get_service(host_ip = "0.0.0.0", **kwargs): return MicroService( name="llm", host=host_ip, - # port=None if kwargs.get("node_id_as_ip") else 9000, - port=9000, + port=9000 if kwargs.get("node_id_as_ip") else 9009, endpoint="/v1/chat/completions", use_remote_service=True, service_type=ServiceType.LLM diff --git a/app-backend/templates/microservices/reranking_tei.py b/app-backend/templates/microservices/reranking_tei.py index 2d9cc7f..c5a271f 100644 --- a/app-backend/templates/microservices/reranking_tei.py +++ b/app-backend/templates/microservices/reranking_tei.py @@ -4,8 +4,7 @@ def get_service(host_ip = "0.0.0.0", **kwargs): return MicroService( name="rerank", host=host_ip, - # port=None if kwargs.get("node_id_as_ip") else 8000, - port=8000, + port=8000 if kwargs.get("node_id_as_ip") else 8009, endpoint="/v1/reranking", use_remote_service=True, service_type=ServiceType.RERANK diff --git a/app-backend/templates/microservices/retriever_redis.py b/app-backend/templates/microservices/retriever_redis.py index 23bf2d8..0950ab5 100644 --- a/app-backend/templates/microservices/retriever_redis.py +++ b/app-backend/templates/microservices/retriever_redis.py @@ -4,8 +4,7 @@ def get_service(host_ip = "0.0.0.0", **kwargs): return MicroService( name="retriever", host=host_ip, - # port=None if kwargs.get("node_id_as_ip") else 7000, - port=7000, + port=7000 if kwargs.get("node_id_as_ip") else 7009, endpoint="/v1/retrieval", use_remote_service=True, service_type=ServiceType.RETRIEVER diff --git a/studio-backend/app/templates/microsvc-composes/data-prep.yaml b/studio-backend/app/templates/microsvc-composes/data-prep.yaml index 69a86da..b1698e2 100644 --- a/studio-backend/app/templates/microsvc-composes/data-prep.yaml +++ b/studio-backend/app/templates/microsvc-composes/data-prep.yaml @@ -2,8 +2,10 @@ image: ${REGISTRY}/dataprep-redis:${TAG} container_name: "{{endpoint}}" depends_on: - - "{{redis_vector_store_endpoint}}" - - "{{tei_endpoint}}" + "{{redis_vector_store_endpoint}}": + condition: service_started + "{{tei_endpoint}}": + condition: service_healthy ports: - 6007:6007 environment: diff --git a/studio-backend/app/templates/microsvc-composes/embedding-usvc.yaml b/studio-backend/app/templates/microsvc-composes/embedding-usvc.yaml index 4720dce..1d0b5e1 100644 --- a/studio-backend/app/templates/microsvc-composes/embedding-usvc.yaml +++ b/studio-backend/app/templates/microsvc-composes/embedding-usvc.yaml @@ -2,9 +2,10 @@ image: ${REGISTRY}/embedding-tei:${TAG} container_name: "{{endpoint}}" depends_on: - - "{{tei_endpoint}}" + "{{tei_endpoint}}": + condition: service_healthy ports: - - 6000:6000 + - 6009:6000 ipc: host environment: no_proxy: ${no_proxy} diff --git a/studio-backend/app/templates/microsvc-composes/llm-uservice.yaml b/studio-backend/app/templates/microsvc-composes/llm-uservice.yaml index 59f4b63..428b118 100644 --- a/studio-backend/app/templates/microsvc-composes/llm-uservice.yaml +++ b/studio-backend/app/templates/microsvc-composes/llm-uservice.yaml @@ -2,9 +2,10 @@ image: ${REGISTRY}/llm-tgi:${TAG} container_name: "{{endpoint}}" depends_on: - - "{{tgi_endpoint}}" + "{{tgi_endpoint}}": + condition: service_healthy ports: - - 9000:9000 + - 9009:9000 ipc: host environment: no_proxy: ${no_proxy} diff --git a/studio-backend/app/templates/microsvc-composes/reranking-usvc.yaml b/studio-backend/app/templates/microsvc-composes/reranking-usvc.yaml index 04c0b4c..6897a5c 100644 --- a/studio-backend/app/templates/microsvc-composes/reranking-usvc.yaml +++ b/studio-backend/app/templates/microsvc-composes/reranking-usvc.yaml @@ -2,9 +2,10 @@ image: ${REGISTRY}/reranking-tei:${TAG} container_name: "{{endpoint}}" depends_on: - - "{{tei_endpoint}}" + "{{tei_endpoint}}": + condition: service_healthy ports: - - 8000:8000 + - 8009:8000 ipc: host environment: no_proxy: ${no_proxy} diff --git a/studio-backend/app/templates/microsvc-composes/retriever-usvc.yaml b/studio-backend/app/templates/microsvc-composes/retriever-usvc.yaml index 648d402..eb55859 100644 --- a/studio-backend/app/templates/microsvc-composes/retriever-usvc.yaml +++ b/studio-backend/app/templates/microsvc-composes/retriever-usvc.yaml @@ -2,10 +2,12 @@ image: ${REGISTRY}/retriever-redis:${TAG} container_name: "{{endpoint}}" depends_on: - - "{{redis_vector_store_endpoint}}" - - "{{tei_endpoint}}" + "{{redis_vector_store_endpoint}}": + condition: service_started + "{{tei_endpoint}}": + condition: service_healthy ports: - - 7000:7000 + - 7009:7000 ipc: host environment: no_proxy: ${no_proxy} diff --git a/studio-backend/app/templates/microsvc-composes/tei.yaml b/studio-backend/app/templates/microsvc-composes/tei.yaml index 71a1482..cb239ad 100644 --- a/studio-backend/app/templates/microsvc-composes/tei.yaml +++ b/studio-backend/app/templates/microsvc-composes/tei.yaml @@ -10,4 +10,9 @@ no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} - command: --model-id {{modelName}} --auto-truncate \ No newline at end of file + entrypoint: /bin/sh -c "apt-get update && apt-get install -y curl && text-embeddings-router --json-output --model-id {{modelName}} --auto-truncate" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:80/health"] + interval: 30s + retries: 20 + timeout: 10s \ No newline at end of file diff --git a/studio-backend/app/templates/microsvc-composes/tgi.yaml b/studio-backend/app/templates/microsvc-composes/tgi.yaml index 6c603cc..1f31a28 100644 --- a/studio-backend/app/templates/microsvc-composes/tgi.yaml +++ b/studio-backend/app/templates/microsvc-composes/tgi.yaml @@ -13,4 +13,9 @@ HF_TOKEN: "{{huggingFaceToken}}" HF_HUB_DISABLE_PROGRESS_BARS: 1 HF_HUB_ENABLE_HF_TRANSFER: 0 - command: --model-id {{modelName}} --cuda-graphs 0 \ No newline at end of file + command: --model-id {{modelName}} --cuda-graphs 0 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:80/health"] + interval: 30s + retries: 20 + timeout: 10s \ No newline at end of file diff --git a/studio-backend/app/templates/microsvc-manifests/data-prep.yaml b/studio-backend/app/templates/microsvc-manifests/data-prep.yaml index 6f03f5e..92bbde6 100644 --- a/studio-backend/app/templates/microsvc-manifests/data-prep.yaml +++ b/studio-backend/app/templates/microsvc-manifests/data-prep.yaml @@ -8,6 +8,7 @@ kind: ConfigMap metadata: name: config-{endpoint} data: + HEALTHCHECK_ENDPOINT: "{tei_endpoint}" TEI_ENDPOINT: "http://{tei_endpoint}" EMBED_MODEL: "" REDIS_URL: "redis://{redis_vector_store_endpoint}:{redis_vector_store_port}" @@ -61,6 +62,13 @@ spec: spec: securityContext: {} + initContainers: + - name: wait-for-remote-service + image: busybox + command: ['sh', '-c', 'until nc -z -v -w30 $HEALTHCHECK_ENDPOINT 80; do echo "Waiting for remote service..."; sleep 5; done'] + envFrom: + - configMapRef: + name: config-{endpoint} containers: - name: data-prep envFrom: diff --git a/studio-backend/app/templates/microsvc-manifests/embedding-usvc.yaml b/studio-backend/app/templates/microsvc-manifests/embedding-usvc.yaml index e0b2e71..cc433a6 100644 --- a/studio-backend/app/templates/microsvc-manifests/embedding-usvc.yaml +++ b/studio-backend/app/templates/microsvc-manifests/embedding-usvc.yaml @@ -8,6 +8,7 @@ kind: ConfigMap metadata: name: config-{endpoint} data: + HEALTHCHECK_ENDPOINT: "{tei_endpoint}" TEI_EMBEDDING_ENDPOINT: "http://{tei_endpoint}" http_proxy: "" https_proxy: "" @@ -54,6 +55,13 @@ spec: spec: securityContext: {} + initContainers: + - name: wait-for-remote-service + image: busybox + command: ['sh', '-c', 'until nc -z -v -w30 $HEALTHCHECK_ENDPOINT 80; do echo "Waiting for remote service..."; sleep 5; done'] + envFrom: + - configMapRef: + name: config-{endpoint} containers: - name: embedding-usvc envFrom: diff --git a/studio-backend/app/templates/microsvc-manifests/llm-uservice.yaml b/studio-backend/app/templates/microsvc-manifests/llm-uservice.yaml index 9ae5004..f6d184c 100644 --- a/studio-backend/app/templates/microsvc-manifests/llm-uservice.yaml +++ b/studio-backend/app/templates/microsvc-manifests/llm-uservice.yaml @@ -8,6 +8,7 @@ kind: ConfigMap metadata: name: config-{endpoint} data: + HEALTHCHECK_ENDPOINT: "{tgi_endpoint}" LLM_ENDPOINT: "http://{tgi_endpoint}" HUGGINGFACEHUB_API_TOKEN: "{tgi_huggingFaceToken}" HF_HOME: "/tmp/.cache/huggingface" @@ -56,6 +57,13 @@ spec: spec: securityContext: {} + initContainers: + - name: wait-for-remote-service + image: busybox + command: ['sh', '-c', 'until nc -z -v -w30 $HEALTHCHECK_ENDPOINT 80; do echo "Waiting for remote service..."; sleep 5; done'] + envFrom: + - configMapRef: + name: config-{endpoint} containers: - name: llm-uservice envFrom: diff --git a/studio-backend/app/templates/microsvc-manifests/readme.MD b/studio-backend/app/templates/microsvc-manifests/readme.MD deleted file mode 100644 index 61a51cb..0000000 --- a/studio-backend/app/templates/microsvc-manifests/readme.MD +++ /dev/null @@ -1,17 +0,0 @@ -## Template Manifests - -The manifest files in this directory were sourced from the following GitHub repository: - -- **Repository**: [opea-project/GenAIEval](https://github.com/opea-project/GenAIInfra) -- **Commit ID**: `325126e30a30790305d10646372f32c6af52fed8` -- **Date Copied**: 2024-09-25 - -These files are used as templates for generating new manifests within our application. - -## Modifications - -Modification: Updated templates with variables - -## License - -The original files were distributed under the [Apache-2.0 License](https://opensource.org/licenses/Apache-2.0). A copy of the license can be found in the LICENSE file at the root of the original repository. \ No newline at end of file diff --git a/studio-backend/app/templates/microsvc-manifests/reranking-usvc.yaml b/studio-backend/app/templates/microsvc-manifests/reranking-usvc.yaml index d91e645..8a8451d 100644 --- a/studio-backend/app/templates/microsvc-manifests/reranking-usvc.yaml +++ b/studio-backend/app/templates/microsvc-manifests/reranking-usvc.yaml @@ -8,6 +8,7 @@ kind: ConfigMap metadata: name: config-{endpoint} data: + HEALTHCHECK_ENDPOINT: "{tei_endpoint}" TEI_RERANKING_ENDPOINT: "http://{tei_endpoint}" http_proxy: "" https_proxy: "" @@ -54,6 +55,13 @@ spec: spec: securityContext: {} + initContainers: + - name: wait-for-remote-service + image: busybox + command: ['sh', '-c', 'until nc -z -v -w30 $HEALTHCHECK_ENDPOINT 80; do echo "Waiting for remote service..."; sleep 5; done'] + envFrom: + - configMapRef: + name: config-{endpoint} containers: - name: reranking-usvc envFrom: diff --git a/studio-backend/app/templates/microsvc-manifests/retriever-usvc.yaml b/studio-backend/app/templates/microsvc-manifests/retriever-usvc.yaml index 17edc0b..2501662 100644 --- a/studio-backend/app/templates/microsvc-manifests/retriever-usvc.yaml +++ b/studio-backend/app/templates/microsvc-manifests/retriever-usvc.yaml @@ -8,6 +8,7 @@ kind: ConfigMap metadata: name: config-{endpoint} data: + HEALTHCHECK_ENDPOINT: "{tei_endpoint}" TEI_EMBEDDING_ENDPOINT: "http://{tei_endpoint}" EMBED_MODEL: "" REDIS_URL: "redis://{redis_vector_store_endpoint}:{redis_vector_store_port}" @@ -60,6 +61,13 @@ spec: spec: securityContext: {} + initContainers: + - name: wait-for-remote-service + image: busybox + command: ['sh', '-c', 'until nc -z -v -w30 $HEALTHCHECK_ENDPOINT 80; do echo "Waiting for remote service..."; sleep 5; done'] + envFrom: + - configMapRef: + name: config-{endpoint} containers: - name: retriever-usvc envFrom: From ab6f973abe1f0d78724e64cca5430a5df114e66e Mon Sep 17 00:00:00 2001 From: wwanarif Date: Thu, 16 Jan 2025 16:29:47 +0000 Subject: [PATCH 11/15] integrate mysql playbook under the setup genai studio playbook Signed-off-by: wwanarif --- setup-scripts/setup-genai-studio/genai-studio.yml | 3 +++ .../setup-genai-studio/playbooks/setup-mysqldb.yml | 4 +++- setup-scripts/setup-genai-studio/readme.md | 10 ++++++---- setup-scripts/setup-genai-studio/vars.yml | 3 ++- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/setup-scripts/setup-genai-studio/genai-studio.yml b/setup-scripts/setup-genai-studio/genai-studio.yml index e4749b5..a15582e 100644 --- a/setup-scripts/setup-genai-studio/genai-studio.yml +++ b/setup-scripts/setup-genai-studio/genai-studio.yml @@ -1,3 +1,6 @@ +- name: Setup mysqldb + import_playbook: playbooks/setup-mysqldb.yml + - name: Deploy monitoring import_playbook: playbooks/deploy-monitoring.yml diff --git a/setup-scripts/setup-genai-studio/playbooks/setup-mysqldb.yml b/setup-scripts/setup-genai-studio/playbooks/setup-mysqldb.yml index 8b5261a..2f64665 100644 --- a/setup-scripts/setup-genai-studio/playbooks/setup-mysqldb.yml +++ b/setup-scripts/setup-genai-studio/playbooks/setup-mysqldb.yml @@ -12,15 +12,17 @@ mysql_user: login_user: root login_password: root + login_unix_socket: /var/run/mysqld/mysqld.sock name: studio check_implicit_admin: yes state: present + check_mode: yes register: studio_user_exists ignore_errors: yes - name: End playbook if MySQL user 'studio' exists meta: end_play - when: studio_user_exists is succeeded + when: studio_user_exists.changed == false - name: Install MySQL server apt: diff --git a/setup-scripts/setup-genai-studio/readme.md b/setup-scripts/setup-genai-studio/readme.md index 0115fea..265a97a 100644 --- a/setup-scripts/setup-genai-studio/readme.md +++ b/setup-scripts/setup-genai-studio/readme.md @@ -2,16 +2,18 @@ The genai-studio playbook script will: -1. Deploy a customized monitoring stack based on prometheus-community/kube-prometheus-stack (which contains both Prometheus and Grafana) in the monitoring namespace with a local-path-provisioner in local-path-storage namespace, for dynamic Persistent Volumes (PVs) provisioning. +1. Install and configure a MySQL server in localhost machine. -2. Deploy the studio-backend, studio-frontend and also a studio-nginx in the studio namespace. +2. Deploy a customized monitoring stack based on prometheus-community/kube-prometheus-stack (which contains both Prometheus and Grafana) in the monitoring namespace with a local-path-provisioner in local-path-storage namespace, for dynamic Persistent Volumes (PVs) provisioning. +3. Deploy the keycloak, studio-backend, studio-frontend and also a studio-nginx in the studio namespace. -### Pre-requisite +### Pre-requisite +- Disclaimer: The Ansible script has been tested on a fresh machine without any pre-existing MySQL server installation, studio, or monitoring deployment. Any other environment might require modifications to the Ansible playbooks accordingly. - Existing kubernetes cluster available. If not, please install by following the [Kubernetes official setup guide](https://kubernetes.io/docs/setup/). Alternatively, you can try out our [setup onpremise kubernetes script](../setup-onpremise-kubernetes/readme.md). -- Update var.yml accordingly +- Update vars.yml accordingly. By default, if you have a locally installed MySQL server, you will need to update the variable `mysql_host` in vars.yml with the external public IP of your localhost. ### Installation steps: diff --git a/setup-scripts/setup-genai-studio/vars.yml b/setup-scripts/setup-genai-studio/vars.yml index 5fdc49c..131d094 100644 --- a/setup-scripts/setup-genai-studio/vars.yml +++ b/setup-scripts/setup-genai-studio/vars.yml @@ -1,4 +1,5 @@ container_registry: 'opea' container_tag: 'latest' http_proxy: '' -no_proxy: '' \ No newline at end of file +no_proxy: '' +mysql_host: 'Your_External_Host_IP' \ No newline at end of file From 4e9b7ed0983cfdb8ee3224502b4a9d686eeef00e Mon Sep 17 00:00:00 2001 From: "Chin, Yi Xiang" Date: Fri, 17 Jan 2025 03:23:36 +0000 Subject: [PATCH 12/15] Fix input for simple llm example Signed-off-by: Chin, Yi Xiang --- app-backend/megaservice.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app-backend/megaservice.py b/app-backend/megaservice.py index ef4b033..f27a79f 100644 --- a/app-backend/megaservice.py +++ b/app-backend/megaservice.py @@ -136,10 +136,13 @@ def align_inputs(self, inputs, *args, **kwargs): next_inputs["model"] = inputs.get("model") or "Intel/neural-chat-7b-v3-3" if inputs.get("inputs"): next_inputs["messages"] = [{"role": "user", "content": inputs["inputs"]}] - else: + elif inputs.get("query") and inputs.get("documents"): # for rag case next_inputs["query"] = inputs["query"] next_inputs["documents"] = inputs.get("documents",[]) + else: + # simple llm case + next_inputs["messages"] = [{"role": "user", "content": next(value for key in ["query", "text", "input", "inputs"] if (value := inputs.get(key)))}] next_inputs["max_tokens"] = llm_parameters_dict["max_tokens"] next_inputs["top_p"] = llm_parameters_dict["top_p"] next_inputs["stream"] = inputs["stream"] From 5b7307af535ee05d647917e12a942d121800a46a Mon Sep 17 00:00:00 2001 From: wwanarif Date: Fri, 17 Jan 2025 08:33:38 +0000 Subject: [PATCH 13/15] temp update mysqlhost in e2e CI Signed-off-by: wwanarif --- .github/workflows/_e2e-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_e2e-test.yml b/.github/workflows/_e2e-test.yml index 652f209..664ed33 100644 --- a/.github/workflows/_e2e-test.yml +++ b/.github/workflows/_e2e-test.yml @@ -53,7 +53,7 @@ jobs: fi sleep 5 sudo apt install ansible -y - ansible-playbook genai-studio.yml -e "container_registry=${OPEA_IMAGE_REPO}opea" -e "container_tag=${{ inputs.tag }}" -e "mysql_host=${OPEA_IMAGE_REPO%%:*}" + ansible-playbook genai-studio.yml -e "container_registry=${OPEA_IMAGE_REPO}opea" -e "container_tag=${{ inputs.tag }}" -e "mysql_host=$(hostname -I | awk '{print $1}')" sleep 5 kubectl wait --for=condition=ready pod --all --namespace=studio --timeout=300s --field-selector=status.phase!=Succeeded kubectl wait --for=condition=ready pod --all --namespace=monitoring --timeout=300s --field-selector=status.phase!=Succeeded From 8d84f16a4168a1dcb57bace199fdd2cec0ca31e4 Mon Sep 17 00:00:00 2001 From: wwanarif Date: Fri, 17 Jan 2025 09:36:29 +0000 Subject: [PATCH 14/15] update microservice image names Signed-off-by: wwanarif --- .../app/templates/microsvc-composes/embedding-usvc.yaml | 2 +- .../app/templates/microsvc-composes/llm-uservice.yaml | 2 +- .../app/templates/microsvc-composes/reranking-usvc.yaml | 2 +- .../app/templates/microsvc-composes/retriever-usvc.yaml | 2 +- .../app/templates/microsvc-manifests/embedding-usvc.yaml | 2 +- .../app/templates/microsvc-manifests/llm-uservice.yaml | 2 +- .../app/templates/microsvc-manifests/reranking-usvc.yaml | 2 +- .../app/templates/microsvc-manifests/retriever-usvc.yaml | 2 +- .../tests/exporter-groundtruth/gt_app-compose.yaml | 8 ++++---- .../exporter-groundtruth/gt_app-manifest-with-nginx.yaml | 8 ++++---- .../tests/exporter-groundtruth/gt_app-manifest.yaml | 8 ++++---- 11 files changed, 20 insertions(+), 20 deletions(-) diff --git a/studio-backend/app/templates/microsvc-composes/embedding-usvc.yaml b/studio-backend/app/templates/microsvc-composes/embedding-usvc.yaml index 1d0b5e1..7baf93b 100644 --- a/studio-backend/app/templates/microsvc-composes/embedding-usvc.yaml +++ b/studio-backend/app/templates/microsvc-composes/embedding-usvc.yaml @@ -1,5 +1,5 @@ "{{endpoint}}": - image: ${REGISTRY}/embedding-tei:${TAG} + image: ${REGISTRY}/embedding:${TAG} container_name: "{{endpoint}}" depends_on: "{{tei_endpoint}}": diff --git a/studio-backend/app/templates/microsvc-composes/llm-uservice.yaml b/studio-backend/app/templates/microsvc-composes/llm-uservice.yaml index 428b118..2047713 100644 --- a/studio-backend/app/templates/microsvc-composes/llm-uservice.yaml +++ b/studio-backend/app/templates/microsvc-composes/llm-uservice.yaml @@ -1,5 +1,5 @@ "{{endpoint}}": - image: ${REGISTRY}/llm-tgi:${TAG} + image: ${REGISTRY}/llm-textgen:${TAG} container_name: "{{endpoint}}" depends_on: "{{tgi_endpoint}}": diff --git a/studio-backend/app/templates/microsvc-composes/reranking-usvc.yaml b/studio-backend/app/templates/microsvc-composes/reranking-usvc.yaml index 6897a5c..b9c65c9 100644 --- a/studio-backend/app/templates/microsvc-composes/reranking-usvc.yaml +++ b/studio-backend/app/templates/microsvc-composes/reranking-usvc.yaml @@ -1,5 +1,5 @@ "{{endpoint}}": - image: ${REGISTRY}/reranking-tei:${TAG} + image: ${REGISTRY}/reranking:${TAG} container_name: "{{endpoint}}" depends_on: "{{tei_endpoint}}": diff --git a/studio-backend/app/templates/microsvc-composes/retriever-usvc.yaml b/studio-backend/app/templates/microsvc-composes/retriever-usvc.yaml index eb55859..416bc43 100644 --- a/studio-backend/app/templates/microsvc-composes/retriever-usvc.yaml +++ b/studio-backend/app/templates/microsvc-composes/retriever-usvc.yaml @@ -1,5 +1,5 @@ "{{endpoint}}": - image: ${REGISTRY}/retriever-redis:${TAG} + image: ${REGISTRY}/retriever:${TAG} container_name: "{{endpoint}}" depends_on: "{{redis_vector_store_endpoint}}": diff --git a/studio-backend/app/templates/microsvc-manifests/embedding-usvc.yaml b/studio-backend/app/templates/microsvc-manifests/embedding-usvc.yaml index cc433a6..a0c96ba 100644 --- a/studio-backend/app/templates/microsvc-manifests/embedding-usvc.yaml +++ b/studio-backend/app/templates/microsvc-manifests/embedding-usvc.yaml @@ -77,7 +77,7 @@ spec: runAsUser: 1000 seccompProfile: type: RuntimeDefault - image: "${REGISTRY}/embedding-tei:${TAG}" + image: "${REGISTRY}/embedding:${TAG}" imagePullPolicy: Always ports: - name: embedding-usvc diff --git a/studio-backend/app/templates/microsvc-manifests/llm-uservice.yaml b/studio-backend/app/templates/microsvc-manifests/llm-uservice.yaml index f6d184c..b8b7b98 100644 --- a/studio-backend/app/templates/microsvc-manifests/llm-uservice.yaml +++ b/studio-backend/app/templates/microsvc-manifests/llm-uservice.yaml @@ -79,7 +79,7 @@ spec: runAsUser: 1000 seccompProfile: type: RuntimeDefault - image: "${REGISTRY}/llm-tgi:${TAG}" + image: "${REGISTRY}/llm-textgen:${TAG}" imagePullPolicy: Always ports: - name: llm-uservice diff --git a/studio-backend/app/templates/microsvc-manifests/reranking-usvc.yaml b/studio-backend/app/templates/microsvc-manifests/reranking-usvc.yaml index 8a8451d..2191348 100644 --- a/studio-backend/app/templates/microsvc-manifests/reranking-usvc.yaml +++ b/studio-backend/app/templates/microsvc-manifests/reranking-usvc.yaml @@ -77,7 +77,7 @@ spec: runAsUser: 1000 seccompProfile: type: RuntimeDefault - image: "${REGISTRY}/reranking-tei:${TAG}" + image: "${REGISTRY}/reranking:${TAG}" imagePullPolicy: Always ports: - name: reranking-usvc diff --git a/studio-backend/app/templates/microsvc-manifests/retriever-usvc.yaml b/studio-backend/app/templates/microsvc-manifests/retriever-usvc.yaml index 2501662..6206d4e 100644 --- a/studio-backend/app/templates/microsvc-manifests/retriever-usvc.yaml +++ b/studio-backend/app/templates/microsvc-manifests/retriever-usvc.yaml @@ -83,7 +83,7 @@ spec: runAsUser: 1000 seccompProfile: type: RuntimeDefault - image: "${REGISTRY}/retriever-redis:${TAG}" + image: "${REGISTRY}/retriever:${TAG}" imagePullPolicy: Always ports: - name: retriever-usvc diff --git a/studio-backend/tests/exporter-groundtruth/gt_app-compose.yaml b/studio-backend/tests/exporter-groundtruth/gt_app-compose.yaml index ba03bb3..0cc47fb 100644 --- a/studio-backend/tests/exporter-groundtruth/gt_app-compose.yaml +++ b/studio-backend/tests/exporter-groundtruth/gt_app-compose.yaml @@ -64,7 +64,7 @@ services: https_proxy: ${https_proxy} command: --model-id BAAI/bge-base-en-v1.5 --auto-truncate embedding-tei-langchain-0: - image: opea/embedding-tei:latest + image: opea/embedding:latest container_name: embedding-tei-langchain-0 depends_on: - tei-0 @@ -78,7 +78,7 @@ services: TEI_EMBEDDING_ENDPOINT: http://${public_host_ip}:2081 restart: unless-stopped llm-tgi-0: - image: opea/llm-tgi:latest + image: opea/llm-textgen:latest container_name: llm-tgi-0 depends_on: - tgi-0 @@ -112,7 +112,7 @@ services: TEI_ENDPOINT: http://${public_host_ip}:2081 HUGGINGFACEHUB_API_TOKEN: NA reranking-tei-0: - image: opea/reranking-tei:latest + image: opea/reranking:latest container_name: reranking-tei-0 depends_on: - tei-1 @@ -129,7 +129,7 @@ services: HF_HUB_ENABLE_HF_TRANSFER: 0 restart: unless-stopped retriever-redis-0: - image: opea/retriever-redis:latest + image: opea/retriever:latest container_name: retriever-redis-0 depends_on: - redis-vector-store-0 diff --git a/studio-backend/tests/exporter-groundtruth/gt_app-manifest-with-nginx.yaml b/studio-backend/tests/exporter-groundtruth/gt_app-manifest-with-nginx.yaml index 08ff0c4..a3d64d6 100644 --- a/studio-backend/tests/exporter-groundtruth/gt_app-manifest-with-nginx.yaml +++ b/studio-backend/tests/exporter-groundtruth/gt_app-manifest-with-nginx.yaml @@ -575,7 +575,7 @@ spec: runAsUser: 1000 seccompProfile: type: RuntimeDefault - image: opea/embedding-tei:latest + image: opea/embedding:latest imagePullPolicy: Always ports: - name: embedding-usvc @@ -674,7 +674,7 @@ spec: runAsUser: 1000 seccompProfile: type: RuntimeDefault - image: opea/llm-tgi:latest + image: opea/llm-textgen:latest imagePullPolicy: Always ports: - name: llm-uservice @@ -875,7 +875,7 @@ spec: runAsUser: 1000 seccompProfile: type: RuntimeDefault - image: opea/reranking-tei:latest + image: opea/reranking:latest imagePullPolicy: Always ports: - name: reranking-usvc @@ -978,7 +978,7 @@ spec: runAsUser: 1000 seccompProfile: type: RuntimeDefault - image: opea/retriever-redis:latest + image: opea/retriever:latest imagePullPolicy: Always ports: - name: retriever-usvc diff --git a/studio-backend/tests/exporter-groundtruth/gt_app-manifest.yaml b/studio-backend/tests/exporter-groundtruth/gt_app-manifest.yaml index a227332..5f8d992 100644 --- a/studio-backend/tests/exporter-groundtruth/gt_app-manifest.yaml +++ b/studio-backend/tests/exporter-groundtruth/gt_app-manifest.yaml @@ -579,7 +579,7 @@ spec: runAsUser: 1000 seccompProfile: type: RuntimeDefault - image: opea/embedding-tei:latest + image: opea/embedding:latest imagePullPolicy: Always ports: - name: embedding-usvc @@ -678,7 +678,7 @@ spec: runAsUser: 1000 seccompProfile: type: RuntimeDefault - image: opea/llm-tgi:latest + image: opea/llm-textgen:latest imagePullPolicy: Always ports: - name: llm-uservice @@ -879,7 +879,7 @@ spec: runAsUser: 1000 seccompProfile: type: RuntimeDefault - image: opea/reranking-tei:latest + image: opea/reranking:latest imagePullPolicy: Always ports: - name: reranking-usvc @@ -982,7 +982,7 @@ spec: runAsUser: 1000 seccompProfile: type: RuntimeDefault - image: opea/retriever-redis:latest + image: opea/retriever:latest imagePullPolicy: Always ports: - name: retriever-usvc From f3ce3af598ea6a1db49945d8580f1dbe4ea04e62 Mon Sep 17 00:00:00 2001 From: wwanarif Date: Fri, 17 Jan 2025 10:21:19 +0000 Subject: [PATCH 15/15] udpate retriever variables Signed-off-by: wwanarif --- .../app/templates/microsvc-composes/retriever-usvc.yaml | 2 ++ .../app/templates/microsvc-manifests/retriever-usvc.yaml | 1 + 2 files changed, 3 insertions(+) diff --git a/studio-backend/app/templates/microsvc-composes/retriever-usvc.yaml b/studio-backend/app/templates/microsvc-composes/retriever-usvc.yaml index 416bc43..11a1e74 100644 --- a/studio-backend/app/templates/microsvc-composes/retriever-usvc.yaml +++ b/studio-backend/app/templates/microsvc-composes/retriever-usvc.yaml @@ -17,4 +17,6 @@ INDEX_NAME: "rag-redis" TEI_EMBEDDING_ENDPOINT: "http://${public_host_ip}:{{tei_port}}" HUGGINGFACEHUB_API_TOKEN: "{{tei_huggingFaceToken}}" + LOGFLAG: ${LOGFLAG} + RETRIEVER_COMPONENT_NAME: "OPEA_RETRIEVER_REDIS" restart: unless-stopped \ No newline at end of file diff --git a/studio-backend/app/templates/microsvc-manifests/retriever-usvc.yaml b/studio-backend/app/templates/microsvc-manifests/retriever-usvc.yaml index 6206d4e..55b5dcb 100644 --- a/studio-backend/app/templates/microsvc-manifests/retriever-usvc.yaml +++ b/studio-backend/app/templates/microsvc-manifests/retriever-usvc.yaml @@ -20,6 +20,7 @@ data: HF_HOME: "/tmp/.cache/huggingface" HUGGINGFACEHUB_API_TOKEN: "{tei_huggingFaceToken}" LOGFLAG: "" + RETRIEVER_COMPONENT_NAME: "OPEA_RETRIEVER_REDIS" --- # Source: retriever-usvc/templates/service.yaml # Copyright (C) 2024 Intel Corporation