From 1876f312f3637eb8c4592b3b0f65677a379c7b29 Mon Sep 17 00:00:00 2001 From: wwanarif Date: Thu, 10 Jul 2025 03:16:50 +0000 Subject: [PATCH 1/4] update setup playbooks Signed-off-by: wwanarif --- setup-scripts/setup-genai-studio/readme.md | 1 + .../playbooks/setup-local-registry.yml | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/setup-scripts/setup-genai-studio/readme.md b/setup-scripts/setup-genai-studio/readme.md index 58d7c18..58815c6 100644 --- a/setup-scripts/setup-genai-studio/readme.md +++ b/setup-scripts/setup-genai-studio/readme.md @@ -21,6 +21,7 @@ The genai-studio playbook script will: Run below commands: ```sh +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash sudo apt install ansible -y ansible-playbook genai-studio.yml ``` diff --git a/setup-scripts/setup-onpremise-kubernetes/playbooks/setup-local-registry.yml b/setup-scripts/setup-onpremise-kubernetes/playbooks/setup-local-registry.yml index 48089af..c14f7c4 100644 --- a/setup-scripts/setup-onpremise-kubernetes/playbooks/setup-local-registry.yml +++ b/setup-scripts/setup-onpremise-kubernetes/playbooks/setup-local-registry.yml @@ -52,11 +52,11 @@ https_proxy: "{{ http_proxy }}" no_proxy: "{{ no_proxy }}" - - name: Install Docker SDK for Python - pip: - name: docker - executable: pip3 + - name: Install Docker SDK for Python via apt + apt: + name: python3-docker state: present + become: yes environment: http_proxy: "{{ http_proxy }}" https_proxy: "{{ http_proxy }}" From c071b770d1db0ca8415bc9bf306e597a5e1dbc76 Mon Sep 17 00:00:00 2001 From: wwanarif Date: Thu, 24 Jul 2025 07:00:02 +0000 Subject: [PATCH 2/4] debuglogs to display the connections between pods Signed-off-by: wwanarif --- studio-backend/app/routers/debuglog_router.py | 192 +++++++++++++++++- .../packages/ui/src/views/debuglogs/index.jsx | 31 ++- 2 files changed, 211 insertions(+), 12 deletions(-) diff --git a/studio-backend/app/routers/debuglog_router.py b/studio-backend/app/routers/debuglog_router.py index 4b4654b..b4b3cdd 100644 --- a/studio-backend/app/routers/debuglog_router.py +++ b/studio-backend/app/routers/debuglog_router.py @@ -1,8 +1,173 @@ from fastapi import APIRouter, HTTPException from kubernetes import client +import re router = APIRouter() +def find_pod_dependencies(pod, all_pods, services, namespace, core_v1_api): + """ + Analyze pod dependencies by checking: + 1. Environment variables pointing to other services + 2. Service selectors matching pod labels + 3. ConfigMaps and Secrets that might reference other pods + 4. Init containers waiting for remote services + """ + dependencies = [] + + # Get pod environment variables from main containers + env_vars = [] + configmap_refs = [] + + if pod.spec.containers: + for container in pod.spec.containers: + # Direct environment variables + if container.env: + for env in container.env: + if env.value: + env_vars.append(env.value) + + # Environment variables from ConfigMaps + if container.env_from: + for env_from in container.env_from: + if env_from.config_map_ref: + configmap_refs.append(env_from.config_map_ref.name) + + # Get environment variables from init containers (for wait-for-remote-service patterns) + init_env_vars = [] + + if pod.spec.init_containers: + for init_container in pod.spec.init_containers: + # Direct environment variables from init containers + if init_container.env: + for env in init_container.env: + if env.value: + init_env_vars.append(env.value) + + # Environment variables from ConfigMaps in init containers + if init_container.env_from: + for env_from in init_container.env_from: + if env_from.config_map_ref and env_from.config_map_ref.name not in configmap_refs: + configmap_refs.append(env_from.config_map_ref.name) + + # Check init container commands for wait-for-remote-service patterns + if init_container.name == "wait-for-remote-service" or "wait-for" in init_container.name.lower(): + if init_container.command: + command_str = " ".join(init_container.command) + # Look for service endpoints in commands like "nc -z -v -w30 $HEALTHCHECK_ENDPOINT" + # Extract service names from HEALTHCHECK_ENDPOINT or similar patterns + for env_var in init_env_vars: + # Pattern like "tei-0:9003" or "redis-vector-store-0:9001" + service_match = re.search(r'([a-zA-Z0-9-]+):\d+', env_var) + if service_match: + service_name = service_match.group(1) + # Find pods that this service targets + for service in services.items: + if service.metadata.name == service_name and service.spec.selector: + for target_pod in all_pods.items: + if target_pod.metadata.name != pod.metadata.name: + target_labels = target_pod.metadata.labels or {} + if all(target_labels.get(k) == v for k, v in service.spec.selector.items()): + if target_pod.metadata.name not in dependencies: + dependencies.append(target_pod.metadata.name) + + # Fetch ConfigMap data to analyze environment variables + configmap_env_vars = [] + for configmap_name in configmap_refs: + try: + configmap = core_v1_api.read_namespaced_config_map(name=configmap_name, namespace=namespace) + if configmap.data: + for key, value in configmap.data.items(): + if value: + configmap_env_vars.append(value) + except Exception as e: + print(f"Error fetching ConfigMap {configmap_name}: {str(e)}") + + # Combine all environment variables for further analysis + all_env_vars = env_vars + init_env_vars + configmap_env_vars + + # Debug output + print(f"Analyzing dependencies for pod: {pod.metadata.name}") + print(f"ConfigMap refs: {configmap_refs}") + print(f"Total env vars found: {len(all_env_vars)}") + if all_env_vars: + print(f"Sample env vars: {all_env_vars[:3]}") # Show first 3 for debugging + + # Check if environment variables reference other services + for service in services.items: + service_name = service.metadata.name + # Check if service name is referenced in environment variables + for env_val in all_env_vars: + if service_name in env_val: + # Find pods that this service targets + if service.spec.selector: + for target_pod in all_pods.items: + if target_pod.metadata.name != pod.metadata.name: + target_labels = target_pod.metadata.labels or {} + if all(target_labels.get(k) == v for k, v in service.spec.selector.items()): + if target_pod.metadata.name not in dependencies: + dependencies.append(target_pod.metadata.name) + + # Check for service endpoint patterns in environment variables + # Pattern like "http://service-name:port" or "service-name:port" + for env_val in all_env_vars: + # HTTP URL pattern + http_matches = re.findall(r'https?://([a-zA-Z0-9-]+):\d+', env_val) + for service_name in http_matches: + for service in services.items: + if service.metadata.name == service_name and service.spec.selector: + for target_pod in all_pods.items: + if target_pod.metadata.name != pod.metadata.name: + target_labels = target_pod.metadata.labels or {} + if all(target_labels.get(k) == v for k, v in service.spec.selector.items()): + if target_pod.metadata.name not in dependencies: + dependencies.append(target_pod.metadata.name) + + # Direct service:port pattern + service_port_matches = re.findall(r'([a-zA-Z0-9-]+):\d+', env_val) + for service_name in service_port_matches: + for service in services.items: + if service.metadata.name == service_name and service.spec.selector: + for target_pod in all_pods.items: + if target_pod.metadata.name != pod.metadata.name: + target_labels = target_pod.metadata.labels or {} + if all(target_labels.get(k) == v for k, v in service.spec.selector.items()): + if target_pod.metadata.name not in dependencies: + dependencies.append(target_pod.metadata.name) + + # Check DNS-based dependencies (common pattern: service-name.namespace.svc.cluster.local) + dns_pattern = r'([a-zA-Z0-9-]+)\.(' + re.escape(namespace) + r')\.svc\.cluster\.local' + for env_val in all_env_vars: + matches = re.findall(dns_pattern, env_val) + for match in matches: + service_name = match[0] + # Find the service and its target pods + for service in services.items: + if service.metadata.name == service_name and service.spec.selector: + for target_pod in all_pods.items: + if target_pod.metadata.name != pod.metadata.name: + target_labels = target_pod.metadata.labels or {} + if all(target_labels.get(k) == v for k, v in service.spec.selector.items()): + if target_pod.metadata.name not in dependencies: + dependencies.append(target_pod.metadata.name) + + # Additional pattern matching for service names that might not include ports + # Look for patterns where service names appear in environment variable values + service_names = [service.metadata.name for service in services.items] + for env_val in all_env_vars: + for service_name in service_names: + # More flexible matching - look for service name as whole word + if re.search(r'\b' + re.escape(service_name) + r'\b', env_val): + for service in services.items: + if service.metadata.name == service_name and service.spec.selector: + for target_pod in all_pods.items: + if target_pod.metadata.name != pod.metadata.name: + target_labels = target_pod.metadata.labels or {} + if all(target_labels.get(k) == v for k, v in service.spec.selector.items()): + if target_pod.metadata.name not in dependencies: + dependencies.append(target_pod.metadata.name) + + return dependencies + @router.get("/podlogs/{namespace}", summary="Fetch all pods in a namespace") async def get_all_pods_in_namespace(namespace: str): core_v1_api = client.CoreV1Api() @@ -11,6 +176,13 @@ async def get_all_pods_in_namespace(namespace: str): if not pods.items: return {"namespace": namespace, "pods": []} + # Fetch all services in the namespace for dependency analysis + try: + services = core_v1_api.list_namespaced_service(namespace=namespace) + except Exception as e: + print(f"Error fetching services: {str(e)}") + services = None + pod_list = [] for pod in pods.items: pod_name = pod.metadata.name @@ -22,10 +194,14 @@ async def get_all_pods_in_namespace(namespace: str): # Fetch logs related to the pod try: pod_logs = core_v1_api.read_namespaced_pod_log(name=pod_name, namespace=namespace, tail_lines=200) - for line in pod_logs.splitlines(): - log_entries.append(line) + if pod_logs and pod_logs.strip(): + for line in pod_logs.splitlines(): + log_entries.append(line) + else: + log_entries.append("** Pod has no logs available") except Exception as e: print(f"Error fetching logs: {str(e)}") + log_entries.append("Unable to fetch logs: Pod may not be running or logs are not accessible") # Fetch events related to the pod try: @@ -38,6 +214,17 @@ async def get_all_pods_in_namespace(namespace: str): except Exception as e: print(f"Error fetching events: {str(e)}") + # Analyze pod dependencies + dependencies = [] + if services: + try: + dependencies = find_pod_dependencies(pod, pods, services, namespace, core_v1_api) + print(f"Pod {pod_name} dependencies: {dependencies}") + except Exception as e: + print(f"Error analyzing dependencies for pod {pod_name}: {str(e)}") + import traceback + traceback.print_exc() + # Determine the Ready and Status of the pod ready_status = "Unknown" pod_status = pod.status.phase @@ -63,6 +250,7 @@ async def get_all_pods_in_namespace(namespace: str): "annotations": pod.metadata.annotations, "logs": log_entries, "events": event_entries, + "dependencies": dependencies, }) return {"namespace": namespace, "pods": pod_list} \ No newline at end of file diff --git a/studio-frontend/packages/ui/src/views/debuglogs/index.jsx b/studio-frontend/packages/ui/src/views/debuglogs/index.jsx index 8ac3172..fb92fc2 100644 --- a/studio-frontend/packages/ui/src/views/debuglogs/index.jsx +++ b/studio-frontend/packages/ui/src/views/debuglogs/index.jsx @@ -151,6 +151,7 @@ export default function PodLogsView() { Pod Name Pod Ready Pod Status + Dependencies Pod Events Pod Logs @@ -161,6 +162,12 @@ export default function PodLogsView() { {pod.name} {pod.ready} {pod.status} + + {pod.dependencies && pod.dependencies.length > 0 + ? pod.dependencies.join(', ') + : 'None' + } + {pod.events.length > 0 ? ( @@ -187,13 +194,15 @@ export default function PodLogsView() { BackdropProps={{ timeout: 500 }} > - + {selectedLogPod?.name} Logs - 
-                            {selectedLogPod?.logs?.join('\n') || 'No logs available'}
-
- + + 
+                                {selectedLogPod?.logs?.join('\n') || 'No logs available'}
+
+ + @@ -208,13 +217,15 @@ export default function PodLogsView() { BackdropProps={{ timeout: 500 }} > - + {selectedEventPod?.name} Events - 
-                            {selectedEventPod?.events?.join('\n') || 'No events available'}
-
- + + 
+                                {selectedEventPod?.events?.join('\n') || 'No events available'}
+
+ + From 865aa901e96c00d4982b6ccf7bec6c415e041891 Mon Sep 17 00:00:00 2001 From: wwanarif Date: Fri, 25 Jul 2025 01:20:46 +0000 Subject: [PATCH 3/4] fix chat history bug and duplicate workflow bug Signed-off-by: wwanarif --- .../react/src/pages/History/HistoryView.tsx | 5 +++- .../src/ui-component/button/FlowListMenu.jsx | 13 ++++++-- .../ui/src/views/canvas/CanvasHeader.jsx | 14 +++++++-- .../packages/ui/src/views/canvas/index.jsx | 30 +++++++++++++++++-- 4 files changed, 54 insertions(+), 8 deletions(-) diff --git a/app-frontend/react/src/pages/History/HistoryView.tsx b/app-frontend/react/src/pages/History/HistoryView.tsx index 3a59e5e..f8885fb 100644 --- a/app-frontend/react/src/pages/History/HistoryView.tsx +++ b/app-frontend/react/src/pages/History/HistoryView.tsx @@ -1,3 +1,4 @@ +import React from "react"; import { Box, Checkbox, @@ -26,6 +27,7 @@ import { SolidButton, TextButton, } from "@root/shared/ActionButtons"; +import { useToWithQuery } from "@utils/navigationAndAxiosWithQuery"; interface HistoryViewProps { shared: boolean; @@ -34,6 +36,7 @@ interface HistoryViewProps { const HistoryView: React.FC = ({ shared }) => { const dispatch = useAppDispatch(); const { name } = useAppSelector(userSelector); + const toWithQuery = useToWithQuery(); const theme = useTheme(); @@ -109,7 +112,7 @@ const HistoryView: React.FC = ({ shared }) => { ) : ( diff --git a/studio-frontend/packages/ui/src/ui-component/button/FlowListMenu.jsx b/studio-frontend/packages/ui/src/ui-component/button/FlowListMenu.jsx index 43f9bca..421d014 100644 --- a/studio-frontend/packages/ui/src/ui-component/button/FlowListMenu.jsx +++ b/studio-frontend/packages/ui/src/ui-component/button/FlowListMenu.jsx @@ -265,8 +265,17 @@ export default function FlowListMenu({ chatflow, isAgentCanvas, setError, update const handleDuplicate = () => { setAnchorEl(null) try { - localStorage.setItem('duplicatedFlowData', chatflow.flowData) - window.open(`${uiBaseURL}/${isAgentCanvas ? 'agentcanvas' : 'canvas'}`, '_blank') + // Store both the flow data and type information + const duplicatedData = { + flowData: chatflow.flowData, + type: chatflow.type || (isAgentCanvas ? 'MULTIAGENT' : 'CHATFLOW') + } + localStorage.setItem('duplicatedFlowData', JSON.stringify(duplicatedData)) + + // Determine canvas type based on original chatflow type + const targetCanvas = chatflow.type === 'OPEA' ? 'opeacanvas' : + chatflow.type === 'MULTIAGENT' ? 'agentcanvas' : 'canvas' + window.open(`${uiBaseURL}/${targetCanvas}`, '_blank') } catch (e) { console.error(e) } diff --git a/studio-frontend/packages/ui/src/views/canvas/CanvasHeader.jsx b/studio-frontend/packages/ui/src/views/canvas/CanvasHeader.jsx index 787d57d..4a0fd93 100644 --- a/studio-frontend/packages/ui/src/views/canvas/CanvasHeader.jsx +++ b/studio-frontend/packages/ui/src/views/canvas/CanvasHeader.jsx @@ -121,8 +121,18 @@ const CanvasHeader = ({ chatflow, isAgentCanvas, isOpeaCanvas, handleSaveFlow, h let flowData = chatflow.flowData const parsedFlowData = JSON.parse(flowData) flowData = JSON.stringify(parsedFlowData) - localStorage.setItem('duplicatedFlowData', flowData) - window.open(`${uiBaseURL}/${isAgentCanvas ? 'agentcanvas' : isOpeaCanvas ? 'opeacanvas' : 'canvas'}`, '_blank') + + // Store both the flow data and type information + const duplicatedData = { + flowData: flowData, + type: chatflow.type || (isAgentCanvas ? 'MULTIAGENT' : isOpeaCanvas ? 'OPEA' : 'CHATFLOW') + } + localStorage.setItem('duplicatedFlowData', JSON.stringify(duplicatedData)) + + // Determine canvas type based on original chatflow type + const targetCanvas = chatflow.type === 'OPEA' ? 'opeacanvas' : + chatflow.type === 'MULTIAGENT' ? 'agentcanvas' : 'canvas' + window.open(`${uiBaseURL}/${targetCanvas}`, '_blank') } catch (e) { console.error(e) } diff --git a/studio-frontend/packages/ui/src/views/canvas/index.jsx b/studio-frontend/packages/ui/src/views/canvas/index.jsx index 58c8ed7..cfb8739 100644 --- a/studio-frontend/packages/ui/src/views/canvas/index.jsx +++ b/studio-frontend/packages/ui/src/views/canvas/index.jsx @@ -232,15 +232,22 @@ const Canvas = () => { console.log (keycloak?.tokenParsed) if (!chatflow.id) { + // Determine type: use duplicated flow type if available, otherwise use current canvas type + const chatflowType = duplicatedFlowType || + (isAgentCanvas ? 'MULTIAGENT' : isOpeaCanvas ? 'OPEA' : 'CHATFLOW') + const newChatflowBody = { name: chatflowName, deployed: false, isPublic: false, flowData, userid: keycloak?.tokenParsed?.email ? keycloak.tokenParsed.email : '', - type: isAgentCanvas ? 'MULTIAGENT' : isOpeaCanvas ? 'OPEA' : 'CHATFLOW' + type: chatflowType } createNewChatflowApi.request(newChatflowBody) + + // Clear the duplicated flow type after using it + setDuplicatedFlowType(null) } else { const updateBody = { name: chatflowName, @@ -468,6 +475,9 @@ const Canvas = () => { // eslint-disable-next-line react-hooks/exhaustive-deps }, [canvasDataStore.chatflow]) + // Store duplicated flow type to be used when saving + const [duplicatedFlowType, setDuplicatedFlowType] = useState(null) + // Initialization useEffect(() => { setIsSyncNodesButtonEnabled(false) @@ -475,8 +485,22 @@ const Canvas = () => { if (chatflowId) { getSpecificChatflowApi.request(chatflowId) } else { - if (localStorage.getItem('duplicatedFlowData')) { - handleLoadFlow(localStorage.getItem('duplicatedFlowData')) + const duplicatedData = localStorage.getItem('duplicatedFlowData') + if (duplicatedData) { + try { + // Try to parse as new format (with type information) + const parsedData = JSON.parse(duplicatedData) + if (parsedData && parsedData.flowData && parsedData.type) { + handleLoadFlow(parsedData.flowData) + setDuplicatedFlowType(parsedData.type) + } else { + // Fallback to old format (just flow data) + handleLoadFlow(duplicatedData) + } + } catch (e) { + // If JSON parsing fails, treat as old format + handleLoadFlow(duplicatedData) + } setTimeout(() => localStorage.removeItem('duplicatedFlowData'), 0) } else { setNodes([]) From c781b585142f9ac440282ed8847a3ddf11046f4b Mon Sep 17 00:00:00 2001 From: wwanarif Date: Fri, 25 Jul 2025 11:06:50 +0000 Subject: [PATCH 4/4] fix genai-studio ansible to include create-ssh-secrets playbook Signed-off-by: wwanarif --- setup-scripts/setup-genai-studio/genai-studio.yml | 3 +++ setup-scripts/setup-genai-studio/helm-values/mysqldb.yaml | 3 +++ .../setup-genai-studio/playbooks/create-ssh-secrets.yml | 3 +++ 3 files changed, 9 insertions(+) diff --git a/setup-scripts/setup-genai-studio/genai-studio.yml b/setup-scripts/setup-genai-studio/genai-studio.yml index fac546c..10fdc80 100644 --- a/setup-scripts/setup-genai-studio/genai-studio.yml +++ b/setup-scripts/setup-genai-studio/genai-studio.yml @@ -1,3 +1,6 @@ +- name: Create ssh secrets + import_playbook: playbooks/create-ssh-secrets.yml + - name: Deploy mysqldb import_playbook: playbooks/deploy-mysqldb.yml diff --git a/setup-scripts/setup-genai-studio/helm-values/mysqldb.yaml b/setup-scripts/setup-genai-studio/helm-values/mysqldb.yaml index 7763537..1a0e46b 100644 --- a/setup-scripts/setup-genai-studio/helm-values/mysqldb.yaml +++ b/setup-scripts/setup-genai-studio/helm-values/mysqldb.yaml @@ -1,6 +1,9 @@ auth: rootPassword: root +image: + tag: "8.0" + primary: extraFlags: "--innodb-use-native-aio=0" diff --git a/setup-scripts/setup-genai-studio/playbooks/create-ssh-secrets.yml b/setup-scripts/setup-genai-studio/playbooks/create-ssh-secrets.yml index 4bf2c8a..a4532e5 100644 --- a/setup-scripts/setup-genai-studio/playbooks/create-ssh-secrets.yml +++ b/setup-scripts/setup-genai-studio/playbooks/create-ssh-secrets.yml @@ -3,6 +3,9 @@ tasks: + - name: Ensure namespace studio exists + shell: kubectl get ns studio || kubectl create ns studio + - name: Check if Kubernetes secret exists command: kubectl -n studio get secret ssh-keys register: kubectl_secret_check