Implement store backend + ee executor router

TracecatHQ · Dec 21, 2024 · 3e34155 · 3e34155
1 parent fed116a
commit 3e34155
Show file tree

Hide file tree

Showing 32 changed files with 1,706 additions and 148 deletions.
diff --git a/.env.example b/.env.example
@@ -131,3 +131,12 @@ TRACECAT__PRELOAD_OSS_MODELS=
 # Ollama settings
 OLLAMA__VERSION=0.3.12
 OLLAMA__API_URL=http://ollama:11434
+
+# --- Object store ---
+# One of 'memory', 'store'
+TRACECAT__RESULTS_BACKEND=memory
+# Minio
+MINIO_ENDPOINT_URL=http://minio:9000
+MINIO_ACCESS_KEY=
+MINIO_SECRET_KEY=
+MINIO_BUCKET=tracecat
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -56,6 +56,11 @@ services:
       # AI
       TRACECAT__PRELOAD_OSS_MODELS: ${TRACECAT__PRELOAD_OSS_MODELS}
       OLLAMA__API_URL: ${OLLAMA__API_URL}
+      # Object store
+      TRACECAT__RESULTS_BACKEND: ${TRACECAT__RESULTS_BACKEND}
+      MINIO_ENDPOINT_URL: ${MINIO_ENDPOINT_URL}
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
     volumes:
       - ./tracecat:/app/tracecat
       - ./registry:/app/registry
@@ -84,6 +89,11 @@ services:
       # Temporal
       TEMPORAL__CLUSTER_URL: ${TEMPORAL__CLUSTER_URL}
       TEMPORAL__CLUSTER_QUEUE: ${TEMPORAL__CLUSTER_QUEUE}
+      # Minio
+      TRACECAT__RESULTS_BACKEND: ${TRACECAT__RESULTS_BACKEND}
+      MINIO_ENDPOINT_URL: ${MINIO_ENDPOINT_URL}
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
     volumes:
       - ./tracecat:/app/tracecat
       - ./registry:/app/registry
@@ -110,6 +120,11 @@ services:
       # AI
       TRACECAT__PRELOAD_OSS_MODELS: ${TRACECAT__PRELOAD_OSS_MODELS}
       OLLAMA__API_URL: ${OLLAMA__API_URL}
+      # Object store
+      TRACECAT__RESULTS_BACKEND: ${TRACECAT__RESULTS_BACKEND}
+      MINIO_ENDPOINT_URL: ${MINIO_ENDPOINT_URL}
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
     volumes:
       - ./tracecat:/app/tracecat
     ports:
@@ -218,7 +233,22 @@ services:
     volumes:
       - ollama:/root/.ollama
 
+  minio:
+    image: minio/minio:latest
+    container_name: minio
+    restart: unless-stopped
+    ports:
+      - 9000:9000 # API port
+      - 9001:9001 # Console port
+    environment:
+      MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minio}
+      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minio123}
+    volumes:
+      - minio-data:/data
+    command: server /data --console-address ":9001"
+
 volumes:
   core-db:
   temporal-db:
   ollama:
+  minio-data:
diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts
@@ -508,13 +508,12 @@ export const $CreateWorkflowExecutionParams = {
         },
         inputs: {
             anyOf: [
-                {
-                    '$ref': '#/components/schemas/JsonValue'
-                },
+                {},
                 {
                     type: 'null'
                 }
-            ]
+            ],
+            title: 'Inputs'
         }
     },
     type: 'object',
@@ -755,13 +754,12 @@ export const $DSLRunArgs = {
         },
         trigger_inputs: {
             anyOf: [
-                {
-                    '$ref': '#/components/schemas/JsonValue'
-                },
+                {},
                 {
                     type: 'null'
                 }
-            ]
+            ],
+            title: 'Trigger Inputs'
         },
         parent_run_context: {
             anyOf: [
@@ -1225,8 +1223,6 @@ export const $JoinStrategy = {
     title: 'JoinStrategy'
 } as const;
 
-export const $JsonValue = {} as const;
-
 export const $OAuth2AuthorizeResponse = {
     properties: {
         authorization_url: {

diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts
@@ -160,7 +160,7 @@ export type status = 'success' | 'failure';
 
 export type CreateWorkflowExecutionParams = {
     workflow_id: string;
-    inputs?: JsonValue | null;
+    inputs?: unknown | null;
 };
 
 export type CreateWorkflowExecutionResponse = {
@@ -268,7 +268,7 @@ export type DSLRunArgs = {
     role: Role;
     dsl?: DSLInput | null;
     wf_id: string;
-    trigger_inputs?: JsonValue | null;
+    trigger_inputs?: unknown | null;
     parent_run_context?: RunContext | null;
     /**
      * Runtime configuration that can be set on workflow entry. Note that this can override the default config in DSLInput.
@@ -383,8 +383,6 @@ export type HTTPValidationError = {
 
 export type JoinStrategy = 'any' | 'all';
 
-export type JsonValue = unknown;
-
 export type OAuth2AuthorizeResponse = {
     authorization_url: string;
 };
@@ -1199,9 +1197,7 @@ export type PublicIncomingWebhookWaitData = {
     secret: string;
 };
 
-export type PublicIncomingWebhookWaitResponse = {
-    [key: string]: unknown;
-};
+export type PublicIncomingWebhookWaitResponse = unknown;
 
 export type WorkspacesListWorkspacesResponse = Array<WorkspaceMetadataResponse>;
 
@@ -1770,9 +1766,7 @@ export type $OpenApiTs = {
                 /**
                  * Successful Response
                  */
-                200: {
-                    [key: string]: unknown;
-                };
+                200: unknown;
                 /**
                  * Validation Error
                  */

diff --git a/pyproject.toml b/pyproject.toml
@@ -72,6 +72,7 @@ dev = [
     "pytest==8.3.2",
     "python-dotenv",
     "respx",
+    "moto[server]==5.0.24",
 ]
 
 [tool.hatch.version]

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -13,6 +13,7 @@
 from tracecat.contexts import ctx_role
 from tracecat.db.engine import get_async_engine, get_async_session_context_manager
 from tracecat.db.schemas import User
+from tracecat.executor.enums import ResultsBackend
 from tracecat.logger import logger
 from tracecat.registry.repositories.models import RegistryRepositoryCreate
 from tracecat.registry.repositories.service import RegistryReposService
@@ -63,6 +64,8 @@ def env_sandbox(monkeysession: pytest.MonkeyPatch):
     )
     # Need this for local unit tests
     monkeysession.setattr(config, "TRACECAT__EXECUTOR_URL", "http://localhost:8001")
+    # Use memory backend for local unit tests
+    monkeysession.setattr(config, "TRACECAT__RESULTS_BACKEND", ResultsBackend.MEMORY)
 
     monkeysession.setenv(
         "TRACECAT__DB_URI",