Support basic semantic search with preset workflow template (#121) (#122

) Signed-off-by: Tyler Ohlsen <[email protected]> (cherry picked from commit d5ba601) Co-authored-by: Tyler Ohlsen <[email protected]>
opensearch-project · Apr 3, 2024 · 8184c39 · 8184c39
1 parent f0cdfc6
commit 8184c39
Show file tree

Hide file tree

Showing 34 changed files with 588 additions and 203 deletions.
diff --git a/common/constants.ts b/common/constants.ts
@@ -6,13 +6,20 @@
 export const PLUGIN_ID = 'flow-framework';
 
 /**
- * BACKEND/CLUSTER APIs
+ * BACKEND FLOW FRAMEWORK APIs
  */
 export const FLOW_FRAMEWORK_API_ROUTE_PREFIX = '/_plugins/_flow_framework';
 export const FLOW_FRAMEWORK_WORKFLOW_ROUTE_PREFIX = `${FLOW_FRAMEWORK_API_ROUTE_PREFIX}/workflow`;
 export const FLOW_FRAMEWORK_SEARCH_WORKFLOWS_ROUTE = `${FLOW_FRAMEWORK_WORKFLOW_ROUTE_PREFIX}/_search`;
 export const FLOW_FRAMEWORK_SEARCH_WORKFLOW_STATE_ROUTE = `${FLOW_FRAMEWORK_WORKFLOW_ROUTE_PREFIX}/state/_search`;
 
+/**
+ * BACKEND ML PLUGIN APIs
+ */
+export const ML_API_ROUTE_PREFIX = '/_plugins/_ml';
+export const ML_MODEL_ROUTE_PREFIX = `${ML_API_ROUTE_PREFIX}/models`;
+export const ML_SEARCH_MODELS_ROUTE = `${ML_MODEL_ROUTE_PREFIX}/_search`;
+
 /**
  * NODE APIs
  */
@@ -31,11 +38,16 @@ export const CREATE_WORKFLOW_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/cre
 export const DELETE_WORKFLOW_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/delete`;
 export const GET_PRESET_WORKFLOWS_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/presets`;
 
+// ML Plugin node APIs
+export const BASE_MODEL_NODE_API_PATH = `${BASE_NODE_API_PATH}/model`;
+export const SEARCH_MODELS_NODE_API_PATH = `${BASE_MODEL_NODE_API_PATH}/search`;
+
 /**
  * MISCELLANEOUS
  */
 export const NEW_WORKFLOW_ID_URL = 'new';
 export const START_FROM_SCRATCH_WORKFLOW_NAME = 'Start From Scratch';
 export const DEFAULT_NEW_WORKFLOW_NAME = 'new_workflow';
+export const DEFAULT_NEW_WORKFLOW_DESCRIPTION = 'My new workflow';
 export const DATE_FORMAT_PATTERN = 'MM/DD/YY hh:mm A';
 export const EMPTY_FIELD_STRING = '--';
diff --git a/common/interfaces.ts b/common/interfaces.ts
@@ -28,6 +28,10 @@ type ReactFlowViewport = {
   zoom: number;
 };
 
+export type UIState = {
+  workspaceFlow: WorkspaceFlowState;
+};
+
 export type WorkspaceFlowState = {
   nodes: ReactFlowComponent[];
   edges: ReactFlowEdge[];
@@ -51,7 +55,8 @@ export type TemplateEdge = {
 };
 
 export type TemplateFlow = {
-  user_params?: Map<string, any>;
+  user_inputs?: Map<string, any>;
+  previous_node_inputs?: Map<string, any>;
   nodes: TemplateNode[];
   edges?: TemplateEdge[];
 };
@@ -69,14 +74,14 @@ export type WorkflowTemplate = {
   // https://github.com/opensearch-project/flow-framework/issues/526
   version: any;
   workflows: TemplateFlows;
+  // UI state and any ReactFlow state may not exist if a workflow is created via API/backend-only.
+  ui_metadata?: UIState;
 };
 
 // An instance of a workflow based on a workflow template
 export type Workflow = WorkflowTemplate & {
   // won't exist until created in backend
   id?: string;
-  // ReactFlow state may not exist if a workflow is created via API/backend-only.
-  workspaceFlowState?: WorkspaceFlowState;
   // won't exist until created in backend
   lastUpdated?: number;
   // won't exist until launched/provisioned in backend
@@ -89,6 +94,14 @@ export enum USE_CASE {
   PROVISION = 'PROVISION',
 }
 
+/**
+ ********** ML PLUGIN TYPES/INTERFACES **********
+ */
+export type Model = {
+  id: string;
+  algorithm: string;
+};
+
 /**
  ********** MISC TYPES/INTERFACES ************
  */
@@ -111,3 +124,7 @@ export enum WORKFLOW_STATE {
 export type WorkflowDict = {
   [workflowId: string]: Workflow;
 };
+
+export type ModelDict = {
+  [modelId: string]: Model;
+};
diff --git a/common/utils.ts b/common/utils.ts
@@ -4,6 +4,7 @@
  */
 
 import moment from 'moment';
+import { MarkerType } from 'reactflow';
 import {
   WorkspaceFlowState,
   ReactFlowComponent,
@@ -17,21 +18,88 @@ import {
   DATE_FORMAT_PATTERN,
   COMPONENT_CATEGORY,
   NODE_CATEGORY,
+  WorkspaceFormValues,
 } from './';
 
 // TODO: implement this and remove hardcoded return values
 /**
- * Converts a ReactFlow workspace flow to a backend-compatible set of ingest and/or search sub-workflows,
- * along with a provision sub-workflow if resources are to be created.
+ * Given a ReactFlow workspace flow and the set of current form values within such flow,
+ * generate a backend-compatible set of sub-workflows.
+ *
  */
 export function toTemplateFlows(
-  workspaceFlow: WorkspaceFlowState
+  workspaceFlow: WorkspaceFlowState,
+  formValues: WorkspaceFormValues
 ): TemplateFlows {
+  const textEmbeddingTransformerNodeId = Object.keys(formValues).find((key) =>
+    key.includes('text_embedding')
+  ) as string;
+  const knnIndexerNodeId = Object.keys(formValues).find((key) =>
+    key.includes('knn')
+  ) as string;
+  const textEmbeddingFields = formValues[textEmbeddingTransformerNodeId];
+  const knnIndexerFields = formValues[knnIndexerNodeId];
+
   return {
     provision: {
-      user_params: {} as Map<string, any>,
-      nodes: [],
-      edges: [],
+      nodes: [
+        {
+          id: 'create_ingest_pipeline',
+          type: 'create_ingest_pipeline',
+          user_inputs: {
+            pipeline_id: 'test-pipeline',
+            model_id: textEmbeddingFields['modelId'],
+            input_field: textEmbeddingFields['inputField'],
+            output_field: textEmbeddingFields['vectorField'],
+            configurations: {
+              description: 'A text embedding ingest pipeline',
+              processors: [
+                {
+                  text_embedding: {
+                    model_id: textEmbeddingFields['modelId'],
+                    field_map: {
+                      [textEmbeddingFields['inputField']]:
+                        textEmbeddingFields['vectorField'],
+                    },
+                  },
+                },
+              ],
+            },
+          },
+        },
+        {
+          id: 'create_index',
+          type: 'create_index',
+          previous_node_inputs: {
+            create_ingest_pipeline: 'pipeline_id',
+          },
+          user_inputs: {
+            index_name: knnIndexerFields['indexName'],
+            configurations: {
+              settings: {
+                default_pipeline: '${{create_ingest_pipeline.pipeline_id}}',
+              },
+              mappings: {
+                properties: {
+                  [textEmbeddingFields['vectorField']]: {
+                    type: 'knn_vector',
+                    dimension: 768,
+                    method: {
+                      engine: 'lucene',
+                      space_type: 'l2',
+                      name: 'hnsw',
+                      parameters: {},
+                    },
+                  },
+                  [textEmbeddingFields['inputField']]: {
+                    type: 'text',
+                  },
+                },
+              },
+            },
+          },
+        },
+      ],
     },
   };
 }
@@ -47,10 +115,8 @@ export function toWorkspaceFlow(
   const ingestId1 = generateId('text_embedding_processor');
   const ingestId2 = generateId('knn_index');
   const ingestGroupId = generateId(COMPONENT_CATEGORY.INGEST);
-
-  const searchId1 = generateId('text_embedding_processor');
-  const searchId2 = generateId('knn_index');
   const searchGroupId = generateId(COMPONENT_CATEGORY.SEARCH);
+  const edgeId = generateId('edge');
 
   const ingestNodes = [
     {
@@ -61,11 +127,10 @@ export function toWorkspaceFlow(
       style: {
         width: 900,
         height: 400,
-        overflowX: 'auto',
-        overflowY: 'auto',
       },
       className: 'reactflow__group-node__ingest',
       selectable: true,
+      deletable: false,
     },
     {
       id: ingestId1,
@@ -78,6 +143,7 @@ export function toWorkspaceFlow(
       parentNode: ingestGroupId,
       extent: 'parent',
       draggable: true,
+      deletable: false,
     },
     {
       id: ingestId2,
@@ -87,6 +153,7 @@ export function toWorkspaceFlow(
       parentNode: ingestGroupId,
       extent: 'parent',
       draggable: true,
+      deletable: false,
     },
   ] as ReactFlowComponent[];
 
@@ -99,38 +166,30 @@ export function toWorkspaceFlow(
       style: {
         width: 900,
         height: 400,
-        overflowX: 'auto',
-        overflowY: 'auto',
       },
       className: 'reactflow__group-node__search',
       selectable: true,
-    },
-    {
-      id: searchId1,
-      position: { x: 100, y: 70 },
-      data: initComponentData(
-        new TextEmbeddingTransformer().toObj(),
-        searchId1
-      ),
-      type: NODE_CATEGORY.CUSTOM,
-      parentNode: searchGroupId,
-      extent: 'parent',
-      draggable: true,
-    },
-    {
-      id: searchId2,
-      position: { x: 500, y: 70 },
-      data: initComponentData(new KnnIndexer().toObj(), searchId2),
-      type: NODE_CATEGORY.CUSTOM,
-      parentNode: searchGroupId,
-      extent: 'parent',
-      draggable: true,
+      deletable: false,
     },
   ] as ReactFlowComponent[];
 
   return {
     nodes: [...ingestNodes, ...searchNodes],
-    edges: [] as ReactFlowEdge[],
+    edges: [
+      {
+        id: edgeId,
+        key: edgeId,
+        source: ingestId1,
+        target: ingestId2,
+        markerEnd: {
+          type: MarkerType.ArrowClosed,
+          width: 20,
+          height: 20,
+        },
+        zIndex: 2,
+        deletable: false,
+      },
+    ] as ReactFlowEdge[],
   };
 }
 

diff --git a/public/component_types/indexer/indexer.ts b/public/component_types/indexer/indexer.ts
@@ -25,7 +25,6 @@ export class Indexer extends BaseComponent {
         // TODO: may need to change to be looser. it should be able to take
         // in other component types
         baseClass: COMPONENT_CLASS.TRANSFORMER,
-        optional: false,
         acceptMultiple: false,
       },
     ];
@@ -34,32 +33,27 @@ export class Indexer extends BaseComponent {
         label: 'Index Name',
         name: 'indexName',
         type: 'select',
-        optional: false,
-        advanced: false,
       },
     ];
     this.createFields = [
       {
         label: 'Index Name',
         name: 'indexName',
         type: 'string',
-        optional: false,
-        advanced: false,
-      },
-      {
-        label: 'Mappings',
-        name: 'indexMappings',
-        type: 'json',
-        placeholder: 'Enter an index mappings JSON blob...',
-        optional: false,
-        advanced: false,
-      },
-    ];
-    this.outputs = [
-      {
-        label: this.label,
-        baseClasses: this.baseClasses,
       },
+      // {
+      //   label: 'Mappings',
+      //   name: 'indexMappings',
+      //   type: 'json',
+      //   placeholder: 'Enter an index mappings JSON blob...',
+      // },
     ];
+    // this.outputs = [
+    //   {
+    //     label: this.label,
+    //     baseClasses: this.baseClasses,
+    //   },
+    // ];
+    this.outputs = [];
   }
 }
diff --git a/public/component_types/indexer/knn_indexer.ts b/public/component_types/indexer/knn_indexer.ts
@@ -17,14 +17,12 @@ export class KnnIndexer extends Indexer {
       // @ts-ignore
       ...this.createFields,
       // TODO: finalize what to expose / what to have for defaults here
-      {
-        label: 'K-NN Settings',
-        name: 'knnSettings',
-        type: 'json',
-        placeholder: 'Enter K-NN settings JSON blob...',
-        optional: false,
-        advanced: false,
-      },
+      // {
+      //   label: 'K-NN Settings',
+      //   name: 'knnSettings',
+      //   type: 'json',
+      //   placeholder: 'Enter K-NN settings JSON blob...',
+      // },
     ];
   }
 }