diff --git a/common/constants.ts b/common/constants.ts index eb10828f..f82a1167 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -6,13 +6,20 @@ export const PLUGIN_ID = 'flow-framework'; /** - * BACKEND/CLUSTER APIs + * BACKEND FLOW FRAMEWORK APIs */ export const FLOW_FRAMEWORK_API_ROUTE_PREFIX = '/_plugins/_flow_framework'; export const FLOW_FRAMEWORK_WORKFLOW_ROUTE_PREFIX = `${FLOW_FRAMEWORK_API_ROUTE_PREFIX}/workflow`; export const FLOW_FRAMEWORK_SEARCH_WORKFLOWS_ROUTE = `${FLOW_FRAMEWORK_WORKFLOW_ROUTE_PREFIX}/_search`; export const FLOW_FRAMEWORK_SEARCH_WORKFLOW_STATE_ROUTE = `${FLOW_FRAMEWORK_WORKFLOW_ROUTE_PREFIX}/state/_search`; +/** + * BACKEND ML PLUGIN APIs + */ +export const ML_API_ROUTE_PREFIX = '/_plugins/_ml'; +export const ML_MODEL_ROUTE_PREFIX = `${ML_API_ROUTE_PREFIX}/models`; +export const ML_SEARCH_MODELS_ROUTE = `${ML_MODEL_ROUTE_PREFIX}/_search`; + /** * NODE APIs */ @@ -31,11 +38,16 @@ export const CREATE_WORKFLOW_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/cre export const DELETE_WORKFLOW_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/delete`; export const GET_PRESET_WORKFLOWS_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/presets`; +// ML Plugin node APIs +export const BASE_MODEL_NODE_API_PATH = `${BASE_NODE_API_PATH}/model`; +export const SEARCH_MODELS_NODE_API_PATH = `${BASE_MODEL_NODE_API_PATH}/search`; + /** * MISCELLANEOUS */ export const NEW_WORKFLOW_ID_URL = 'new'; export const START_FROM_SCRATCH_WORKFLOW_NAME = 'Start From Scratch'; export const DEFAULT_NEW_WORKFLOW_NAME = 'new_workflow'; +export const DEFAULT_NEW_WORKFLOW_DESCRIPTION = 'My new workflow'; export const DATE_FORMAT_PATTERN = 'MM/DD/YY hh:mm A'; export const EMPTY_FIELD_STRING = '--'; diff --git a/common/interfaces.ts b/common/interfaces.ts index b559d47e..eb412f1e 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -28,6 +28,10 @@ type ReactFlowViewport = { zoom: number; }; +export type UIState = { + workspaceFlow: WorkspaceFlowState; +}; + export type WorkspaceFlowState = { nodes: ReactFlowComponent[]; edges: ReactFlowEdge[]; @@ -51,7 +55,8 @@ export type TemplateEdge = { }; export type TemplateFlow = { - user_params?: Map; + user_inputs?: Map; + previous_node_inputs?: Map; nodes: TemplateNode[]; edges?: TemplateEdge[]; }; @@ -69,14 +74,14 @@ export type WorkflowTemplate = { // https://github.com/opensearch-project/flow-framework/issues/526 version: any; workflows: TemplateFlows; + // UI state and any ReactFlow state may not exist if a workflow is created via API/backend-only. + ui_metadata?: UIState; }; // An instance of a workflow based on a workflow template export type Workflow = WorkflowTemplate & { // won't exist until created in backend id?: string; - // ReactFlow state may not exist if a workflow is created via API/backend-only. - workspaceFlowState?: WorkspaceFlowState; // won't exist until created in backend lastUpdated?: number; // won't exist until launched/provisioned in backend @@ -89,6 +94,14 @@ export enum USE_CASE { PROVISION = 'PROVISION', } +/** + ********** ML PLUGIN TYPES/INTERFACES ********** + */ +export type Model = { + id: string; + algorithm: string; +}; + /** ********** MISC TYPES/INTERFACES ************ */ @@ -111,3 +124,7 @@ export enum WORKFLOW_STATE { export type WorkflowDict = { [workflowId: string]: Workflow; }; + +export type ModelDict = { + [modelId: string]: Model; +}; diff --git a/common/utils.ts b/common/utils.ts index 2e5bf2d9..0a01b684 100644 --- a/common/utils.ts +++ b/common/utils.ts @@ -4,6 +4,7 @@ */ import moment from 'moment'; +import { MarkerType } from 'reactflow'; import { WorkspaceFlowState, ReactFlowComponent, @@ -17,21 +18,88 @@ import { DATE_FORMAT_PATTERN, COMPONENT_CATEGORY, NODE_CATEGORY, + WorkspaceFormValues, } from './'; // TODO: implement this and remove hardcoded return values /** - * Converts a ReactFlow workspace flow to a backend-compatible set of ingest and/or search sub-workflows, - * along with a provision sub-workflow if resources are to be created. + * Given a ReactFlow workspace flow and the set of current form values within such flow, + * generate a backend-compatible set of sub-workflows. + * */ export function toTemplateFlows( - workspaceFlow: WorkspaceFlowState + workspaceFlow: WorkspaceFlowState, + formValues: WorkspaceFormValues ): TemplateFlows { + const textEmbeddingTransformerNodeId = Object.keys(formValues).find((key) => + key.includes('text_embedding') + ) as string; + const knnIndexerNodeId = Object.keys(formValues).find((key) => + key.includes('knn') + ) as string; + const textEmbeddingFields = formValues[textEmbeddingTransformerNodeId]; + const knnIndexerFields = formValues[knnIndexerNodeId]; + return { provision: { - user_params: {} as Map, - nodes: [], - edges: [], + nodes: [ + { + id: 'create_ingest_pipeline', + type: 'create_ingest_pipeline', + user_inputs: { + pipeline_id: 'test-pipeline', + model_id: textEmbeddingFields['modelId'], + input_field: textEmbeddingFields['inputField'], + output_field: textEmbeddingFields['vectorField'], + configurations: { + description: 'A text embedding ingest pipeline', + processors: [ + { + text_embedding: { + model_id: textEmbeddingFields['modelId'], + field_map: { + [textEmbeddingFields['inputField']]: + textEmbeddingFields['vectorField'], + }, + }, + }, + ], + }, + }, + }, + { + id: 'create_index', + type: 'create_index', + previous_node_inputs: { + create_ingest_pipeline: 'pipeline_id', + }, + user_inputs: { + index_name: knnIndexerFields['indexName'], + configurations: { + settings: { + default_pipeline: '${{create_ingest_pipeline.pipeline_id}}', + }, + mappings: { + properties: { + [textEmbeddingFields['vectorField']]: { + type: 'knn_vector', + dimension: 768, + method: { + engine: 'lucene', + space_type: 'l2', + name: 'hnsw', + parameters: {}, + }, + }, + [textEmbeddingFields['inputField']]: { + type: 'text', + }, + }, + }, + }, + }, + }, + ], }, }; } @@ -47,10 +115,8 @@ export function toWorkspaceFlow( const ingestId1 = generateId('text_embedding_processor'); const ingestId2 = generateId('knn_index'); const ingestGroupId = generateId(COMPONENT_CATEGORY.INGEST); - - const searchId1 = generateId('text_embedding_processor'); - const searchId2 = generateId('knn_index'); const searchGroupId = generateId(COMPONENT_CATEGORY.SEARCH); + const edgeId = generateId('edge'); const ingestNodes = [ { @@ -61,11 +127,10 @@ export function toWorkspaceFlow( style: { width: 900, height: 400, - overflowX: 'auto', - overflowY: 'auto', }, className: 'reactflow__group-node__ingest', selectable: true, + deletable: false, }, { id: ingestId1, @@ -78,6 +143,7 @@ export function toWorkspaceFlow( parentNode: ingestGroupId, extent: 'parent', draggable: true, + deletable: false, }, { id: ingestId2, @@ -87,6 +153,7 @@ export function toWorkspaceFlow( parentNode: ingestGroupId, extent: 'parent', draggable: true, + deletable: false, }, ] as ReactFlowComponent[]; @@ -99,38 +166,30 @@ export function toWorkspaceFlow( style: { width: 900, height: 400, - overflowX: 'auto', - overflowY: 'auto', }, className: 'reactflow__group-node__search', selectable: true, - }, - { - id: searchId1, - position: { x: 100, y: 70 }, - data: initComponentData( - new TextEmbeddingTransformer().toObj(), - searchId1 - ), - type: NODE_CATEGORY.CUSTOM, - parentNode: searchGroupId, - extent: 'parent', - draggable: true, - }, - { - id: searchId2, - position: { x: 500, y: 70 }, - data: initComponentData(new KnnIndexer().toObj(), searchId2), - type: NODE_CATEGORY.CUSTOM, - parentNode: searchGroupId, - extent: 'parent', - draggable: true, + deletable: false, }, ] as ReactFlowComponent[]; return { nodes: [...ingestNodes, ...searchNodes], - edges: [] as ReactFlowEdge[], + edges: [ + { + id: edgeId, + key: edgeId, + source: ingestId1, + target: ingestId2, + markerEnd: { + type: MarkerType.ArrowClosed, + width: 20, + height: 20, + }, + zIndex: 2, + deletable: false, + }, + ] as ReactFlowEdge[], }; } diff --git a/public/component_types/indexer/indexer.ts b/public/component_types/indexer/indexer.ts index e87a81b0..781b1f27 100644 --- a/public/component_types/indexer/indexer.ts +++ b/public/component_types/indexer/indexer.ts @@ -25,7 +25,6 @@ export class Indexer extends BaseComponent { // TODO: may need to change to be looser. it should be able to take // in other component types baseClass: COMPONENT_CLASS.TRANSFORMER, - optional: false, acceptMultiple: false, }, ]; @@ -34,8 +33,6 @@ export class Indexer extends BaseComponent { label: 'Index Name', name: 'indexName', type: 'select', - optional: false, - advanced: false, }, ]; this.createFields = [ @@ -43,23 +40,20 @@ export class Indexer extends BaseComponent { label: 'Index Name', name: 'indexName', type: 'string', - optional: false, - advanced: false, - }, - { - label: 'Mappings', - name: 'indexMappings', - type: 'json', - placeholder: 'Enter an index mappings JSON blob...', - optional: false, - advanced: false, - }, - ]; - this.outputs = [ - { - label: this.label, - baseClasses: this.baseClasses, }, + // { + // label: 'Mappings', + // name: 'indexMappings', + // type: 'json', + // placeholder: 'Enter an index mappings JSON blob...', + // }, ]; + // this.outputs = [ + // { + // label: this.label, + // baseClasses: this.baseClasses, + // }, + // ]; + this.outputs = []; } } diff --git a/public/component_types/indexer/knn_indexer.ts b/public/component_types/indexer/knn_indexer.ts index 93f55017..65023604 100644 --- a/public/component_types/indexer/knn_indexer.ts +++ b/public/component_types/indexer/knn_indexer.ts @@ -17,14 +17,12 @@ export class KnnIndexer extends Indexer { // @ts-ignore ...this.createFields, // TODO: finalize what to expose / what to have for defaults here - { - label: 'K-NN Settings', - name: 'knnSettings', - type: 'json', - placeholder: 'Enter K-NN settings JSON blob...', - optional: false, - advanced: false, - }, + // { + // label: 'K-NN Settings', + // name: 'knnSettings', + // type: 'json', + // placeholder: 'Enter K-NN settings JSON blob...', + // }, ]; } } diff --git a/public/component_types/interfaces.ts b/public/component_types/interfaces.ts index e9349cd3..94e12952 100644 --- a/public/component_types/interfaces.ts +++ b/public/component_types/interfaces.ts @@ -11,6 +11,7 @@ import { COMPONENT_CATEGORY, COMPONENT_CLASS } from '../utils'; * ************ Types ************************* */ export type FieldType = 'string' | 'json' | 'select'; +export type SelectType = 'model'; // TODO: this may expand to more types in the future. Formik supports 'any' so we can too. // For now, limiting scope to expected types. export type FieldValue = string | {}; @@ -25,20 +26,18 @@ export type WorkspaceSchema = ObjectSchema; /** * Represents a single base class as an input handle for a component. - * It may be optional. It may also accept multiples of that class. + * It may accept multiples of that class. */ export interface IComponentInput { id: string; label: string; baseClass: COMPONENT_CLASS; - optional: boolean; acceptMultiple: boolean; } /** * An input field for a component. Specifies enough configuration for the - * UI node to render it properly within the component (show it as optional, - * put it in advanced settings, placeholder values, etc.) + * UI node to render it properly (help text, links, etc.) */ export interface IComponentField { label: string; @@ -46,8 +45,9 @@ export interface IComponentField { name: string; value?: FieldValue; placeholder?: string; - optional?: boolean; - advanced?: boolean; + helpText?: string; + helpLink?: string; + selectType?: SelectType; } /** diff --git a/public/component_types/transformer/text_embedding_transformer.ts b/public/component_types/transformer/text_embedding_transformer.ts index dbdce094..28e07582 100644 --- a/public/component_types/transformer/text_embedding_transformer.ts +++ b/public/component_types/transformer/text_embedding_transformer.ts @@ -14,27 +14,34 @@ export class TextEmbeddingTransformer extends MLTransformer { this.label = 'Text Embedding Transformer'; this.description = 'A specialized ML transformer for embedding text'; this.inputs = []; - this.fields = [ + this.createFields = [ { label: 'Model ID', name: 'modelId', - type: 'string', - optional: false, - advanced: false, + type: 'select', + selectType: 'model', + helpText: 'The deployed text embedding model to use for embedding.', + helpLink: + 'https://opensearch.org/docs/latest/ml-commons-plugin/integrating-ml-models/#choosing-a-model', }, { label: 'Input Field', name: 'inputField', type: 'string', - optional: false, - advanced: false, + helpText: + 'The name of the field from which to obtain text for generating text embeddings.', + helpLink: + 'https://opensearch.org/docs/latest/ingest-pipelines/processors/text-embedding/', }, + { - label: 'Output Field', - name: 'outputField', + label: 'Vector Field', + name: 'vectorField', type: 'string', - optional: false, - advanced: false, + helpText: + ' The name of the vector field in which to store the generated text embeddings.', + helpLink: + 'https://opensearch.org/docs/latest/ingest-pipelines/processors/text-embedding/', }, ]; this.outputs = [ diff --git a/public/pages/workflow_detail/component_details/component_inputs.tsx b/public/pages/workflow_detail/component_details/component_inputs.tsx index 3d9f076e..6e8ad9b2 100644 --- a/public/pages/workflow_detail/component_details/component_inputs.tsx +++ b/public/pages/workflow_detail/component_details/component_inputs.tsx @@ -3,10 +3,11 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React from 'react'; +import React, { useState } from 'react'; import { EuiSpacer, EuiText, EuiTitle } from '@elastic/eui'; import { InputFieldList } from './input_field_list'; import { NODE_CATEGORY, ReactFlowComponent } from '../../../../common'; +import { NewOrExistingTabs } from '../workspace/workspace_components/new_or_existing_tabs'; interface ComponentInputsProps { selectedComponent: ReactFlowComponent; @@ -14,6 +15,13 @@ interface ComponentInputsProps { } export function ComponentInputs(props: ComponentInputsProps) { + // Tab state + enum TAB { + NEW = 'new', + EXISTING = 'existing', + } + const [selectedTabId, setSelectedTabId] = useState(TAB.NEW); + // Have custom layouts for parent/group flows if (props.selectedComponent.type === NODE_CATEGORY.INGEST_GROUP) { return ( @@ -47,9 +55,19 @@ export function ComponentInputs(props: ComponentInputsProps) {

{props.selectedComponent.data.label || ''}

 + + diff --git a/public/pages/workflow_detail/component_details/input_field_list.tsx b/public/pages/workflow_detail/component_details/input_field_list.tsx index f7aed9d8..ca13d59c 100644 --- a/public/pages/workflow_detail/component_details/input_field_list.tsx +++ b/public/pages/workflow_detail/component_details/input_field_list.tsx @@ -6,7 +6,7 @@ import React from 'react'; import { EuiFlexItem, EuiSpacer } from '@elastic/eui'; import { TextField, JsonField, SelectField } from './input_fields'; -import { ReactFlowComponent } from '../../../../common'; +import { IComponentField } from '../../../../common'; /** * A helper component to format all of the input fields for a component. Dynamically @@ -14,12 +14,15 @@ import { ReactFlowComponent } from '../../../../common'; */ interface InputFieldListProps { - selectedComponent: ReactFlowComponent; + componentId: string; + componentFields: IComponentField[] | undefined; onFormChange: () => void; } +const INPUT_FIELD_SPACER_SIZE = 'm'; + export function InputFieldList(props: InputFieldListProps) { - const inputFields = props.selectedComponent.data.fields || []; + const inputFields = props.componentFields || []; return ( {inputFields.map((field, idx) => { @@ -30,10 +33,10 @@ export function InputFieldList(props: InputFieldListProps) { - + ); break; @@ -43,9 +46,10 @@ export function InputFieldList(props: InputFieldListProps) { + ); break; @@ -57,6 +61,7 @@ export function InputFieldList(props: InputFieldListProps) { label={field.label} placeholder={field.placeholder || ''} /> + ); break; diff --git a/public/pages/workflow_detail/component_details/input_fields/select_field.tsx b/public/pages/workflow_detail/component_details/input_fields/select_field.tsx index f76567a8..008ece7f 100644 --- a/public/pages/workflow_detail/component_details/input_fields/select_field.tsx +++ b/public/pages/workflow_detail/component_details/input_fields/select_field.tsx @@ -3,35 +3,23 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React from 'react'; +import React, { useEffect, useState } from 'react'; +import { useSelector } from 'react-redux'; +import { Field, FieldProps, useFormikContext } from 'formik'; import { EuiFormRow, + EuiLink, EuiSuperSelect, EuiSuperSelectOption, EuiText, } from '@elastic/eui'; -import { Field, FieldProps, useFormikContext } from 'formik'; import { IComponentField, WorkspaceFormValues, getInitialValue, isFieldInvalid, } from '../../../../../common'; - -// TODO: Should be fetched from global state. -// Need to have a way to determine where to fetch this dynamic data. -const existingIndices = [ - { - value: 'index-1', - inputDisplay: my-index-1, - disabled: false, - }, - { - value: 'index-2', - inputDisplay: my-index-2, - disabled: false, - }, -] as Array>; +import { AppState } from '../../../../store'; interface SelectFieldProps { field: IComponentField; @@ -44,7 +32,22 @@ interface SelectFieldProps { * options. */ export function SelectField(props: SelectFieldProps) { - const options = existingIndices; + // Redux store state + // Initial store is fetched when loading base page. We don't + // re-fetch here as it could overload client-side if user clicks back and forth / + // keeps re-rendering this component (and subsequently re-fetching data) as they're building flows + const models = useSelector((state: AppState) => state.models.models); + + // Options state + const [options, setOptions] = useState([]); + + // Populate options depending on the select type + useEffect(() => { + if (props.field.selectType === 'model' && models) { + setOptions(Object.keys(models)); + } + }, [models]); + const formField = `${props.componentId}.${props.field.name}`; const { errors, touched } = useFormikContext(); @@ -52,9 +55,28 @@ export function SelectField(props: SelectFieldProps) { {({ field, form }: FieldProps) => { return ( - + + + Learn more + + + ) : undefined + } + helpText={props.field.helpText || undefined} + > + ({ + value: option, + inputDisplay: {option}, + disabled: false, + } as EuiSuperSelectOption) + )} valueOfSelected={field.value || getInitialValue(props.field.type)} onChange={(option) => { form.setFieldValue(formField, option); diff --git a/public/pages/workflow_detail/component_details/input_fields/text_field.tsx b/public/pages/workflow_detail/component_details/input_fields/text_field.tsx index d3866a7e..78ea27ec 100644 --- a/public/pages/workflow_detail/component_details/input_fields/text_field.tsx +++ b/public/pages/workflow_detail/component_details/input_fields/text_field.tsx @@ -5,7 +5,7 @@ import React from 'react'; import { Field, FieldProps, useFormikContext } from 'formik'; -import { EuiFieldText, EuiFormRow } from '@elastic/eui'; +import { EuiFieldText, EuiFormRow, EuiLink, EuiText } from '@elastic/eui'; import { IComponentField, WorkspaceFormValues, @@ -34,6 +34,16 @@ export function TextField(props: TextFieldProps) { + + Learn more + + + ) : undefined + } + helpText={props.field.helpText || undefined} error={getFieldError(props.componentId, props.field.name, errors)} isInvalid={isFieldInvalid( props.componentId, diff --git a/public/pages/workflow_detail/utils/utils.ts b/public/pages/workflow_detail/utils/utils.ts index b3037a1c..5108c519 100644 --- a/public/pages/workflow_detail/utils/utils.ts +++ b/public/pages/workflow_detail/utils/utils.ts @@ -3,15 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { Workflow, ReactFlowComponent } from '../../../../common'; - -export function saveWorkflow(workflow?: Workflow): void { - if (workflow && workflow.id) { - // TODO: implement connection to update workflow API - } else { - // TODO: implement connection to create workflow API - } -} +import { ReactFlowComponent } from '../../../../common'; // Process the raw ReactFlow nodes to only persist the fields we need export function processNodes( diff --git a/public/pages/workflow_detail/workflow_detail.tsx b/public/pages/workflow_detail/workflow_detail.tsx index 5907d75d..43664bf5 100644 --- a/public/pages/workflow_detail/workflow_detail.tsx +++ b/public/pages/workflow_detail/workflow_detail.tsx @@ -12,7 +12,7 @@ import { EuiPage, EuiPageBody } from '@elastic/eui'; import { BREADCRUMBS } from '../../utils'; import { getCore } from '../../services'; import { WorkflowDetailHeader } from './components'; -import { AppState, searchWorkflows } from '../../store'; +import { AppState, searchModels, searchWorkflows } from '../../store'; import { ResizableWorkspace } from './workspace'; import { Launches } from './launches'; import { Prototype } from './prototype'; @@ -97,11 +97,13 @@ export function WorkflowDetail(props: WorkflowDetailProps) { // On initial load: // - fetch workflow, if there is an existing workflow ID + // - fetch available models as their IDs may be used when building flows useEffect(() => { if (!isNewWorkflow) { // TODO: can optimize to only fetch a single workflow dispatch(searchWorkflows({ query: { match_all: {} } })); } + dispatch(searchModels({ query: { match_all: {} } })); }, []); const tabs = [ @@ -137,17 +139,19 @@ export function WorkflowDetail(props: WorkflowDetailProps) { return ( - + {selectedTabId === WORKFLOW_DETAILS_TAB.EDITOR && ( - + + + )} {selectedTabId === WORKFLOW_DETAILS_TAB.LAUNCHES && } {selectedTabId === WORKFLOW_DETAILS_TAB.PROTOTYPE && ( diff --git a/public/pages/workflow_detail/workspace/reactflow-styles.scss b/public/pages/workflow_detail/workspace/reactflow-styles.scss index 900bdc45..c4fc599d 100644 --- a/public/pages/workflow_detail/workspace/reactflow-styles.scss +++ b/public/pages/workflow_detail/workspace/reactflow-styles.scss @@ -25,8 +25,6 @@ $handle-color-invalid: $euiColorDanger; .reactflow__group-node { width: 1200px; height: 700px; - overflow-x: auto; - overflow-y: auto; border: 'none'; &__ingest { diff --git a/public/pages/workflow_detail/workspace/resizable_workspace.tsx b/public/pages/workflow_detail/workspace/resizable_workspace.tsx index 11f93956..b1168228 100644 --- a/public/pages/workflow_detail/workspace/resizable_workspace.tsx +++ b/public/pages/workflow_detail/workspace/resizable_workspace.tsx @@ -5,7 +5,7 @@ import React, { useRef, useState, useEffect } from 'react'; import { useDispatch, useSelector } from 'react-redux'; -import { ReactFlowProvider, useReactFlow } from 'reactflow'; +import { useReactFlow } from 'reactflow'; import { Form, Formik } from 'formik'; import * as yup from 'yup'; import { cloneDeep } from 'lodash'; @@ -29,11 +29,19 @@ import { validateWorkspaceFlow, WorkspaceFlowState, toTemplateFlows, + DEFAULT_NEW_WORKFLOW_NAME, + DEFAULT_NEW_WORKFLOW_DESCRIPTION, + USE_CASE, } from '../../../../common'; -import { AppState, removeDirty, setDirty } from '../../../store'; +import { + AppState, + createWorkflow, + removeDirty, + setDirty, +} from '../../../store'; import { Workspace } from './workspace'; import { ComponentDetails } from '../component_details'; -import { processNodes, saveWorkflow } from '../utils'; +import { processNodes } from '../utils'; // styling import './workspace-styles.scss'; @@ -107,22 +115,35 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { } } - // Hook to update the workflow's flow state, if applicable. It may not exist if - // it is a backend-only-created workflow, or a new, unsaved workflow. If so, - // generate a default one based on the 'workflows' JSON field. + // Hook to update some default values for the workflow, if applicable. Flow state + // may not exist if it is a backend-only-created workflow, or a new, unsaved workflow. + // Metadata fields (name/description/use_case/etc.) may not exist if the user + // cold reloads the page on a new, unsaved workflow. useEffect(() => { - const workflowCopy = { ...props.workflow } as Workflow; - if (workflowCopy) { - if (!workflowCopy.workspaceFlowState) { - workflowCopy.workspaceFlowState = toWorkspaceFlow( - workflowCopy.workflows - ); - console.debug( - `There is no saved UI flow for workflow: ${workflowCopy.name}. Generating a default one.` - ); - } - setWorkflow(workflowCopy); + let workflowCopy = { ...props.workflow } as Workflow; + if (!workflowCopy.ui_metadata || !workflowCopy.ui_metadata.workspaceFlow) { + workflowCopy.ui_metadata = { + ...(workflowCopy.ui_metadata || {}), + workspaceFlow: toWorkspaceFlow(workflowCopy.workflows), + }; + console.debug( + `There is no saved UI flow for workflow: ${workflowCopy.name}. Generating a default one.` + ); } + + // TODO: tune some of the defaults, like use_case and version as these will change + workflowCopy = { + ...workflowCopy, + name: workflowCopy.name || DEFAULT_NEW_WORKFLOW_NAME, + description: workflowCopy.description || DEFAULT_NEW_WORKFLOW_DESCRIPTION, + use_case: workflowCopy.use_case || USE_CASE.PROVISION, + version: workflowCopy.version || { + template: '1.0.0', + compatibility: ['2.12.0', '3.0.0'], + }, + }; + + setWorkflow(workflowCopy); }, [props.workflow]); // Hook to updated the selected ReactFlow component @@ -140,10 +161,10 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { // Initialize the form state to an existing workflow, if applicable. useEffect(() => { - if (workflow?.workspaceFlowState) { + if (workflow?.ui_metadata?.workspaceFlow) { const initFormValues = {} as WorkspaceFormValues; const initSchemaObj = {} as WorkspaceSchemaObj; - workflow.workspaceFlowState.nodes.forEach((node) => { + workflow.ui_metadata.workspaceFlow.nodes.forEach((node) => { initFormValues[node.id] = componentDataToFormik(node.data); initSchemaObj[node.id] = getComponentSchema(node.data); }); @@ -250,7 +271,6 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { setFormValidOnSubmit(false); } else { setFormValidOnSubmit(true); - // @ts-ignore let curFlowState = reactFlowInstance.toObject() as WorkspaceFlowState; curFlowState = { ...curFlowState, @@ -260,10 +280,20 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { setFlowValidOnSubmit(true); const updatedWorkflow = { ...workflow, - workspaceFlowState: curFlowState, - workflows: toTemplateFlows(curFlowState), + ui_metadata: { + ...workflow?.ui_metadata, + workspaceFlow: curFlowState, + }, + workflows: toTemplateFlows( + curFlowState, + formikProps.values + ), } as Workflow; - saveWorkflow(updatedWorkflow); + if (updatedWorkflow.id) { + // TODO: add update workflow API + } else { + dispatch(createWorkflow(updatedWorkflow)); + } } else { setFlowValidOnSubmit(false); } @@ -302,14 +332,12 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { className="workspace-panel" > - - - + diff --git a/public/pages/workflow_detail/workspace/workspace-styles.scss b/public/pages/workflow_detail/workspace/workspace-styles.scss index 0831a31a..e977c728 100644 --- a/public/pages/workflow_detail/workspace/workspace-styles.scss +++ b/public/pages/workflow_detail/workspace/workspace-styles.scss @@ -1,5 +1,5 @@ .workspace-panel { // this ratio will allow the workspace to render on a standard // laptop (3024 x 1964) without introducing overflow/scrolling - height: 60vh; + height: 55vh; } diff --git a/public/pages/workflow_detail/workspace/workspace.tsx b/public/pages/workflow_detail/workspace/workspace.tsx index 72162e66..b5af1b0c 100644 --- a/public/pages/workflow_detail/workspace/workspace.tsx +++ b/public/pages/workflow_detail/workspace/workspace.tsx @@ -15,6 +15,7 @@ import ReactFlow, { useStore, useReactFlow, useOnSelectionChange, + MarkerType, } from 'reactflow'; import { EuiFlexItem, EuiFlexGroup } from '@elastic/eui'; import { setDirty } from '../../../store'; @@ -82,7 +83,19 @@ export function Workspace(props: WorkspaceProps) { ...params, type: 'customEdge', }; - setEdges((eds) => addEdge(edge, eds)); + setEdges((eds) => + addEdge( + { + ...edge, + markerEnd: { + type: MarkerType.ArrowClosed, + width: 20, + height: 20, + }, + }, + eds + ) + ); dispatch(setDirty()); }, [setEdges] @@ -91,9 +104,9 @@ export function Workspace(props: WorkspaceProps) { // Initialization. Set the nodes and edges to an existing workflow state, useEffect(() => { const workflow = { ...props.workflow }; - if (workflow && workflow.workspaceFlowState) { - setNodes(workflow.workspaceFlowState.nodes); - setEdges(workflow.workspaceFlowState.edges); + if (workflow?.ui_metadata?.workspaceFlow) { + setNodes(workflow.ui_metadata.workspaceFlow.nodes); + setEdges(workflow.ui_metadata.workspaceFlow.edges); } }, [props.workflow]); @@ -115,7 +128,8 @@ export function Workspace(props: WorkspaceProps) { nodes={nodes} edges={edges} nodeTypes={nodeTypes} - edgeTypes={edgeTypes} + // TODO: add custom edge types back if we want to support custom deletable buttons + // edgeTypes={edgeTypes} onNodesChange={onNodesChange} onEdgesChange={onEdgesChange} onConnect={onConnect} diff --git a/public/pages/workflow_detail/workspace/workspace_components/new_or_existing_tabs.tsx b/public/pages/workflow_detail/workspace/workspace_components/new_or_existing_tabs.tsx index 3bc7d1e2..f5a936fc 100644 --- a/public/pages/workflow_detail/workspace/workspace_components/new_or_existing_tabs.tsx +++ b/public/pages/workflow_detail/workspace/workspace_components/new_or_existing_tabs.tsx @@ -16,16 +16,16 @@ interface NewOrExistingTabsProps { } const inputTabs = [ - { - id: 'existing', - name: 'Existing', - disabled: false, - }, { id: 'new', name: 'New', disabled: false, }, + { + id: 'existing', + name: 'Existing', + disabled: true, + }, ]; export function NewOrExistingTabs(props: NewOrExistingTabsProps) { diff --git a/public/pages/workflow_detail/workspace/workspace_edge/deletable-edge-styles.scss b/public/pages/workflow_detail/workspace/workspace_edge/deletable-edge-styles.scss index a249a641..62ed2776 100644 --- a/public/pages/workflow_detail/workspace/workspace_edge/deletable-edge-styles.scss +++ b/public/pages/workflow_detail/workspace/workspace_edge/deletable-edge-styles.scss @@ -1,8 +1,8 @@ .delete-edge-button { width: 20px; height: 20px; - background: #eee; - border: 1px solid #fff; + background: #000000; + border: 1px solid #000000; cursor: pointer; border-radius: 50%; font-size: 12px; @@ -10,5 +10,5 @@ } .delete-edge-button:hover { - box-shadow: 0 0 6px 2px rgba(0, 0, 0, 0.08); + box-shadow: 0 0 6px 2px rgba(0, 0, 0, 0.5); } diff --git a/public/pages/workflow_detail/workspace/workspace_edge/deletable_edge.tsx b/public/pages/workflow_detail/workspace/workspace_edge/deletable_edge.tsx index 81ca6063..d9c9ff4d 100644 --- a/public/pages/workflow_detail/workspace/workspace_edge/deletable_edge.tsx +++ b/public/pages/workflow_detail/workspace/workspace_edge/deletable_edge.tsx @@ -9,6 +9,7 @@ import { Edge, EdgeLabelRenderer, EdgeProps, + MarkerType, getBezierPath, useReactFlow, } from 'reactflow'; @@ -64,7 +65,7 @@ export function DeletableEdge(props: DeletableEdgeProps) { transform: `translate(-50%, -50%) translate(${labelX}px,${labelY}px)`, fontSize: 12, pointerEvents: 'all', - zIndex: 1, + zIndex: 3, }} className="nodrag nopan" > diff --git a/public/route_service.ts b/public/route_service.ts index 6130e2a6..78faddd9 100644 --- a/public/route_service.ts +++ b/public/route_service.ts @@ -12,6 +12,7 @@ import { GET_WORKFLOW_STATE_NODE_API_PATH, SEARCH_WORKFLOWS_NODE_API_PATH, GET_PRESET_WORKFLOWS_NODE_API_PATH, + SEARCH_MODELS_NODE_API_PATH, } from '../common'; /** @@ -25,10 +26,14 @@ export interface RouteService { getWorkflow: (workflowId: string) => Promise; searchWorkflows: (body: {}) => Promise; getWorkflowState: (workflowId: string) => Promise; - createWorkflow: (body: {}) => Promise; + createWorkflow: ( + body: {}, + provision?: boolean + ) => Promise; deleteWorkflow: (workflowId: string) => Promise; getWorkflowPresets: () => Promise; catIndices: (pattern: string) => Promise; + searchModels: (body: {}) => Promise; } export function configureRoutes(core: CoreStart): RouteService { @@ -66,10 +71,10 @@ export function configureRoutes(core: CoreStart): RouteService { return e as HttpFetchError; } }, - createWorkflow: async (body: {}) => { + createWorkflow: async (body: {}, provision: boolean = false) => { try { const response = await core.http.post<{ respString: string }>( - CREATE_WORKFLOW_NODE_API_PATH, + `${CREATE_WORKFLOW_NODE_API_PATH}/${provision}`, { body: JSON.stringify(body), } @@ -109,5 +114,18 @@ export function configureRoutes(core: CoreStart): RouteService { return e as HttpFetchError; } }, + searchModels: async (body: {}) => { + try { + const response = await core.http.post<{ respString: string }>( + SEARCH_MODELS_NODE_API_PATH, + { + body: JSON.stringify(body), + } + ); + return response; + } catch (e: any) { + return e as HttpFetchError; + } + }, }; } diff --git a/public/store/reducers/index.ts b/public/store/reducers/index.ts index 522987d9..ef82cf2c 100644 --- a/public/store/reducers/index.ts +++ b/public/store/reducers/index.ts @@ -7,3 +7,4 @@ export * from './workspace_reducer'; export * from './opensearch_reducer'; export * from './workflows_reducer'; export * from './presets_reducer'; +export * from './models_reducer'; diff --git a/public/store/reducers/models_reducer.ts b/public/store/reducers/models_reducer.ts new file mode 100644 index 00000000..a43ccb79 --- /dev/null +++ b/public/store/reducers/models_reducer.ts @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { createAsyncThunk, createSlice } from '@reduxjs/toolkit'; +import { ModelDict } from '../../../common'; +import { HttpFetchError } from '../../../../../src/core/public'; +import { getRouteService } from '../../services'; + +const initialState = { + loading: false, + errorMessage: '', + models: {} as ModelDict, +}; + +const MODELS_ACTION_PREFIX = 'models'; +const SEARCH_MODELS_ACTION = `${MODELS_ACTION_PREFIX}/searchModels`; + +export const searchModels = createAsyncThunk( + SEARCH_MODELS_ACTION, + async (body: {}, { rejectWithValue }) => { + const response: any | HttpFetchError = await getRouteService().searchModels( + body + ); + if (response instanceof HttpFetchError) { + return rejectWithValue( + 'Error searching models: ' + response.body.message + ); + } else { + return response; + } + } +); + +const modelsSlice = createSlice({ + name: 'models', + initialState, + reducers: {}, + extraReducers: (builder) => { + builder + // Pending states + .addCase(searchModels.pending, (state, action) => { + state.loading = true; + state.errorMessage = ''; + }) + // Fulfilled states + .addCase(searchModels.fulfilled, (state, action) => { + const { models } = action.payload as { models: ModelDict }; + state.models = models; + state.loading = false; + state.errorMessage = ''; + }) + // Rejected states + .addCase(searchModels.rejected, (state, action) => { + state.errorMessage = action.payload as string; + state.loading = false; + }); + }, +}); + +export const modelsReducer = modelsSlice.reducer; diff --git a/public/store/reducers/workflows_reducer.ts b/public/store/reducers/workflows_reducer.ts index 1092d605..f748f08b 100644 --- a/public/store/reducers/workflows_reducer.ts +++ b/public/store/reducers/workflows_reducer.ts @@ -4,18 +4,7 @@ */ import { createAsyncThunk, createSlice } from '@reduxjs/toolkit'; -import { - Workflow, - ReactFlowComponent, - ReactFlowEdge, - KnnIndexer, - TextEmbeddingTransformer, - generateId, - initComponentData, - WORKFLOW_STATE, - WorkflowDict, - WorkflowTemplate, -} from '../../../common'; +import { Workflow, WorkflowDict } from '../../../common'; import { HttpFetchError } from '../../../../../src/core/public'; import { getRouteService } from '../../services'; @@ -85,10 +74,13 @@ export const getWorkflowState = createAsyncThunk( export const createWorkflow = createAsyncThunk( CREATE_WORKFLOW_ACTION, - async (body: {}, { rejectWithValue }) => { + async (workflowBody: {}, { rejectWithValue }) => { const response: | any - | HttpFetchError = await getRouteService().createWorkflow(body); + | HttpFetchError = await getRouteService().createWorkflow( + workflowBody, + false + ); if (response instanceof HttpFetchError) { return rejectWithValue( 'Error creating workflow: ' + response.body.message diff --git a/public/store/store.ts b/public/store/store.ts index fa906b99..06d7d2ea 100644 --- a/public/store/store.ts +++ b/public/store/store.ts @@ -10,12 +10,14 @@ import { opensearchReducer, workflowsReducer, presetsReducer, + modelsReducer, } from './reducers'; const rootReducer = combineReducers({ workspace: workspaceReducer, workflows: workflowsReducer, presets: presetsReducer, + models: modelsReducer, opensearch: opensearchReducer, }); diff --git a/public/utils/utils.ts b/public/utils/utils.ts index 5eaa77c1..1a24f6ea 100644 --- a/public/utils/utils.ts +++ b/public/utils/utils.ts @@ -42,10 +42,12 @@ export function initComponentData( **************** Formik (form) utils ********************** */ +// TODO: below, we are hardcoding to only persisting and validating create fields. +// If we support both, we will need to dynamically update. // Converting stored values in component data to initial formik values export function componentDataToFormik(data: IComponentData): FormikValues { const formikValues = {} as FormikValues; - data.fields?.forEach((field) => { + data.createFields?.forEach((field) => { formikValues[field.name] = field.value || getInitialValue(field.type); }); return formikValues; @@ -99,9 +101,11 @@ export function getFieldError( **************** Yup (validation) utils ********************** */ +// TODO: below, we are hardcoding to only persisting and validating create fields. +// If we support both, we will need to dynamically update. export function getComponentSchema(data: IComponentData): ObjectSchema { const schemaObj = {} as { [key: string]: Schema }; - data.fields?.forEach((field) => { + data.createFields?.forEach((field) => { schemaObj[field.name] = getFieldSchema(field); }); return yup.object(schemaObj); @@ -122,9 +126,13 @@ function getFieldSchema(field: IComponentField): Schema { break; } } - return field.optional - ? baseSchema.optional() - : baseSchema.required('Required'); + + // TODO: make optional schema if we support optional fields in the future + // return field.optional + // ? baseSchema.optional() + // : baseSchema.required('Required'); + + return baseSchema.required('Required'); } export function getStateOptions(): EuiFilterSelectItem[] { diff --git a/server/cluster/flow_framework_plugin.ts b/server/cluster/flow_framework_plugin.ts index 3b7616dc..2e649a4b 100644 --- a/server/cluster/flow_framework_plugin.ts +++ b/server/cluster/flow_framework_plugin.ts @@ -67,7 +67,13 @@ export function flowFrameworkPlugin(Client: any, config: any, components: any) { flowFramework.createWorkflow = ca({ url: { - fmt: FLOW_FRAMEWORK_WORKFLOW_ROUTE_PREFIX, + fmt: `${FLOW_FRAMEWORK_WORKFLOW_ROUTE_PREFIX}?provision=<%=provision%>`, + req: { + provision: { + type: 'boolean', + required: true, + }, + }, }, needBody: true, method: 'POST', diff --git a/server/cluster/index.ts b/server/cluster/index.ts index 5ff70614..a29304ea 100644 --- a/server/cluster/index.ts +++ b/server/cluster/index.ts @@ -4,3 +4,4 @@ */ export * from './flow_framework_plugin'; +export * from './ml_plugin'; diff --git a/server/cluster/ml_plugin.ts b/server/cluster/ml_plugin.ts new file mode 100644 index 00000000..4601a9dd --- /dev/null +++ b/server/cluster/ml_plugin.ts @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { ML_SEARCH_MODELS_ROUTE } from '../../common'; + +/** + * Used during the plugin's setup() lifecycle phase to register various client actions + * representing ML plugin APIs. These are then exposed and used on the + * server-side when processing node APIs - see server/routes/ml_routes_service + * for examples. + */ +export function mlPlugin(Client: any, config: any, components: any) { + const ca = components.clientAction.factory; + + Client.prototype.mlClient = components.clientAction.namespaceFactory(); + const mlClient = Client.prototype.mlClient.prototype; + + mlClient.searchModels = ca({ + url: { + fmt: ML_SEARCH_MODELS_ROUTE, + }, + needBody: true, + method: 'POST', + }); +} diff --git a/server/plugin.ts b/server/plugin.ts index 2772d5aa..62c8ae27 100644 --- a/server/plugin.ts +++ b/server/plugin.ts @@ -11,7 +11,7 @@ import { Logger, } from '../../../src/core/server'; import { first } from 'rxjs/operators'; -import { flowFrameworkPlugin } from './cluster'; +import { flowFrameworkPlugin, mlPlugin } from './cluster'; import { FlowFrameworkDashboardsPluginSetup, FlowFrameworkDashboardsPluginStart, @@ -21,6 +21,8 @@ import { registerFlowFrameworkRoutes, OpenSearchRoutesService, FlowFrameworkRoutesService, + registerMLRoutes, + MLRoutesService, } from './routes'; import { ILegacyClusterClient } from '../../../src/core/server/'; @@ -50,17 +52,19 @@ export class FlowFrameworkDashboardsPlugin const client: ILegacyClusterClient = core.opensearch.legacy.createClient( 'flow_framework', { - plugins: [flowFrameworkPlugin], + plugins: [flowFrameworkPlugin, mlPlugin], ...globalConfig.opensearch, } ); const opensearchRoutesService = new OpenSearchRoutesService(client); const flowFrameworkRoutesService = new FlowFrameworkRoutesService(client); + const mlRoutesService = new MLRoutesService(client); // Register server side APIs with the corresponding service functions registerOpenSearchRoutes(router, opensearchRoutesService); registerFlowFrameworkRoutes(router, flowFrameworkRoutesService); + registerMLRoutes(router, mlRoutesService); return {}; } diff --git a/server/routes/flow_framework_routes_service.ts b/server/routes/flow_framework_routes_service.ts index 19c802a5..4eda47b5 100644 --- a/server/routes/flow_framework_routes_service.ts +++ b/server/routes/flow_framework_routes_service.ts @@ -69,9 +69,12 @@ export function registerFlowFrameworkRoutes( router.post( { - path: CREATE_WORKFLOW_NODE_API_PATH, + path: `${CREATE_WORKFLOW_NODE_API_PATH}/{provision}`, validate: { body: schema.any(), + params: schema.object({ + provision: schema.boolean(), + }), }, }, flowFrameworkRoutesService.createWorkflow @@ -180,11 +183,11 @@ export class FlowFrameworkRoutesService { res: OpenSearchDashboardsResponseFactory ): Promise> => { const body = req.body; - + const { provision } = req.params as { provision: boolean }; try { const response = await this.client .asScoped(req) - .callAsCurrentUser('flowFramework.createWorkflow', { body }); + .callAsCurrentUser('flowFramework.createWorkflow', { body, provision }); return res.ok({ body: { id: response._id } }); } catch (err: any) { return generateCustomError(res, err); diff --git a/server/routes/helpers.ts b/server/routes/helpers.ts index 8fddfe8f..5b912b49 100644 --- a/server/routes/helpers.ts +++ b/server/routes/helpers.ts @@ -3,7 +3,13 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { WORKFLOW_STATE, Workflow, WorkflowDict } from '../../common'; +import { + Model, + ModelDict, + WORKFLOW_STATE, + Workflow, + WorkflowDict, +} from '../../common'; // OSD does not provide an interface for this response, but this is following the suggested // implementations. To prevent typescript complaining, leaving as loosely-typed 'any' @@ -61,3 +67,17 @@ export function getWorkflowsFromResponses( }); return workflowDict; } + +export function getModelsFromResponses(modelHits: any[]): ModelDict { + const modelDict = {} as ModelDict; + modelHits.forEach((modelHit: any) => { + const modelId = modelHit._source?.model_id; + // in case of schema changes from ML plugin, this may crash. That is ok, as the error + // produced will help expose the root cause + modelDict[modelId] = { + id: modelId, + algorithm: modelHit._source?.algorithm, + } as Model; + }); + return modelDict; +} diff --git a/server/routes/index.ts b/server/routes/index.ts index 8b55b8cb..db0e21c8 100644 --- a/server/routes/index.ts +++ b/server/routes/index.ts @@ -5,3 +5,4 @@ export * from './opensearch_routes_service'; export * from './flow_framework_routes_service'; +export * from './ml_routes_service'; diff --git a/server/routes/ml_routes_service.ts b/server/routes/ml_routes_service.ts new file mode 100644 index 00000000..874d84d3 --- /dev/null +++ b/server/routes/ml_routes_service.ts @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { schema } from '@osd/config-schema'; +import { + IRouter, + IOpenSearchDashboardsResponse, + RequestHandlerContext, + OpenSearchDashboardsRequest, + OpenSearchDashboardsResponseFactory, +} from '../../../../src/core/server'; +import { SEARCH_MODELS_NODE_API_PATH } from '../../common'; +import { generateCustomError, getModelsFromResponses } from './helpers'; + +/** + * Server-side routes to process ml-plugin-related node API calls and execute the + * corresponding API calls against the OpenSearch cluster. + */ +export function registerMLRoutes( + router: IRouter, + mlRoutesService: MLRoutesService +): void { + router.post( + { + path: SEARCH_MODELS_NODE_API_PATH, + validate: { + body: schema.any(), + }, + }, + mlRoutesService.searchModels + ); +} + +export class MLRoutesService { + private client: any; + + constructor(client: any) { + this.client = client; + } + + searchModels = async ( + context: RequestHandlerContext, + req: OpenSearchDashboardsRequest, + res: OpenSearchDashboardsResponseFactory + ): Promise> => { + const body = req.body; + try { + const modelsResponse = await this.client + .asScoped(req) + .callAsCurrentUser('mlClient.searchModels', { body }); + const modelHits = modelsResponse.hits.hits as any[]; + const modelDict = getModelsFromResponses(modelHits); + + return res.ok({ body: { models: modelDict } }); + } catch (err: any) { + return generateCustomError(res, err); + } + }; +}