From f506b86df83049b0300a95c971f8d602bc3fd506 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 12 Mar 2024 09:12:01 -0700 Subject: [PATCH] Refactor component types and component inheritance (#110) Signed-off-by: Tyler Ohlsen (cherry picked from commit 616b1cbfc6ebd48dce14ab49ed94f2ba871d22b7) --- opensearch_dashboards.json | 3 +- public/component_types/base_component.tsx | 26 +++++- public/component_types/index.ts | 4 +- .../{indices => indexer}/index.ts | 3 +- public/component_types/indexer/indexer.ts | 65 +++++++++++++ public/component_types/indexer/knn_indexer.ts | 30 ++++++ public/component_types/indices/knn_index.ts | 92 ------------------ public/component_types/interfaces.ts | 10 +- .../processors/text_embedding_processor.ts | 76 --------------- .../transformer/base_transformer.ts | 21 +++++ .../{processors => transformer}/index.ts | 3 +- .../transformer/ml_transformer.ts | 20 ++++ .../transformer/text_embedding_transformer.ts | 47 ++++++++++ public/store/reducers/workflows_reducer.ts | 93 ++++++++----------- public/utils/constants.ts | 21 ++++- 15 files changed, 271 insertions(+), 243 deletions(-) rename public/component_types/{indices => indexer}/index.ts (59%) create mode 100644 public/component_types/indexer/indexer.ts create mode 100644 public/component_types/indexer/knn_indexer.ts delete mode 100644 public/component_types/indices/knn_index.ts delete mode 100644 public/component_types/processors/text_embedding_processor.ts create mode 100644 public/component_types/transformer/base_transformer.ts rename public/component_types/{processors => transformer}/index.ts (51%) create mode 100644 public/component_types/transformer/ml_transformer.ts create mode 100644 public/component_types/transformer/text_embedding_transformer.ts diff --git a/opensearch_dashboards.json b/opensearch_dashboards.json index f098d4f5..c49701ed 100644 --- a/opensearch_dashboards.json +++ b/opensearch_dashboards.json @@ -6,7 +6,8 @@ "ui": true, "requiredBundles": [], "requiredPlugins": [ - "navigation" + "navigation", + "opensearchDashboardsUtils" ], "optionalPlugins": [] } \ No newline at end of file diff --git a/public/component_types/base_component.tsx b/public/component_types/base_component.tsx index 1e6dcd4f..c2a511bd 100644 --- a/public/component_types/base_component.tsx +++ b/public/component_types/base_component.tsx @@ -3,10 +3,32 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { COMPONENT_CATEGORY, COMPONENT_CLASS } from '../utils'; +import { + IComponent, + IComponentField, + IComponentInput, + IComponentOutput, +} from './interfaces'; + /** - * A base component class. + * A base UI drag-and-drop component class. */ -export abstract class BaseComponent { +export abstract class BaseComponent implements IComponent { + type: COMPONENT_CLASS; + label: string; + description: string; + categories: COMPONENT_CATEGORY[]; + allowsCreation: boolean; + baseClasses: COMPONENT_CLASS[]; + inputs?: IComponentInput[]; + fields?: IComponentField[]; + createFields?: IComponentField[]; + outputs?: IComponentOutput[]; + + // No-op constructor. If there are general / defaults for field values, add in here. + constructor() {} + // Persist a standard toObj() fn that all component classes can use. This is necessary // so we have standard JS Object when serializing comoponent state in redux. toObj() { diff --git a/public/component_types/index.ts b/public/component_types/index.ts index bc3be5cb..f21f6aec 100644 --- a/public/component_types/index.ts +++ b/public/component_types/index.ts @@ -4,5 +4,5 @@ */ export * from './interfaces'; -export * from './processors'; -export * from './indices'; +export * from './transformer'; +export * from './indexer'; diff --git a/public/component_types/indices/index.ts b/public/component_types/indexer/index.ts similarity index 59% rename from public/component_types/indices/index.ts rename to public/component_types/indexer/index.ts index cc5778c9..815396af 100644 --- a/public/component_types/indices/index.ts +++ b/public/component_types/indexer/index.ts @@ -3,4 +3,5 @@ * SPDX-License-Identifier: Apache-2.0 */ -export * from './knn_index'; +export * from './indexer'; +export * from './knn_indexer'; diff --git a/public/component_types/indexer/indexer.ts b/public/component_types/indexer/indexer.ts new file mode 100644 index 00000000..e87a81b0 --- /dev/null +++ b/public/component_types/indexer/indexer.ts @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { COMPONENT_CATEGORY, COMPONENT_CLASS } from '../../utils'; +import { BaseComponent } from '../base_component'; + +/** + * A base indexer UI component + */ +export class Indexer extends BaseComponent { + constructor() { + super(); + this.type = COMPONENT_CLASS.INDEXER; + this.label = 'Indexer'; + this.description = 'A general indexer'; + this.categories = [COMPONENT_CATEGORY.INGEST, COMPONENT_CATEGORY.SEARCH]; + this.allowsCreation = true; + this.baseClasses = [this.type]; + this.inputs = [ + { + id: 'transformer', + label: 'Transformer', + // TODO: may need to change to be looser. it should be able to take + // in other component types + baseClass: COMPONENT_CLASS.TRANSFORMER, + optional: false, + acceptMultiple: false, + }, + ]; + this.fields = [ + { + label: 'Index Name', + name: 'indexName', + type: 'select', + optional: false, + advanced: false, + }, + ]; + this.createFields = [ + { + label: 'Index Name', + name: 'indexName', + type: 'string', + optional: false, + advanced: false, + }, + { + label: 'Mappings', + name: 'indexMappings', + type: 'json', + placeholder: 'Enter an index mappings JSON blob...', + optional: false, + advanced: false, + }, + ]; + this.outputs = [ + { + label: this.label, + baseClasses: this.baseClasses, + }, + ]; + } +} diff --git a/public/component_types/indexer/knn_indexer.ts b/public/component_types/indexer/knn_indexer.ts new file mode 100644 index 00000000..93f55017 --- /dev/null +++ b/public/component_types/indexer/knn_indexer.ts @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { Indexer } from './indexer'; + +/** + * A specialized indexer component for vector/K-NN indices + */ +export class KnnIndexer extends Indexer { + constructor() { + super(); + this.label = 'K-NN Indexer'; + this.description = 'A specialized indexer for K-NN indices'; + this.createFields = [ + // @ts-ignore + ...this.createFields, + // TODO: finalize what to expose / what to have for defaults here + { + label: 'K-NN Settings', + name: 'knnSettings', + type: 'json', + placeholder: 'Enter K-NN settings JSON blob...', + optional: false, + advanced: false, + }, + ]; + } +} diff --git a/public/component_types/indices/knn_index.ts b/public/component_types/indices/knn_index.ts deleted file mode 100644 index e56e6dd2..00000000 --- a/public/component_types/indices/knn_index.ts +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { COMPONENT_CATEGORY, COMPONENT_CLASS } from '../../utils'; -import { BaseComponent } from '../base_component'; -import { - IComponent, - IComponentField, - IComponentInput, - IComponentOutput, - UIFlow, -} from '../interfaces'; - -/** - * A k-NN index UI component - */ -export class KnnIndex extends BaseComponent implements IComponent { - type: COMPONENT_CLASS; - label: string; - description: string; - category: COMPONENT_CATEGORY; - allowsCreation: boolean; - isApplicationStep: boolean; - allowedFlows: UIFlow[]; - baseClasses: COMPONENT_CLASS[]; - inputs: IComponentInput[]; - fields: IComponentField[]; - createFields: IComponentField[]; - outputs: IComponentOutput[]; - - constructor() { - super(); - this.type = COMPONENT_CLASS.KNN_INDEX; - this.label = 'k-NN Index'; - this.description = 'A k-NN Index to be used as a vector store'; - this.category = COMPONENT_CATEGORY.INDICES; - this.allowsCreation = true; - this.isApplicationStep = false; - // TODO: 'other' may not be how this is stored. the idea is 'other' allows - // for placement outside of the ingest or query flows- typically something - // that will be referenced/used as input across multiple flows - this.allowedFlows = ['Ingest', 'Query', 'Other']; - this.baseClasses = [this.type]; - this.inputs = [ - { - id: 'text-embedding-processor', - label: 'Text embedding processor', - baseClass: COMPONENT_CLASS.TEXT_EMBEDDING_PROCESSOR, - optional: false, - acceptMultiple: false, - }, - ]; - this.fields = [ - { - label: 'Index Name', - name: 'indexName', - type: 'select', - optional: false, - advanced: false, - }, - ]; - this.createFields = [ - { - label: 'Index Name', - name: 'indexName', - type: 'string', - optional: false, - advanced: false, - }, - // we don't need to expose "settings" here since it will be index.knn by default - // just let users customize the mappings - // TODO: figure out how to handle defaults for all of these values. maybe toggle between - // simple form inputs vs. complex JSON editor - { - label: 'Mappings', - name: 'indexMappings', - type: 'json', - placeholder: 'Enter an index mappings JSON blob...', - optional: false, - advanced: false, - }, - ]; - this.outputs = [ - { - label: this.label, - baseClasses: this.baseClasses, - }, - ]; - } -} diff --git a/public/component_types/interfaces.ts b/public/component_types/interfaces.ts index 6aea8cd9..e9349cd3 100644 --- a/public/component_types/interfaces.ts +++ b/public/component_types/interfaces.ts @@ -10,7 +10,6 @@ import { COMPONENT_CATEGORY, COMPONENT_CLASS } from '../utils'; /** * ************ Types ************************* */ -export type UIFlow = string; export type FieldType = 'string' | 'json' | 'select'; // TODO: this may expand to more types in the future. Formik supports 'any' so we can too. // For now, limiting scope to expected types. @@ -68,17 +67,12 @@ export interface IComponent { label: string; description: string; // will be used for grouping together in the drag-and-drop component library - category: COMPONENT_CATEGORY; + // and determining which flows the component can be drug into the workspace flows + categories: COMPONENT_CATEGORY[]; // determines if this component allows for new creation. this means to // allow a "create" option on the UI component, as well as potentially // include in the use case template construction ('provisioning' flow) allowsCreation: boolean; - // determines if this is something that will be included in the use - // case template construction (query or ingest flows). provisioning flow - // is handled by the allowsCreation flag above. - isApplicationStep: boolean; - // the set of allowed flows this component can be drug into the workspace - allowedFlows: UIFlow[]; // the list of base classes that will be used in the component output baseClasses?: COMPONENT_CLASS[]; inputs?: IComponentInput[]; diff --git a/public/component_types/processors/text_embedding_processor.ts b/public/component_types/processors/text_embedding_processor.ts deleted file mode 100644 index 4f5b16c5..00000000 --- a/public/component_types/processors/text_embedding_processor.ts +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { COMPONENT_CATEGORY, COMPONENT_CLASS } from '../../utils'; -import { BaseComponent } from '../base_component'; -import { - IComponent, - IComponentField, - IComponentInput, - IComponentOutput, - UIFlow, -} from '../interfaces'; - -/** - * A text embedding processor UI component - */ -export class TextEmbeddingProcessor - extends BaseComponent - implements IComponent { - type: COMPONENT_CLASS; - label: string; - description: string; - category: COMPONENT_CATEGORY; - allowsCreation: boolean; - isApplicationStep: boolean; - allowedFlows: UIFlow[]; - baseClasses: COMPONENT_CLASS[]; - inputs: IComponentInput[]; - fields: IComponentField[]; - outputs: IComponentOutput[]; - - constructor() { - super(); - this.type = COMPONENT_CLASS.TEXT_EMBEDDING_PROCESSOR; - this.label = 'Text Embedding Processor'; - this.description = - 'A text embedding ingest processor to be used in an ingest pipeline'; - this.category = COMPONENT_CATEGORY.INGEST_PROCESSORS; - this.allowsCreation = false; - this.isApplicationStep = false; - this.allowedFlows = ['Ingest']; - this.baseClasses = [this.type]; - this.inputs = []; - this.fields = [ - { - label: 'Model ID', - name: 'modelId', - type: 'string', - optional: false, - advanced: false, - }, - { - label: 'Input Field', - name: 'inputField', - type: 'string', - optional: false, - advanced: false, - }, - { - label: 'Output Field', - name: 'outputField', - type: 'string', - optional: false, - advanced: false, - }, - ]; - this.outputs = [ - { - label: this.label, - baseClasses: this.baseClasses, - }, - ]; - } -} diff --git a/public/component_types/transformer/base_transformer.ts b/public/component_types/transformer/base_transformer.ts new file mode 100644 index 00000000..c46ee89c --- /dev/null +++ b/public/component_types/transformer/base_transformer.ts @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { COMPONENT_CATEGORY, COMPONENT_CLASS } from '../../utils'; +import { BaseComponent } from '../base_component'; + +/** + * A base transformer UI component + */ +export abstract class BaseTransformer extends BaseComponent { + constructor() { + super(); + this.type = COMPONENT_CLASS.TRANSFORMER; + this.label = 'Transformer'; + this.categories = [COMPONENT_CATEGORY.INGEST, COMPONENT_CATEGORY.SEARCH]; + this.allowsCreation = false; + this.baseClasses = [this.type]; + } +} diff --git a/public/component_types/processors/index.ts b/public/component_types/transformer/index.ts similarity index 51% rename from public/component_types/processors/index.ts rename to public/component_types/transformer/index.ts index 364e51bc..5fe06d42 100644 --- a/public/component_types/processors/index.ts +++ b/public/component_types/transformer/index.ts @@ -3,4 +3,5 @@ * SPDX-License-Identifier: Apache-2.0 */ -export * from './text_embedding_processor'; +export * from './ml_transformer'; +export * from './text_embedding_transformer'; diff --git a/public/component_types/transformer/ml_transformer.ts b/public/component_types/transformer/ml_transformer.ts new file mode 100644 index 00000000..09a51fba --- /dev/null +++ b/public/component_types/transformer/ml_transformer.ts @@ -0,0 +1,20 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { COMPONENT_CLASS } from '../../utils'; +import { BaseTransformer } from './base_transformer'; + +/** + * A generic ML transformer UI component + */ +export class MLTransformer extends BaseTransformer { + constructor() { + super(); + this.type = COMPONENT_CLASS.ML_TRANSFORMER; + this.label = 'ML Transformer'; + this.description = 'A general ML transformer'; + this.baseClasses = [...this.baseClasses, this.type]; + } +} diff --git a/public/component_types/transformer/text_embedding_transformer.ts b/public/component_types/transformer/text_embedding_transformer.ts new file mode 100644 index 00000000..dbdce094 --- /dev/null +++ b/public/component_types/transformer/text_embedding_transformer.ts @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { MLTransformer } from '.'; + +/** + * A specialized text embedding ML transformer UI component + */ +export class TextEmbeddingTransformer extends MLTransformer { + constructor() { + super(); + this.label = 'Text Embedding Transformer'; + this.description = 'A specialized ML transformer for embedding text'; + this.inputs = []; + this.fields = [ + { + label: 'Model ID', + name: 'modelId', + type: 'string', + optional: false, + advanced: false, + }, + { + label: 'Input Field', + name: 'inputField', + type: 'string', + optional: false, + advanced: false, + }, + { + label: 'Output Field', + name: 'outputField', + type: 'string', + optional: false, + advanced: false, + }, + ]; + this.outputs = [ + { + label: this.label, + baseClasses: this.baseClasses, + }, + ]; + } +} diff --git a/public/store/reducers/workflows_reducer.ts b/public/store/reducers/workflows_reducer.ts index d06eac97..4dddf786 100644 --- a/public/store/reducers/workflows_reducer.ts +++ b/public/store/reducers/workflows_reducer.ts @@ -8,8 +8,8 @@ import { Workflow, ReactFlowComponent, ReactFlowEdge, - KnnIndex, - TextEmbeddingProcessor, + KnnIndexer, + TextEmbeddingTransformer, generateId, initComponentData, WORKFLOW_STATE, @@ -18,61 +18,31 @@ import { import { HttpFetchError } from '../../../../../src/core/public'; import { getRouteService } from '../../services'; -// TODO: remove hardcoded initial state below after fetching from server side, +// TODO: remove hardcoded dummy node data below after fetching from server side, // and workflow data model interface is more defined. -// const id1 = generateId('text_embedding_processor'); -// const id2 = generateId('text_embedding_processor'); -// const id3 = generateId('knn_index'); -// const dummyNodes = [ -// { -// id: id1, -// position: { x: 0, y: 500 }, -// data: initComponentData(new TextEmbeddingProcessor().toObj(), id1), -// type: 'customComponent', -// }, -// { -// id: id2, -// position: { x: 0, y: 200 }, -// data: initComponentData(new TextEmbeddingProcessor().toObj(), id2), -// type: 'customComponent', -// }, -// { -// id: id3, -// position: { x: 500, y: 500 }, -// data: initComponentData(new KnnIndex().toObj(), id3), -// type: 'customComponent', -// }, -// ] as ReactFlowComponent[]; - -// let workflows = {} as { [workflowId: string]: Workflow }; -// workflows['workflow-1-id'] = { -// name: 'Workflow-1', -// id: 'workflow-1-id', -// description: 'description for workflow 1', -// state: WORKFLOW_STATE.SUCCEEDED, -// workspaceFlowState: { -// nodes: dummyNodes, -// edges: [] as ReactFlowEdge[], -// }, -// template: {}, -// } as Workflow; -// workflows['workflow-2-id'] = { -// name: 'Workflow-2', -// id: 'workflow-2-id', -// description: 'description for workflow 2', -// state: WORKFLOW_STATE.FAILED, -// workspaceFlowState: { -// nodes: dummyNodes, -// edges: [] as ReactFlowEdge[], -// }, -// template: {}, -// } as Workflow; - -// const initialState = { -// loading: false, -// errorMessage: '', -// workflows, -// }; +const id1 = generateId('text_embedding_processor'); +const id2 = generateId('text_embedding_processor'); +const id3 = generateId('knn_index'); +const dummyNodes = [ + { + id: id1, + position: { x: 0, y: 500 }, + data: initComponentData(new TextEmbeddingTransformer().toObj(), id1), + type: 'customComponent', + }, + { + id: id2, + position: { x: 0, y: 200 }, + data: initComponentData(new TextEmbeddingTransformer().toObj(), id2), + type: 'customComponent', + }, + { + id: id3, + position: { x: 500, y: 500 }, + data: initComponentData(new KnnIndexer().toObj(), id3), + type: 'customComponent', + }, +] as ReactFlowComponent[]; const initialState = { loading: false, @@ -225,6 +195,17 @@ const workflowsSlice = createSlice({ }) .addCase(searchWorkflows.fulfilled, (state, action) => { const { workflows } = action.payload as { workflows: WorkflowDict }; + + // TODO: remove hardcoded workspace flow state. For testing purposes only + Object.entries(workflows).forEach(([workflowId, workflow]) => { + workflows[workflowId] = { + ...workflows[workflowId], + workspaceFlowState: { + nodes: dummyNodes, + edges: [] as ReactFlowEdge[], + }, + }; + }); state.workflows = workflows; state.loading = false; state.errorMessage = ''; diff --git a/public/utils/constants.ts b/public/utils/constants.ts index 430de186..eb078018 100644 --- a/public/utils/constants.ts +++ b/public/utils/constants.ts @@ -22,12 +22,25 @@ export const BREADCRUMBS = Object.freeze({ WORKFLOWS: { text: 'Workflows', href: `#${APP_PATH.WORKFLOWS}` }, }); +/** + * The static set of available categories that can be used to organize + * the component library. Sets guardrails on what components can be + * drag-and-dropped into the ingest and/or search flows. + */ export enum COMPONENT_CATEGORY { - INGEST_PROCESSORS = 'Ingest Processors', - INDICES = 'Indices', + INGEST = 'Ingest', + SEARCH = 'Search', } +// TODO: subject to change +/** + * A base set of component classes / types. + */ export enum COMPONENT_CLASS { - KNN_INDEX = 'knn_index', - TEXT_EMBEDDING_PROCESSOR = 'text_embedding_processor', + INDEXER = 'indexer', + RETRIEVER = 'retriever', + TRANSFORMER = 'transformer', + JSON_TO_JSON_TRANSFORMER = 'json_to_json_transformer', + ML_TRANSFORMER = 'ml_transformer', + QUERY = 'query', }