forked from opensearch-project/ml-commons
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
support question answering model (opensearch-project#2208) (opensearc…
…h-project#2224) * support question answering model Signed-off-by: Bhavana Ramaram <[email protected]> (cherry picked from commit c560fcc)
- Loading branch information
Showing
21 changed files
with
1,114 additions
and
49 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
51 changes: 51 additions & 0 deletions
51
common/src/main/java/org/opensearch/ml/common/dataset/QuestionAnsweringInputDataSet.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
| package org.opensearch.ml.common.dataset; | ||
|
|
||
| import lombok.AccessLevel; | ||
| import lombok.Builder; | ||
| import lombok.Getter; | ||
| import lombok.experimental.FieldDefaults; | ||
| import org.opensearch.core.common.io.stream.StreamInput; | ||
| import org.opensearch.core.common.io.stream.StreamOutput; | ||
| import org.opensearch.ml.common.annotation.InputDataSet; | ||
|
|
||
| import java.io.IOException; | ||
|
|
||
| @Getter | ||
| @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) | ||
| @InputDataSet(MLInputDataType.QUESTION_ANSWERING) | ||
| public class QuestionAnsweringInputDataSet extends MLInputDataset { | ||
|
|
||
| String question; | ||
|
|
||
| String context; | ||
|
|
||
| @Builder(toBuilder = true) | ||
| public QuestionAnsweringInputDataSet(String question, String context) { | ||
| super(MLInputDataType.QUESTION_ANSWERING); | ||
| if(question == null) { | ||
| throw new IllegalArgumentException("Question is not provided"); | ||
| } | ||
| if(context == null) { | ||
| throw new IllegalArgumentException("Context is not provided"); | ||
| } | ||
| this.question = question; | ||
| this.context = context; | ||
| } | ||
|
|
||
| public QuestionAnsweringInputDataSet(StreamInput in) throws IOException { | ||
| super(MLInputDataType.QUESTION_ANSWERING); | ||
| this.question = in.readString(); | ||
| this.context = in.readString(); | ||
| } | ||
|
|
||
| @Override | ||
| public void writeTo(StreamOutput out) throws IOException { | ||
| super.writeTo(out); | ||
| out.writeString(question); | ||
| out.writeString(context); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
91 changes: 91 additions & 0 deletions
91
common/src/main/java/org/opensearch/ml/common/input/nlp/QuestionAnsweringMLInput.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
| package org.opensearch.ml.common.input.nlp; | ||
|
|
||
| import org.opensearch.core.common.io.stream.StreamInput; | ||
| import org.opensearch.core.common.io.stream.StreamOutput; | ||
| import org.opensearch.core.xcontent.XContentBuilder; | ||
| import org.opensearch.core.xcontent.XContentParser; | ||
| import org.opensearch.ml.common.FunctionName; | ||
| import org.opensearch.ml.common.dataset.MLInputDataset; | ||
| import org.opensearch.ml.common.dataset.QuestionAnsweringInputDataSet; | ||
| import org.opensearch.ml.common.input.MLInput; | ||
|
|
||
| import java.io.IOException; | ||
|
|
||
| import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; | ||
|
|
||
|
|
||
| /** | ||
| * MLInput which supports a question answering algorithm | ||
| * Inputs are question and context. Output is the answer | ||
| */ | ||
| @org.opensearch.ml.common.annotation.MLInput(functionNames = {FunctionName.QUESTION_ANSWERING}) | ||
| public class QuestionAnsweringMLInput extends MLInput { | ||
|
|
||
| public QuestionAnsweringMLInput(FunctionName algorithm, MLInputDataset dataset) { | ||
| super(algorithm, null, dataset); | ||
| } | ||
|
|
||
| public QuestionAnsweringMLInput(StreamInput in) throws IOException { | ||
| super(in); | ||
| } | ||
|
|
||
| @Override | ||
| public void writeTo(StreamOutput out) throws IOException { | ||
| super.writeTo(out); | ||
| } | ||
|
|
||
| @Override | ||
| public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { | ||
| builder.startObject(); | ||
| builder.field(ALGORITHM_FIELD, algorithm.name()); | ||
| if(parameters != null) { | ||
| builder.field(ML_PARAMETERS_FIELD, parameters); | ||
| } | ||
| if(inputDataset != null) { | ||
| QuestionAnsweringInputDataSet ds = (QuestionAnsweringInputDataSet) this.inputDataset; | ||
| String question = ds.getQuestion(); | ||
| String context = ds.getContext(); | ||
| builder.field(QUESTION_FIELD, question); | ||
| builder.field(CONTEXT_FIELD, context); | ||
| } | ||
| builder.endObject(); | ||
| return builder; | ||
| } | ||
|
|
||
| public QuestionAnsweringMLInput(XContentParser parser, FunctionName functionName) throws IOException { | ||
| super(); | ||
| this.algorithm = functionName; | ||
| String question = null; | ||
| String context = null; | ||
|
|
||
| ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); | ||
| while (parser.nextToken() != XContentParser.Token.END_OBJECT) { | ||
| String fieldName = parser.currentName(); | ||
| parser.nextToken(); | ||
|
|
||
| switch (fieldName) { | ||
| case QUESTION_FIELD: | ||
| question = parser.text(); | ||
| break; | ||
| case CONTEXT_FIELD: | ||
| context = parser.text(); | ||
| break; | ||
| default: | ||
| parser.skipChildren(); | ||
| break; | ||
| } | ||
| } | ||
| if(question == null) { | ||
| throw new IllegalArgumentException("Question is not provided"); | ||
| } | ||
| if(context == null) { | ||
| throw new IllegalArgumentException("Context is not provided"); | ||
| } | ||
| inputDataset = new QuestionAnsweringInputDataSet(question, context); | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.