diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f815f792..fdfedc126 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Validate zero vector when using cosine metric [#1501](https://github.com/opensearch-project/k-NN/pull/1501) * Persist model definition in model metadata [#1527] (https://github.com/opensearch-project/k-NN/pull/1527) * Added Inner Product Space type support for Lucene Engine [#1551](https://github.com/opensearch-project/k-NN/pull/1551) +* Add Range Validation for Faiss SQFP16 [#1493](https://github.com/opensearch-project/k-NN/pull/1493) +* SQFP16 Range Validation for Faiss IVF Models [#1557](https://github.com/opensearch-project/k-NN/pull/1557) ### Bug Fixes * Disable sdc table for HNSWPQ read-only indices [#1518](https://github.com/opensearch-project/k-NN/pull/1518) * Switch SpaceType.INNERPRODUCT's vector similarity function to MAXIMUM_INNER_PRODUCT [#1532](https://github.com/opensearch-project/k-NN/pull/1532) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/FaissSQIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/FaissSQIT.java new file mode 100644 index 000000000..b6f1697bb --- /dev/null +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/FaissSQIT.java @@ -0,0 +1,390 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.bwc; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.hc.core5.http.ParseException; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.opensearch.client.Response; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.knn.KNNResult; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.KNNMethod; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.query.KNNQueryBuilder; +import org.opensearch.knn.index.util.KNNEngine; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.TreeMap; + +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_CLIP; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_TYPE; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.MODEL_ID; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; +import static org.opensearch.knn.common.KNNConstants.NAME; + +public class FaissSQIT extends AbstractRestartUpgradeTestCase { + private static final String TEST_FIELD = "test-field"; + private static final String TRAIN_TEST_FIELD = "train-test-field"; + private static final String TRAIN_INDEX = "train-index"; + private static final String TEST_MODEL = "test-model"; + private static final int DIMENSION = 128; + private static final int NUM_DOCS = 100; + + public void testHNSWSQFP16_onUpgradeWhenIndexedAndQueried_thenSucceed() throws Exception { + if (!isRunningAgainstOldCluster()) { + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + SpaceType[] spaceTypes = { SpaceType.L2, SpaceType.INNER_PRODUCT }; + Random random = new Random(); + SpaceType spaceType = spaceTypes[random.nextInt(spaceTypes.length)]; + + List mValues = ImmutableList.of(16, 32, 64, 128); + List efConstructionValues = ImmutableList.of(16, 32, 64, 128); + List efSearchValues = ImmutableList.of(16, 32, 64, 128); + + // Create an index + /** + * "properties": { + * "test-field": { + * "type": "knn_vector", + * "dimension": 128, + * "method": { + * "name": "hnsw", + * "space_type": "l2", + * "engine": "faiss", + * "parameters": { + * "m": 16, + * "ef_construction": 128, + * "ef_search": 128, + * "encoder": { + * "name": "sq", + * "parameters": { + * "type": "fp16" + * } + * } + * } + * } + * } + * } + */ + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(TEST_FIELD) + .field("type", "knn_vector") + .field("dimension", DIMENSION) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field( + KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, + efConstructionValues.get(random().nextInt(efConstructionValues.size())) + ) + .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + + createKnnIndex(testIndex, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(testIndex))); + indexTestData(testIndex, TEST_FIELD, DIMENSION, NUM_DOCS); + queryTestData(testIndex, TEST_FIELD, DIMENSION, NUM_DOCS); + deleteKNNIndex(testIndex); + validateGraphEviction(); + } + } + + public void testHNSWSQFP16_onUpgradeWhenClipToFp16isTrueAndIndexedWithOutOfFP16Range_thenSucceed() throws Exception { + if (!isRunningAgainstOldCluster()) { + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + new Random(); + + List mValues = ImmutableList.of(16, 32, 64, 128); + List efConstructionValues = ImmutableList.of(16, 32, 64, 128); + List efSearchValues = ImmutableList.of(16, 32, 64, 128); + + int dimension = 2; + + // Create an index + /** + * "properties": { + * "test-field": { + * "type": "knn_vector", + * "dimension": 128, + * "method": { + * "name": "hnsw", + * "space_type": "l2", + * "engine": "faiss", + * "parameters": { + * "m": 16, + * "ef_construction": 128, + * "ef_search": 128, + * "encoder": { + * "name": "sq", + * "parameters": { + * "type": "fp16", + * "clip": true + * } + * } + * } + * } + * } + * } + */ + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(TEST_FIELD) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field( + KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, + efConstructionValues.get(random().nextInt(efConstructionValues.size())) + ) + .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .field(FAISS_SQ_CLIP, true) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + + createKnnIndex(testIndex, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(testIndex))); + Float[] vector1 = { -65523.76f, 65504.2f }; + Float[] vector2 = { -270.85f, 65514.2f }; + Float[] vector3 = { -150.9f, 65504.0f }; + Float[] vector4 = { -20.89f, 100000000.0f }; + addKnnDoc(testIndex, "1", TEST_FIELD, vector1); + addKnnDoc(testIndex, "2", TEST_FIELD, vector2); + addKnnDoc(testIndex, "3", TEST_FIELD, vector3); + addKnnDoc(testIndex, "4", TEST_FIELD, vector4); + + float[] queryVector = { -10.5f, 25.48f }; + int k = 4; + Response searchResponse = searchKNNIndex(testIndex, new KNNQueryBuilder(TEST_FIELD, queryVector, k), k); + List results = parseSearchResponse(EntityUtils.toString(searchResponse.getEntity()), TEST_FIELD); + assertEquals(k, results.size()); + for (int i = 0; i < k; i++) { + assertEquals(k - i, Integer.parseInt(results.get(i).getDocId())); + } + deleteKNNIndex(testIndex); + validateGraphEviction(); + } + } + + public void testIVFSQFP16_onUpgradeWhenIndexedAndQueried_thenSucceed() throws Exception { + if (!isRunningAgainstOldCluster()) { + + // Add training data + createBasicKnnIndex(TRAIN_INDEX, TRAIN_TEST_FIELD, DIMENSION); + int trainingDataCount = 200; + bulkIngestRandomVectors(TRAIN_INDEX, TRAIN_TEST_FIELD, trainingDataCount, DIMENSION); + + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .field(METHOD_PARAMETER_SPACE_TYPE, "l2") + .startObject(PARAMETERS) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .endObject() + .endObject() + .endObject() + .endObject(); + Map method = xContentBuilderToMap(builder); + + trainModel(TEST_MODEL, TRAIN_INDEX, TRAIN_TEST_FIELD, DIMENSION, method, "faiss ivf sqfp16 test description"); + + // Make sure training succeeds after 30 seconds + assertTrainingSucceeds(TEST_MODEL, 30, 1000); + + // Create knn index from model + String indexMapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(TEST_FIELD) + .field("type", "knn_vector") + .field(MODEL_ID, TEST_MODEL) + .endObject() + .endObject() + .endObject() + .toString(); + + createKnnIndex(testIndex, getKNNDefaultIndexSettings(), indexMapping); + + indexTestData(testIndex, TEST_FIELD, DIMENSION, NUM_DOCS); + queryTestData(testIndex, TEST_FIELD, DIMENSION, NUM_DOCS); + deleteKNNIndex(TRAIN_INDEX); + deleteKNNIndex(testIndex); + deleteModel(TEST_MODEL); + validateGraphEviction(); + } + } + + public void testIVFSQFP16_onUpgradeWhenClipToFp16isTrueAndIndexedWithOutOfFP16Range_thenSucceed() throws Exception { + if (!isRunningAgainstOldCluster()) { + int dimension = 2; + + // Add training data + createBasicKnnIndex(TRAIN_INDEX, TRAIN_TEST_FIELD, dimension); + int trainingDataCount = 200; + bulkIngestRandomVectors(TRAIN_INDEX, TRAIN_TEST_FIELD, trainingDataCount, dimension); + + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .field(METHOD_PARAMETER_SPACE_TYPE, "l2") + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NLIST, 1) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .field(FAISS_SQ_CLIP, true) + .endObject() + .endObject() + .endObject() + .endObject(); + Map method = xContentBuilderToMap(builder); + + trainModel(TEST_MODEL, TRAIN_INDEX, TRAIN_TEST_FIELD, dimension, method, "faiss ivf sqfp16 test description"); + + // Make sure training succeeds after 30 seconds + assertTrainingSucceeds(TEST_MODEL, 30, 1000); + + // Create knn index from model + String indexMapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(TEST_FIELD) + .field("type", "knn_vector") + .field(MODEL_ID, TEST_MODEL) + .endObject() + .endObject() + .endObject() + .toString(); + + createKnnIndex(testIndex, getKNNDefaultIndexSettings(), indexMapping); + Float[] vector1 = { -65523.76f, 65504.2f }; + Float[] vector2 = { -270.85f, 65514.2f }; + Float[] vector3 = { -150.9f, 65504.0f }; + Float[] vector4 = { -20.89f, 100000000.0f }; + addKnnDoc(testIndex, "1", TEST_FIELD, vector1); + addKnnDoc(testIndex, "2", TEST_FIELD, vector2); + addKnnDoc(testIndex, "3", TEST_FIELD, vector3); + addKnnDoc(testIndex, "4", TEST_FIELD, vector4); + + float[] queryVector = { -10.5f, 25.48f }; + int k = 4; + Response searchResponse = searchKNNIndex(testIndex, new KNNQueryBuilder(TEST_FIELD, queryVector, k), k); + List results = parseSearchResponse(EntityUtils.toString(searchResponse.getEntity()), TEST_FIELD); + assertEquals(k, results.size()); + for (int i = 0; i < k; i++) { + assertEquals(k - i, Integer.parseInt(results.get(i).getDocId())); + } + + deleteKNNIndex(testIndex); + deleteKNNIndex(TRAIN_INDEX); + deleteModel(TEST_MODEL); + validateGraphEviction(); + } + } + + private void validateGraphEviction() throws Exception { + // Search every 5 seconds 14 times to confirm graph gets evicted + int intervals = 14; + for (int i = 0; i < intervals; i++) { + if (getTotalGraphsInCache() == 0) { + return; + } + + Thread.sleep(5 * 1000); + } + + fail("Graphs are not getting evicted"); + } + + private void queryTestData(final String indexName, final String fieldName, final int dimension, final int numDocs) throws IOException, + ParseException { + float[] queryVector = new float[dimension]; + Arrays.fill(queryVector, (float) numDocs); + int k = 10; + + Response searchResponse = searchKNNIndex(indexName, new KNNQueryBuilder(fieldName, queryVector, k), k); + List results = parseSearchResponse(EntityUtils.toString(searchResponse.getEntity()), fieldName); + assertEquals(k, results.size()); + for (int i = 0; i < k; i++) { + assertEquals(numDocs - i - 1, Integer.parseInt(results.get(i).getDocId())); + } + } + + private void indexTestData(final String indexName, final String fieldName, final int dimension, final int numDocs) throws Exception { + for (int i = 0; i < numDocs; i++) { + float[] indexVector = new float[dimension]; + Arrays.fill(indexVector, (float) i); + addKnnDocWithAttributes(indexName, Integer.toString(i), fieldName, indexVector, ImmutableMap.of("rating", String.valueOf(i))); + } + + // Assert that all docs are ingested + refreshAllNonSystemIndices(); + assertEquals(numDocs, getDocCount(indexName)); + } + +} diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index 314d7d4d1..46d82b2bf 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -97,6 +97,7 @@ public class KNNConstants { public static final String FAISS_SQ_TYPE = "type"; public static final String FAISS_SQ_ENCODER_FP16 = "fp16"; public static final List FAISS_SQ_ENCODER_TYPES = List.of(FAISS_SQ_ENCODER_FP16); + public static final String FAISS_SQ_CLIP = "clip"; // Parameter defaults/limits public static final Integer ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT = 1; @@ -111,6 +112,9 @@ public class KNNConstants { public static final Integer MODEL_CACHE_CAPACITY_ATROPHY_THRESHOLD_IN_MINUTES = 30; public static final Integer MODEL_CACHE_EXPIRE_AFTER_ACCESS_TIME_MINUTES = 30; + public static final Float FP16_MAX_VALUE = 65504.0f; + public static final Float FP16_MIN_VALUE = -65504.0f; + // Lib names private static final String JNI_LIBRARY_PREFIX = "opensearchknn_"; public static final String FAISS_JNI_LIBRARY_NAME = JNI_LIBRARY_PREFIX + FAISS_NAME; diff --git a/src/main/java/org/opensearch/knn/index/Parameter.java b/src/main/java/org/opensearch/knn/index/Parameter.java index bef5a33e9..e223909d5 100644 --- a/src/main/java/org/opensearch/knn/index/Parameter.java +++ b/src/main/java/org/opensearch/knn/index/Parameter.java @@ -66,6 +66,31 @@ public T getDefaultValue() { */ public abstract ValidationException validate(Object value); + /** + * Boolean method parameter + */ + public static class BooleanParameter extends Parameter { + public BooleanParameter(String name, Boolean defaultValue, Predicate validator) { + super(name, defaultValue, validator); + } + + @Override + public ValidationException validate(Object value) { + ValidationException validationException = null; + if (!(value instanceof Boolean)) { + validationException = new ValidationException(); + validationException.addValidationError(String.format("value not of type Boolean for Boolean parameter [%s].", getName())); + return validationException; + } + + if (!validator.test((Boolean) value)) { + validationException = new ValidationException(); + validationException.addValidationError(String.format("parameter validation failed for Boolean parameter [%s].", getName())); + } + return validationException; + } + } + /** * Integer method parameter */ diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 2369a6937..a36a4222b 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -44,6 +44,7 @@ import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.KNNVectorIndexFieldData; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; import org.opensearch.knn.index.util.KNNEngine; @@ -51,14 +52,31 @@ import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.lookup.SearchLookup; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Supplier; + import static org.opensearch.knn.common.KNNConstants.DEFAULT_VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_CLIP; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_TYPE; import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.common.KNNValidationUtil.validateByteVectorValue; import static org.opensearch.knn.common.KNNValidationUtil.validateFloatVectorValue; import static org.opensearch.knn.common.KNNValidationUtil.validateVectorDimension; import static org.opensearch.knn.index.KNNSettings.KNN_INDEX; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.addStoredFieldForVectorField; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.clipVectorValueToFP16Range; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateFP16VectorValue; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateVectorDataTypeWithEngine; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateVectorDataTypeWithKnnIndexSetting; @@ -511,10 +529,23 @@ protected String contentType() { @Override protected void parseCreateField(ParseContext context) throws IOException { - parseCreateField(context, fieldType().getDimension(), fieldType().getSpaceType()); + parseCreateField( + context, + fieldType().getDimension(), + fieldType().getSpaceType(), + getMethodComponentContext(fieldType().getKnnMethodContext()) + ); } - protected void parseCreateField(ParseContext context, int dimension, SpaceType spaceType) throws IOException { + private MethodComponentContext getMethodComponentContext(KNNMethodContext knnMethodContext) { + if (Objects.isNull(knnMethodContext)) { + return null; + } + return knnMethodContext.getMethodComponentContext(); + } + + protected void parseCreateField(ParseContext context, int dimension, SpaceType spaceType, MethodComponentContext methodComponentContext) + throws IOException { validateIfKNNPluginEnabled(); validateIfCircuitBreakerIsNotTriggered(); @@ -532,7 +563,7 @@ protected void parseCreateField(ParseContext context, int dimension, SpaceType s context.doc().add(point); addStoredFieldForVectorField(context, fieldType, name(), point.toString()); } else if (VectorDataType.FLOAT == vectorDataType) { - Optional floatsArrayOptional = getFloatsFromContext(context, dimension); + Optional floatsArrayOptional = getFloatsFromContext(context, dimension, methodComponentContext); if (floatsArrayOptional.isEmpty()) { return; @@ -551,6 +582,47 @@ protected void parseCreateField(ParseContext context, int dimension, SpaceType s context.path().remove(); } + // Verify mapping and return true if it is a "faiss" Index using "sq" encoder of type "fp16" + protected boolean isFaissSQfp16(MethodComponentContext methodComponentContext) { + if (Objects.isNull(methodComponentContext)) { + return false; + } + + if (methodComponentContext.getParameters().size() == 0) { + return false; + } + + Map methodComponentParams = methodComponentContext.getParameters(); + + // The method component parameters should have an encoder + if (!methodComponentParams.containsKey(METHOD_ENCODER_PARAMETER)) { + return false; + } + + // Validate if the object is of type MethodComponentContext before casting it later + if (!(methodComponentParams.get(METHOD_ENCODER_PARAMETER) instanceof MethodComponentContext)) { + return false; + } + + MethodComponentContext encoderMethodComponentContext = (MethodComponentContext) methodComponentParams.get(METHOD_ENCODER_PARAMETER); + + // returns true if encoder name is "sq" and type is "fp16" + return ENCODER_SQ.equals(encoderMethodComponentContext.getName()) + && FAISS_SQ_ENCODER_FP16.equals( + encoderMethodComponentContext.getParameters().getOrDefault(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + ); + + } + + // Verify mapping and return the value of "clip" parameter(default false) for a "faiss" Index + // using "sq" encoder of type "fp16". + protected boolean isFaissSQClipToFP16RangeEnabled(MethodComponentContext methodComponentContext) { + if (Objects.nonNull(methodComponentContext)) { + return (boolean) methodComponentContext.getParameters().getOrDefault(FAISS_SQ_CLIP, false); + } + return false; + } + void validateIfCircuitBreakerIsNotTriggered() { if (KNNSettings.isCircuitBreakerTriggered()) { throw new IllegalStateException( @@ -600,9 +672,22 @@ Optional getBytesFromContext(ParseContext context, int dimension) throws return Optional.of(array); } - Optional getFloatsFromContext(ParseContext context, int dimension) throws IOException { + Optional getFloatsFromContext(ParseContext context, int dimension, MethodComponentContext methodComponentContext) + throws IOException { context.path().add(simpleName()); + // Returns an optional array of float values where each value in the vector is parsed as a float and validated + // if it is a finite number and within the fp16 range of [-65504 to 65504] by default if Faiss encoder is SQ and type is 'fp16'. + // If the encoder parameter, "clip" is set to True, if the vector value is outside the FP16 range then it will be + // clipped to FP16 range. + boolean isFaissSQfp16Flag = isFaissSQfp16(methodComponentContext); + boolean clipVectorValueToFP16RangeFlag = false; + if (isFaissSQfp16Flag) { + clipVectorValueToFP16RangeFlag = isFaissSQClipToFP16RangeEnabled( + (MethodComponentContext) methodComponentContext.getParameters().get(METHOD_ENCODER_PARAMETER) + ); + } + ArrayList vector = new ArrayList<>(); XContentParser.Token token = context.parser().currentToken(); float value; @@ -610,13 +695,30 @@ Optional getFloatsFromContext(ParseContext context, int dimension) thro token = context.parser().nextToken(); while (token != XContentParser.Token.END_ARRAY) { value = context.parser().floatValue(); - validateFloatVectorValue(value); + if (isFaissSQfp16Flag) { + if (clipVectorValueToFP16RangeFlag) { + value = clipVectorValueToFP16Range(value); + } else { + validateFP16VectorValue(value); + } + } else { + validateFloatVectorValue(value); + } + vector.add(value); token = context.parser().nextToken(); } } else if (token == XContentParser.Token.VALUE_NUMBER) { value = context.parser().floatValue(); - validateFloatVectorValue(value); + if (isFaissSQfp16Flag) { + if (clipVectorValueToFP16RangeFlag) { + value = clipVectorValueToFP16Range(value); + } else { + validateFP16VectorValue(value); + } + } else { + validateFloatVectorValue(value); + } vector.add(value); context.parser().nextToken(); } else if (token == XContentParser.Token.VALUE_NULL) { diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java index b525b9dc6..283d35f00 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java @@ -23,12 +23,55 @@ import java.util.Locale; +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; +import static org.opensearch.knn.common.KNNConstants.FP16_MAX_VALUE; +import static org.opensearch.knn.common.KNNConstants.FP16_MIN_VALUE; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.common.KNNValidationUtil.validateFloatVectorValue; @NoArgsConstructor(access = AccessLevel.PRIVATE) public class KNNVectorFieldMapperUtil { + + /** + * Validate the float vector value and throw exception if it is not a number or not in the finite range + * or is not within the FP16 range of [-65504 to 65504]. + * + * @param value float vector value + */ + public static void validateFP16VectorValue(float value) { + validateFloatVectorValue(value); + + if (value < FP16_MIN_VALUE || value > FP16_MAX_VALUE) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", + ENCODER_SQ, + FAISS_SQ_ENCODER_FP16, + FP16_MIN_VALUE, + FP16_MAX_VALUE + ) + ); + } + } + + /** + * Validate the float vector value and if it is outside FP16 range, + * then it will be clipped to FP16 range of [-65504 to 65504]. + * + * @param value float vector value + * @return vector value clipped to FP16 range + */ + public static float clipVectorValueToFP16Range(float value) { + validateFloatVectorValue(value); + if (value < FP16_MIN_VALUE) return FP16_MIN_VALUE; + if (value > FP16_MAX_VALUE) return FP16_MAX_VALUE; + return value; + } + /** * Validates and throws exception if data_type field is set in the index mapping * using any VectorDataType (other than float, which is default) because other diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java index 81c7216bf..185ab3dc4 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -18,6 +18,7 @@ import org.opensearch.common.Explicit; import org.opensearch.index.mapper.ParseContext; import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; @@ -75,7 +76,8 @@ public class LuceneFieldMapper extends KNNVectorFieldMapper { } @Override - protected void parseCreateField(ParseContext context, int dimension, SpaceType spaceType) throws IOException { + protected void parseCreateField(ParseContext context, int dimension, SpaceType spaceType, MethodComponentContext methodComponentContext) + throws IOException { validateIfKNNPluginEnabled(); validateIfCircuitBreakerIsNotTriggered(); @@ -96,7 +98,7 @@ protected void parseCreateField(ParseContext context, int dimension, SpaceType s context.doc().add(new VectorField(name(), array, vectorFieldType)); } } else if (VectorDataType.FLOAT == vectorDataType) { - Optional floatsArrayOptional = getFloatsFromContext(context, dimension); + Optional floatsArrayOptional = getFloatsFromContext(context, dimension, methodComponentContext); if (floatsArrayOptional.isEmpty()) { return; diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java index 2367d7422..ce92d2967 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java @@ -61,6 +61,6 @@ protected void parseCreateField(ParseContext context) throws IOException { ); } - parseCreateField(context, modelMetadata.getDimension(), modelMetadata.getSpaceType()); + parseCreateField(context, modelMetadata.getDimension(), modelMetadata.getSpaceType(), modelMetadata.getMethodComponentContext()); } } diff --git a/src/main/java/org/opensearch/knn/index/util/Faiss.java b/src/main/java/org/opensearch/knn/index/util/Faiss.java index 3b21488b9..563311c49 100644 --- a/src/main/java/org/opensearch/knn/index/util/Faiss.java +++ b/src/main/java/org/opensearch/knn/index/util/Faiss.java @@ -32,6 +32,7 @@ import static org.opensearch.knn.common.KNNConstants.FAISS_HNSW_DESCRIPTION; import static org.opensearch.knn.common.KNNConstants.FAISS_IVF_DESCRIPTION; import static org.opensearch.knn.common.KNNConstants.FAISS_PQ_DESCRIPTION; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_CLIP; import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_DESCRIPTION; import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_TYPES; @@ -90,6 +91,7 @@ class Faiss extends NativeLibrary { FAISS_SQ_TYPE, new Parameter.StringParameter(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16, FAISS_SQ_ENCODER_TYPES::contains) ) + .addParameter(FAISS_SQ_CLIP, new Parameter.BooleanParameter(FAISS_SQ_CLIP, false, Objects::nonNull)) .setMapGenerator( ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( FAISS_SQ_DESCRIPTION, diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index f00942068..13d1bc64d 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -20,6 +20,7 @@ import org.opensearch.client.Response; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.settings.Settings; +import org.opensearch.client.ResponseException; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.knn.KNNRestTestCase; @@ -34,6 +35,7 @@ import java.net.URL; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Random; import java.util.TreeMap; @@ -44,8 +46,11 @@ import static org.opensearch.knn.common.KNNConstants.ENCODER_PQ; import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_CLIP; import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_TYPE; +import static org.opensearch.knn.common.KNNConstants.FP16_MAX_VALUE; +import static org.opensearch.knn.common.KNNConstants.FP16_MIN_VALUE; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; @@ -391,6 +396,356 @@ public void testIVFSQFP16_whenIndexedAndQueried_thenSucceed() { validateGraphEviction(); } + @SneakyThrows + public void testHNSWSQFP16_whenIndexedWithOutOfFP16Range_thenThrowException() { + String indexName = "test-index-sqfp16"; + String fieldName = "test-field-sqfp16"; + + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + SpaceType[] spaceTypes = { SpaceType.L2, SpaceType.INNER_PRODUCT }; + Random random = new Random(); + SpaceType spaceType = spaceTypes[random.nextInt(spaceTypes.length)]; + + List mValues = ImmutableList.of(16, 32, 64, 128); + List efConstructionValues = ImmutableList.of(16, 32, 64, 128); + List efSearchValues = ImmutableList.of(16, 32, 64, 128); + + int dimension = 2; + + // Create an index + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + + createKnnIndex(indexName, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); + Float[] vector = { -10.76f, 65504.2f }; + + ResponseException ex = expectThrows(ResponseException.class, () -> addKnnDoc(indexName, "1", fieldName, vector)); + assertTrue( + ex.getMessage() + .contains( + String.format( + Locale.ROOT, + "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", + ENCODER_SQ, + FAISS_SQ_ENCODER_FP16, + FP16_MIN_VALUE, + FP16_MAX_VALUE + ) + ) + ); + + Float[] vector1 = { -65506.84f, 12.56f }; + + ResponseException ex1 = expectThrows(ResponseException.class, () -> addKnnDoc(indexName, "2", fieldName, vector1)); + assertTrue( + ex1.getMessage() + .contains( + String.format( + Locale.ROOT, + "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", + ENCODER_SQ, + FAISS_SQ_ENCODER_FP16, + FP16_MIN_VALUE, + FP16_MAX_VALUE + ) + ) + ); + + Float[] vector2 = { -65526.4567f, 65526.4567f }; + + ResponseException ex2 = expectThrows(ResponseException.class, () -> addKnnDoc(indexName, "3", fieldName, vector2)); + assertTrue( + ex2.getMessage() + .contains( + String.format( + Locale.ROOT, + "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", + ENCODER_SQ, + FAISS_SQ_ENCODER_FP16, + FP16_MIN_VALUE, + FP16_MAX_VALUE + ) + ) + ); + deleteKNNIndex(indexName); + validateGraphEviction(); + } + + @SneakyThrows + public void testHNSWSQFP16_whenClipToFp16isTrueAndIndexedWithOutOfFP16Range_thenSucceed() { + String indexName = "test-index-sqfp16-clip-fp16"; + String fieldName = "test-field-sqfp16"; + + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + Random random = new Random(); + + List mValues = ImmutableList.of(16, 32, 64, 128); + List efConstructionValues = ImmutableList.of(16, 32, 64, 128); + List efSearchValues = ImmutableList.of(16, 32, 64, 128); + + int dimension = 2; + + // Create an index + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .field(FAISS_SQ_CLIP, true) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + + createKnnIndex(indexName, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); + Float[] vector1 = { -65523.76f, 65504.2f }; + Float[] vector2 = { -270.85f, 65514.2f }; + Float[] vector3 = { -150.9f, 65504.0f }; + Float[] vector4 = { -20.89f, 100000000.0f }; + addKnnDoc(indexName, "1", fieldName, vector1); + addKnnDoc(indexName, "2", fieldName, vector2); + addKnnDoc(indexName, "3", fieldName, vector3); + addKnnDoc(indexName, "4", fieldName, vector4); + + float[] queryVector = { -10.5f, 25.48f }; + int k = 4; + Response searchResponse = searchKNNIndex(indexName, new KNNQueryBuilder(fieldName, queryVector, k), k); + List results = parseSearchResponse(EntityUtils.toString(searchResponse.getEntity()), fieldName); + assertEquals(k, results.size()); + for (int i = 0; i < k; i++) { + assertEquals(k - i, Integer.parseInt(results.get(i).getDocId())); + } + + deleteKNNIndex(indexName); + validateGraphEviction(); + } + + @SneakyThrows + public void testIVFSQFP16_whenIndexedWithOutOfFP16Range_thenThrowException() { + String modelId = "test-model-ivf-sqfp16"; + int dimension = 128; + + String trainingIndexName = "train-index-ivf-sqfp16"; + String trainingFieldName = "train-field-ivf-sqfp16"; + + // Add training data + createBasicKnnIndex(trainingIndexName, trainingFieldName, dimension); + int trainingDataCount = 200; + bulkIngestRandomVectors(trainingIndexName, trainingFieldName, trainingDataCount, dimension); + + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .field(METHOD_PARAMETER_SPACE_TYPE, "l2") + .startObject(PARAMETERS) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .endObject() + .endObject() + .endObject() + .endObject(); + Map method = xContentBuilderToMap(builder); + + trainModel(modelId, trainingIndexName, trainingFieldName, dimension, method, "faiss ivf sqfp16 test description"); + + // Make sure training succeeds after 30 seconds + assertTrainingSucceeds(modelId, 30, 1000); + + // Create knn index from model + String fieldName = "test-field-name-ivf-sqfp16"; + String indexName = "test-index-name-ivf-sqfp16"; + String indexMapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field(MODEL_ID, modelId) + .endObject() + .endObject() + .endObject() + .toString(); + + createKnnIndex(indexName, getKNNDefaultIndexSettings(), indexMapping); + Float[] vector = { -10.76f, 65504.2f }; + + ResponseException ex = expectThrows(ResponseException.class, () -> addKnnDoc(indexName, "1", fieldName, vector)); + assertTrue( + ex.getMessage() + .contains( + String.format( + Locale.ROOT, + "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", + ENCODER_SQ, + FAISS_SQ_ENCODER_FP16, + FP16_MIN_VALUE, + FP16_MAX_VALUE + ) + ) + ); + + Float[] vector1 = { -65506.84f, 12.56f }; + + ResponseException ex1 = expectThrows(ResponseException.class, () -> addKnnDoc(indexName, "2", fieldName, vector1)); + assertTrue( + ex1.getMessage() + .contains( + String.format( + Locale.ROOT, + "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", + ENCODER_SQ, + FAISS_SQ_ENCODER_FP16, + FP16_MIN_VALUE, + FP16_MAX_VALUE + ) + ) + ); + + Float[] vector2 = { -65526.4567f, 65526.4567f }; + + ResponseException ex2 = expectThrows(ResponseException.class, () -> addKnnDoc(indexName, "3", fieldName, vector2)); + assertTrue( + ex2.getMessage() + .contains( + String.format( + Locale.ROOT, + "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", + ENCODER_SQ, + FAISS_SQ_ENCODER_FP16, + FP16_MIN_VALUE, + FP16_MAX_VALUE + ) + ) + ); + deleteKNNIndex(indexName); + deleteKNNIndex(trainingIndexName); + deleteModel(modelId); + } + + @SneakyThrows + public void testIVFSQFP16_whenClipToFp16isTrueAndIndexedWithOutOfFP16Range_thenSucceed() { + String modelId = "test-model-ivf-sqfp16"; + int dimension = 2; + + String trainingIndexName = "train-index-ivf-sqfp16"; + String trainingFieldName = "train-field-ivf-sqfp16"; + + // Add training data + createBasicKnnIndex(trainingIndexName, trainingFieldName, dimension); + int trainingDataCount = 200; + bulkIngestRandomVectors(trainingIndexName, trainingFieldName, trainingDataCount, dimension); + + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .field(METHOD_PARAMETER_SPACE_TYPE, "l2") + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NLIST, 1) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .field(FAISS_SQ_CLIP, true) + .endObject() + .endObject() + .endObject() + .endObject(); + Map method = xContentBuilderToMap(builder); + + trainModel(modelId, trainingIndexName, trainingFieldName, dimension, method, "faiss ivf sqfp16 test description"); + + // Make sure training succeeds after 30 seconds + assertTrainingSucceeds(modelId, 30, 1000); + + // Create knn index from model + String fieldName = "test-field-name-ivf-sqfp16"; + String indexName = "test-index-name-ivf-sqfp16"; + String indexMapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field(MODEL_ID, modelId) + .endObject() + .endObject() + .endObject() + .toString(); + + createKnnIndex(indexName, getKNNDefaultIndexSettings(), indexMapping); + Float[] vector1 = { -65523.76f, 65504.2f }; + Float[] vector2 = { -270.85f, 65514.2f }; + Float[] vector3 = { -150.9f, 65504.0f }; + Float[] vector4 = { -20.89f, 100000000.0f }; + addKnnDoc(indexName, "1", fieldName, vector1); + addKnnDoc(indexName, "2", fieldName, vector2); + addKnnDoc(indexName, "3", fieldName, vector3); + addKnnDoc(indexName, "4", fieldName, vector4); + + float[] queryVector = { -10.5f, 25.48f }; + int k = 4; + Response searchResponse = searchKNNIndex(indexName, new KNNQueryBuilder(fieldName, queryVector, k), k); + List results = parseSearchResponse(EntityUtils.toString(searchResponse.getEntity()), fieldName); + assertEquals(k, results.size()); + for (int i = 0; i < k; i++) { + assertEquals(k - i, Integer.parseInt(results.get(i).getDocId())); + } + + deleteKNNIndex(indexName); + deleteKNNIndex(trainingIndexName); + deleteModel(modelId); + validateGraphEviction(); + } + @SneakyThrows public void testEndToEnd_whenMethodIsHNSWPQAndHyperParametersNotSet_thenSucceed() { String indexName = "test-index"; diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 824b61a9f..a9b65878f 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -53,6 +53,10 @@ import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.opensearch.knn.common.KNNConstants.DIMENSION; +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; +import static org.opensearch.knn.common.KNNConstants.FP16_MAX_VALUE; +import static org.opensearch.knn.common.KNNConstants.FP16_MIN_VALUE; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; @@ -68,6 +72,8 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.clipVectorValueToFP16Range; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateFP16VectorValue; public class KNNVectorFieldMapperTests extends KNNTestCase { @@ -733,10 +739,16 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { when(parseContext.path()).thenReturn(contentPath); LuceneFieldMapper luceneFieldMapper = Mockito.spy(new LuceneFieldMapper(inputBuilder.build())); - doReturn(Optional.of(TEST_VECTOR)).when(luceneFieldMapper).getFloatsFromContext(parseContext, TEST_DIMENSION); + doReturn(Optional.of(TEST_VECTOR)).when(luceneFieldMapper) + .getFloatsFromContext(parseContext, TEST_DIMENSION, new MethodComponentContext(METHOD_HNSW, Collections.emptyMap())); doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); - luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, luceneFieldMapper.fieldType().spaceType); + luceneFieldMapper.parseCreateField( + parseContext, + TEST_DIMENSION, + luceneFieldMapper.fieldType().spaceType, + luceneFieldMapper.fieldType().knnMethodContext.getMethodComponentContext() + ); // Document should have 2 fields: one for VectorField (binary doc values) and one for KnnVectorField List fields = document.getFields(); @@ -770,11 +782,17 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { inputBuilder.hasDocValues(false); luceneFieldMapper = Mockito.spy(new LuceneFieldMapper(inputBuilder.build())); - doReturn(Optional.of(TEST_VECTOR)).when(luceneFieldMapper).getFloatsFromContext(parseContext, TEST_DIMENSION); + doReturn(Optional.of(TEST_VECTOR)).when(luceneFieldMapper) + .getFloatsFromContext(parseContext, TEST_DIMENSION, new MethodComponentContext(METHOD_HNSW, Collections.emptyMap())); doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); - luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, luceneFieldMapper.fieldType().spaceType); + luceneFieldMapper.parseCreateField( + parseContext, + TEST_DIMENSION, + luceneFieldMapper.fieldType().spaceType, + luceneFieldMapper.fieldType().knnMethodContext.getMethodComponentContext() + ); // Document should have 1 field: one for KnnVectorField fields = document.getFields(); @@ -803,7 +821,12 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); - luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, luceneFieldMapper.fieldType().spaceType); + luceneFieldMapper.parseCreateField( + parseContext, + TEST_DIMENSION, + luceneFieldMapper.fieldType().spaceType, + luceneFieldMapper.fieldType().knnMethodContext.getMethodComponentContext() + ); // Document should have 2 fields: one for VectorField (binary doc values) and one for KnnByteVectorField List fields = document.getFields(); @@ -840,7 +863,12 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); - luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, luceneFieldMapper.fieldType().spaceType); + luceneFieldMapper.parseCreateField( + parseContext, + TEST_DIMENSION, + luceneFieldMapper.fieldType().spaceType, + luceneFieldMapper.fieldType().knnMethodContext.getMethodComponentContext() + ); // Document should have 1 field: one for KnnByteVectorField fields = document.getFields(); @@ -851,6 +879,45 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { assertArrayEquals(TEST_BYTE_VECTOR, knnByteVectorField.vectorValue()); } + public void testValidateFp16VectorValue_outOfRange_throwsException() { + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> validateFP16VectorValue(65505.25f)); + assertTrue( + ex.getMessage() + .contains( + String.format( + Locale.ROOT, + "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", + ENCODER_SQ, + FAISS_SQ_ENCODER_FP16, + FP16_MIN_VALUE, + FP16_MAX_VALUE + ) + ) + ); + + IllegalArgumentException ex1 = expectThrows(IllegalArgumentException.class, () -> validateFP16VectorValue(-65525.65f)); + assertTrue( + ex1.getMessage() + .contains( + String.format( + Locale.ROOT, + "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", + ENCODER_SQ, + FAISS_SQ_ENCODER_FP16, + FP16_MIN_VALUE, + FP16_MAX_VALUE + ) + ) + ); + } + + public void testClipVectorValuetoFP16Range_succeed() { + assertEquals(65504.0f, clipVectorValueToFP16Range(65504.10f), 0.0f); + assertEquals(65504.0f, clipVectorValueToFP16Range(1000000.89f), 0.0f); + assertEquals(-65504.0f, clipVectorValueToFP16Range(-65504.10f), 0.0f); + assertEquals(-65504.0f, clipVectorValueToFP16Range(-1000000.89f), 0.0f); + } + private LuceneFieldMapper.CreateLuceneFieldMapperInput.CreateLuceneFieldMapperInputBuilder createLuceneFieldMapperInputBuilder( VectorDataType vectorDataType ) {