Skip to content

Commit

Permalink
Add nested field synthetic and tests for update, search, reindex, nested
Browse files Browse the repository at this point in the history
  • Loading branch information
luyuncheng committed May 14, 2024
1 parent 469fbf3 commit 256c32a
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 12 deletions.
3 changes: 1 addition & 2 deletions src/main/java/org/opensearch/knn/index/fetch/KNNFetchSubPhase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ protected void syntheticNestedDocValues(MapperService mapperService, HitContext
// nested array in one nested path
ArrayList nestedDocList = (ArrayList) nestedObj;

log.info(
log.debug(
"object mapper: nested:"
+ objectMapper.nested().isNested()
+ " Value:"
Expand Down Expand Up @@ -192,7 +192,6 @@ protected void syntheticNestedDocValues(MapperService mapperService, HitContext
}
if (offset < nestedDocList.size()) {
Object o2 = nestedDocList.get(offset);
log.info("arraylist value:" + o2.getClass().getName());
if (o2 instanceof Map) {
Map<String, Object> o2map = (Map<String, Object>) o2;
String suffix = f.field.substring(path.length() + 1);
Expand Down
77 changes: 67 additions & 10 deletions src/test/java/org/opensearch/knn/index/KNNSyntheticSourceIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,6 @@ public void testNestedFieldSyntheticSourceSearch_whenEnabledSynthetic_thenReturn
XContentBuilder builder = constructNestedMappingBuilder();

String mapping = builder.toString();
System.out.println("mapping debug:" + mapping);
Settings indexSettingWithSynthetic = Settings.builder()
.put("number_of_shards", 1)
.put("number_of_replicas", 0)
Expand All @@ -178,7 +177,6 @@ public void testNestedFieldSyntheticSourceSearch_whenEnabledSynthetic_thenReturn

Response responseWithSynthetic = queryNestedField(indexNameWithSynthetic, 10, vector);
String resp1 = EntityUtils.toString(responseWithSynthetic.getEntity());
System.out.println(resp1);
assertTrue(resp1.contains("\"nested-field\":[{\"test-nested-field-1\":[6.0,6.0]},{\"test-nested-field-1\":[6.0,6.0]}]"));
}

Expand All @@ -188,7 +186,6 @@ public void testNestedFieldSyntheticSourceSearch_whenDisabledSynthetic_thenRetur
XContentBuilder builder = constructNestedMappingBuilder();

String mapping = builder.toString();
System.out.println("mapping debug:" + mapping);
Settings indexSettingWithSynthetic = Settings.builder()
.put("number_of_shards", 1)
.put("number_of_replicas", 0)
Expand All @@ -203,7 +200,6 @@ public void testNestedFieldSyntheticSourceSearch_whenDisabledSynthetic_thenRetur

Response responseWithSynthetic = queryNestedField(indexNameWithSynthetic, 10, vector);
String resp1 = EntityUtils.toString(responseWithSynthetic.getEntity());
System.out.println(resp1);
assertFalse(resp1.contains("\"nested-field\":[{\"test-nested-field-1\":[6.0,6.0]},{\"test-nested-field-1\":[6.0,6.0]}]"));
}

Expand All @@ -213,7 +209,6 @@ public void testMultiNestedField_whenEnabledSynthetic_thenReturnSuccess() throws
XContentBuilder builder = constructNestedMappingBuilder();

String mapping = builder.toString();
System.out.println("mapping debug:" + mapping);
Settings indexSettingWithSynthetic = Settings.builder()
.put("number_of_shards", 1)
.put("number_of_replicas", 0)
Expand Down Expand Up @@ -249,7 +244,6 @@ public void testMultiNestedField_whenEnabledSynthetic_thenReturnSuccess() throws
Response responseWithSynthetic = queryNestedField(indexNameWithSynthetic, 10, vector, null, null, null, RestStatus.OK);

String resp1 = EntityUtils.toString(responseWithSynthetic.getEntity());
System.out.println("Resp" + resp1);
assertTrue(
resp1.contains(
"{\"nested-field\":[{\"nested_numeric\":1.0,\"test-nested-field-1\":[6.0,6.0]},{\"nested_numeric\":2.0,\"test-nested-field-1\":[6.0,6.0]}]}}]}"
Expand All @@ -263,7 +257,6 @@ public void testMultiNestedFieldWithNull_whenEnabledSynthetic_thenReturnFailed()
XContentBuilder builder = constructNestedMappingBuilder();

String mapping = builder.toString();
System.out.println("mapping debug:" + mapping);
Settings indexSettingWithSynthetic = Settings.builder()
.put("number_of_shards", 1)
.put("number_of_replicas", 0)
Expand Down Expand Up @@ -372,6 +365,72 @@ public void testSyntheticSourceUpdate_whenEnabledSynthetic_thenReturnSource() th
assertTrue(respUpdate.contains("\"test-field-1\":[8.0,8.0]"));
}

public void testSyntheticSourceUpdateOtherField_whenEnabledSynthetic_thenReturnNothing() throws IOException, ParseException {
String indexNameWithSynthetic = "test-index-synthetic";
String fieldName = "test-field-1";
String fieldName2 = "test-field-2";
Integer dimension = 2;

KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW);
SpaceType spaceType = SpaceType.L2;

// Create an index
XContentBuilder builder = XContentFactory.jsonBuilder()
.startObject()
.startObject("_source")
.startArray("excludes")
.value(fieldName)
.endArray()
.endObject()
.startObject("properties")
.startObject(fieldName)
.field("type", "knn_vector")
.field("dimension", dimension)
.startObject(KNNConstants.KNN_METHOD)
.field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName())
.field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue())
.field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName())
.endObject()
.endObject()
.startObject(fieldName2)
.field("type", "knn_vector")
.field("dimension", dimension)
.startObject(KNNConstants.KNN_METHOD)
.field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName())
.field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue())
.field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName())
.endObject()
.endObject()
.endObject()
.endObject();

String mapping = builder.toString();
Settings indexSettingWithSynthetic = Settings.builder()
.put("number_of_shards", 1)
.put("number_of_replicas", 0)
.put("index.knn.synthetic_source.enabled", true)
.put("index.knn", true)
.build();

createKnnIndex(indexNameWithSynthetic, indexSettingWithSynthetic, mapping);

Float[] vector = { 6.0f, 6.0f };
addKnnDoc(indexNameWithSynthetic, "1", fieldName, vector);
float[] queryVector = { 6.0f, 6.0f };

Response responseWithSynthetic = searchKNNIndex(indexNameWithSynthetic, new KNNQueryBuilder(fieldName, queryVector, 10), 10);
String resp1 = EntityUtils.toString(responseWithSynthetic.getEntity());
assertTrue(resp1.contains("\"test-field-1\":[6.0,6.0]"));

Float[] vector2 = { 8.0f, 8.0f };
updateKnnDoc(indexNameWithSynthetic, "1", fieldName2, vector2);
float[] queryVector2 = { 8.0f, 8.0f };
Response responseAfterUpdate = searchKNNIndex(indexNameWithSynthetic, new KNNQueryBuilder(fieldName2, queryVector2, 10), 10);
String respUpdate = EntityUtils.toString(responseAfterUpdate.getEntity());
assertTrue(respUpdate.contains("\"test-field-2\":[8.0,8.0]"));
assertFalse(respUpdate.contains("\"test-field-1\""));
}

private Response queryNestedField(final String index, final int k, final Object[] vector) throws IOException {
return queryNestedField(index, k, vector, null, null, null, RestStatus.OK);
}
Expand Down Expand Up @@ -407,8 +466,7 @@ private Response queryNestedField(
builder.endObject().endObject().endObject().endObject().endObject().endObject();
String requestStr = builder.toString();
Request request = new Request("POST", "/" + index + "/_search");
System.out.println("DEBUG Query DSL:" + requestStr);
request.setJsonEntity(builder.toString());
request.setJsonEntity(requestStr);
Response response;

response = client().performRequest(request);
Expand Down Expand Up @@ -449,7 +507,6 @@ private void addKnnDocWithBuilder(String index, String docId, XContentBuilder bu

Request request = new Request("POST", "/" + index + "/_doc/" + docId + "?refresh=true");
String docStr = builder.toString();
System.out.println("DEBUG index:" + docStr);
request.setJsonEntity(docStr);
client().performRequest(request);

Expand Down

0 comments on commit 256c32a

Please sign in to comment.