From 781a2d4f6e68ecf4bb0f62170c7bb6471f3b73c1 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Tue, 9 Jul 2024 21:31:24 +0530 Subject: [PATCH] doc values file format Signed-off-by: Sarthak Aggarwal --- .../Composite99DocValuesConsumer.java | 68 ++++++++ .../lucene90/StarTree99DocValuesProducer.java | 50 +++--- .../lucene/index/BaseStarTreeBuilder.java | 148 ++++++++++++++++-- .../composite/Composite90DocValuesReader.java | 42 +++-- .../composite/Composite90DocValuesWriter.java | 49 +++--- .../aggregators/MetricAggregatorInfo.java | 6 +- .../builder/OnHeapStarTreeBuilder.java | 10 +- .../startree/builder/StarTreeBuilder.java | 23 ++- .../startree/builder/StarTreesBuilder.java | 23 ++- .../startree/node/OffHeapStarTree.java | 4 +- .../startree/node/OffHeapStarTreeNode.java | 7 +- .../startree/utils/StarTreeConstants.java | 28 ---- .../startree/utils/StarTreeHelper.java | 38 +++++ .../utils/StarTreeMetaSerializer.java | 6 +- .../builder/BaseStarTreeBuilderTests.java | 8 +- .../builder/OnHeapStarTreeBuilderTests.java | 12 +- 16 files changed, 392 insertions(+), 130 deletions(-) create mode 100644 server/src/main/java/org/apache/lucene/codecs/lucene90/Composite99DocValuesConsumer.java delete mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeConstants.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeHelper.java diff --git a/server/src/main/java/org/apache/lucene/codecs/lucene90/Composite99DocValuesConsumer.java b/server/src/main/java/org/apache/lucene/codecs/lucene90/Composite99DocValuesConsumer.java new file mode 100644 index 0000000000000..c0a22a0b32fd2 --- /dev/null +++ b/server/src/main/java/org/apache/lucene/codecs/lucene90/Composite99DocValuesConsumer.java @@ -0,0 +1,68 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.apache.lucene.codecs.lucene90; + +import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.SegmentWriteState; + +import java.io.IOException; + +/** + * This class is an abstraction of the {@link DocValuesConsumer} for the Star Tree index structure. + * It is responsible to consume various types of document values (numeric, binary, sorted, sorted numeric, + * and sorted set) for fields in the Star Tree index. + * + * @opensearch.experimental + */ +public class Composite99DocValuesConsumer extends DocValuesConsumer { + + Lucene90DocValuesConsumer lucene90DocValuesConsumer; + + public Composite99DocValuesConsumer( + SegmentWriteState state, + String dataCodec, + String dataExtension, + String metaCodec, + String metaExtension + ) throws IOException { + lucene90DocValuesConsumer = new Lucene90DocValuesConsumer(state, dataCodec, dataExtension, metaCodec, metaExtension); + } + + @Override + public void close() throws IOException { + lucene90DocValuesConsumer.close(); + } + + @Override + public void addNumericField(FieldInfo fieldInfo, DocValuesProducer docValuesProducer) throws IOException { + lucene90DocValuesConsumer.addNumericField(fieldInfo, docValuesProducer); + } + + @Override + public void addBinaryField(FieldInfo fieldInfo, DocValuesProducer docValuesProducer) throws IOException { + lucene90DocValuesConsumer.addNumericField(fieldInfo, docValuesProducer); + } + + @Override + public void addSortedField(FieldInfo fieldInfo, DocValuesProducer docValuesProducer) throws IOException { + lucene90DocValuesConsumer.addSortedField(fieldInfo, docValuesProducer); + } + + @Override + public void addSortedNumericField(FieldInfo fieldInfo, DocValuesProducer docValuesProducer) throws IOException { + lucene90DocValuesConsumer.addSortedNumericField(fieldInfo, docValuesProducer); + } + + @Override + public void addSortedSetField(FieldInfo fieldInfo, DocValuesProducer docValuesProducer) throws IOException { + lucene90DocValuesConsumer.addSortedSetField(fieldInfo, docValuesProducer); + } +} diff --git a/server/src/main/java/org/apache/lucene/codecs/lucene90/StarTree99DocValuesProducer.java b/server/src/main/java/org/apache/lucene/codecs/lucene90/StarTree99DocValuesProducer.java index 2d96d5a505694..154f4626bc472 100644 --- a/server/src/main/java/org/apache/lucene/codecs/lucene90/StarTree99DocValuesProducer.java +++ b/server/src/main/java/org/apache/lucene/codecs/lucene90/StarTree99DocValuesProducer.java @@ -21,15 +21,15 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricEntry; -import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeConstants; import java.io.IOException; -import java.util.ArrayList; import java.util.Collections; import java.util.List; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeHelper.fullFieldNameForStarTreeDimensionsDocValues; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeHelper.fullFieldNameForStarTreeMetricsDocValues; + /** * This class is a custom abstraction of the {@link DocValuesProducer} for the Star Tree index structure. * It is responsible for providing access to various types of document values (numeric, binary, sorted, sorted numeric, @@ -41,7 +41,7 @@ public class StarTree99DocValuesProducer extends DocValuesProducer { Lucene90DocValuesProducer lucene90DocValuesProducer; private final List dimensions; - private final List metrics; + private final List metrics; private final FieldInfos fieldInfos; public StarTree99DocValuesProducer( @@ -55,15 +55,10 @@ public StarTree99DocValuesProducer( String compositeFieldName ) throws IOException { this.dimensions = dimensions; - this.metrics = new ArrayList<>(); - for (MetricEntry metricEntry : metricEntries) { - this.metrics.add( - MetricAggregatorInfo.toFieldName(compositeFieldName, metricEntry.getMetricName(), metricEntry.getMetricStat().getTypeName()) - ); - } + this.metrics = metricEntries; // populates the dummy list of field infos to fetch doc id set iterators for respective fields. - this.fieldInfos = new FieldInfos(getFieldInfoList()); + this.fieldInfos = new FieldInfos(getFieldInfoList(compositeFieldName)); SegmentReadState segmentReadState = new SegmentReadState(state.directory, state.segmentInfo, fieldInfos, state.context); lucene90DocValuesProducer = new Lucene90DocValuesProducer(segmentReadState, dataCodec, dataExtension, metaCodec, metaExtension); } @@ -108,35 +103,36 @@ public void close() throws IOException { this.lucene90DocValuesProducer.close(); } - private FieldInfo[] getFieldInfoList() { + private FieldInfo[] getFieldInfoList(String compositeFieldName) { FieldInfo[] fieldInfoList = new FieldInfo[this.dimensions.size() + metrics.size()]; + // field number is not really used. We depend on unique field names to get the desired iterator int fieldNumber = 0; for (FieldInfo dimension : this.dimensions) { fieldInfoList[fieldNumber] = new FieldInfo( - dimension.getName() + StarTreeConstants.DIMENSION_SUFFIX, + fullFieldNameForStarTreeDimensionsDocValues(compositeFieldName, dimension.getName()), fieldNumber, false, - dimension.omitsNorms(), - dimension.hasPayloads(), - dimension.getIndexOptions(), - dimension.getDocValuesType(), + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, -1, - dimension.attributes(), - dimension.getPointDimensionCount(), - dimension.getPointIndexDimensionCount(), - dimension.getPointNumBytes(), - dimension.getVectorDimension(), - dimension.getVectorEncoding(), - dimension.getVectorSimilarityFunction(), + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, false, - dimension.isParentField() + false ); fieldNumber++; } - for (String metric : metrics) { + for (MetricEntry metric : metrics) { fieldInfoList[fieldNumber] = new FieldInfo( - metric + StarTreeConstants.METRIC_SUFFIX, + fullFieldNameForStarTreeMetricsDocValues(compositeFieldName, metric.getMetricName(), metric.getMetricStat().getTypeName()), fieldNumber, false, false, diff --git a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java index 07037c188a14c..545e50f3129b7 100644 --- a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java @@ -9,8 +9,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.Counter; +import org.apache.lucene.util.NumericUtils; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; @@ -32,12 +35,17 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeHelper.fullFieldNameForStarTreeDimensionsDocValues; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeHelper.fullFieldNameForStarTreeMetricsDocValues; /** * Builder for star tree. Defines the algorithm to construct star-tree @@ -160,15 +168,9 @@ public List getMetricReaders(SegmentWriteState stat for (MetricStat metricType : metric.getMetrics()) { SequentialDocValuesIterator metricReader; FieldInfo metricFieldInfo = state.fieldInfos.fieldInfo(metric.getField()); - if (metricType != MetricStat.COUNT) { - // Need not initialize the metric reader with relevant doc id set iterator for COUNT metric type metricReader = new SequentialDocValuesIterator( fieldProducerMap.get(metricFieldInfo.name).getSortedNumeric(metricFieldInfo) ); - } else { - metricReader = new SequentialDocValuesIterator(); - } - metricReaders.add(metricReader); } } @@ -178,10 +180,16 @@ public List getMetricReaders(SegmentWriteState stat /** * Builds the star tree from the original segment documents * - * @param fieldProducerMap contain s the docValues producer to get docValues associated with each field + * @param fieldProducerMap contain s the docValues producer to get docValues associated with each field + * @param fieldNumberAcrossStarTrees + * @param starTreeDocValuesConsumer * @throws IOException when we are unable to build star-tree */ - public void build(Map fieldProducerMap) throws IOException { + public void build( + Map fieldProducerMap, + AtomicInteger fieldNumberAcrossStarTrees, + DocValuesConsumer starTreeDocValuesConsumer + ) throws IOException { long startTime = System.currentTimeMillis(); logger.debug("Star-tree build is a go with star tree field {}", starTreeField.getName()); @@ -201,17 +209,23 @@ public void build(Map fieldProducerMap) throws IOExce metricReaders ); logger.debug("Sorting and aggregating star-tree in ms : {}", (System.currentTimeMillis() - startTime)); - build(starTreeDocumentIterator); + build(starTreeDocumentIterator, fieldNumberAcrossStarTrees, starTreeDocValuesConsumer); logger.debug("Finished Building star-tree in ms : {}", (System.currentTimeMillis() - startTime)); } /** * Builds the star tree using sorted and aggregated star-tree Documents * - * @param starTreeDocumentIterator contains the sorted and aggregated documents + * @param starTreeDocumentIterator contains the sorted and aggregated documents + * @param fieldNumberAcrossStarTrees + * @param starTreeDocValuesConsumer * @throws IOException when we are unable to build star-tree */ - public void build(Iterator starTreeDocumentIterator) throws IOException { + public void build( + Iterator starTreeDocumentIterator, + AtomicInteger fieldNumberAcrossStarTrees, + DocValuesConsumer starTreeDocValuesConsumer + ) throws IOException { int numSegmentStarTreeDocument = totalSegmentDocs; while (starTreeDocumentIterator.hasNext()) { @@ -238,12 +252,11 @@ public void build(Iterator starTreeDocumentIterator) throws IO int numAggregatedStarTreeDocument = numStarTreeDocs - numStarTreeDocument - numStarTreeDocumentUnderStarNode; logger.debug("Finished creating aggregated documents : {}", numAggregatedStarTreeDocument); - // TODO: When StarTree Codec is ready // Create doc values indices in disk + createSortedDocValuesIndices(starTreeDocValuesConsumer, fieldNumberAcrossStarTrees); + // serialize star-tree serializeStarTree(numSegmentStarTreeDocument); - - // Write star tree metadata for off heap implementation } private void serializeStarTree(int numSegmentStarTreeDocument) throws IOException { @@ -263,6 +276,113 @@ private void serializeStarTree(int numSegmentStarTreeDocument) throws IOExceptio ); } + private void createSortedDocValuesIndices(DocValuesConsumer docValuesConsumer, AtomicInteger fieldNumberAcrossStarTrees) + throws IOException { + List dimensionWriters = new ArrayList<>(); + List metricWriters = new ArrayList<>(); + FieldInfo[] dimensionFieldInfoList = new FieldInfo[starTreeField.getDimensionsOrder().size()]; + FieldInfo[] metricFieldInfoList = new FieldInfo[metricAggregatorInfos.size()]; + + for (int i = 0; i < dimensionFieldInfoList.length; i++) { + final FieldInfo fi = new FieldInfo( + fullFieldNameForStarTreeDimensionsDocValues(starTreeField.getName(), starTreeField.getDimensionsOrder().get(i).getField()), + fieldNumberAcrossStarTrees.getAndIncrement(), + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + dimensionFieldInfoList[i] = fi; + dimensionWriters.add(new SortedNumericDocValuesWriter(fi, Counter.newCounter())); + } + for (int i = 0; i < metricAggregatorInfos.size(); i++) { + FieldInfo fi = new FieldInfo( + fullFieldNameForStarTreeMetricsDocValues( + starTreeField.getName(), + metricAggregatorInfos.get(i).getField(), + metricAggregatorInfos.get(i).getMetricStat().getTypeName() + ), + fieldNumberAcrossStarTrees.getAndIncrement(), + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + metricFieldInfoList[i] = fi; + metricWriters.add(new SortedNumericDocValuesWriter(fi, Counter.newCounter())); + } + + for (int docId = 0; docId < numStarTreeDocs; docId++) { + StarTreeDocument starTreeDocument = getStarTreeDocument(docId); + for (int i = 0; i < starTreeDocument.dimensions.length; i++) { + Long val = starTreeDocument.dimensions[i]; + if (val != null) { + dimensionWriters.get(i).addValue(docId, val); + } + } + + for (int i = 0; i < starTreeDocument.metrics.length; i++) { + try { + switch (metricAggregatorInfos.get(i).getValueAggregators().getAggregatedValueType()) { + case LONG: + metricWriters.get(i).addValue(docId, (Long) starTreeDocument.metrics[i]); + break; + case DOUBLE: + metricWriters.get(i).addValue(docId, NumericUtils.doubleToSortableLong((Double) starTreeDocument.metrics[i])); + break; + default: + throw new IllegalStateException("Unknown metric doc value type"); + } + } catch (IllegalArgumentException e) { + logger.info("could not parse the value, exiting creation of star tree"); + } + } + } + + addStarTreeDocValueFields(docValuesConsumer, dimensionWriters, dimensionFieldInfoList, starTreeField.getDimensionsOrder().size()); + addStarTreeDocValueFields(docValuesConsumer, metricWriters, metricFieldInfoList, metricAggregatorInfos.size()); + } + + private void addStarTreeDocValueFields( + DocValuesConsumer docValuesConsumer, + List docValuesWriters, + FieldInfo[] fieldInfoList, + int fieldCount + ) throws IOException { + for (int i = 0; i < fieldCount; i++) { + final int increment = i; + DocValuesProducer docValuesProducer = new EmptyDocValuesProducer() { + @Override + public SortedNumericDocValues getSortedNumeric(FieldInfo field) { + return docValuesWriters.get(increment).getDocValues(); + } + }; + docValuesConsumer.addSortedNumericField(fieldInfoList[i], docValuesProducer); + } + } + /** * Adds a document to the star-tree. * diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesReader.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesReader.java index 0c9c5d1d91063..ad8ea581a4f29 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesReader.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesReader.java @@ -25,8 +25,8 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.IOUtils; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; import org.opensearch.index.compositeindex.CompositeIndexMetadata; import org.opensearch.index.compositeindex.datacube.Dimension; @@ -34,13 +34,12 @@ import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricEntry; import org.opensearch.index.compositeindex.datacube.startree.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.node.OffHeapStarTree; import org.opensearch.index.compositeindex.datacube.startree.node.StarTree; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; -import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeConstants; +import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeHelper; import java.io.IOException; import java.util.ArrayList; @@ -61,8 +60,8 @@ public class Composite90DocValuesReader extends DocValuesProducer implements Com private static final Logger logger = LogManager.getLogger(CompositeIndexMetadata.class); private final DocValuesProducer delegate; - private final IndexInput dataIn; - private final ChecksumIndexInput metaIn; + private IndexInput dataIn; + private ChecksumIndexInput metaIn; private final Map starTreeMap = new LinkedHashMap<>(); private final Map compositeIndexMetadataMap = new LinkedHashMap<>(); private final Map compositeDocValuesProducerMap = new LinkedHashMap<>(); @@ -195,16 +194,30 @@ public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException { @Override public void checkIntegrity() throws IOException { delegate.checkIntegrity(); - CodecUtil.checksumEntireFile(metaIn); CodecUtil.checksumEntireFile(dataIn); } @Override public void close() throws IOException { delegate.close(); - starTreeMap.clear(); - compositeIndexMetadataMap.clear(); - compositeDocValuesProducerMap.clear(); + boolean success = false; + try { + IOUtils.close(metaIn, dataIn); + for (DocValuesProducer docValuesProducer : compositeDocValuesProducerMap.values()) { + IOUtils.close(docValuesProducer); + } + success = true; + } finally { + if (!success) { + + IOUtils.closeWhileHandlingException(metaIn, dataIn); + } + starTreeMap.clear(); + compositeIndexMetadataMap.clear(); + compositeDocValuesProducerMap.clear(); + metaIn = null; + dataIn = null; + } } @Override @@ -214,7 +227,8 @@ public List getCompositeIndexFields() { } @Override - public CompositeIndexValues getCompositeIndexValues(CompositeIndexFieldInfo compositeIndexFieldInfo) throws IOException { + public CompositeIndexValues getCompositeIndexValues(CompositeIndexFieldInfo compositeIndexFieldInfo) throws + IOException { switch (compositeIndexFieldInfo.getType()) { case STAR_TREE: @@ -263,13 +277,15 @@ public CompositeIndexValues getCompositeIndexValues(CompositeIndexFieldInfo comp for (String dimension : dimensions) { dimensionsDocIdSetIteratorMap.put( dimension, - starTree99DocValuesProducer.getSortedNumeric(dimension + StarTreeConstants.DIMENSION_SUFFIX) + starTree99DocValuesProducer.getSortedNumeric( + StarTreeHelper.fullFieldNameForStarTreeDimensionsDocValues(starTreeField.getName(), dimension) + ) ); } for (MetricEntry metricEntry : starTreeMetadata.getMetricEntries()) { - String metricFullName = MetricAggregatorInfo.toFieldName( - compositeIndexFieldInfo.getField(), + String metricFullName = StarTreeHelper.fullFieldNameForStarTreeMetricsDocValues( + starTreeField.getName(), metricEntry.getMetricName(), metricEntry.getMetricStat().getTypeName() ); diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesWriter.java index 97dec4bd55c88..68740cad60a14 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesWriter.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesWriter.java @@ -13,13 +13,14 @@ import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.codecs.lucene90.Composite99DocValuesConsumer; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.MergeState; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.IOUtils; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreesBuilder; @@ -27,7 +28,7 @@ import org.opensearch.index.mapper.MapperService; import java.io.IOException; -import java.util.Collections; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -48,6 +49,7 @@ public class Composite90DocValuesWriter extends DocValuesConsumer { private MergeState mergeState = null; private final Set compositeMappedFieldTypes; private final Set compositeFieldSet; + private DocValuesConsumer composite99DocValuesConsumer; public IndexOutput dataOut; public IndexOutput metaOut; @@ -58,8 +60,25 @@ public class Composite90DocValuesWriter extends DocValuesConsumer { public Composite90DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState segmentWriteState, MapperService mapperService) throws IOException { + this.delegate = delegate; + this.state = segmentWriteState; + this.mapperService = mapperService; + this.compositeMappedFieldTypes = mapperService.getCompositeFieldTypes(); + compositeFieldSet = new HashSet<>(); + for (CompositeMappedFieldType type : compositeMappedFieldTypes) { + compositeFieldSet.addAll(type.fields()); + } + boolean success = false; try { + this.composite99DocValuesConsumer = new Composite99DocValuesConsumer( + segmentWriteState, + Composite90DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite90DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite90DocValuesFormat.META_DOC_VALUES_CODEC, + Composite90DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + String dataFileName = IndexFileNames.segmentFileName( segmentWriteState.segmentInfo.name, segmentWriteState.segmentSuffix, @@ -94,15 +113,6 @@ public Composite90DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState IOUtils.closeWhileHandlingException(this); } } - - this.delegate = delegate; - this.state = segmentWriteState; - this.mapperService = mapperService; - this.compositeMappedFieldTypes = mapperService.getCompositeFieldTypes(); - compositeFieldSet = new HashSet<>(); - for (CompositeMappedFieldType type : compositeMappedFieldTypes) { - compositeFieldSet.addAll(type.fields()); - } } @Override @@ -146,14 +156,16 @@ public void close() throws IOException { if (dataOut != null) { CodecUtil.writeFooter(dataOut); // write checksum } + success = true; } finally { if (success) { - IOUtils.close(dataOut, metaOut); + IOUtils.close(dataOut, metaOut, composite99DocValuesConsumer); } else { - IOUtils.closeWhileHandlingException(dataOut, metaOut); + IOUtils.closeWhileHandlingException(dataOut, metaOut, composite99DocValuesConsumer); } metaOut = dataOut = null; + composite99DocValuesConsumer = null; } } @@ -167,11 +179,10 @@ private void createCompositeIndicesIfPossible(DocValuesProducer valuesProducer, if (compositeFieldSet.isEmpty()) { for (CompositeMappedFieldType mappedType : compositeMappedFieldTypes) { if (mappedType.getCompositeIndexType().equals(CompositeMappedFieldType.CompositeFieldType.STAR_TREE)) { - // TODO : Call StarTree builder + StarTreesBuilder starTreesBuilder = new StarTreesBuilder(state, mapperService); + starTreesBuilder.build(metaOut, dataOut, fieldProducerMap, composite99DocValuesConsumer); } } - StarTreesBuilder starTreesBuilder = new StarTreesBuilder(state, mapperService); - starTreesBuilder.build(metaOut, dataOut, fieldProducerMap); } } @@ -216,7 +227,7 @@ private void mergeStarTreeFields(MergeState mergeState) throws IOException { if (fieldInfo.getType().equals(CompositeMappedFieldType.CompositeFieldType.STAR_TREE)) { CompositeIndexValues compositeIndexValues = reader.getCompositeIndexValues(fieldInfo); if (compositeIndexValues instanceof StarTreeValues) { - List fieldsList = starTreeSubsPerField.getOrDefault(fieldInfo.getField(), Collections.emptyList()); + List fieldsList = starTreeSubsPerField.getOrDefault(fieldInfo.getField(), new ArrayList<>()); if (!starTreeFieldMap.containsKey(fieldInfo.getField())) { starTreeFieldMap.put(fieldInfo.getField(), ((StarTreeValues) compositeIndexValues).getStarTreeField()); } @@ -232,7 +243,7 @@ private void mergeStarTreeFields(MergeState mergeState) throws IOException { } } } - final StarTreesBuilder starTreesBuilder = new StarTreesBuilder(state, mapperService); - starTreesBuilder.buildDuringMerge(metaOut, dataOut, starTreeFieldMap, starTreeSubsPerField); + StarTreesBuilder starTreesBuilder = new StarTreesBuilder(state, mapperService); + starTreesBuilder.buildDuringMerge(metaOut, dataOut, starTreeFieldMap, starTreeSubsPerField, composite99DocValuesConsumer); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java index a3aa2f2fdde62..9b3c633280b08 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java @@ -84,8 +84,8 @@ public String toFieldName() { } - public static String toFieldName(String starFieldName, String field, String typeName) { - return starFieldName + DELIMITER + field + DELIMITER + typeName; + public static String toFieldName(String starFieldName, String field, String metricName) { + return starFieldName + DELIMITER + field + DELIMITER + metricName; } @Override @@ -100,7 +100,7 @@ public boolean equals(Object obj) { } if (obj instanceof MetricAggregatorInfo) { MetricAggregatorInfo anotherPair = (MetricAggregatorInfo) obj; - return metricStat == anotherPair.metricStat && field.equals(anotherPair.field); + return metricStat.equals(anotherPair.metricStat) && field.equals(anotherPair.field); } return false; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index 00cce31dc0771..d28607c067a61 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -7,6 +7,7 @@ */ package org.opensearch.index.compositeindex.datacube.startree.builder; +import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.index.BaseStarTreeBuilder; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.search.DocIdSetIterator; @@ -26,6 +27,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; /** * On heap based single tree builder @@ -61,8 +63,12 @@ public void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOE } @Override - public void build(List starTreeValuesSubs) throws IOException { - build(mergeStarTrees(starTreeValuesSubs)); + public void build( + List starTreeValuesSubs, + AtomicInteger fieldNumberAcrossStarTrees, + DocValuesConsumer starTreeDocValuesConsumer + ) throws IOException { + build(mergeStarTrees(starTreeValuesSubs), fieldNumberAcrossStarTrees, starTreeDocValuesConsumer); } /** diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java index 988379d7400fe..4c0e8ad8938e7 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java @@ -8,6 +8,7 @@ package org.opensearch.index.compositeindex.datacube.startree.builder; +import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; @@ -16,6 +17,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; /** * A star-tree builder that builds a single star-tree. @@ -28,18 +30,29 @@ public interface StarTreeBuilder extends Closeable { /** * Builds the star tree from the original segment documents * - * @param fieldProducerMap contains the docValues producer to get docValues associated with each field - * + * @param fieldProducerMap contains the docValues producer to get docValues associated with each field + * @param fieldNumberAcrossStarTrees maintains the unique field number across the fields in the star tree + * @param starTreeDocValuesConsumer * @throws IOException when we are unable to build star-tree */ - void build(Map fieldProducerMap) throws IOException; + void build( + Map fieldProducerMap, + AtomicInteger fieldNumberAcrossStarTrees, + DocValuesConsumer starTreeDocValuesConsumer + ) throws IOException; /** * Builds the star tree using StarTree values from multiple segments * - * @param starTreeValuesSubs contains the star tree values from multiple segments + * @param starTreeValuesSubs contains the star tree values from multiple segments + * @param fieldNumberAcrossStarTrees maintains the unique field number across the fields in the star tree + * @param starTreeDocValuesConsumer * @throws IOException when we are unable to build star-tree */ - void build(List starTreeValuesSubs) throws IOException; + void build( + List starTreeValuesSubs, + AtomicInteger fieldNumberAcrossStarTrees, + DocValuesConsumer starTreeDocValuesConsumer + ) throws IOException; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java index 6cb4ca81e07c8..e4bf365332e7a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.store.IndexOutput; @@ -26,6 +27,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; /** * Builder to construct star-trees based on multiple star-tree fields. @@ -40,6 +42,7 @@ public class StarTreesBuilder implements Closeable { private final List starTreeFields; private final SegmentWriteState state; private final MapperService mapperService; + private AtomicInteger fieldNumberAcrossStarTrees; public StarTreesBuilder(SegmentWriteState segmentWriteState, MapperService mapperService) { List starTreeFields = new ArrayList<>(); @@ -59,12 +62,18 @@ public StarTreesBuilder(SegmentWriteState segmentWriteState, MapperService mappe this.starTreeFields = starTreeFields; this.state = segmentWriteState; this.mapperService = mapperService; + this.fieldNumberAcrossStarTrees = new AtomicInteger(); } /** * Builds the star-trees. */ - public void build(IndexOutput metaOut, IndexOutput dataOut, Map fieldProducerMap) throws IOException { + public void build( + IndexOutput metaOut, + IndexOutput dataOut, + Map fieldProducerMap, + DocValuesConsumer starTreeDocValuesConsumer + ) throws IOException { if (starTreeFields.isEmpty()) { logger.debug("no star-tree fields found, returning from star-tree builder"); return; @@ -78,7 +87,7 @@ public void build(IndexOutput metaOut, IndexOutput dataOut, Map starTreeFieldMap, - final Map> starTreeValuesSubsPerField + final Map> starTreeValuesSubsPerField, + DocValuesConsumer starTreeDocValuesConsumer ) throws IOException { for (Map.Entry> entry : starTreeValuesSubsPerField.entrySet()) { List starTreeValuesList = entry.getValue(); StarTreeField starTreeField = starTreeFieldMap.get(entry.getKey()); StarTreeBuilder builder = getSingleTreeBuilder(metaOut, dataOut, starTreeField, state, mapperService); - builder.build(starTreeValuesList); + builder.build(starTreeValuesList, fieldNumberAcrossStarTrees, starTreeDocValuesConsumer); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/OffHeapStarTree.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/OffHeapStarTree.java index 1f16bd924dac6..cfc280643d202 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/OffHeapStarTree.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/OffHeapStarTree.java @@ -42,8 +42,8 @@ public OffHeapStarTree(IndexInput data, StarTreeMetadata starTreeMetadata) throw numNodes = data.readInt(); // num nodes RandomAccessInput in = data.randomAccessSlice( - starTreeMetadata.getDataStartFilePointer(), - starTreeMetadata.getDataStartFilePointer() + starTreeMetadata.getDataLength() + data.getFilePointer(), + starTreeMetadata.getDataLength() ); root = new OffHeapStarTreeNode(in, 0); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/OffHeapStarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/OffHeapStarTreeNode.java index 91ad56ef70eb3..ec30f14f044d4 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/OffHeapStarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/OffHeapStarTreeNode.java @@ -18,16 +18,17 @@ * @opensearch.experimental */ public class OffHeapStarTreeNode implements StarTreeNode { - public static final int NUM_INT_SERIALIZABLE_FIELDS = 7; + public static final int NUM_INT_SERIALIZABLE_FIELDS = 6; public static final int NUM_LONG_SERIALIZABLE_FIELDS = 1; + public static final int NUM_BYTE_SERIALIZABLE_FIELDS = 1; public static final long SERIALIZABLE_DATA_SIZE_IN_BYTES = (Integer.BYTES * NUM_INT_SERIALIZABLE_FIELDS) + (Long.BYTES - * NUM_LONG_SERIALIZABLE_FIELDS); + * NUM_LONG_SERIALIZABLE_FIELDS) + (NUM_BYTE_SERIALIZABLE_FIELDS * Byte.BYTES); private static final int DIMENSION_ID_OFFSET = 0; private static final int DIMENSION_VALUE_OFFSET = DIMENSION_ID_OFFSET + Integer.BYTES; private static final int START_DOC_ID_OFFSET = DIMENSION_VALUE_OFFSET + Long.BYTES; private static final int END_DOC_ID_OFFSET = START_DOC_ID_OFFSET + Integer.BYTES; private static final int AGGREGATE_DOC_ID_OFFSET = END_DOC_ID_OFFSET + Integer.BYTES; - private static final int IS_STAR_NODE_OFFSET = AGGREGATE_DOC_ID_OFFSET + Integer.BYTES; + private static final int IS_STAR_NODE_OFFSET = AGGREGATE_DOC_ID_OFFSET + Byte.BYTES; private static final int FIRST_CHILD_ID_OFFSET = IS_STAR_NODE_OFFSET + Integer.BYTES; private static final int LAST_CHILD_ID_OFFSET = FIRST_CHILD_ID_OFFSET + Integer.BYTES; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeConstants.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeConstants.java deleted file mode 100644 index 2670b7b41f4df..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeConstants.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.compositeindex.datacube.startree.utils; - -/** - * This class contains constant values used throughout the Star Tree index implementation. - * - * @opensearch.experimental - */ -public class StarTreeConstants { - - /** - * The suffix appended to dimension field names in the Star Tree index. - */ - public static final String DIMENSION_SUFFIX = "_dim"; - - /** - * The suffix appended to metric field names in the Star Tree index. - */ - public static final String METRIC_SUFFIX = "_metric"; - -} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeHelper.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeHelper.java new file mode 100644 index 0000000000000..8cbb7508ee1ef --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeHelper.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.utils; + +import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; + +/** + * This class contains helper methods used throughout the Star Tree index implementation. + * + * @opensearch.experimental + */ +public class StarTreeHelper { + + /** + * The suffix appended to dimension field names in the Star Tree index. + */ + public static final String DIMENSION_SUFFIX = "dim"; + + /** + * The suffix appended to metric field names in the Star Tree index. + */ + public static final String METRIC_SUFFIX = "metric"; + + public static String fullFieldNameForStarTreeDimensionsDocValues(String starTreeFieldName, String dimensionName) { + return starTreeFieldName + "_" + dimensionName + "_" + DIMENSION_SUFFIX; + } + + public static String fullFieldNameForStarTreeMetricsDocValues(String name, String fieldName, String metricName) { + return MetricAggregatorInfo.toFieldName(name, fieldName, metricName) + "_" + METRIC_SUFFIX; + } + +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeMetaSerializer.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeMetaSerializer.java index dc91c97fb224f..e1091ba35d3f8 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeMetaSerializer.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeMetaSerializer.java @@ -100,7 +100,7 @@ private static long computeMetricEntriesSizeInBytes(List m long totalMetricEntriesSize = 0; for (MetricAggregatorInfo metricAggregatorInfo : metricAggregatorInfos) { - totalMetricEntriesSize += metricAggregatorInfo.getMetric().getBytes(UTF_8).length; + totalMetricEntriesSize += metricAggregatorInfo.getField().getBytes(UTF_8).length; totalMetricEntriesSize += metricAggregatorInfo.getMetricStat().getTypeName().getBytes(UTF_8).length; } @@ -189,7 +189,7 @@ private static void writeMeta( // metric - metric stat pair for (MetricAggregatorInfo metricAggregatorInfo : metricAggregatorInfos) { - String metricName = metricAggregatorInfo.getMetric(); + String metricName = metricAggregatorInfo.getField(); String metricStatName = metricAggregatorInfo.getMetricStat().getTypeName(); metaOut.writeString(metricName); metaOut.writeString(metricStatName); @@ -202,7 +202,7 @@ private static void writeMeta( metaOut.writeInt(starTreeField.getStarTreeConfig().maxLeafDocs()); // number of skip star node creation dimensions - metaOut.writeInt(starTreeField.getStarTreeConfig().maxLeafDocs()); + metaOut.writeInt(starTreeField.getStarTreeConfig().getSkipStarNodeCreationInDims().size()); // skip star node creations for (String dimension : starTreeField.getStarTreeConfig().getSkipStarNodeCreationInDims()) { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java index 8de583f5aefc5..b3b528eb55803 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java @@ -8,6 +8,7 @@ package org.opensearch.index.compositeindex.datacube.startree.builder; +import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.index.BaseStarTreeBuilder; @@ -55,6 +56,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -180,7 +182,11 @@ public static void setup() throws IOException { builder = new BaseStarTreeBuilder(metaOut, dataOut, starTreeField, writeState, mapperService) { @Override - public void build(List starTreeValuesSubs) throws IOException {} + public void build( + List starTreeValuesSubs, + AtomicInteger fieldNumberAcrossStarTrees, + DocValuesConsumer starTreeDocValuesConsumer + ) throws IOException {} @Override public void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException {} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java index 7ccb9bcd91c30..1e655d7b3d455 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java @@ -8,6 +8,7 @@ package org.opensearch.index.compositeindex.datacube.startree.builder; +import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.index.DocValuesType; @@ -56,6 +57,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -85,6 +87,7 @@ public class OnHeapStarTreeBuilderTests extends OpenSearchTestCase { private SegmentWriteState writeState; private IndexOutput dataOut; private IndexOutput metaOut; + private DocValuesConsumer docValuesConsumer; @Before public void setup() throws IOException { @@ -103,6 +106,7 @@ public void setup() throws IOException { ); DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); + docValuesConsumer = mock(DocValuesConsumer.class); compositeField = new StarTreeField( "test", @@ -487,7 +491,7 @@ public void test_build_halfFloatMetrics() throws IOException { } Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments); - builder.build(segmentStarTreeDocumentIterator); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); List resultStarTreeDocuments = builder.getStarTreeDocuments(); assertEquals(8, resultStarTreeDocuments.size()); @@ -552,7 +556,7 @@ public void test_build_floatMetrics() throws IOException { } Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments); - builder.build(segmentStarTreeDocumentIterator); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); List resultStarTreeDocuments = builder.getStarTreeDocuments(); assertEquals(8, resultStarTreeDocuments.size()); @@ -606,7 +610,7 @@ public void test_build_longMetrics() throws IOException { } Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments); - builder.build(segmentStarTreeDocumentIterator); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); List resultStarTreeDocuments = builder.getStarTreeDocuments(); assertEquals(8, resultStarTreeDocuments.size()); @@ -650,7 +654,7 @@ public void test_build() throws IOException { } Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments); - builder.build(segmentStarTreeDocumentIterator); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); List resultStarTreeDocuments = builder.getStarTreeDocuments(); assertEquals(8, resultStarTreeDocuments.size());