Skip to content

Commit

Permalink
Star Tree Meta and Data Writers (opensearch-project#15295)
Browse files Browse the repository at this point in the history
---------
Signed-off-by: Sarthak Aggarwal <[email protected]>
  • Loading branch information
sarthakaggarwal97 authored Aug 29, 2024
1 parent 5663b4a commit 8629279
Show file tree
Hide file tree
Showing 30 changed files with 2,418 additions and 92 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -34,17 +34,15 @@ public static DocValuesConsumer getDocValuesConsumerForCompositeCodec(
String metaCodec,
String metaExtension
) throws IOException {
try (
Lucene90DocValuesConsumerWrapper lucene90DocValuesConsumerWrapper = new Lucene90DocValuesConsumerWrapper(
state,
dataCodec,
dataExtension,
metaCodec,
metaExtension
)
) {
return lucene90DocValuesConsumerWrapper.getLucene90DocValuesConsumer();
}
Lucene90DocValuesConsumerWrapper lucene90DocValuesConsumerWrapper = new Lucene90DocValuesConsumerWrapper(
state,
dataCodec,
dataExtension,
metaCodec,
metaExtension
);
return lucene90DocValuesConsumerWrapper.getLucene90DocValuesConsumer();

}

}
Original file line number Diff line number Diff line change
Expand Up @@ -40,17 +40,14 @@ public static DocValuesProducer getDocValuesProducerForCompositeCodec(

switch (compositeCodec) {
case Composite99Codec.COMPOSITE_INDEX_CODEC_NAME:
try (
Lucene90DocValuesProducerWrapper lucene90DocValuesProducerWrapper = new Lucene90DocValuesProducerWrapper(
state,
dataCodec,
dataExtension,
metaCodec,
metaExtension
)
) {
return lucene90DocValuesProducerWrapper.getLucene90DocValuesProducer();
}
Lucene90DocValuesProducerWrapper lucene90DocValuesProducerWrapper = new Lucene90DocValuesProducerWrapper(
state,
dataCodec,
dataExtension,
metaCodec,
metaExtension
);
return lucene90DocValuesProducerWrapper.getLucene90DocValuesProducer();
default:
throw new IllegalStateException("Invalid composite codec " + "[" + compositeCodec + "]");
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.index.compositeindex;

/**
* This class contains constants used in the Composite Index implementation.
*/
public class CompositeIndexConstants {

/**
* The magic marker value used for sanity checks in the Composite Index implementation.
*/
public static final long COMPOSITE_FIELD_MARKER = 0xC0950513F1E1DL; // Composite Field

/**
* Represents the key to fetch number of non-star aggregated segment documents.
*/
public static final String SEGMENT_DOCS_COUNT = "segmentDocsCount";

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.index.compositeindex;

import org.opensearch.index.mapper.CompositeMappedFieldType;

/**
* This class represents the metadata of a Composite Index, which includes information about
* the composite field name, type, and the specific metadata for the type of composite field
* (e.g., Star Tree metadata).
*
* @opensearch.experimental
*/
public class CompositeIndexMetadata {

private final String compositeFieldName;
private final CompositeMappedFieldType.CompositeFieldType compositeFieldType;

/**
* Constructs a CompositeIndexMetadata object with the provided composite field name and type.
*
* @param compositeFieldName the name of the composite field
* @param compositeFieldType the type of the composite field
*/
public CompositeIndexMetadata(String compositeFieldName, CompositeMappedFieldType.CompositeFieldType compositeFieldType) {
this.compositeFieldName = compositeFieldName;
this.compositeFieldType = compositeFieldType;
}

/**
* Returns the name of the composite field.
*
* @return the composite field name
*/
public String getCompositeFieldName() {
return compositeFieldName;
}

/**
* Returns the type of the composite field.
*
* @return the composite field type
*/
public CompositeMappedFieldType.CompositeFieldType getCompositeFieldType() {
return compositeFieldType;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,37 +20,43 @@
*/
@ExperimentalApi
public enum MetricStat {
VALUE_COUNT("value_count"),
SUM("sum"),
MIN("min"),
MAX("max"),
AVG("avg", VALUE_COUNT, SUM),
DOC_COUNT("doc_count", true);
VALUE_COUNT("value_count", 0),
SUM("sum", 1),
MIN("min", 2),
MAX("max", 3),
AVG("avg", 4, VALUE_COUNT, SUM),
DOC_COUNT("doc_count", true, 5);

private final String typeName;
private final MetricStat[] baseMetrics;
private final int metricOrdinal;

// System field stats cannot be used as input for user metric types
private final boolean isSystemFieldStat;

MetricStat(String typeName) {
this(typeName, false);
MetricStat(String typeName, int metricOrdinal) {
this(typeName, false, metricOrdinal);
}

MetricStat(String typeName, MetricStat... baseMetrics) {
this(typeName, false, baseMetrics);
MetricStat(String typeName, int metricOrdinal, MetricStat... baseMetrics) {
this(typeName, false, metricOrdinal, baseMetrics);
}

MetricStat(String typeName, boolean isSystemFieldStat, MetricStat... baseMetrics) {
MetricStat(String typeName, boolean isSystemFieldStat, int metricOrdinal, MetricStat... baseMetrics) {
this.typeName = typeName;
this.isSystemFieldStat = isSystemFieldStat;
this.baseMetrics = baseMetrics;
this.metricOrdinal = metricOrdinal;
}

public String getTypeName() {
return typeName;
}

public int getMetricOrdinal() {
return metricOrdinal;
}

/**
* Return the list of metrics that this metric is derived from
* For example, AVG is derived from COUNT and SUM
Expand All @@ -76,4 +82,13 @@ public static MetricStat fromTypeName(String typeName) {
}
throw new IllegalArgumentException("Invalid metric stat: " + typeName);
}

public static MetricStat fromMetricOrdinal(int metricOrdinal) {
for (MetricStat metric : MetricStat.values()) {
if (metric.getMetricOrdinal() == metricOrdinal) {
return metric;
}
}
throw new IllegalArgumentException("Invalid metric stat: " + metricOrdinal);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.index.compositeindex.datacube;

import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.index.mapper.CompositeDataCubeFieldType;

import java.io.IOException;
import java.util.Objects;

/**
* Represents a dimension for reconstructing StarTreeField from file formats during searches and merges.
*
* @opensearch.experimental
*/
public class ReadDimension implements Dimension {
public static final String READ = "read";
private final String field;

public ReadDimension(String field) {
this.field = field;
}

public String getField() {
return field;
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(CompositeDataCubeFieldType.NAME, field);
builder.field(CompositeDataCubeFieldType.TYPE, READ);
builder.endObject();
return builder;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ReadDimension dimension = (ReadDimension) o;
return Objects.equals(field, dimension.getField());
}

@Override
public int hashCode() {
return Objects.hash(field);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -56,19 +56,25 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
@ExperimentalApi
public enum StarTreeBuildMode {
// TODO : remove onheap support unless this proves useful
ON_HEAP("onheap"),
OFF_HEAP("offheap");
ON_HEAP("onheap", (byte) 0),
OFF_HEAP("offheap", (byte) 1);

private final String typeName;
private final byte buildModeOrdinal;

StarTreeBuildMode(String typeName) {
StarTreeBuildMode(String typeName, byte buildModeOrdinal) {
this.typeName = typeName;
this.buildModeOrdinal = buildModeOrdinal;
}

public String getTypeName() {
return typeName;
}

public byte getBuildModeOrdinal() {
return buildModeOrdinal;
}

public static StarTreeBuildMode fromTypeName(String typeName) {
for (StarTreeBuildMode starTreeBuildMode : StarTreeBuildMode.values()) {
if (starTreeBuildMode.getTypeName().equalsIgnoreCase(typeName)) {
Expand All @@ -77,6 +83,16 @@ public static StarTreeBuildMode fromTypeName(String typeName) {
}
throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid star tree build mode: [%s] ", typeName));
}

public static StarTreeBuildMode fromBuildModeOrdinal(byte buildModeOrdinal) {
for (StarTreeBuildMode starTreeBuildMode : StarTreeBuildMode.values()) {
if (starTreeBuildMode.getBuildModeOrdinal() == buildModeOrdinal) {
return starTreeBuildMode;
}
}
throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid star tree build mode: [%s] ", buildModeOrdinal));
}

}

public int maxLeafDocs() {
Expand Down
Loading

0 comments on commit 8629279

Please sign in to comment.