From 44fd2bb9bcf4e51bdc983f87855db1bd0309ec04 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 12 Jul 2023 13:11:53 -0700 Subject: [PATCH] Add release notes for 2.9 (#1857) (#1858) Signed-off-by: Max Ksyunz (cherry picked from commit e186cf7d263a87a04c69875e6ad04d1f75e03d6a) Co-authored-by: Max Ksyunz --- ...faultQueryRangeFunctionResponseHandle.java | 74 +++++++++---------- .../response/PrometheusResponse.java | 25 +------ .../storage/PrometheusMetricScan.java | 7 +- .../storage/PrometheusMetricTable.java | 4 - .../opensearch-sql.release-notes-2.9.0.0.md | 44 +++++++++++ 5 files changed, 84 insertions(+), 70 deletions(-) create mode 100644 release-notes/opensearch-sql.release-notes-2.9.0.0.md diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/response/DefaultQueryRangeFunctionResponseHandle.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/response/DefaultQueryRangeFunctionResponseHandle.java index 7f261360f7..799440f93b 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/response/DefaultQueryRangeFunctionResponseHandle.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/response/DefaultQueryRangeFunctionResponseHandle.java @@ -5,6 +5,8 @@ package org.opensearch.sql.prometheus.functions.response; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.LABELS; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; import java.time.Instant; @@ -12,9 +14,9 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; -import org.jetbrains.annotations.NotNull; import org.json.JSONArray; import org.json.JSONObject; +import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTimestampValue; @@ -22,7 +24,6 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.executor.ExecutionEngine; -import org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants; /** * Default implementation of QueryRangeFunctionResponseHandle. @@ -40,63 +41,62 @@ public class DefaultQueryRangeFunctionResponseHandle implements QueryRangeFuncti */ public DefaultQueryRangeFunctionResponseHandle(JSONObject responseObject) { this.responseObject = responseObject; - constructIteratorAndSchema(); + constructSchema(); + constructIterator(); } - private void constructIteratorAndSchema() { + private void constructIterator() { List result = new ArrayList<>(); List columnList = new ArrayList<>(); if ("matrix".equals(responseObject.getString("resultType"))) { JSONArray itemArray = responseObject.getJSONArray("result"); for (int i = 0; i < itemArray.length(); i++) { + LinkedHashMap linkedHashMap = new LinkedHashMap<>(); JSONObject item = itemArray.getJSONObject(i); - JSONObject metric = item.getJSONObject("metric"); - JSONArray values = item.getJSONArray("values"); - if (i == 0) { - columnList = getColumnList(metric); - } - for (int j = 0; j < values.length(); j++) { - LinkedHashMap linkedHashMap = - extractRow(metric, values.getJSONArray(j), columnList); - result.add(new ExprTupleValue(linkedHashMap)); - } + linkedHashMap.put(LABELS, extractLabels(item.getJSONObject("metric"))); + extractTimestampAndValues(item.getJSONArray("values"), linkedHashMap); + result.add(new ExprTupleValue(linkedHashMap)); } } else { throw new RuntimeException(String.format("Unexpected Result Type: %s during Prometheus " + "Response Parsing. 'matrix' resultType is expected", responseObject.getString("resultType"))); } - this.schema = new ExecutionEngine.Schema(columnList); this.responseIterator = result.iterator(); } - @NotNull - private static LinkedHashMap extractRow(JSONObject metric, - JSONArray values, List columnList) { - LinkedHashMap linkedHashMap = new LinkedHashMap<>(); - for (ExecutionEngine.Schema.Column column : columnList) { - if (PrometheusFieldConstants.TIMESTAMP.equals(column.getName())) { - linkedHashMap.put(PrometheusFieldConstants.TIMESTAMP, - new ExprTimestampValue(Instant.ofEpochMilli((long) (values.getDouble(0) * 1000)))); - } else if (column.getName().equals(VALUE)) { - linkedHashMap.put(VALUE, new ExprDoubleValue(values.getDouble(1))); - } else { - linkedHashMap.put(column.getName(), - new ExprStringValue(metric.getString(column.getName()))); - } + private static void extractTimestampAndValues(JSONArray values, + LinkedHashMap linkedHashMap) { + List timestampList = new ArrayList<>(); + List valueList = new ArrayList<>(); + for(int j=0; j labelsMap = new LinkedHashMap<>(); + metric.keySet().forEach(key + -> labelsMap.put(key, new ExprStringValue(metric.getString(key)))); + return new ExprTupleValue(labelsMap); } - private List getColumnList(JSONObject metric) { + private List getColumnList() { List columnList = new ArrayList<>(); - columnList.add(new ExecutionEngine.Schema.Column(PrometheusFieldConstants.TIMESTAMP, - PrometheusFieldConstants.TIMESTAMP, ExprCoreType.TIMESTAMP)); - columnList.add(new ExecutionEngine.Schema.Column(VALUE, VALUE, ExprCoreType.DOUBLE)); - for (String key : metric.keySet()) { - columnList.add(new ExecutionEngine.Schema.Column(key, key, ExprCoreType.STRING)); - } + columnList.add(new ExecutionEngine.Schema.Column(TIMESTAMP, TIMESTAMP, ExprCoreType.ARRAY)); + columnList.add(new ExecutionEngine.Schema.Column(VALUE, VALUE, ExprCoreType.ARRAY)); + columnList.add(new ExecutionEngine.Schema.Column(LABELS, LABELS, ExprCoreType.STRUCT)); return columnList; } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java index 331605b1d5..bd9e36ccdc 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java @@ -35,8 +35,6 @@ public class PrometheusResponse implements Iterable { private final PrometheusResponseFieldNames prometheusResponseFieldNames; - private final Boolean isQueryRangeFunctionScan; - /** * Constructor. * @@ -46,11 +44,9 @@ public class PrometheusResponse implements Iterable { * and timestamp fieldName. */ public PrometheusResponse(JSONObject responseObject, - PrometheusResponseFieldNames prometheusResponseFieldNames, - Boolean isQueryRangeFunctionScan) { + PrometheusResponseFieldNames prometheusResponseFieldNames) { this.responseObject = responseObject; this.prometheusResponseFieldNames = prometheusResponseFieldNames; - this.isQueryRangeFunctionScan = isQueryRangeFunctionScan; } @NonNull @@ -70,24 +66,7 @@ public Iterator iterator() { new ExprTimestampValue(Instant.ofEpochMilli((long) (val.getDouble(0) * 1000)))); linkedHashMap.put(prometheusResponseFieldNames.getValueFieldName(), getValue(val, 1, prometheusResponseFieldNames.getValueType())); - // Concept: - // {\"instance\":\"localhost:9090\",\"__name__\":\"up\",\"job\":\"prometheus\"}" - // This is the label string in the prometheus response. - // Q: how do we map this to columns in a table. - // For queries like source = prometheus.metric_name | .... - // we can get the labels list in prior as we know which metric we are working on. - // In case of commands like source = prometheus.query_range('promQL'); - // Any arbitrary command can be written and we don't know the labels - // in the prometheus response in prior. - // So for PPL like commands...output structure is @value, @timestamp - // and each label is treated as a separate column where as in case of query_range - // function irrespective of promQL, the output structure is - // @value, @timestamp, @labels [jsonfied string of all the labels for a data point] - if (isQueryRangeFunctionScan) { - linkedHashMap.put(LABELS, new ExprStringValue(metric.toString())); - } else { - insertLabels(linkedHashMap, metric); - } + insertLabels(linkedHashMap, metric); result.add(new ExprTupleValue(linkedHashMap)); } } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScan.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScan.java index 8611ae04f1..7f75cb3c07 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScan.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScan.java @@ -40,10 +40,6 @@ public class PrometheusMetricScan extends TableScanOperator { private Iterator iterator; - @Setter - @Getter - private Boolean isQueryRangeFunctionScan = Boolean.FALSE; - @Setter private PrometheusResponseFieldNames prometheusResponseFieldNames; @@ -69,8 +65,7 @@ public void open() { JSONObject responseObject = prometheusClient.queryRange( request.getPromQl(), request.getStartTime(), request.getEndTime(), request.getStep()); - return new PrometheusResponse(responseObject, prometheusResponseFieldNames, - isQueryRangeFunctionScan).iterator(); + return new PrometheusResponse(responseObject, prometheusResponseFieldNames).iterator(); } catch (IOException e) { LOG.error(e.getMessage()); throw new RuntimeException("Error fetching data from prometheus server. " + e.getMessage()); diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTable.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTable.java index a03d69bc41..b3b63327d0 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTable.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTable.java @@ -97,10 +97,6 @@ public Map getFieldTypes() { public PhysicalPlan implement(LogicalPlan plan) { PrometheusMetricScan metricScan = new PrometheusMetricScan(prometheusClient); - if (prometheusQueryRequest != null) { - metricScan.setRequest(prometheusQueryRequest); - metricScan.setIsQueryRangeFunctionScan(Boolean.TRUE); - } return plan.accept(new PrometheusDefaultImplementor(), metricScan); } diff --git a/release-notes/opensearch-sql.release-notes-2.9.0.0.md b/release-notes/opensearch-sql.release-notes-2.9.0.0.md new file mode 100644 index 0000000000..a374676cf8 --- /dev/null +++ b/release-notes/opensearch-sql.release-notes-2.9.0.0.md @@ -0,0 +1,44 @@ +Compatible with OpenSearch and OpenSearch Dashboards Version 2.9.0 + +### Features + +* Enable Table Function and PromQL function ([#1719](https://github.com/opensearch-project/sql/pull/1719)) +* Add spark connector ([#1780](https://github.com/opensearch-project/sql/pull/1780)) + +### Enhancements + +* Pagination: Support WHERE clause, column list in SELECT clause and for functions and expressions in the query ([#1500](https://github.com/opensearch-project/sql/pull/1500)) +* Pagination: Support ORDER BY clauses and queries without FROM clause ([#1599](https://github.com/opensearch-project/sql/pull/1599)) +* Remove backticks on by field in stats ([#1728](https://github.com/opensearch-project/sql/pull/1728)) +* Support Array and ExprValue Parsing With Inner Hits ([#1737](https://github.com/opensearch-project/sql/pull/1737)) +* Add Support for Nested Function in Order By Clause ([#1789](https://github.com/opensearch-project/sql/pull/1789)) +* Add Support for Field Star in Nested Function ([#1773](https://github.com/opensearch-project/sql/pull/1773)) +* Guarantee datasource read api is strong consistent read (compatibility with segment replication) ([#1815](https://github.com/opensearch-project/sql/pull/1815)) +* Added new datetime functions and aliases to PPL ([#1807](https://github.com/opensearch-project/sql/pull/1807)) +* Support user-defined and incomplete date formats ([#1821](https://github.com/opensearch-project/sql/pull/1821)) +* Add _routing to SQL includes list ([#1771](https://github.com/opensearch-project/sql/pull/1771)) +* Disable read of plugins.query.datasources.encryption.masterkey from cluster settings GET API ([#1825](https://github.com/opensearch-project/sql/pull/1825)) +* Add EMR client to spark connector ([#1790](https://github.com/opensearch-project/sql/pull/1790)) +* Improved error codes in case of data sourcde API security exception ([#1753](https://github.com/opensearch-project/sql/pull/1753)) +* Remove Default master encryption key from settings ([#1851](https://github.com/opensearch-project/sql/pull/1851)) + +### Bug Fixes + +* Fixed bug of byte/short not handling 0 denominator in divide/modulus equations ([#1716](https://github.com/opensearch-project/sql/pull/1716)) +* Fix CSV/RAW output header being application/json rather than plain/text ([#1779](https://github.com/opensearch-project/sql/pull/1779)) + +### Documentation + +* Updated documentation of round function return type ([#1725](https://github.com/opensearch-project/sql/pull/1725)) +* Updated `protocol.rst` with new wording for error message ([#1662](https://github.com/opensearch-project/sql/pull/1662)) + +### Infrastructure + +* stopPrometheus task in doctest build.gradle now runs upon project failure in startOpenSearch ([#1747](https://github.com/opensearch-project/sql/pull/1747)) +* Upgrade guava to 32.0.1 +* Disable CrossClusterSearchIT test ([#1814](https://github.com/opensearch-project/sql/pull/1814)) +* fix flakytest when tests.locale=tr ([#1827](https://github.com/opensearch-project/sql/pull/1827)) + +### Refactoring + +* Simplify OpenSearchIndexScanBuilder ([#1738](https://github.com/opensearch-project/sql/pull/1738)) \ No newline at end of file