Initial steps of implementing Derivative using linear regression

Author: pabloem

x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DerivGroupingAggregatorFunction.java

@@ -0,0 +1,45 @@
+package org.elasticsearch.compute.aggregation;
+
+class SimpleLinearRegressionWithTimeseries {
+    long count;
+    double sumVal;
+    long sumTs;
+    double sumTsVal;
+    long sumTsSq;
+
+    SimpleLinearRegressionWithTimeseries() {
+        this.count = 0;
+        this.sumVal = 0.0;
+        this.sumTs = 0;
+        this.sumTsVal = 0.0;
+        this.sumTsSq = 0;
+    }
+
+    void add(long ts, double val) {
+        count++;
+        sumVal += val;
+        sumTs += ts;
+        sumTsVal += ts * val;
+        sumTsSq += ts * ts;
+    }
+
+    double slope() {
+        if (count <= 1) {
+            return Double.NaN;
+        }
+        double numerator = count * sumTsVal - sumTs * sumVal;
+        double denominator = count * sumTsSq - sumTs * sumTs;
+        if (denominator == 0) {
+            return Double.NaN;
+        }
+        return numerator / denominator;
+    }
+
+    double intercept() {
+        if (count == 0) {
+            return 0.0; // or handle as needed
+        }
+        return (sumVal - slope() * sumTs) / count;
+    }
+
+}

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/SimpleLinearRegressionWithTimeseries.java

@@ -593,3 +593,25 @@ max_cost:integer | pod:keyword | time_bucket:datetime     | max_cluster_cost:int
 1209             | three       | 2024-05-10T00:00:00.000Z | 1209                     | staging
 973              | two         | 2024-05-10T00:00:00.000Z | 1209                     | staging
 ;
+
+derivative_of_gauge_metric
+required_capability: ts_command_v0
+
+TS k8s
+| STATS max_deriv = max(deriv(network.cost)) BY time_bucket = bucket(@timestamp,5minute), pod
+| SORT pod, time_bucket
+| LIMIT 10
+;
+
+max_deriv:double       | time_bucket:datetime     | pod:keyword
+2.7573529411764707E-5  | 2024-05-10T00:00:00.000Z | one
+-9.657960185083621E-6  | 2024-05-10T00:05:00.000Z | one
+2.6771965095388827E-5  | 2024-05-10T00:10:00.000Z | one
+4.405946549223768E-5   | 2024-05-10T00:15:00.000Z | one
+8.564814814814814E-5   | 2024-05-10T00:20:00.000Z | one
+9.27740599107712E-5    | 2024-05-10T00:00:00.000Z | three
+3.80263223304832E-5    | 2024-05-10T00:05:00.000Z | three
+-1.9699890788329952E-5 | 2024-05-10T00:10:00.000Z | three
+-1.2087599544937429E-6 | 2024-05-10T00:15:00.000Z | three
+NaN                    | 2024-05-10T00:20:00.000Z | three
+;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec

@@ -27,6 +27,7 @@
 import org.elasticsearch.xpack.esql.expression.function.aggregate.CountDistinctOverTime;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.CountOverTime;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Delta;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.Deriv;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.First;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.FirstOverTime;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Idelta;
@@ -523,6 +524,7 @@ private static FunctionDefinition[][] functions() {
                 def(Irate.class, uni(Irate::new), "irate"),
                 def(Idelta.class, uni(Idelta::new), "idelta"),
                 def(Delta.class, uni(Delta::new), "delta"),
+                def(Deriv.class, uni(Deriv::new), "deriv"),
                 def(Increase.class, uni(Increase::new), "increase"),
                 def(MaxOverTime.class, uni(MaxOverTime::new), "max_over_time"),
                 def(MinOverTime.class, uni(MinOverTime::new), "min_over_time"),

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java

@@ -29,6 +29,7 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
             Idelta.ENTRY,
             Increase.ENTRY,
             Delta.ENTRY,
+            Deriv.ENTRY,
             Sample.ENTRY,
             SpatialCentroid.ENTRY,
             SpatialExtent.ENTRY,