Timestamps: Fix timestamp handling and release 1.2.1 (#64)

* fixed timestamp parsing

* removed unnecessary mutliplier

* adapted tests

* queries should be made in UTC

* fixed tests not using utc

* added version upgrade

* update changelog

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 All notable changes to this project will be documented in this file.
 
+## v1.2.1
+### Bug fixes
+- Fixed timestamps in the backend beeing handled wrong (#31)
+- Fixed timestamps in the frontend being assumed as local, whereas they should be UTC (#21, #67)
 ## v1.2.0
 ### Features and enhancements
 - Upgrade of `@grafana/data`, `@grafana/ui`, `@grafana/runtime`, `@grafana/toolkit` to 8.5.5 (#35, #41)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "grafana-opensearch-datasource",
-  "version": "1.2.0",
+  "version": "1.2.1",
   "description": "",
   "scripts": {
     "build": "grafana-toolkit plugin:build",

pkg/opensearch/models.go

@@ -113,6 +113,6 @@ const (
 
 // PPL date time type formats
 const (
-	pplTSFormat   = "2006-01-02 15:04:05.000000"
+	pplTSFormat   = "2006-01-02 15:04:05.999999"
 	pplDateFormat = "2006-01-02"
 )

pkg/opensearch/ppl_handler.go

@@ -20,8 +20,8 @@ var newPPLHandler = func(client es.Client, req *backend.QueryDataRequest) *pplHa
 }
 
 func (h *pplHandler) processQuery(q *Query) error {
-	from := h.req.Queries[0].TimeRange.From.Local().Format("2006-01-02 15:04:05")
-	to := h.req.Queries[0].TimeRange.To.Local().Format("2006-01-02 15:04:05")
+	from := h.req.Queries[0].TimeRange.From.UTC().Format("2006-01-02 15:04:05")
+	to := h.req.Queries[0].TimeRange.To.UTC().Format("2006-01-02 15:04:05")
 
 	builder := h.client.PPL()
 	builder.AddPPLQueryString(h.client.GetTimeField(), to, from, q.RawQuery)

pkg/opensearch/ppl_response_parser.go

@@ -84,7 +84,8 @@ func (rp *pplResponseParser) addDatarow(frame *data.Frame, i int, datarow es.Dat
 	if err != nil {
 		return err
 	}
-	frame.Set(0, i, nullFloatToNullableTime(timestamp))
+
+	frame.Set(0, i, utils.NullFloatToNullableTime(timestamp))
 	if value.Valid {
 		frame.Set(1, i, &value.Float64)
 	} else {

pkg/opensearch/ppl_response_parser_test.go

@@ -43,8 +43,8 @@ func TestPPLResponseParser(t *testing.T) {
 				frame := queryRes.Frames[0]
 				So(frame.Name, ShouldEqual, "testMetric")
 				So(frame.Rows(), ShouldEqual, 2)
-				So(floatAt(frame, 0, 0), ShouldEqual, 100000)
-				So(floatAt(frame, 0, 1), ShouldEqual, 200000)
+				So(floatAt(frame, 0, 0), ShouldEqual, 100)
+				So(floatAt(frame, 0, 1), ShouldEqual, 200)
 				So(floatAt(frame, 1, 0), ShouldEqual, 10)
 				So(floatAt(frame, 1, 1), ShouldEqual, 15)
 			})
@@ -77,8 +77,8 @@ func TestPPLResponseParser(t *testing.T) {
 				frame := queryRes.Frames[0]
 				So(frame.Name, ShouldEqual, "testMetric")
 				So(frame.Rows(), ShouldEqual, 2)
-				So(floatAt(frame, 0, 0), ShouldEqual, 100000)
-				So(floatAt(frame, 0, 1), ShouldEqual, 200000)
+				So(floatAt(frame, 0, 0), ShouldEqual, 100)
+				So(floatAt(frame, 0, 1), ShouldEqual, 200)
 				So(floatAt(frame, 1, 0), ShouldEqual, 20)
 				So(floatAt(frame, 1, 1), ShouldEqual, 25)
 			})
@@ -141,7 +141,7 @@ func TestPPLResponseParser(t *testing.T) {
 				frame := queryRes.Frames[0]
 				So(frame.Name, ShouldEqual, "testMetric")
 				So(frame.Rows(), ShouldEqual, 1)
-				So(floatAt(frame, 0, 0), ShouldEqual, 100000)
+				So(floatAt(frame, 0, 0), ShouldEqual, 100)
 				So(floatAt(frame, 1, 0), ShouldEqual, 10)
 			})
 
@@ -156,7 +156,7 @@ func TestPPLResponseParser(t *testing.T) {
 				frame := queryRes.Frames[0]
 				So(frame.Name, ShouldEqual, "testMetric")
 				So(frame.Rows(), ShouldEqual, 1)
-				So(floatAt(frame, 0, 0), ShouldEqual, 100000)
+				So(floatAt(frame, 0, 0), ShouldEqual, 100)
 				So(floatAt(frame, 1, 0), ShouldEqual, 10)
 			})
 

pkg/opensearch/response_parser.go

@@ -2,12 +2,10 @@ package opensearch
 
 import (
 	"errors"
-	"math"
 	"regexp"
 	"sort"
 	"strconv"
 	"strings"
-	"time"
 
 	simplejson "github.com/bitly/go-simplejson"
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
@@ -636,23 +634,12 @@ func getErrorFromOpenSearchResponse(response *es.SearchResponse) error {
 	return err
 }
 
-func setFrameRow(frame *data.Frame, i int, key, value null.Float) {
-	frame.Set(0, i, nullFloatToNullableTime(key))
+func setFrameRow(frame *data.Frame, i int, ntime null.Float, value null.Float) {
+	frame.Set(0, i, utils.NullFloatToNullableTime(ntime))
 
 	if value.Valid {
 		frame.Set(1, i, &value.Float64)
 	} else {
 		frame.Set(1, i, nil)
 	}
 }
-
-func nullFloatToNullableTime(ts null.Float) *time.Time {
-	if !ts.Valid {
-		return nil
-	}
-
-	sec, fract := math.Modf(ts.Float64)
-	nsec := int64(fract * float64(time.Second))
-	timestamp := time.Unix(int64(sec), nsec)
-	return &timestamp
-}

pkg/utils/utils.go

@@ -2,8 +2,10 @@ package utils
 
 import (
 	"encoding/json"
+	"time"
 
 	"github.com/bitly/go-simplejson"
+	"github.com/grafana/opensearch-datasource/pkg/null"
 )
 
 func NewJsonFromAny(data interface{}) *simplejson.Json {
@@ -18,3 +20,12 @@ func NewRawJsonFromAny(data interface{}) []byte {
 	dataJsonRaw, _ := dataJson.MarshalJSON()
 	return dataJsonRaw
 }
+
+func NullFloatToNullableTime(ts null.Float) *time.Time {
+	if !ts.Valid {
+		return nil
+	}
+
+	timestamp := time.UnixMilli(int64(ts.Float64)).UTC()
+	return &timestamp
+}

src/OpenSearchResponse.ts

@@ -3,7 +3,6 @@ import flatten from './dependencies/flatten';
 import * as queryDef from './query_def';
 import TableModel from './dependencies/table_model';
 import {
-  dateTime,
   DataQueryResponse,
   DataFrame,
   toDataFrame,
@@ -755,7 +754,7 @@ const getPPLDatapoints = (response: any): { datapoints: any; targetVal: any; inv
   const datapoints = _.map(response.datarows, datarow => {
     const newDatarow = _.clone(datarow);
     const [timestamp] = newDatarow.splice(timeFieldIndex, 1);
-    newDatarow.push(dateTime(timestamp).unix() * 1000);
+    newDatarow.push(toUtc(timestamp).unix() * 1000);
     return newDatarow;
   });
 

src/datasource.test.ts

@@ -646,7 +646,7 @@ describe('OpenSearchDatasource', function(this: any) {
         timezone: '',
         app: CoreApp.Dashboard,
         startTime: 0,
-        range: createTimeRange(dateTime([2015, 4, 30, 10]), dateTime([2015, 5, 1, 10])),
+        range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])),
         targets,
       };
 

src/datasource.ts

@@ -520,8 +520,12 @@ export class OpenSearchDatasource extends DataSourceApi<OpenSearchQuery, OpenSea
     for (const target of targets) {
       let payload = this.createPPLQuery(target, options);
 
-      const rangeFrom = dateTime(options.range.from.valueOf()).format('YYYY-MM-DD HH:mm:ss');
-      const rangeTo = dateTime(options.range.to.valueOf()).format('YYYY-MM-DD HH:mm:ss');
+      const rangeFrom = dateTime(options.range.from.valueOf())
+        .utc()
+        .format('YYYY-MM-DD HH:mm:ss');
+      const rangeTo = dateTime(options.range.to.valueOf())
+        .utc()
+        .format('YYYY-MM-DD HH:mm:ss');
       // Replace the range here for actual values.
       payload = payload.replace(/\$timeTo/g, rangeTo);
       payload = payload.replace(/\$timeFrom/g, rangeFrom);