From df9db17e29667707e20fe44f59c6f2ec9e68ac40 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Sun, 9 Oct 2022 12:57:32 +0800 Subject: [PATCH 01/47] init - make success init repo. make (using mingw32-make) success. The binary is executable on windows. --- .gitignore | 2 ++ go.sum | 1 + 2 files changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index caf0afa9a..42a55f1ef 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ # High Dynamic Range (HDR) Histogram files *.hdr + +bin/ diff --git a/go.sum b/go.sum index 6b3ac40ef..9e09456f8 100644 --- a/go.sum +++ b/go.sum @@ -902,6 +902,7 @@ github.com/timescale/promscale v0.0.0-20201006153045-6a66a36f5c84/go.mod h1:rkhy github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= +github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= From 7021966410cab28e48d36aa3f5631a55ea07d6a8 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Sat, 22 Oct 2022 19:26:51 +0800 Subject: [PATCH 02/47] Init iotdb, part Query not completed Init iotdb, part Query not completed 1. pkg\query\iotdb.go added, but not completed yet 2. cmd\...\iotdb\common.go added, but not completed yet 3. cmd\...\iotdb\devops.go added, but not completed yet 4. go.mod and go.sum modified --- .../databases/iotdb/common.go | 26 +++++ .../databases/iotdb/devops.go | 100 ++++++++++++++++++ go.mod | 1 + go.sum | 16 +-- pkg/query/iotdb.go | 78 ++++++++++++++ 5 files changed, 215 insertions(+), 6 deletions(-) create mode 100644 cmd/tsbs_generate_queries/databases/iotdb/common.go create mode 100644 cmd/tsbs_generate_queries/databases/iotdb/devops.go create mode 100644 pkg/query/iotdb.go diff --git a/cmd/tsbs_generate_queries/databases/iotdb/common.go b/cmd/tsbs_generate_queries/databases/iotdb/common.go new file mode 100644 index 000000000..5172fd0ed --- /dev/null +++ b/cmd/tsbs_generate_queries/databases/iotdb/common.go @@ -0,0 +1,26 @@ +package iotdb + +import ( + "github.com/timescale/tsbs/pkg/query" +) + +// BaseGenerator contains settings specific for IoTDB +type BaseGenerator struct { + BasicPath string // e.g. "root.sg" is basic path of "root.sg.device" + BasicPathLevel int32 // e.g. 0 for "root", 1 for "root.device" +} + +// GenerateEmptyQuery returns an empty query.Mongo. +func (g *BaseGenerator) GenerateEmptyQuery() query.Query { + return query.NewIoTDB() +} + +// fillInQuery fills the query struct with data. +func (g *BaseGenerator) fillInQuery(qi query.Query, humanLabel, humanDesc, tableName, sql string) { + q := qi.(*query.IoTDB) + q.HumanLabel = []byte(humanLabel) + q.HumanDescription = []byte(humanDesc) + q.TableName = []byte(tableName) + q.SqlQuery = []byte(sql) + // CRTODO: 在修改了结构之后，这里是否还需要更多的东西？ +} diff --git a/cmd/tsbs_generate_queries/databases/iotdb/devops.go b/cmd/tsbs_generate_queries/databases/iotdb/devops.go new file mode 100644 index 000000000..0dc914fa1 --- /dev/null +++ b/cmd/tsbs_generate_queries/databases/iotdb/devops.go @@ -0,0 +1,100 @@ +package iotdb + +import ( + "fmt" + "strings" + "time" + + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/devops" + "github.com/timescale/tsbs/pkg/query" +) + +// TODO: Remove the need for this by continuing to bubble up errors +func panicIfErr(err error) { + if err != nil { + panic(err.Error()) + } +} + +// Devops produces IoTDB-specific queries for all the devops query types. +type Devops struct { + *BaseGenerator + *devops.Core +} + +// getHostFromString gets multiple random hostnames and creates a FROM SQL statement for these hostnames. +// e.g. A storage group "root.cpu" has two devices, named "host1" and "host2" +// Two paths for them are "root.cpu.host1" and "root.cpu.host2" +// This function returns "root.cpu.host1, root.cpu.host2" (without "FROM") +func (d *Devops) getHostFromString(nHosts int) string { + hostnames, err := d.GetRandomHosts(nHosts) + panicIfErr(err) + var hostnameClauses []string + + for _, hostname := range hostnames { + hostnameClauses = append(hostnameClauses, fmt.Sprintf("%s.cpu.%s", d.BasicPath, hostname)) + } + + return strings.Join(hostnameClauses, ", ") +} + +// getSelectClausesAggMetrics gets clauses for aggregate functions. +func (d *Devops) getSelectClausesAggMetrics(agg string, metrics []string) []string { + selectClauses := make([]string, len(metrics)) + for i, m := range metrics { + selectClauses[i] = fmt.Sprintf("%s(%s)", agg, m) + } + + return selectClauses +} + +// GroupByTime selects the MAX for numMetrics metrics under 'cpu', +// per minute for nhosts hosts, +// e.g. in pseudo-SQL: +// +// SELECT minute, max(metric1), ..., max(metricN) +// FROM cpu +// WHERE (hostname = '$HOSTNAME_1' OR ... OR hostname = '$HOSTNAME_N') +// AND time >= '$HOUR_START' AND time < '$HOUR_END' +// GROUP BY minute ORDER BY minute ASC +func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange time.Duration) { + interval := d.Interval.MustRandWindow(timeRange) + metrics, err := devops.GetCPUMetricsSlice(numMetrics) + panicIfErr(err) + selectClauses := d.getSelectClausesAggMetrics("MAX_VALUE", metrics) + fromHosts := d.getHostFromString(nHosts) + + humanLabel := fmt.Sprintf("IoTDB %d cpu metric(s), random %4d hosts, random %s by 1m", numMetrics, nHosts, timeRange) + humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) + sql := "" + sql = sql + fmt.Sprintf("SELECT %s", strings.Join(selectClauses, ", ")) + sql = sql + fmt.Sprintf(" FROM %s", fromHosts) + // sql = sql + fmt.Sprintf(" WHERE time >= %s AND time < %s", interval.StartString(), interval.EndString()) + sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1m)", interval.StartString(), interval.EndString()) + + d.fillInQuery(qi, humanLabel, humanDesc, devops.TableName, sql) +} + +// GroupByTimeAndPrimaryTag selects the AVG of numMetrics metrics under 'cpu' per device per hour for a day, +// e.g. in pseudo-SQL: +// +// SELECT AVG(metric1), ..., AVG(metricN) +// FROM cpu +// WHERE time >= '$HOUR_START' AND time < '$HOUR_END' +// GROUP BY hour, hostname ORDER BY hour +func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) { + metrics, err := devops.GetCPUMetricsSlice(numMetrics) + panicIfErr(err) + interval := d.Interval.MustRandWindow(devops.DoubleGroupByDuration) + selectClauses := d.getSelectClausesAggMetrics("AVG", metrics) + + humanLabel := devops.GetDoubleGroupByLabel("IoTDB", numMetrics) + humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) + sql := "" + sql = sql + fmt.Sprintf("SELECT %s", strings.Join(selectClauses, ", ")) + sql = sql + fmt.Sprintf(" FROM %s.cpu.*", d.BasicPath) + // sql = sql + fmt.Sprintf(" WHERE time >= %s AND time < %s", interval.StartString(), interval.EndString()) + sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1m), LEVEL = %d", interval.StartString(), interval.EndString(), d.BasicPathLevel+2) + + d.fillInQuery(qi, humanLabel, humanDesc, devops.TableName, sql) +} diff --git a/go.mod b/go.mod index 1106116ed..a53d6bd32 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/SiriDB/go-siridb-connector v0.0.0-20190110105621-86b34c44c921 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 + github.com/apache/iotdb-client-go v0.13.0 github.com/aws/aws-sdk-go v1.35.13 github.com/blagojts/viper v1.6.3-0.20200313094124-068f44cf5e69 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 diff --git a/go.sum b/go.sum index 9e09456f8..6c6f9c3de 100644 --- a/go.sum +++ b/go.sum @@ -93,8 +93,12 @@ github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDa github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= +github.com/apache/iotdb-client-go v0.13.0 h1:b1608bn+s8Oqd1UTJbeR0H+P+B46+zTUE+g8J1bovvw= +github.com/apache/iotdb-client-go v0.13.0/go.mod h1:kaergHbc+hEtIST6Zgz1JQjukP3MIewLD9gaNDDQ/v4= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.14.1 h1:Yh8v0hpCj63p5edXOLaqTJW0IJ1p+eMW6+YSOqw1d6s= +github.com/apache/thrift v0.14.1/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -828,11 +832,8 @@ github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs= github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= -github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= -github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v3.21.3+incompatible h1:uenXGGa8ESCQq+dbgtl916dmg6PSAz2cXov0uORQ9v8= github.com/shirou/gopsutil v3.21.3+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc h1:jUIKcSPO9MoMJBbEoyE/RJoE8vz7Mb8AjvifMMwSyvY= @@ -883,13 +884,16 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM= @@ -1137,7 +1141,6 @@ golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200821140526-fda516888d29/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200908134130-d2e65c121b96 h1:gJciq3lOg0eS9fSZJcoHfv7q1BfC6cJfnmSSKL1yu3Q= golang.org/x/sys v0.0.0-20200908134130-d2e65c121b96/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa h1:ZYxPR6aca/uhfRJyaOAtflSHjJYiktO7QnJC5ut7iY4= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1349,8 +1352,9 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v0.0.0-20181223230014-1083505acf35/go.mod h1:R//lfYlUuTOTfblYI3lGoAAAebUdzjvbmQsuB7Ykd90= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/query/iotdb.go b/pkg/query/iotdb.go new file mode 100644 index 000000000..5500b0590 --- /dev/null +++ b/pkg/query/iotdb.go @@ -0,0 +1,78 @@ +package query + +import ( + "fmt" + "sync" + + "github.com/apache/iotdb-client-go/client" +) + +// IoTDB encodes a IoTDB request. This will be serialized for use +// by the tsbs_run_queries_iotdb program. +type IoTDB struct { + HumanLabel []byte + HumanDescription []byte + + // CRTODO: 设计好这里的结构，这里可能还需要其他东西 + ClientSession client.Session + TableName []byte + SqlQuery []byte + id uint64 +} + +// IoTDBPool is a sync.Pool of IoTDB Query types +var IoTDBPool = sync.Pool{ + New: func() interface{} { + return &IoTDB{ + HumanLabel: []byte{}, + HumanDescription: []byte{}, + + TableName: []byte{}, + SqlQuery: []byte{}, + } + }, +} + +// NewIoTDB returns a new IoTDB Query instance +func NewIoTDB() *IoTDB { + return IoTDBPool.Get().(*IoTDB) +} + +// GetID returns the ID of this Query +func (q *IoTDB) GetID() uint64 { + return q.id +} + +// SetID sets the ID for this Query +func (q *IoTDB) SetID(id uint64) { + q.id = id +} + +// String produces a debug-ready description of a Query. +func (q *IoTDB) String() string { + return fmt.Sprintf( + "HumanLabel: %s, HumanDescription: %s, Query: %s", + q.HumanLabel, q.HumanDescription, q.SqlQuery, + ) +} + +// HumanLabelName returns the human readable name of this Query +func (q *IoTDB) HumanLabelName() []byte { + return q.HumanLabel +} + +// HumanDescriptionName returns the human readable description of this Query +func (q *IoTDB) HumanDescriptionName() []byte { + return q.HumanDescription +} + +// Release resets and returns this Query to its pool +func (q *IoTDB) Release() { + // CRTODO: 弄清楚这里是否还需要其他的操作 + q.HumanLabel = q.HumanLabel[:0] + q.HumanDescription = q.HumanDescription[:0] + q.id = 0 + q.SqlQuery = q.SqlQuery[:0] + + IoTDBPool.Put(q) +} From 425da8c19136d52892e1c471afa80a730fe24c7c Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Sat, 22 Oct 2022 21:05:21 +0800 Subject: [PATCH 03/47] complete devops.go; fix typo 1. complete devops.go. without devops_test.go Other files are not completed so it may cannot generate queries till now. 2. fix some typo in internal\utils\time_interval.go --- .../databases/iotdb/common.go | 3 +- .../databases/iotdb/devops.go | 87 ++++++++++++++++++- internal/utils/time_interval.go | 4 +- pkg/query/iotdb.go | 4 +- 4 files changed, 89 insertions(+), 9 deletions(-) diff --git a/cmd/tsbs_generate_queries/databases/iotdb/common.go b/cmd/tsbs_generate_queries/databases/iotdb/common.go index 5172fd0ed..28aefb73a 100644 --- a/cmd/tsbs_generate_queries/databases/iotdb/common.go +++ b/cmd/tsbs_generate_queries/databases/iotdb/common.go @@ -16,11 +16,10 @@ func (g *BaseGenerator) GenerateEmptyQuery() query.Query { } // fillInQuery fills the query struct with data. -func (g *BaseGenerator) fillInQuery(qi query.Query, humanLabel, humanDesc, tableName, sql string) { +func (g *BaseGenerator) fillInQuery(qi query.Query, humanLabel, humanDesc, sql string) { q := qi.(*query.IoTDB) q.HumanLabel = []byte(humanLabel) q.HumanDescription = []byte(humanDesc) - q.TableName = []byte(tableName) q.SqlQuery = []byte(sql) // CRTODO: 在修改了结构之后，这里是否还需要更多的东西？ } diff --git a/cmd/tsbs_generate_queries/databases/iotdb/devops.go b/cmd/tsbs_generate_queries/databases/iotdb/devops.go index 0dc914fa1..17d2b9e8f 100644 --- a/cmd/tsbs_generate_queries/databases/iotdb/devops.go +++ b/cmd/tsbs_generate_queries/databases/iotdb/devops.go @@ -72,7 +72,7 @@ func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange t // sql = sql + fmt.Sprintf(" WHERE time >= %s AND time < %s", interval.StartString(), interval.EndString()) sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1m)", interval.StartString(), interval.EndString()) - d.fillInQuery(qi, humanLabel, humanDesc, devops.TableName, sql) + d.fillInQuery(qi, humanLabel, humanDesc, sql) } // GroupByTimeAndPrimaryTag selects the AVG of numMetrics metrics under 'cpu' per device per hour for a day, @@ -96,5 +96,88 @@ func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) { // sql = sql + fmt.Sprintf(" WHERE time >= %s AND time < %s", interval.StartString(), interval.EndString()) sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1m), LEVEL = %d", interval.StartString(), interval.EndString(), d.BasicPathLevel+2) - d.fillInQuery(qi, humanLabel, humanDesc, devops.TableName, sql) + d.fillInQuery(qi, humanLabel, humanDesc, sql) +} + +// LastPointPerHost finds the last row for every host in the dataset +func (d *Devops) LastPointPerHost(qi query.Query) { + humanLabel := "IoTDB last row per host" + humanDesc := humanLabel + ": cpu" + + sql := fmt.Sprintf("SELECT LAST * FROM %s.cpu.* ", d.BasicPath) + d.fillInQuery(qi, humanLabel, humanDesc, sql) +} + +// MaxAllCPU selects the MAX of all metrics under 'cpu' per hour for nhosts hosts, +// e.g. in pseudo-SQL: +// +// SELECT MAX(metric1), ..., MAX(metricN) +// FROM cpu WHERE (hostname = '$HOSTNAME_1' OR ... OR hostname = '$HOSTNAME_N') +// AND time >= '$HOUR_START' AND time < '$HOUR_END' +// GROUP BY hour ORDER BY hour +func (d *Devops) MaxAllCPU(qi query.Query, nHosts int, duration time.Duration) { + interval := d.Interval.MustRandWindow(duration) + fromHosts := d.getHostFromString(nHosts) + selectClauses := d.getSelectClausesAggMetrics("MAX_VALUE", devops.GetAllCPUMetrics()) + + humanLabel := devops.GetMaxAllLabel("IoTDB", nHosts) + humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) + sql := "" + sql = sql + fmt.Sprintf("SELECT %s", strings.Join(selectClauses, ", ")) + sql = sql + fmt.Sprintf(" FROM %s", fromHosts) + // sql = sql + fmt.Sprintf(" WHERE time >= %s AND time < %s", interval.StartString(), interval.EndString()) + sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1h)", interval.StartString(), interval.EndString()) + + d.fillInQuery(qi, humanLabel, humanDesc, sql) +} + +// GroupByOrderByLimit benchmarks a query that has a time WHERE clause, that groups by a truncated date, orders by that date, and takes a limit: +// SELECT date_trunc('minute', time) AS t, MAX(cpu) FROM cpu +// WHERE time < '$TIME' +// GROUP BY t ORDER BY t DESC +// LIMIT $LIMIT +func (d *Devops) GroupByOrderByLimit(qi query.Query) { + interval := d.Interval.MustRandWindow(time.Hour) + selectClauses := d.getSelectClausesAggMetrics("MAX_VALUE", []string{"usage_user"}) + + sql := "" + sql = sql + fmt.Sprintf("SELECT %s", strings.Join(selectClauses, ", ")) + sql = sql + fmt.Sprintf(" FROM %s.cpu.*", d.BasicPath) + // sql = sql + fmt.Sprintf(" WHERE time < %s", interval.EndString()) + sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1m)", interval.StartString(), interval.EndString()) + sql = sql + " LIMIT 5" + + humanLabel := "IoTDB max cpu over last 5 min-intervals (random end)" + humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) + + d.fillInQuery(qi, humanLabel, humanDesc, sql) +} + +// HighCPUForHosts populates a query that gets CPU metrics when the CPU has high +// usage between a time period for a number of hosts (if 0, it will search all hosts), +// e.g. in pseudo-SQL: +// +// SELECT * FROM cpu +// WHERE usage_user > 90.0 +// AND time >= '$TIME_START' AND time < '$TIME_END' +// AND (hostname = '$HOST' OR hostname = '$HOST2'...) +func (d *Devops) HighCPUForHosts(qi query.Query, nHosts int) { + interval := d.Interval.MustRandWindow(devops.HighCPUDuration) + + var fromHosts string + if nHosts <= 0 { + fromHosts = fmt.Sprintf("%s.cpu.*", d.BasicPath) + } else { + fromHosts = d.getHostFromString(nHosts) + } + + humanLabel, err := devops.GetHighCPULabel("IoTDB", nHosts) + panicIfErr(err) + humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) + + sql := "SELECT *" + sql = sql + fmt.Sprintf(" FROM %s", fromHosts) + sql = sql + fmt.Sprintf(" WHERE usage_user > 90.0 AND time >= %s AND time < %s", interval.StartString(), interval.EndString()) + + d.fillInQuery(qi, humanLabel, humanDesc, sql) } diff --git a/internal/utils/time_interval.go b/internal/utils/time_interval.go index e2b271541..276b16824 100644 --- a/internal/utils/time_interval.go +++ b/internal/utils/time_interval.go @@ -118,12 +118,12 @@ func (ti *TimeInterval) StartUnixMillis() int64 { return ti.start.UTC().UnixNano() / int64(time.Millisecond) } -// StartString formats the start of the TimeInterval according to RFC3339. +// StartString formats the start of the TimeInterval according to RFC3339. func (ti *TimeInterval) StartString() string { return ti.start.Format(time.RFC3339) } -// End returns the starting time in UTC. +// End returns the end time in UTC. func (ti *TimeInterval) End() time.Time { return ti.end } diff --git a/pkg/query/iotdb.go b/pkg/query/iotdb.go index 5500b0590..c96cf20fc 100644 --- a/pkg/query/iotdb.go +++ b/pkg/query/iotdb.go @@ -15,7 +15,6 @@ type IoTDB struct { // CRTODO: 设计好这里的结构，这里可能还需要其他东西 ClientSession client.Session - TableName []byte SqlQuery []byte id uint64 } @@ -27,8 +26,7 @@ var IoTDBPool = sync.Pool{ HumanLabel: []byte{}, HumanDescription: []byte{}, - TableName: []byte{}, - SqlQuery: []byte{}, + SqlQuery: []byte{}, } }, } From 2fbc8e21b2f373c097da9249a1ebe1da1eec6d31 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Sat, 22 Oct 2022 22:28:43 +0800 Subject: [PATCH 04/47] devops query generate completed. test file added. 1. devops query generate completed. 2. test file added, but NOT completed --- .../databases/iotdb/common.go | 20 +++ .../databases/iotdb/devops.go | 60 ++++++-- .../databases/iotdb/devops_test.go | 140 ++++++++++++++++++ go.mod | 1 + 4 files changed, 206 insertions(+), 15 deletions(-) create mode 100644 cmd/tsbs_generate_queries/databases/iotdb/devops_test.go diff --git a/cmd/tsbs_generate_queries/databases/iotdb/common.go b/cmd/tsbs_generate_queries/databases/iotdb/common.go index 28aefb73a..65b5d2332 100644 --- a/cmd/tsbs_generate_queries/databases/iotdb/common.go +++ b/cmd/tsbs_generate_queries/databases/iotdb/common.go @@ -1,6 +1,10 @@ package iotdb import ( + "time" + + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/devops" + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/utils" "github.com/timescale/tsbs/pkg/query" ) @@ -23,3 +27,19 @@ func (g *BaseGenerator) fillInQuery(qi query.Query, humanLabel, humanDesc, sql s q.SqlQuery = []byte(sql) // CRTODO: 在修改了结构之后，这里是否还需要更多的东西？ } + +// NewDevops creates a new devops use case query generator. +func (g *BaseGenerator) NewDevops(start, end time.Time, scale int) (utils.QueryGenerator, error) { + core, err := devops.NewCore(start, end, scale) + + if err != nil { + return nil, err + } + + devops := &Devops{ + BaseGenerator: g, + Core: core, + } + + return devops, nil +} diff --git a/cmd/tsbs_generate_queries/databases/iotdb/devops.go b/cmd/tsbs_generate_queries/databases/iotdb/devops.go index 17d2b9e8f..31524114d 100644 --- a/cmd/tsbs_generate_queries/databases/iotdb/devops.go +++ b/cmd/tsbs_generate_queries/databases/iotdb/devops.go @@ -22,6 +22,35 @@ type Devops struct { *devops.Core } +// modifyHostnames makes sure IP address can appear in the path. +// Node names in path can NOT contain "." unless enclosing it within either single quote (') or double quote ("). +// In this case, quotes are recognized as part of the node name to avoid ambiguity. +func (d *Devops) modifyHostnames(hostnames []string) []string { + for i, hostname := range hostnames { + if strings.Contains(hostname, ".") { + if !(hostname[:1] == "\"" && hostname[len(hostname)-1:] == "\"") { + // not modified yet + hostnames[i] = "\"" + hostnames[i] + "\"" + } + + } + } + return hostnames +} + +// getHostFromWithHostnames creates FROM SQL statement for multiple hostnames. +// e.g. A storage group "root.cpu" has two devices. +// Two hostnames are "host1" and "host2" +// This function returns "root.cpu.host1, root.cpu.host2" (without "FROM") +func (d *Devops) getHostFromWithHostnames(hostnames []string) string { + hostnames = d.modifyHostnames(hostnames) + var hostnameClauses []string + for _, hostname := range hostnames { + hostnameClauses = append(hostnameClauses, fmt.Sprintf("%s.cpu.%s", d.BasicPath, hostname)) + } + return strings.Join(hostnameClauses, ", ") +} + // getHostFromString gets multiple random hostnames and creates a FROM SQL statement for these hostnames. // e.g. A storage group "root.cpu" has two devices, named "host1" and "host2" // Two paths for them are "root.cpu.host1" and "root.cpu.host2" @@ -29,13 +58,8 @@ type Devops struct { func (d *Devops) getHostFromString(nHosts int) string { hostnames, err := d.GetRandomHosts(nHosts) panicIfErr(err) - var hostnameClauses []string - - for _, hostname := range hostnames { - hostnameClauses = append(hostnameClauses, fmt.Sprintf("%s.cpu.%s", d.BasicPath, hostname)) - } - - return strings.Join(hostnameClauses, ", ") + fromClauses := d.getHostFromWithHostnames(hostnames) + return fromClauses } // getSelectClausesAggMetrics gets clauses for aggregate functions. @@ -48,6 +72,12 @@ func (d *Devops) getSelectClausesAggMetrics(agg string, metrics []string) []stri return selectClauses } +// getSelectClausesAggMetricsString gets a whole select clause for aggregate functions. +func (d *Devops) getSelectClausesAggMetricsString(agg string, metrics []string) string { + selectClauses := d.getSelectClausesAggMetrics(agg, metrics) + return strings.Join(selectClauses, ", ") +} + // GroupByTime selects the MAX for numMetrics metrics under 'cpu', // per minute for nhosts hosts, // e.g. in pseudo-SQL: @@ -61,13 +91,13 @@ func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange t interval := d.Interval.MustRandWindow(timeRange) metrics, err := devops.GetCPUMetricsSlice(numMetrics) panicIfErr(err) - selectClauses := d.getSelectClausesAggMetrics("MAX_VALUE", metrics) + selectClause := d.getSelectClausesAggMetricsString("MAX_VALUE", metrics) fromHosts := d.getHostFromString(nHosts) humanLabel := fmt.Sprintf("IoTDB %d cpu metric(s), random %4d hosts, random %s by 1m", numMetrics, nHosts, timeRange) humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) sql := "" - sql = sql + fmt.Sprintf("SELECT %s", strings.Join(selectClauses, ", ")) + sql = sql + fmt.Sprintf("SELECT %s", selectClause) sql = sql + fmt.Sprintf(" FROM %s", fromHosts) // sql = sql + fmt.Sprintf(" WHERE time >= %s AND time < %s", interval.StartString(), interval.EndString()) sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1m)", interval.StartString(), interval.EndString()) @@ -86,12 +116,12 @@ func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) { metrics, err := devops.GetCPUMetricsSlice(numMetrics) panicIfErr(err) interval := d.Interval.MustRandWindow(devops.DoubleGroupByDuration) - selectClauses := d.getSelectClausesAggMetrics("AVG", metrics) + selectClause := d.getSelectClausesAggMetrics("AVG", metrics) humanLabel := devops.GetDoubleGroupByLabel("IoTDB", numMetrics) humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) sql := "" - sql = sql + fmt.Sprintf("SELECT %s", strings.Join(selectClauses, ", ")) + sql = sql + fmt.Sprintf("SELECT %s", selectClause) sql = sql + fmt.Sprintf(" FROM %s.cpu.*", d.BasicPath) // sql = sql + fmt.Sprintf(" WHERE time >= %s AND time < %s", interval.StartString(), interval.EndString()) sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1m), LEVEL = %d", interval.StartString(), interval.EndString(), d.BasicPathLevel+2) @@ -118,12 +148,12 @@ func (d *Devops) LastPointPerHost(qi query.Query) { func (d *Devops) MaxAllCPU(qi query.Query, nHosts int, duration time.Duration) { interval := d.Interval.MustRandWindow(duration) fromHosts := d.getHostFromString(nHosts) - selectClauses := d.getSelectClausesAggMetrics("MAX_VALUE", devops.GetAllCPUMetrics()) + selectClause := d.getSelectClausesAggMetrics("MAX_VALUE", devops.GetAllCPUMetrics()) humanLabel := devops.GetMaxAllLabel("IoTDB", nHosts) humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) sql := "" - sql = sql + fmt.Sprintf("SELECT %s", strings.Join(selectClauses, ", ")) + sql = sql + fmt.Sprintf("SELECT %s", selectClause) sql = sql + fmt.Sprintf(" FROM %s", fromHosts) // sql = sql + fmt.Sprintf(" WHERE time >= %s AND time < %s", interval.StartString(), interval.EndString()) sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1h)", interval.StartString(), interval.EndString()) @@ -138,10 +168,10 @@ func (d *Devops) MaxAllCPU(qi query.Query, nHosts int, duration time.Duration) { // LIMIT $LIMIT func (d *Devops) GroupByOrderByLimit(qi query.Query) { interval := d.Interval.MustRandWindow(time.Hour) - selectClauses := d.getSelectClausesAggMetrics("MAX_VALUE", []string{"usage_user"}) + selectClause := d.getSelectClausesAggMetrics("MAX_VALUE", []string{"usage_user"}) sql := "" - sql = sql + fmt.Sprintf("SELECT %s", strings.Join(selectClauses, ", ")) + sql = sql + fmt.Sprintf("SELECT %s", selectClause) sql = sql + fmt.Sprintf(" FROM %s.cpu.*", d.BasicPath) // sql = sql + fmt.Sprintf(" WHERE time < %s", interval.EndString()) sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1m)", interval.StartString(), interval.EndString()) diff --git a/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go b/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go new file mode 100644 index 000000000..23c83addd --- /dev/null +++ b/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go @@ -0,0 +1,140 @@ +package iotdb + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestModifyHostnames(t *testing.T) { + cases := []struct { + description string + hostnames []string + expected []string + }{ + { + description: "normal node names", + hostnames: []string{"hostname", "hello_world"}, + expected: []string{"hostname", "hello_world"}, + }, + { + description: "IP address or URL as hostnames", + hostnames: []string{"192.168.1.1", "8.8.8.8", "iotdb.apache.org"}, + expected: []string{"\"192.168.1.1\"", "\"8.8.8.8\"", "\"iotdb.apache.org\""}, + }, + { + description: "already modified case", + hostnames: []string{"\"192.168.1.1\"", "\"8.8.8.8\"", "\"iotdb.apache.org\""}, + expected: []string{"\"192.168.1.1\"", "\"8.8.8.8\"", "\"iotdb.apache.org\""}, + }, + { + description: "mixed host names", + hostnames: []string{"192.168.1.1", "hostname", "iotdb.apache.org", "\"8.8.8.8\""}, + expected: []string{"\"192.168.1.1\"", "hostname", "\"iotdb.apache.org\"", "\"8.8.8.8\""}, + }, + } + + for _, c := range cases { + t.Run(c.description, func(t *testing.T) { + b := BaseGenerator{BasicPath: "root", BasicPathLevel: 0} + dq, err := b.NewDevops(time.Now(), time.Now(), 10) + require.NoError(t, err, "Error while creating devops generator") + d := dq.(*Devops) + + actual := d.modifyHostnames(c.hostnames) + require.EqualValues(t, c.expected, actual) + }) + } +} + +func TestDevopsGetHostFromString(t *testing.T) { + cases := []struct { + description string + basicPath string + basicPathLevel int32 + hostnames []string + expected string + }{ + { + description: "single host", + basicPath: "root", + basicPathLevel: 0, + hostnames: []string{"host1"}, + expected: "root.cpu.host1", + }, + { + description: "multi host (2)", + basicPath: "root", + basicPathLevel: 0, + hostnames: []string{"host1", "host2"}, + expected: "root.cpu.host1, root.cpu.host2", + }, + { + description: "multi host (2) with storage group", + basicPath: "root.ln", + basicPathLevel: 1, + hostnames: []string{"host1", "host2"}, + expected: "root.ln.cpu.host1, root.ln.cpu.host2", + }, + { + description: "multi host (3) with special node names", + basicPath: "root", + basicPathLevel: 0, + hostnames: []string{"host1", "192.168.1.1", "\"iotdb.apache.org\""}, + expected: "root.cpu.host1, root.cpu.\"192.168.1.1\", root.cpu.\"iotdb.apache.org\"", + }, + } + + for _, c := range cases { + t.Run(c.description, func(t *testing.T) { + b := BaseGenerator{BasicPath: c.basicPath, BasicPathLevel: c.basicPathLevel} + dq, err := b.NewDevops(time.Now(), time.Now(), 10) + require.NoError(t, err, "Error while creating devops generator") + d := dq.(*Devops) + + actual := d.getHostFromWithHostnames(c.hostnames) + require.EqualValues(t, c.expected, actual) + }) + } +} + +func TestDevopsGetSelectClausesAggMetricsString(t *testing.T) { + cases := []struct { + description string + agg string + metrics []string + expected string + }{ + { + description: "single metric - max", + agg: "MAX_VALUE", + metrics: []string{"value"}, + expected: "MAX_VALUE(value)", + }, + { + description: "multiple metric - max", + agg: "MAX_VALUE", + metrics: []string{"temperature", "frequency"}, + expected: "MAX_VALUE(temperature), MAX_VALUE(frequency)", + }, + { + description: "multiple metric - avg", + agg: "AVG", + metrics: []string{"temperature", "frequency"}, + expected: "AVG(temperature), AVG(frequency)", + }, + } + + for _, c := range cases { + t.Run(c.description, func(t *testing.T) { + b := BaseGenerator{BasicPath: "root", BasicPathLevel: 0} + dq, err := b.NewDevops(time.Now(), time.Now(), 10) + require.NoError(t, err, "Error while creating devops generator") + d := dq.(*Devops) + + actual := d.getSelectClausesAggMetricsString(c.agg, c.metrics) + require.EqualValues(t, c.expected, actual) + }) + } +} diff --git a/go.mod b/go.mod index a53d6bd32..531d291fe 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/shirou/gopsutil v3.21.3+incompatible github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 + github.com/stretchr/testify v1.8.0 github.com/timescale/promscale v0.0.0-20201006153045-6a66a36f5c84 github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/transceptor-technology/go-qpack v0.0.0-20190116123619-49a14b216a45 From 401fe511a01ba64159ebb2f9a32e783bb56429ca Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Mon, 24 Oct 2022 20:19:26 +0800 Subject: [PATCH 05/47] queries test complete queries test complete --- .../databases/iotdb/common.go | 2 + .../databases/iotdb/devops.go | 18 +- .../databases/iotdb/devops_test.go | 246 +++++++++++++++++- 3 files changed, 249 insertions(+), 17 deletions(-) diff --git a/cmd/tsbs_generate_queries/databases/iotdb/common.go b/cmd/tsbs_generate_queries/databases/iotdb/common.go index 65b5d2332..d8263dd5c 100644 --- a/cmd/tsbs_generate_queries/databases/iotdb/common.go +++ b/cmd/tsbs_generate_queries/databases/iotdb/common.go @@ -8,6 +8,8 @@ import ( "github.com/timescale/tsbs/pkg/query" ) +const iotdbTimeFmt = "2006-01-02 15:04:05" + // BaseGenerator contains settings specific for IoTDB type BaseGenerator struct { BasicPath string // e.g. "root.sg" is basic path of "root.sg.device" diff --git a/cmd/tsbs_generate_queries/databases/iotdb/devops.go b/cmd/tsbs_generate_queries/databases/iotdb/devops.go index 31524114d..fb4fa4970 100644 --- a/cmd/tsbs_generate_queries/databases/iotdb/devops.go +++ b/cmd/tsbs_generate_queries/databases/iotdb/devops.go @@ -100,7 +100,7 @@ func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange t sql = sql + fmt.Sprintf("SELECT %s", selectClause) sql = sql + fmt.Sprintf(" FROM %s", fromHosts) // sql = sql + fmt.Sprintf(" WHERE time >= %s AND time < %s", interval.StartString(), interval.EndString()) - sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1m)", interval.StartString(), interval.EndString()) + sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1m)", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt)) d.fillInQuery(qi, humanLabel, humanDesc, sql) } @@ -116,7 +116,7 @@ func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) { metrics, err := devops.GetCPUMetricsSlice(numMetrics) panicIfErr(err) interval := d.Interval.MustRandWindow(devops.DoubleGroupByDuration) - selectClause := d.getSelectClausesAggMetrics("AVG", metrics) + selectClause := d.getSelectClausesAggMetricsString("AVG", metrics) humanLabel := devops.GetDoubleGroupByLabel("IoTDB", numMetrics) humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) @@ -124,7 +124,7 @@ func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) { sql = sql + fmt.Sprintf("SELECT %s", selectClause) sql = sql + fmt.Sprintf(" FROM %s.cpu.*", d.BasicPath) // sql = sql + fmt.Sprintf(" WHERE time >= %s AND time < %s", interval.StartString(), interval.EndString()) - sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1m), LEVEL = %d", interval.StartString(), interval.EndString(), d.BasicPathLevel+2) + sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1m), LEVEL = %d", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt), d.BasicPathLevel+2) d.fillInQuery(qi, humanLabel, humanDesc, sql) } @@ -134,7 +134,7 @@ func (d *Devops) LastPointPerHost(qi query.Query) { humanLabel := "IoTDB last row per host" humanDesc := humanLabel + ": cpu" - sql := fmt.Sprintf("SELECT LAST * FROM %s.cpu.* ", d.BasicPath) + sql := fmt.Sprintf("SELECT LAST * FROM %s.cpu.*", d.BasicPath) d.fillInQuery(qi, humanLabel, humanDesc, sql) } @@ -148,7 +148,7 @@ func (d *Devops) LastPointPerHost(qi query.Query) { func (d *Devops) MaxAllCPU(qi query.Query, nHosts int, duration time.Duration) { interval := d.Interval.MustRandWindow(duration) fromHosts := d.getHostFromString(nHosts) - selectClause := d.getSelectClausesAggMetrics("MAX_VALUE", devops.GetAllCPUMetrics()) + selectClause := d.getSelectClausesAggMetricsString("MAX_VALUE", devops.GetAllCPUMetrics()) humanLabel := devops.GetMaxAllLabel("IoTDB", nHosts) humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) @@ -156,7 +156,7 @@ func (d *Devops) MaxAllCPU(qi query.Query, nHosts int, duration time.Duration) { sql = sql + fmt.Sprintf("SELECT %s", selectClause) sql = sql + fmt.Sprintf(" FROM %s", fromHosts) // sql = sql + fmt.Sprintf(" WHERE time >= %s AND time < %s", interval.StartString(), interval.EndString()) - sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1h)", interval.StartString(), interval.EndString()) + sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1h)", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt)) d.fillInQuery(qi, humanLabel, humanDesc, sql) } @@ -168,13 +168,13 @@ func (d *Devops) MaxAllCPU(qi query.Query, nHosts int, duration time.Duration) { // LIMIT $LIMIT func (d *Devops) GroupByOrderByLimit(qi query.Query) { interval := d.Interval.MustRandWindow(time.Hour) - selectClause := d.getSelectClausesAggMetrics("MAX_VALUE", []string{"usage_user"}) + selectClause := d.getSelectClausesAggMetricsString("MAX_VALUE", []string{"usage_user"}) sql := "" sql = sql + fmt.Sprintf("SELECT %s", selectClause) sql = sql + fmt.Sprintf(" FROM %s.cpu.*", d.BasicPath) // sql = sql + fmt.Sprintf(" WHERE time < %s", interval.EndString()) - sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1m)", interval.StartString(), interval.EndString()) + sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1m)", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt)) sql = sql + " LIMIT 5" humanLabel := "IoTDB max cpu over last 5 min-intervals (random end)" @@ -207,7 +207,7 @@ func (d *Devops) HighCPUForHosts(qi query.Query, nHosts int) { sql := "SELECT *" sql = sql + fmt.Sprintf(" FROM %s", fromHosts) - sql = sql + fmt.Sprintf(" WHERE usage_user > 90.0 AND time >= %s AND time < %s", interval.StartString(), interval.EndString()) + sql = sql + fmt.Sprintf(" WHERE usage_user > 90.0 AND time >= %s AND time < %s", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt)) d.fillInQuery(qi, humanLabel, humanDesc, sql) } diff --git a/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go b/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go index 23c83addd..e8ab4827c 100644 --- a/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go +++ b/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go @@ -1,10 +1,13 @@ package iotdb import ( + "fmt" + "math/rand" "testing" "time" "github.com/stretchr/testify/require" + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/devops" ) func TestModifyHostnames(t *testing.T) { @@ -38,11 +41,11 @@ func TestModifyHostnames(t *testing.T) { for _, c := range cases { t.Run(c.description, func(t *testing.T) { b := BaseGenerator{BasicPath: "root", BasicPathLevel: 0} - dq, err := b.NewDevops(time.Now(), time.Now(), 10) + queryGenerator, err := b.NewDevops(time.Now(), time.Now(), 10) require.NoError(t, err, "Error while creating devops generator") - d := dq.(*Devops) + dp := queryGenerator.(*Devops) - actual := d.modifyHostnames(c.hostnames) + actual := dp.modifyHostnames(c.hostnames) require.EqualValues(t, c.expected, actual) }) } @@ -89,11 +92,11 @@ func TestDevopsGetHostFromString(t *testing.T) { for _, c := range cases { t.Run(c.description, func(t *testing.T) { b := BaseGenerator{BasicPath: c.basicPath, BasicPathLevel: c.basicPathLevel} - dq, err := b.NewDevops(time.Now(), time.Now(), 10) + queryGenerator, err := b.NewDevops(time.Now(), time.Now(), 10) require.NoError(t, err, "Error while creating devops generator") - d := dq.(*Devops) + dp := queryGenerator.(*Devops) - actual := d.getHostFromWithHostnames(c.hostnames) + actual := dp.getHostFromWithHostnames(c.hostnames) require.EqualValues(t, c.expected, actual) }) } @@ -129,12 +132,239 @@ func TestDevopsGetSelectClausesAggMetricsString(t *testing.T) { for _, c := range cases { t.Run(c.description, func(t *testing.T) { b := BaseGenerator{BasicPath: "root", BasicPathLevel: 0} - dq, err := b.NewDevops(time.Now(), time.Now(), 10) + queryGenerator, err := b.NewDevops(time.Now(), time.Now(), 10) require.NoError(t, err, "Error while creating devops generator") - d := dq.(*Devops) + d := queryGenerator.(*Devops) actual := d.getSelectClausesAggMetricsString(c.agg, c.metrics) require.EqualValues(t, c.expected, actual) }) } } + +func TestGroupByTime(t *testing.T) { + rand.Seed(123) // Setting seed for testing purposes. + start := time.Unix(0, 0) + end := start.Add(time.Hour) + base := BaseGenerator{BasicPath: "root.sg", BasicPathLevel: 1} + queryGenerator, err := base.NewDevops(start, end, 10) + require.NoError(t, err, "Error while creating devops generator") + dp := queryGenerator.(*Devops) + + metrics := 1 + nHosts := 1 + duration := time.Second + + actual := dp.GenerateEmptyQuery() + expected := dp.GenerateEmptyQuery() + dp.fillInQuery(expected, + "IoTDB 1 cpu metric(s), random 1 hosts, random 1s by 1m", + "IoTDB 1 cpu metric(s), random 1 hosts, random 1s by 1m: 1970-01-01T00:05:58Z", + "SELECT MAX_VALUE(usage_user) FROM root.sg.cpu.host_9 GROUP BY ([1970-01-01 00:05:58, 1970-01-01 00:05:59), 1m)", + ) + dp.GroupByTime(actual, nHosts, metrics, duration) + + require.EqualValues(t, expected, actual) +} + +func TestGroupByTimeAndPrimaryTag(t *testing.T) { + cases := []struct { + description string + numMetrics int + baseGenerator BaseGenerator + expectedHumanLabel string + expectedHumanDesc string + expectedSQLQuery string + }{ + { + description: "1 metric with storage group 'root.sg'", + numMetrics: 1, + baseGenerator: BaseGenerator{BasicPath: "root.sg", BasicPathLevel: 1}, + expectedHumanLabel: "IoTDB mean of 1 metrics, all hosts, random 12h0m0s by 1h", + expectedHumanDesc: "IoTDB mean of 1 metrics, all hosts, random 12h0m0s by 1h: 1970-01-01T00:16:22Z", + expectedSQLQuery: "SELECT AVG(usage_user) FROM root.sg.cpu.* GROUP BY ([1970-01-01 00:16:22, 1970-01-01 12:16:22), 1m), LEVEL = 3", + }, + { + description: "5 metric with storage group 'root'", + numMetrics: 5, + baseGenerator: BaseGenerator{BasicPath: "root", BasicPathLevel: 0}, + expectedHumanLabel: "IoTDB mean of 5 metrics, all hosts, random 12h0m0s by 1h", + expectedHumanDesc: "IoTDB mean of 5 metrics, all hosts, random 12h0m0s by 1h: 1970-01-01T00:16:22Z", + expectedSQLQuery: "SELECT AVG(usage_user), AVG(usage_system), AVG(usage_idle), AVG(usage_nice), AVG(usage_iowait) FROM root.cpu.* GROUP BY ([1970-01-01 00:16:22, 1970-01-01 12:16:22), 1m), LEVEL = 2", + }, + } + + start := time.Unix(0, 0) + end := start.Add(devops.DoubleGroupByDuration).Add(time.Hour) + + for _, c := range cases { + t.Run(c.description, func(t *testing.T) { + rand.Seed(123) // Setting seed for testing purposes. + b := c.baseGenerator + queryGenerator, err := b.NewDevops(start, end, 10) + require.NoError(t, err, "Error while creating devops generator") + dp := queryGenerator.(*Devops) + + actual := dp.GenerateEmptyQuery() + expected := dp.GenerateEmptyQuery() + + dp.fillInQuery(expected, c.expectedHumanLabel, c.expectedHumanDesc, c.expectedSQLQuery) + dp.GroupByTimeAndPrimaryTag(actual, c.numMetrics) + + require.EqualValues(t, expected, actual) + }) + } +} + +func TestLastPointPerHost(t *testing.T) { + rand.Seed(123) // Setting seed for testing purposes. + base := BaseGenerator{BasicPath: "root.sg", BasicPathLevel: 1} + queryGenerator, err := base.NewDevops(time.Now(), time.Now(), 10) + require.NoError(t, err, "Error while creating devops generator") + dp := queryGenerator.(*Devops) + + actual := dp.GenerateEmptyQuery() + expected := dp.GenerateEmptyQuery() + dp.fillInQuery(expected, + "IoTDB last row per host", + "IoTDB last row per host: cpu", + "SELECT LAST * FROM root.sg.cpu.*", + ) + dp.LastPointPerHost(actual) + + require.EqualValues(t, expected, actual) +} + +func TestMaxAllCPU(t *testing.T) { + cases := []struct { + description string + nHosts int + baseGenerator BaseGenerator + expectedHumanLabel string + expectedHumanDesc string + expectedSQLQuery string + }{ + { + description: "1 host with storage group 'root'", + nHosts: 1, + baseGenerator: BaseGenerator{BasicPath: "root", BasicPathLevel: 0}, + expectedHumanLabel: "IoTDB max of all CPU metrics, random 1 hosts, random 8h0m0s by 1h", + expectedHumanDesc: "IoTDB max of all CPU metrics, random 1 hosts, random 8h0m0s by 1h: 1970-01-01T02:16:22Z", + expectedSQLQuery: "SELECT MAX_VALUE(usage_user), MAX_VALUE(usage_system), MAX_VALUE(usage_idle), " + + "MAX_VALUE(usage_nice), MAX_VALUE(usage_iowait), MAX_VALUE(usage_irq), MAX_VALUE(usage_softirq), " + + "MAX_VALUE(usage_steal), MAX_VALUE(usage_guest), MAX_VALUE(usage_guest_nice) " + + "FROM root.cpu.host_9 GROUP BY ([1970-01-01 02:16:22, 1970-01-01 10:16:22), 1h)", + }, + { + description: "3 hosts with storage group 'root'", + nHosts: 3, + baseGenerator: BaseGenerator{BasicPath: "root", BasicPathLevel: 0}, + expectedHumanLabel: "IoTDB max of all CPU metrics, random 3 hosts, random 8h0m0s by 1h", + expectedHumanDesc: "IoTDB max of all CPU metrics, random 3 hosts, random 8h0m0s by 1h: 1970-01-01T02:16:22Z", + expectedSQLQuery: "SELECT MAX_VALUE(usage_user), MAX_VALUE(usage_system), MAX_VALUE(usage_idle), " + + "MAX_VALUE(usage_nice), MAX_VALUE(usage_iowait), MAX_VALUE(usage_irq), MAX_VALUE(usage_softirq), " + + "MAX_VALUE(usage_steal), MAX_VALUE(usage_guest), MAX_VALUE(usage_guest_nice) " + + "FROM root.cpu.host_9, root.cpu.host_3, root.cpu.host_5 GROUP BY ([1970-01-01 02:16:22, 1970-01-01 10:16:22), 1h)", + }, + } + + start := time.Unix(0, 0) + end := start.Add(devops.DoubleGroupByDuration).Add(time.Hour) + + for _, c := range cases { + t.Run(c.description, func(t *testing.T) { + rand.Seed(123) // Setting seed for testing purposes. + b := c.baseGenerator + queryGenerator, err := b.NewDevops(start, end, 10) + require.NoError(t, err, "Error while creating devops generator") + dp := queryGenerator.(*Devops) + + actual := dp.GenerateEmptyQuery() + expected := dp.GenerateEmptyQuery() + + dp.fillInQuery(expected, c.expectedHumanLabel, c.expectedHumanDesc, c.expectedSQLQuery) + dp.MaxAllCPU(actual, c.nHosts, devops.MaxAllDuration) + + require.EqualValues(t, expected, actual) + }) + } +} + +func TestGroupByOrderByLimit(t *testing.T) { + rand.Seed(123) // Setting seed for testing purposes. + base := BaseGenerator{BasicPath: "root.ln", BasicPathLevel: 1} + start := time.Unix(0, 0) + end := start.Add(2 * time.Hour) + queryGenerator, err := base.NewDevops(start, end, 10) + require.NoError(t, err, "Error while creating devops generator") + dp := queryGenerator.(*Devops) + + actual := dp.GenerateEmptyQuery() + expected := dp.GenerateEmptyQuery() + dp.fillInQuery(expected, + "IoTDB max cpu over last 5 min-intervals (random end)", + "IoTDB max cpu over last 5 min-intervals (random end): 1970-01-01T00:16:22Z", + "SELECT MAX_VALUE(usage_user) FROM root.ln.cpu.* GROUP BY ([1970-01-01 00:16:22, 1970-01-01 01:16:22), 1m) LIMIT 5", + ) + dp.GroupByOrderByLimit(actual) + + require.EqualValues(t, expected, actual) +} + +func TestHighCPUForHosts(t *testing.T) { + cases := []struct { + description string + nHosts int + baseGenerator BaseGenerator + expectedHumanLabel string + expectedHumanDesc string + expectedSQLQuery string + }{ + { + description: "ALL host with storage group 'root'", + nHosts: 0, + baseGenerator: BaseGenerator{BasicPath: "root", BasicPathLevel: 0}, + expectedHumanLabel: "IoTDB CPU over threshold, all hosts", + expectedHumanDesc: "IoTDB CPU over threshold, all hosts: 1970-01-01T00:16:22Z", + expectedSQLQuery: "SELECT * FROM root.cpu.* WHERE usage_user > 90.0 AND time >= 1970-01-01 00:16:22 AND time < 1970-01-01 12:16:22", + }, + { + description: "1 host with storage group 'root.sg.abc'", + nHosts: 1, + baseGenerator: BaseGenerator{BasicPath: "root.sg.abc", BasicPathLevel: 2}, + expectedHumanLabel: "IoTDB CPU over threshold, 1 host(s)", + expectedHumanDesc: "IoTDB CPU over threshold, 1 host(s): 1970-01-01T00:16:22Z", + expectedSQLQuery: "SELECT * FROM root.sg.abc.cpu.host_9 WHERE usage_user > 90.0 AND time >= 1970-01-01 00:16:22 AND time < 1970-01-01 12:16:22", + }, + { + description: "5 host2 with storage group 'root.ln'", + nHosts: 5, + baseGenerator: BaseGenerator{BasicPath: "root.ln", BasicPathLevel: 1}, + expectedHumanLabel: "IoTDB CPU over threshold, 5 host(s)", + expectedHumanDesc: "IoTDB CPU over threshold, 5 host(s): 1970-01-01T00:16:22Z", + expectedSQLQuery: "SELECT * FROM root.ln.cpu.host_9, root.ln.cpu.host_3, root.ln.cpu.host_5, root.ln.cpu.host_1, root.ln.cpu.host_7 WHERE usage_user > 90.0 AND time >= 1970-01-01 00:16:22 AND time < 1970-01-01 12:16:22", + }, + } + + start := time.Unix(0, 0) + end := start.Add(devops.HighCPUDuration).Add(time.Hour) + + for _, c := range cases { + t.Run(c.description, func(t *testing.T) { + rand.Seed(123) // Setting seed for testing purposes. + b := c.baseGenerator + queryGenerator, err := b.NewDevops(start, end, 10) + require.NoError(t, err, "Error while creating devops generator") + dp := queryGenerator.(*Devops) + + actual := dp.GenerateEmptyQuery() + expected := dp.GenerateEmptyQuery() + + dp.fillInQuery(expected, c.expectedHumanLabel, c.expectedHumanDesc, c.expectedSQLQuery) + dp.HighCPUForHosts(actual, c.nHosts) + fmt.Println(actual) + + require.EqualValues(t, expected, actual) + }) + } +} From 48b77d1fbb4d629e13396bc84718f00396b2c996 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Sun, 30 Oct 2022 19:04:02 +0800 Subject: [PATCH 06/47] serializer complete serializer complete --- cmd/tsbs_load_iotdb/main.go | 5 ++ pkg/targets/iotdb/serializer.go | 90 ++++++++++++++++++++++++++++ pkg/targets/iotdb/serializer_test.go | 51 ++++++++++++++++ 3 files changed, 146 insertions(+) create mode 100644 cmd/tsbs_load_iotdb/main.go create mode 100644 pkg/targets/iotdb/serializer.go create mode 100644 pkg/targets/iotdb/serializer_test.go diff --git a/cmd/tsbs_load_iotdb/main.go b/cmd/tsbs_load_iotdb/main.go new file mode 100644 index 000000000..9de471651 --- /dev/null +++ b/cmd/tsbs_load_iotdb/main.go @@ -0,0 +1,5 @@ +// tsbs_load_iotdb loads an IoTDB daemon with data from stdin. +// +// The caller is responsible for assuring that the database is empty before +// tsbs load. +package main diff --git a/pkg/targets/iotdb/serializer.go b/pkg/targets/iotdb/serializer.go new file mode 100644 index 000000000..8f6672f16 --- /dev/null +++ b/pkg/targets/iotdb/serializer.go @@ -0,0 +1,90 @@ +package iotdb + +import ( + "fmt" + "io" + "strings" + + "github.com/timescale/tsbs/pkg/data" + "github.com/timescale/tsbs/pkg/data/serialize" +) + +// Serializer writes a Point in a serialized form for MongoDB +type Serializer struct{} + +// Serialize writes Point p to the given Writer w, so it can be +// loaded by the IoTDB loader. The format is CSV with two lines per Point, +// with the first row being the names of fields and the second row being the +// field values. +// +// e.g., +// time,deviceID,,,,... +// ,,,,,... +// +// time,deviceID,hostname,tag2 +// 2022-10-26 16:44:55,root.cpu.host_1,host_1,44.0 +// time,deviceID,hostname,tag2 +// 1666281600000,root.cpu.host_1,host_1,44.0 +func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { + // Tag row first, prefixed with 'time,path' + buf1 := make([]byte, 0, 1024) + buf1 = append(buf1, []byte("time,deviceID")...) + tempBuf := make([]byte, 0, 1024) + var hostname string + foundHostname := false + tagKeys := p.TagKeys() + tagValues := p.TagValues() + for i, v := range tagValues { + if keyStr := string(tagKeys[i]); keyStr == "hostname" { + foundHostname = true + hostname = v.(string) + } else { + buf1 = append(buf1, ',') + buf1 = append(buf1, tagKeys[i]...) + tempBuf = append(tempBuf, ',') + tempBuf = serialize.FastFormatAppend(v, tempBuf) + } + } + if !foundHostname { + // errMsg := "IoTDB Serialize Error, 'hostname' tag not found.\n Tags are:" + // for i, _ := range tagKeys { + // errMsg += fmt.Sprintf("%s, ", string(tagKeys[i])) + // } + // return fmt.Errorf("%s", errMsg) + hostname = "unknown" + } + buf2 := make([]byte, 0, 1024) + buf2 = append(buf2, []byte(fmt.Sprintf("%d,", p.Timestamp().UTC().UnixMicro()))...) + buf2 = append(buf2, []byte(fmt.Sprintf("root.%s.%s", modifyHostname(string(p.MeasurementName())), hostname))...) + buf2 = append(buf2, tempBuf...) + // Fields + fieldKeys := p.FieldKeys() + fieldValues := p.FieldValues() + for i, v := range fieldValues { + buf1 = append(buf1, ',') + buf1 = append(buf1, fieldKeys[i]...) + buf2 = append(buf2, ',') + buf2 = serialize.FastFormatAppend(v, buf2) + } + buf1 = append(buf1, '\n') + buf2 = append(buf2, '\n') + _, err := w.Write(buf1) + if err == nil { + _, err = w.Write(buf2) + } + return err +} + +// modifyHostnames makes sure IP address can appear in the path. +// Node names in path can NOT contain "." unless enclosing it within either single quote (') or double quote ("). +// In this case, quotes are recognized as part of the node name to avoid ambiguity. +func modifyHostname(hostname string) string { + if strings.Contains(hostname, ".") { + if !(hostname[:1] == "\"" && hostname[len(hostname)-1:] == "\"") { + // not modified yet + hostname = "\"" + hostname + "\"" + } + + } + return hostname +} diff --git a/pkg/targets/iotdb/serializer_test.go b/pkg/targets/iotdb/serializer_test.go new file mode 100644 index 000000000..0115b618a --- /dev/null +++ b/pkg/targets/iotdb/serializer_test.go @@ -0,0 +1,51 @@ +package iotdb + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" + "github.com/timescale/tsbs/pkg/data" + "github.com/timescale/tsbs/pkg/data/serialize" +) + +func TestSerialize_001(t *testing.T) { + cases := []struct { + description string + inputPoint *data.Point + expected string + }{ + // { + // description: "a regular point", + // inputPoint: serialize.TestPointDefault(), + // expected: "time,deviceID,region,datacenter,usage_guest_nice\n1451606400000000,root.cpu.host_0,eu-west-1,eu-west-1b,38.24311829\n", + // }, + // { + // description: "a regular Point using int as value", + // inputPoint: serialize.TestPointInt(), + // expected: "time,deviceID,region,datacenter,usage_guest\n1451606400000000,root.cpu.host_0,eu-west-1,eu-west-1b,38\n", + // }, + // { + // description: "a regular Point with multiple fields", + // inputPoint: serialize.TestPointMultiField(), + // expected: "time,deviceID,region,datacenter,big_usage_guest,usage_guest,usage_guest_nice\n1451606400000000,root.cpu.host_0,eu-west-1,eu-west-1b,5000000000,38,38.24311829\n", + // }, + { + description: "a Point with no tags", + inputPoint: serialize.TestPointNoTags(), + expected: "time,deviceID,usage_guest_nice\n1451606400000000,root.cpu.host_0,eu-west-1,eu-west-1b,38.24311829\n", + }, + } + for _, c := range cases { + t.Run(c.description, func(t *testing.T) { + ps := &Serializer{} + b := new(bytes.Buffer) + err := ps.Serialize(c.inputPoint, b) + require.NoError(t, err) + actual := b.String() + + require.EqualValues(t, c.expected, actual) + }) + } + +} From 1444627b33ed35aa6cbda99222962ce55c113ed2 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Sun, 30 Oct 2022 22:58:56 +0800 Subject: [PATCH 07/47] working on testcases loading cmd/tsbs_load_iotdb --- cmd/tsbs_load_iotdb/benchmark.go | 41 ++++++++++ cmd/tsbs_load_iotdb/creator.go | 1 + cmd/tsbs_load_iotdb/main.go | 78 +++++++++++++++++++ cmd/tsbs_load_iotdb/processor.go | 1 + cmd/tsbs_load_iotdb/scan.go | 49 ++++++++++++ pkg/targets/constants/constants.go | 2 + .../initializers/target_initializers.go | 6 +- pkg/targets/iotdb/implemented_target.go | 37 +++++++++ 8 files changed, 214 insertions(+), 1 deletion(-) create mode 100644 cmd/tsbs_load_iotdb/benchmark.go create mode 100644 cmd/tsbs_load_iotdb/creator.go create mode 100644 cmd/tsbs_load_iotdb/processor.go create mode 100644 cmd/tsbs_load_iotdb/scan.go create mode 100644 pkg/targets/iotdb/implemented_target.go diff --git a/cmd/tsbs_load_iotdb/benchmark.go b/cmd/tsbs_load_iotdb/benchmark.go new file mode 100644 index 000000000..3a61c3fd8 --- /dev/null +++ b/cmd/tsbs_load_iotdb/benchmark.go @@ -0,0 +1,41 @@ +package main + +import ( + "bufio" + + "github.com/apache/iotdb-client-go/client" + "github.com/timescale/tsbs/load" + "github.com/timescale/tsbs/pkg/targets" +) + +func newBenchmark(clientConfig client.Config, loaderConfig load.BenchmarkRunnerConfig) (targets.Benchmark, error) { + return &iotdbBenchmark{ + cilentConfig: clientConfig, + loaderConfig: loaderConfig, + }, nil +} + +type iotdbBenchmark struct { + cilentConfig client.Config + loaderConfig load.BenchmarkRunnerConfig +} + +func (b *iotdbBenchmark) GetDataSource() targets.DataSource { + return &fileDataSource{scanner: bufio.NewScanner(load.GetBufferedReader(b.loaderConfig.FileName))} +} + +func (b *iotdbBenchmark) GetBatchFactory() targets.BatchFactory { + return &factory{} +} + +func (b *iotdbBenchmark) GetPointIndexer(maxPartitions uint) targets.PointIndexer { + return &targets.ConstantIndexer{} +} + +func (b *iotdbBenchmark) GetProcessor() targets.Processor { + return &processor{} +} + +func (b *iotdbBenchmark) GetDBCreator() targets.DBCreator { + return &dbCreator{} +} diff --git a/cmd/tsbs_load_iotdb/creator.go b/cmd/tsbs_load_iotdb/creator.go new file mode 100644 index 000000000..06ab7d0f9 --- /dev/null +++ b/cmd/tsbs_load_iotdb/creator.go @@ -0,0 +1 @@ +package main diff --git a/cmd/tsbs_load_iotdb/main.go b/cmd/tsbs_load_iotdb/main.go index 9de471651..1d5730be9 100644 --- a/cmd/tsbs_load_iotdb/main.go +++ b/cmd/tsbs_load_iotdb/main.go @@ -3,3 +3,81 @@ // The caller is responsible for assuring that the database is empty before // tsbs load. package main + +import ( + "bytes" + "fmt" + "log" + "sync" + + "github.com/blagojts/viper" + "github.com/spf13/pflag" + "github.com/timescale/tsbs/internal/utils" + "github.com/timescale/tsbs/load" + "github.com/timescale/tsbs/pkg/targets" + "github.com/timescale/tsbs/pkg/targets/constants" + "github.com/timescale/tsbs/pkg/targets/initializers" + + "github.com/apache/iotdb-client-go/client" +) + +// database option vars +var ( + clientConfig client.Config +) + +// Global vars +var ( + target targets.ImplementedTarget + // CRTODO 如果没用就移除它 + loaderConfig load.BenchmarkRunnerConfig + loader load.BenchmarkRunner + bufPool sync.Pool +) + +// allows for testing +var fatal = log.Fatalf // CRTODO 如果没用就移除它 + +// Parse args: +func init() { + target = initializers.GetTarget(constants.FormatIoTDB) + loaderConfig = load.BenchmarkRunnerConfig{} + loaderConfig.AddToFlagSet(pflag.CommandLine) + target.TargetSpecificFlags("", pflag.CommandLine) + pflag.Parse() + + err := utils.SetupConfigFile() + + if err != nil { + panic(fmt.Errorf("fatal error config file: %s", err)) + } + if err := viper.Unmarshal(&loaderConfig); err != nil { + panic(fmt.Errorf("unable to decode config: %s", err)) + } + + host := viper.GetString("host") + port := viper.GetString("port") + user := viper.GetString("user") + password := viper.GetString("password") + + clientConfig = client.Config{ + Host: host, + Port: port, + UserName: user, + Password: password, + } + + loader = load.GetBenchmarkRunner(loaderConfig) +} + +func main() { + bufPool = sync.Pool{ + New: func() interface{} { + return bytes.NewBuffer(make([]byte, 0, 4*1024*1024)) + }, + } + var benchmark targets.Benchmark + benchmark = newBenchmark(clientConfig, loaderConfig) + + loader.RunBenchmark(benchmark) +} diff --git a/cmd/tsbs_load_iotdb/processor.go b/cmd/tsbs_load_iotdb/processor.go new file mode 100644 index 000000000..06ab7d0f9 --- /dev/null +++ b/cmd/tsbs_load_iotdb/processor.go @@ -0,0 +1 @@ +package main diff --git a/cmd/tsbs_load_iotdb/scan.go b/cmd/tsbs_load_iotdb/scan.go new file mode 100644 index 000000000..b766e16ae --- /dev/null +++ b/cmd/tsbs_load_iotdb/scan.go @@ -0,0 +1,49 @@ +package main + +import ( + "bufio" + "bytes" + + "github.com/timescale/tsbs/pkg/data" + "github.com/timescale/tsbs/pkg/data/usecases/common" + "github.com/timescale/tsbs/pkg/targets" +) + +type fileDataSource struct { + scanner *bufio.Scanner +} + +func (d *fileDataSource) NextItem() data.LoadedPoint { + ok := d.scanner.Scan() + if !ok && d.scanner.Err() == nil { // nothing scanned & no error = EOF + return data.LoadedPoint{} + } else if !ok { + fatal("scan error: %v", d.scanner.Err()) + return data.LoadedPoint{} + } + return data.NewLoadedPoint(d.scanner.Bytes()) +} + +func (d *fileDataSource) Headers() *common.GeneratedDataHeaders { return nil } + +type batch struct { + buf *bytes.Buffer + rows uint + metrics uint64 +} + +func (b *batch) Len() uint { + return b.rows +} + +func (b *batch) Append(item data.LoadedPoint) { + // CRTODO 可能需要解析内容 + // CRINTERFACE + b.buf.Write([]byte("\n")) +} + +type factory struct{} + +func (f *factory) New() targets.Batch { + return &batch{buf: bufPool.Get().(*bytes.Buffer)} +} diff --git a/pkg/targets/constants/constants.go b/pkg/targets/constants/constants.go index 516093d09..7bfe4423b 100644 --- a/pkg/targets/constants/constants.go +++ b/pkg/targets/constants/constants.go @@ -14,6 +14,7 @@ const ( FormatVictoriaMetrics = "victoriametrics" FormatTimestream = "timestream" FormatQuestDB = "questdb" + FormatIoTDB = "iotdb" ) func SupportedFormats() []string { @@ -30,5 +31,6 @@ func SupportedFormats() []string { FormatVictoriaMetrics, FormatTimestream, FormatQuestDB, + FormatIoTDB, } } diff --git a/pkg/targets/initializers/target_initializers.go b/pkg/targets/initializers/target_initializers.go index a7a31677e..9afaf0843 100644 --- a/pkg/targets/initializers/target_initializers.go +++ b/pkg/targets/initializers/target_initializers.go @@ -2,6 +2,8 @@ package initializers import ( "fmt" + "strings" + "github.com/timescale/tsbs/pkg/targets" "github.com/timescale/tsbs/pkg/targets/akumuli" "github.com/timescale/tsbs/pkg/targets/cassandra" @@ -9,6 +11,7 @@ import ( "github.com/timescale/tsbs/pkg/targets/constants" "github.com/timescale/tsbs/pkg/targets/crate" "github.com/timescale/tsbs/pkg/targets/influx" + "github.com/timescale/tsbs/pkg/targets/iotdb" "github.com/timescale/tsbs/pkg/targets/mongo" "github.com/timescale/tsbs/pkg/targets/prometheus" "github.com/timescale/tsbs/pkg/targets/questdb" @@ -16,7 +19,6 @@ import ( "github.com/timescale/tsbs/pkg/targets/timescaledb" "github.com/timescale/tsbs/pkg/targets/timestream" "github.com/timescale/tsbs/pkg/targets/victoriametrics" - "strings" ) func GetTarget(format string) targets.ImplementedTarget { @@ -45,6 +47,8 @@ func GetTarget(format string) targets.ImplementedTarget { return timestream.NewTarget() case constants.FormatQuestDB: return questdb.NewTarget() + case constants.FormatIoTDB: + return iotdb.NewTarget() } supportedFormatsStr := strings.Join(constants.SupportedFormats(), ",") diff --git a/pkg/targets/iotdb/implemented_target.go b/pkg/targets/iotdb/implemented_target.go new file mode 100644 index 000000000..280acb5c3 --- /dev/null +++ b/pkg/targets/iotdb/implemented_target.go @@ -0,0 +1,37 @@ +package iotdb + +import ( + "github.com/blagojts/viper" + "github.com/spf13/pflag" + "github.com/timescale/tsbs/pkg/data/serialize" + "github.com/timescale/tsbs/pkg/data/source" + "github.com/timescale/tsbs/pkg/targets" + "github.com/timescale/tsbs/pkg/targets/constants" +) + +func NewTarget() targets.ImplementedTarget { + return &iotdbTarget{} +} + +type iotdbTarget struct { +} + +func (t *iotdbTarget) TargetSpecificFlags(flagPrefix string, flagSet *pflag.FlagSet) { + flagSet.String(flagPrefix+"host", "localhost", "Hostname of IoTDB instance") + flagSet.String(flagPrefix+"port", "6667", "Which port to connect to on the database host") + flagSet.String(flagPrefix+"user", "root", "The user who connect to IoTDB") + flagSet.String(flagPrefix+"password", "root", "The password for user connecting to IoTDB") +} + +func (t *iotdbTarget) TargetName() string { + return constants.FormatIoTDB +} + +func (t *iotdbTarget) Serializer() serialize.PointSerializer { + return &Serializer{} +} + +func (t *iotdbTarget) Benchmark(string, *source.DataSourceConfig, *viper.Viper) (targets.Benchmark, error) { + // CRTODO: 搞清楚这个是干什么的，MongoDB和questdb都没做这个事情 + panic("not implemented") +} From b320cbab7a0c08b5a3c4564468c72f85410de7c3 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Wed, 2 Nov 2022 00:24:58 +0800 Subject: [PATCH 08/47] tsbs_load_iotdb complete (without unit tests) tsbs_load_iotdb is completed, but no unit tests added in this part --- Makefile | 3 +- cmd/tsbs_load_iotdb/benchmark.go | 4 +- cmd/tsbs_load_iotdb/creator.go | 76 +++++++++++++++++++++++ cmd/tsbs_load_iotdb/main.go | 6 +- cmd/tsbs_load_iotdb/process.go | 40 +++++++++++++ cmd/tsbs_load_iotdb/processor.go | 1 - cmd/tsbs_load_iotdb/scan.go | 90 +++++++++++++++++++++++----- pkg/targets/iotdb/serializer.go | 57 ++++++++++++++---- pkg/targets/iotdb/serializer_test.go | 32 +++++----- 9 files changed, 260 insertions(+), 49 deletions(-) create mode 100644 cmd/tsbs_load_iotdb/process.go delete mode 100644 cmd/tsbs_load_iotdb/processor.go diff --git a/Makefile b/Makefile index f6a73c0c3..534105b87 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,8 @@ loaders: tsbs_load \ tsbs_load_siridb \ tsbs_load_timescaledb \ tsbs_load_victoriametrics \ - tsbs_load_questdb + tsbs_load_questdb \ + tsbs_load_iotdb runners: tsbs_run_queries_akumuli \ tsbs_run_queries_cassandra \ diff --git a/cmd/tsbs_load_iotdb/benchmark.go b/cmd/tsbs_load_iotdb/benchmark.go index 3a61c3fd8..cdee29173 100644 --- a/cmd/tsbs_load_iotdb/benchmark.go +++ b/cmd/tsbs_load_iotdb/benchmark.go @@ -8,11 +8,11 @@ import ( "github.com/timescale/tsbs/pkg/targets" ) -func newBenchmark(clientConfig client.Config, loaderConfig load.BenchmarkRunnerConfig) (targets.Benchmark, error) { +func newBenchmark(clientConfig client.Config, loaderConfig load.BenchmarkRunnerConfig) targets.Benchmark { return &iotdbBenchmark{ cilentConfig: clientConfig, loaderConfig: loaderConfig, - }, nil + } } type iotdbBenchmark struct { diff --git a/cmd/tsbs_load_iotdb/creator.go b/cmd/tsbs_load_iotdb/creator.go index 06ab7d0f9..1f0062db2 100644 --- a/cmd/tsbs_load_iotdb/creator.go +++ b/cmd/tsbs_load_iotdb/creator.go @@ -1 +1,77 @@ package main + +import ( + "fmt" + + "github.com/apache/iotdb-client-go/client" +) + +// DBCreator is an interface for a benchmark to do the initial setup of a database +// in preparation for running a benchmark against it. + +type dbCreator struct { + session client.Session +} + +func (d *dbCreator) Init() { + d.session = client.NewSession(&clientConfig) + if err := d.session.Open(false, timeoutInNs); err != nil { + errMsg := fmt.Sprintf("dbCreator init error, session is not open: %v\n", err) + errMsg = errMsg + fmt.Sprintf("Maybe your configuration is inappropriate, please check: %v", clientConfig) + fatal(errMsg) + } +} + +// get all Storage Group +func (d *dbCreator) getAllStorageGroup() ([]string, error) { + var sql = "show storage group" + sessionDataSet, err := d.session.ExecuteStatement(sql) + if err != nil { + return []string{}, err + } + + var sgList []string + + for next, err := sessionDataSet.Next(); err == nil && next; next, err = sessionDataSet.Next() { + for i := 0; i < sessionDataSet.GetColumnCount(); i++ { + columnName := sessionDataSet.GetColumnName(i) + switch sessionDataSet.GetColumnDataType(i) { + case client.TEXT: + sgList = append(sgList, sessionDataSet.GetText(columnName)) + default: + } + } + } + return sgList, nil +} + +func (d *dbCreator) DBExists(dbName string) bool { + // d.session = client.NewSession(&clientConfig) + // defer d.session.Close() + + sgList, err := d.getAllStorageGroup() + if err != nil { + fatal("DBExists error: %v", err) + return false + } + sg := fmt.Sprintf("root.%s", dbName) + for _, thisSG := range sgList { + if thisSG == sg { + return true + } + } + return false +} + +func (d *dbCreator) CreateDB(dbName string) error { + return nil +} + +func (d *dbCreator) RemoveOldDB(dbName string) error { + // d.session = client.NewSession(&clientConfig) + // defer d.session.Close() + + sg := fmt.Sprintf("root.%s", dbName) + _, err := d.session.DeleteStorageGroup(sg) + return err +} diff --git a/cmd/tsbs_load_iotdb/main.go b/cmd/tsbs_load_iotdb/main.go index 1d5730be9..7faab53ee 100644 --- a/cmd/tsbs_load_iotdb/main.go +++ b/cmd/tsbs_load_iotdb/main.go @@ -9,6 +9,7 @@ import ( "fmt" "log" "sync" + "time" "github.com/blagojts/viper" "github.com/spf13/pflag" @@ -24,6 +25,7 @@ import ( // database option vars var ( clientConfig client.Config + timeoutInNs int ) // Global vars @@ -40,6 +42,7 @@ var fatal = log.Fatalf // CRTODO 如果没用就移除它 // Parse args: func init() { + timeoutInNs = int(2 * time.Second) target = initializers.GetTarget(constants.FormatIoTDB) loaderConfig = load.BenchmarkRunnerConfig{} loaderConfig.AddToFlagSet(pflag.CommandLine) @@ -76,8 +79,7 @@ func main() { return bytes.NewBuffer(make([]byte, 0, 4*1024*1024)) }, } - var benchmark targets.Benchmark - benchmark = newBenchmark(clientConfig, loaderConfig) + benchmark := newBenchmark(clientConfig, loaderConfig) loader.RunBenchmark(benchmark) } diff --git a/cmd/tsbs_load_iotdb/process.go b/cmd/tsbs_load_iotdb/process.go new file mode 100644 index 000000000..7d87347d6 --- /dev/null +++ b/cmd/tsbs_load_iotdb/process.go @@ -0,0 +1,40 @@ +package main + +import ( + "fmt" + + "github.com/apache/iotdb-client-go/client" + "github.com/timescale/tsbs/pkg/targets" +) + +type processor struct { + session client.Session +} + +func (p *processor) Init(_ int, doLoad, _ bool) { + if !doLoad { + return + } + p.session = client.NewSession(&clientConfig) + if err := p.session.Open(false, timeoutInNs); err != nil { + errMsg := fmt.Sprintf("dbCreator init error, session is not open: %v\n", err) + errMsg = errMsg + fmt.Sprintf("Maybe your configuration is inappropriate, please check: %v", clientConfig) + fatal(errMsg) + } +} + +func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (metricCount, rowCount uint64) { + batch := b.(*iotdbBatch) + + // Write records + if doLoad { + for _, row := range batch.points { + sql := row.generateInsertStatement() + p.session.ExecuteUpdateStatement(sql) + } + } + + metricCount = batch.metrics + rowCount = uint64(batch.rows) + return metricCount, rowCount +} diff --git a/cmd/tsbs_load_iotdb/processor.go b/cmd/tsbs_load_iotdb/processor.go deleted file mode 100644 index 06ab7d0f9..000000000 --- a/cmd/tsbs_load_iotdb/processor.go +++ /dev/null @@ -1 +0,0 @@ -package main diff --git a/cmd/tsbs_load_iotdb/scan.go b/cmd/tsbs_load_iotdb/scan.go index b766e16ae..158a41fcc 100644 --- a/cmd/tsbs_load_iotdb/scan.go +++ b/cmd/tsbs_load_iotdb/scan.go @@ -2,48 +2,108 @@ package main import ( "bufio" - "bytes" + "fmt" + "strings" "github.com/timescale/tsbs/pkg/data" "github.com/timescale/tsbs/pkg/data/usecases/common" "github.com/timescale/tsbs/pkg/targets" ) +// iotdbPoint is a single record(row) of data +type iotdbPoint struct { + deviceID string // the deviceID(path) of this record, e.g. "root.cpu.host_0" + fieldKeyStr string // the keys of fields, e.g. "timestamp,value,str" + fieldValueStr string // the values of fields in string, e.g. "1666281600000,44.0,\"host_1\"" + fieldsCnt uint64 +} + +func (p *iotdbPoint) generateInsertStatement() string { + sql := fmt.Sprintf("INSERT INTO %s(%s) VALUES(%s)", p.deviceID, p.fieldKeyStr, p.fieldValueStr) + return sql +} + type fileDataSource struct { scanner *bufio.Scanner } -func (d *fileDataSource) NextItem() data.LoadedPoint { +// read new two line, which store one data point +// e.g., +// deviceID,timestamp,,,,... +// ,,,,,... +// +// deviceID,timestamp,hostname,tag2 +// root.cpu.host_1,1666281600000,'host_1',44.0 +// +// return : bool -> true means got one point, else reaches EOF or error happens +func (d *fileDataSource) nextTwoLines() (bool, string, string, error) { ok := d.scanner.Scan() if !ok && d.scanner.Err() == nil { // nothing scanned & no error = EOF - return data.LoadedPoint{} + return false, "", "", nil + } else if !ok { + return false, "", "", fmt.Errorf("scan error: %v", d.scanner.Err()) + } + line1 := d.scanner.Text() + line_ok := strings.HasPrefix(line1, "deviceID,timestamp,") + if !line_ok { + return false, line1, "", fmt.Errorf("scan error, illegal line: %s", line1) + } + ok = d.scanner.Scan() + if !ok && d.scanner.Err() == nil { // nothing scanned & no error = EOF + return false, "", "", nil } else if !ok { - fatal("scan error: %v", d.scanner.Err()) - return data.LoadedPoint{} + return false, "", "", fmt.Errorf("scan error: %v", d.scanner.Err()) } - return data.NewLoadedPoint(d.scanner.Bytes()) + line2 := d.scanner.Text() + return true, line1, line2, nil +} + +func (d *fileDataSource) NextItem() data.LoadedPoint { + scan_ok, line1, line2, err := d.nextTwoLines() + if !scan_ok { + if err == nil { // End of file + return data.LoadedPoint{} + } else { // Some error occurred + fatal("scan error: %v", err) + return data.LoadedPoint{} + } + } + line1_parts := strings.SplitN(line1, ",", 2) // 'deviceID' and rest keys of fields + line2_parts := strings.SplitN(line2, ",", 2) // deviceID and rest values of fields + return data.NewLoadedPoint( + &iotdbPoint{ + deviceID: line2_parts[0], + fieldKeyStr: line1_parts[1], + fieldValueStr: line2_parts[1], + fieldsCnt: uint64(len(strings.Split(line1_parts[1], ","))), + }) } func (d *fileDataSource) Headers() *common.GeneratedDataHeaders { return nil } -type batch struct { - buf *bytes.Buffer - rows uint - metrics uint64 +// A struct that storages data points +type iotdbBatch struct { + points []*iotdbPoint + rows uint // count of records(rows) + metrics uint64 // total count of all metrics in this batch } -func (b *batch) Len() uint { +func (b *iotdbBatch) Len() uint { return b.rows } -func (b *batch) Append(item data.LoadedPoint) { +func (b *iotdbBatch) Append(item data.LoadedPoint) { // CRTODO 可能需要解析内容 - // CRINTERFACE - b.buf.Write([]byte("\n")) + b.rows++ + b.points = append(b.points, item.Data.(*iotdbPoint)) + b.metrics += item.Data.(*iotdbPoint).fieldsCnt } type factory struct{} func (f *factory) New() targets.Batch { - return &batch{buf: bufPool.Get().(*bytes.Buffer)} + return &iotdbBatch{ + rows: 0, + metrics: 0, + } } diff --git a/pkg/targets/iotdb/serializer.go b/pkg/targets/iotdb/serializer.go index 8f6672f16..5c965390a 100644 --- a/pkg/targets/iotdb/serializer.go +++ b/pkg/targets/iotdb/serializer.go @@ -3,10 +3,10 @@ package iotdb import ( "fmt" "io" + "strconv" "strings" "github.com/timescale/tsbs/pkg/data" - "github.com/timescale/tsbs/pkg/data/serialize" ) // Serializer writes a Point in a serialized form for MongoDB @@ -18,17 +18,15 @@ type Serializer struct{} // field values. // // e.g., -// time,deviceID,,,,... -// ,,,,,... +// deviceID,timestamp,,,,... +// ,,,,,... // -// time,deviceID,hostname,tag2 -// 2022-10-26 16:44:55,root.cpu.host_1,host_1,44.0 -// time,deviceID,hostname,tag2 -// 1666281600000,root.cpu.host_1,host_1,44.0 +// deviceID,timestamp,hostname,tag2 +// root.cpu.host_1,1666281600000,'host_1',44.0 func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { // Tag row first, prefixed with 'time,path' buf1 := make([]byte, 0, 1024) - buf1 = append(buf1, []byte("time,deviceID")...) + buf1 = append(buf1, []byte("deviceID,timestamp")...) tempBuf := make([]byte, 0, 1024) var hostname string foundHostname := false @@ -42,7 +40,7 @@ func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { buf1 = append(buf1, ',') buf1 = append(buf1, tagKeys[i]...) tempBuf = append(tempBuf, ',') - tempBuf = serialize.FastFormatAppend(v, tempBuf) + tempBuf = iotdbFormatAppend(v, tempBuf) } } if !foundHostname { @@ -54,8 +52,8 @@ func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { hostname = "unknown" } buf2 := make([]byte, 0, 1024) - buf2 = append(buf2, []byte(fmt.Sprintf("%d,", p.Timestamp().UTC().UnixMicro()))...) - buf2 = append(buf2, []byte(fmt.Sprintf("root.%s.%s", modifyHostname(string(p.MeasurementName())), hostname))...) + buf2 = append(buf2, []byte(fmt.Sprintf("root.%s.%s,", modifyHostname(string(p.MeasurementName())), hostname))...) + buf2 = append(buf2, []byte(fmt.Sprintf("%d", p.Timestamp().UTC().UnixMicro()))...) buf2 = append(buf2, tempBuf...) // Fields fieldKeys := p.FieldKeys() @@ -64,7 +62,7 @@ func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { buf1 = append(buf1, ',') buf1 = append(buf1, fieldKeys[i]...) buf2 = append(buf2, ',') - buf2 = serialize.FastFormatAppend(v, buf2) + buf2 = iotdbFormatAppend(v, buf2) } buf1 = append(buf1, '\n') buf2 = append(buf2, '\n') @@ -88,3 +86,38 @@ func modifyHostname(hostname string) string { } return hostname } + +// Utility function for appending various data types to a byte string +func iotdbFormatAppend(v interface{}, buf []byte) []byte { + switch v.(type) { + case int: + return strconv.AppendInt(buf, int64(v.(int)), 10) + case int64: + return strconv.AppendInt(buf, v.(int64), 10) + case float64: + // Why -1 ? + // From Golang source on genericFtoa (called by AppendFloat): 'Negative precision means "only as much as needed to be exact."' + // Using this instead of an exact number for precision ensures we preserve the precision passed in to the function, allowing us + // to use different precision for different use cases. + return strconv.AppendFloat(buf, v.(float64), 'f', -1, 64) + case float32: + return strconv.AppendFloat(buf, float64(v.(float32)), 'f', -1, 32) + case bool: + return strconv.AppendBool(buf, v.(bool)) + case []byte: + buf = append(buf, []byte("'")...) + buf = append(buf, v.([]byte)...) + buf = append(buf, []byte("'")...) + return buf + case string: + // buf = append(buf, []byte(fmt.Sprintf("\"%s\"", v.(string)))...) + buf = append(buf, []byte("'")...) + buf = append(buf, v.(string)...) + buf = append(buf, []byte("'")...) + return buf + case nil: + return buf + default: + panic(fmt.Sprintf("unknown field type for %#v", v)) + } +} diff --git a/pkg/targets/iotdb/serializer_test.go b/pkg/targets/iotdb/serializer_test.go index 0115b618a..d8e7610d7 100644 --- a/pkg/targets/iotdb/serializer_test.go +++ b/pkg/targets/iotdb/serializer_test.go @@ -15,25 +15,25 @@ func TestSerialize_001(t *testing.T) { inputPoint *data.Point expected string }{ - // { - // description: "a regular point", - // inputPoint: serialize.TestPointDefault(), - // expected: "time,deviceID,region,datacenter,usage_guest_nice\n1451606400000000,root.cpu.host_0,eu-west-1,eu-west-1b,38.24311829\n", - // }, - // { - // description: "a regular Point using int as value", - // inputPoint: serialize.TestPointInt(), - // expected: "time,deviceID,region,datacenter,usage_guest\n1451606400000000,root.cpu.host_0,eu-west-1,eu-west-1b,38\n", - // }, - // { - // description: "a regular Point with multiple fields", - // inputPoint: serialize.TestPointMultiField(), - // expected: "time,deviceID,region,datacenter,big_usage_guest,usage_guest,usage_guest_nice\n1451606400000000,root.cpu.host_0,eu-west-1,eu-west-1b,5000000000,38,38.24311829\n", - // }, + { + description: "a regular point ", + inputPoint: serialize.TestPointDefault(), + expected: "deviceID,timestamp,region,datacenter,usage_guest_nice\nroot.cpu.host_0,1451606400000000,'eu-west-1','eu-west-1b',38.24311829\n", + }, + { + description: "a regular Point using int as value", + inputPoint: serialize.TestPointInt(), + expected: "deviceID,timestamp,region,datacenter,usage_guest\nroot.cpu.host_0,1451606400000000,'eu-west-1','eu-west-1b',38\n", + }, + { + description: "a regular Point with multiple fields", + inputPoint: serialize.TestPointMultiField(), + expected: "deviceID,timestamp,region,datacenter,big_usage_guest,usage_guest,usage_guest_nice\nroot.cpu.host_0,1451606400000000,'eu-west-1','eu-west-1b',5000000000,38,38.24311829\n", + }, { description: "a Point with no tags", inputPoint: serialize.TestPointNoTags(), - expected: "time,deviceID,usage_guest_nice\n1451606400000000,root.cpu.host_0,eu-west-1,eu-west-1b,38.24311829\n", + expected: "deviceID,timestamp,usage_guest_nice\nroot.cpu.unknown,1451606400000000,38.24311829\n", }, } for _, c := range cases { From 2419b000e0b60bad9e71b209ded509ec6e19ce46 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Thu, 3 Nov 2022 21:18:52 +0800 Subject: [PATCH 09/47] update iotdb-client-go version update iotdb-client-go version, use timeoutInMs --- cmd/tsbs_load_iotdb/creator.go | 2 +- cmd/tsbs_load_iotdb/main.go | 19 ++++++++----------- cmd/tsbs_load_iotdb/process.go | 2 +- go.mod | 4 ++-- go.sum | 13 ++++++++----- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/cmd/tsbs_load_iotdb/creator.go b/cmd/tsbs_load_iotdb/creator.go index 1f0062db2..b3ca05b09 100644 --- a/cmd/tsbs_load_iotdb/creator.go +++ b/cmd/tsbs_load_iotdb/creator.go @@ -15,7 +15,7 @@ type dbCreator struct { func (d *dbCreator) Init() { d.session = client.NewSession(&clientConfig) - if err := d.session.Open(false, timeoutInNs); err != nil { + if err := d.session.Open(false, timeoutInMs); err != nil { errMsg := fmt.Sprintf("dbCreator init error, session is not open: %v\n", err) errMsg = errMsg + fmt.Sprintf("Maybe your configuration is inappropriate, please check: %v", clientConfig) fatal(errMsg) diff --git a/cmd/tsbs_load_iotdb/main.go b/cmd/tsbs_load_iotdb/main.go index 7faab53ee..5c8891c66 100644 --- a/cmd/tsbs_load_iotdb/main.go +++ b/cmd/tsbs_load_iotdb/main.go @@ -5,11 +5,8 @@ package main import ( - "bytes" "fmt" "log" - "sync" - "time" "github.com/blagojts/viper" "github.com/spf13/pflag" @@ -25,7 +22,7 @@ import ( // database option vars var ( clientConfig client.Config - timeoutInNs int + timeoutInMs int // 0 for no timeout ) // Global vars @@ -34,7 +31,7 @@ var ( // CRTODO 如果没用就移除它 loaderConfig load.BenchmarkRunnerConfig loader load.BenchmarkRunner - bufPool sync.Pool + // bufPool sync.Pool ) // allows for testing @@ -42,7 +39,7 @@ var fatal = log.Fatalf // CRTODO 如果没用就移除它 // Parse args: func init() { - timeoutInNs = int(2 * time.Second) + timeoutInMs = int(2000) target = initializers.GetTarget(constants.FormatIoTDB) loaderConfig = load.BenchmarkRunnerConfig{} loaderConfig.AddToFlagSet(pflag.CommandLine) @@ -74,11 +71,11 @@ func init() { } func main() { - bufPool = sync.Pool{ - New: func() interface{} { - return bytes.NewBuffer(make([]byte, 0, 4*1024*1024)) - }, - } + // bufPool = sync.Pool{ + // New: func() interface{} { + // return bytes.NewBuffer(make([]byte, 0, 4*1024*1024)) + // }, + // } benchmark := newBenchmark(clientConfig, loaderConfig) loader.RunBenchmark(benchmark) diff --git a/cmd/tsbs_load_iotdb/process.go b/cmd/tsbs_load_iotdb/process.go index 7d87347d6..3f5ee1999 100644 --- a/cmd/tsbs_load_iotdb/process.go +++ b/cmd/tsbs_load_iotdb/process.go @@ -16,7 +16,7 @@ func (p *processor) Init(_ int, doLoad, _ bool) { return } p.session = client.NewSession(&clientConfig) - if err := p.session.Open(false, timeoutInNs); err != nil { + if err := p.session.Open(false, timeoutInMs); err != nil { errMsg := fmt.Sprintf("dbCreator init error, session is not open: %v\n", err) errMsg = errMsg + fmt.Sprintf("Maybe your configuration is inappropriate, please check: %v", clientConfig) fatal(errMsg) diff --git a/go.mod b/go.mod index 531d291fe..0ec959d0a 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/SiriDB/go-siridb-connector v0.0.0-20190110105621-86b34c44c921 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 - github.com/apache/iotdb-client-go v0.13.0 + github.com/apache/iotdb-client-go v0.12.2-0.20221102040630-2731a9e19fc9 github.com/aws/aws-sdk-go v1.35.13 github.com/blagojts/viper v1.6.3-0.20200313094124-068f44cf5e69 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 @@ -26,7 +26,7 @@ require ( github.com/shirou/gopsutil v3.21.3+incompatible github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.8.1 github.com/timescale/promscale v0.0.0-20201006153045-6a66a36f5c84 github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/transceptor-technology/go-qpack v0.0.0-20190116123619-49a14b216a45 diff --git a/go.sum b/go.sum index 6c6f9c3de..4d4f7ad12 100644 --- a/go.sum +++ b/go.sum @@ -93,12 +93,12 @@ github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDa github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= -github.com/apache/iotdb-client-go v0.13.0 h1:b1608bn+s8Oqd1UTJbeR0H+P+B46+zTUE+g8J1bovvw= -github.com/apache/iotdb-client-go v0.13.0/go.mod h1:kaergHbc+hEtIST6Zgz1JQjukP3MIewLD9gaNDDQ/v4= +github.com/apache/iotdb-client-go v0.12.2-0.20221102040630-2731a9e19fc9 h1:0djW7kEV/yYbjj+DYKqNQnx5P9RRFAuvCYJjFDeYLy8= +github.com/apache/iotdb-client-go v0.12.2-0.20221102040630-2731a9e19fc9/go.mod h1:4Az8byJmeA6Nolvl0tYTvuFVP+6AJcF++SZ+gNSeaIA= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.14.1 h1:Yh8v0hpCj63p5edXOLaqTJW0IJ1p+eMW6+YSOqw1d6s= -github.com/apache/thrift v0.14.1/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.15.0 h1:aGvdaR0v1t9XLgjtBYwxcBvBOTMqClzwE26CHOgjW1Y= +github.com/apache/thrift v0.15.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -363,6 +363,7 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -885,6 +886,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -892,8 +894,9 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM= From c5869c95292cc0f87ccd52be671a4458ba263c9f Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Thu, 3 Nov 2022 23:21:51 +0800 Subject: [PATCH 10/47] complete scan test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit complete scan test 也许应该做更多努力来增强batch的性能 --- cmd/tsbs_load_iotdb/scan.go | 2 +- cmd/tsbs_load_iotdb/scan_test.go | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 cmd/tsbs_load_iotdb/scan_test.go diff --git a/cmd/tsbs_load_iotdb/scan.go b/cmd/tsbs_load_iotdb/scan.go index 158a41fcc..33de5f5be 100644 --- a/cmd/tsbs_load_iotdb/scan.go +++ b/cmd/tsbs_load_iotdb/scan.go @@ -14,7 +14,7 @@ import ( type iotdbPoint struct { deviceID string // the deviceID(path) of this record, e.g. "root.cpu.host_0" fieldKeyStr string // the keys of fields, e.g. "timestamp,value,str" - fieldValueStr string // the values of fields in string, e.g. "1666281600000,44.0,\"host_1\"" + fieldValueStr string // the values of fields in string, e.g. "1666281600000,44.0,'host_1'" fieldsCnt uint64 } diff --git a/cmd/tsbs_load_iotdb/scan_test.go b/cmd/tsbs_load_iotdb/scan_test.go new file mode 100644 index 000000000..935cd64e6 --- /dev/null +++ b/cmd/tsbs_load_iotdb/scan_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGenerateInsertStatement(t *testing.T) { + cases := []struct { + description string + point iotdbPoint + expected string + }{ + { + description: "one point(1)", + point: iotdbPoint{ + deviceID: "root.cpu.host_0", + fieldKeyStr: "timestamp,value,str", + fieldValueStr: "123456,999,'abc'", + fieldsCnt: 3, + }, + expected: "INSERT INTO root.cpu.host_0(timestamp,value,str) VALUES(123456,999,'abc')", + }, + { + description: "one point(2)", + point: iotdbPoint{ + deviceID: "root.cpu.host_9", + fieldKeyStr: "timestamp,floatValue,str,intValue", + fieldValueStr: "123456,4321.9,'abc',45621", + fieldsCnt: 4, + }, + expected: "INSERT INTO root.cpu.host_9(timestamp,floatValue,str,intValue) VALUES(123456,4321.9,'abc',45621)", + }, + } + + for _, c := range cases { + t.Run(c.description, func(t *testing.T) { + actual := c.point.generateInsertStatement() + require.EqualValues(t, c.expected, actual) + }) + } +} From 6a925207b4cd951dbe154ac4dc4a1b8539d25c87 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Fri, 4 Nov 2022 11:35:00 +0800 Subject: [PATCH 11/47] update some user friendly tips update some user friendly tips. timeout info, workers info. --- cmd/tsbs_load_iotdb/creator.go | 2 +- cmd/tsbs_load_iotdb/main.go | 5 +++++ cmd/tsbs_load_iotdb/process.go | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cmd/tsbs_load_iotdb/creator.go b/cmd/tsbs_load_iotdb/creator.go index b3ca05b09..416899d5a 100644 --- a/cmd/tsbs_load_iotdb/creator.go +++ b/cmd/tsbs_load_iotdb/creator.go @@ -17,7 +17,7 @@ func (d *dbCreator) Init() { d.session = client.NewSession(&clientConfig) if err := d.session.Open(false, timeoutInMs); err != nil { errMsg := fmt.Sprintf("dbCreator init error, session is not open: %v\n", err) - errMsg = errMsg + fmt.Sprintf("Maybe your configuration is inappropriate, please check: %v", clientConfig) + errMsg = errMsg + fmt.Sprintf("timeout setting: %d ms", timeoutInMs) fatal(errMsg) } } diff --git a/cmd/tsbs_load_iotdb/main.go b/cmd/tsbs_load_iotdb/main.go index 5c8891c66..f9f822ef1 100644 --- a/cmd/tsbs_load_iotdb/main.go +++ b/cmd/tsbs_load_iotdb/main.go @@ -59,6 +59,11 @@ func init() { port := viper.GetString("port") user := viper.GetString("user") password := viper.GetString("password") + workers := viper.GetUint("workers") + log.Printf("tsbs_load_iotdb loading with %d workers.\n", workers) + if workers < 5 { + log.Println("Insertion throughput is strongly related to the number of threads. Use more workers for better performance.") + } clientConfig = client.Config{ Host: host, diff --git a/cmd/tsbs_load_iotdb/process.go b/cmd/tsbs_load_iotdb/process.go index 3f5ee1999..9dc43d668 100644 --- a/cmd/tsbs_load_iotdb/process.go +++ b/cmd/tsbs_load_iotdb/process.go @@ -17,8 +17,8 @@ func (p *processor) Init(_ int, doLoad, _ bool) { } p.session = client.NewSession(&clientConfig) if err := p.session.Open(false, timeoutInMs); err != nil { - errMsg := fmt.Sprintf("dbCreator init error, session is not open: %v\n", err) - errMsg = errMsg + fmt.Sprintf("Maybe your configuration is inappropriate, please check: %v", clientConfig) + errMsg := fmt.Sprintf("processor init error, session is not open: %v\n", err) + errMsg = errMsg + fmt.Sprintf("timeout setting: %d ms", timeoutInMs) fatal(errMsg) } } From 68a0aaf7b570fb10a95c9a607a1a612497c960b5 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Fri, 4 Nov 2022 11:55:20 +0800 Subject: [PATCH 12/47] Users can now specify timeout Users can now specify timeout for session opening check --- cmd/tsbs_load_iotdb/main.go | 10 ++++++++-- pkg/targets/iotdb/implemented_target.go | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/cmd/tsbs_load_iotdb/main.go b/cmd/tsbs_load_iotdb/main.go index f9f822ef1..d80dd1446 100644 --- a/cmd/tsbs_load_iotdb/main.go +++ b/cmd/tsbs_load_iotdb/main.go @@ -39,7 +39,6 @@ var fatal = log.Fatalf // CRTODO 如果没用就移除它 // Parse args: func init() { - timeoutInMs = int(2000) target = initializers.GetTarget(constants.FormatIoTDB) loaderConfig = load.BenchmarkRunnerConfig{} loaderConfig.AddToFlagSet(pflag.CommandLine) @@ -60,7 +59,14 @@ func init() { user := viper.GetString("user") password := viper.GetString("password") workers := viper.GetUint("workers") - log.Printf("tsbs_load_iotdb loading with %d workers.\n", workers) + timeoutInMs = viper.GetInt("timeout") + + timeoutStr := fmt.Sprintf("timeout for session opening check: %d ms", timeoutInMs) + if timeoutInMs <= 0 { + timeoutInMs = 0 // + timeoutStr = "no timeout for session opening check" + } + log.Printf("tsbs_load_iotdb target: %s:%s, %s. Loading with %d workers.\n", host, port, timeoutStr, workers) if workers < 5 { log.Println("Insertion throughput is strongly related to the number of threads. Use more workers for better performance.") } diff --git a/pkg/targets/iotdb/implemented_target.go b/pkg/targets/iotdb/implemented_target.go index 280acb5c3..a0e47d2bd 100644 --- a/pkg/targets/iotdb/implemented_target.go +++ b/pkg/targets/iotdb/implemented_target.go @@ -21,6 +21,7 @@ func (t *iotdbTarget) TargetSpecificFlags(flagPrefix string, flagSet *pflag.Flag flagSet.String(flagPrefix+"port", "6667", "Which port to connect to on the database host") flagSet.String(flagPrefix+"user", "root", "The user who connect to IoTDB") flagSet.String(flagPrefix+"password", "root", "The password for user connecting to IoTDB") + flagSet.Int(flagPrefix+"timeout", 0, "Session timeout check in millisecond. Use 0 for no timeout.") } func (t *iotdbTarget) TargetName() string { From 7c179891685bd93e00484dd31e3738948cce2728 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Wed, 9 Nov 2022 09:17:38 +0800 Subject: [PATCH 13/47] tsbs_run_queries_iotdb has been completed. tsbs_run_queries_iotdb has been completed. Some CRTODO comments need to be removed --- Makefile | 3 +- .../databases/iotdb/common.go | 2 +- cmd/tsbs_run_queries_iotdb/main.go | 107 ++++++++++++++++++ pkg/query/factories/init_factories.go | 5 + pkg/query/iotdb.go | 10 +- 5 files changed, 120 insertions(+), 7 deletions(-) create mode 100644 cmd/tsbs_run_queries_iotdb/main.go diff --git a/Makefile b/Makefile index 534105b87..1f7388f65 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,8 @@ runners: tsbs_run_queries_akumuli \ tsbs_run_queries_timescaledb \ tsbs_run_queries_timestream \ tsbs_run_queries_victoriametrics \ - tsbs_run_queries_questdb + tsbs_run_queries_questdb \ + tsbs_run_queries_iotdb test: $(GOTEST) -v ./... diff --git a/cmd/tsbs_generate_queries/databases/iotdb/common.go b/cmd/tsbs_generate_queries/databases/iotdb/common.go index d8263dd5c..35f0bd1ca 100644 --- a/cmd/tsbs_generate_queries/databases/iotdb/common.go +++ b/cmd/tsbs_generate_queries/databases/iotdb/common.go @@ -12,7 +12,7 @@ const iotdbTimeFmt = "2006-01-02 15:04:05" // BaseGenerator contains settings specific for IoTDB type BaseGenerator struct { - BasicPath string // e.g. "root.sg" is basic path of "root.sg.device" + BasicPath string // e.g. "root.sg" is basic path of "root.sg.device". default : "root" BasicPathLevel int32 // e.g. 0 for "root", 1 for "root.device" } diff --git a/cmd/tsbs_run_queries_iotdb/main.go b/cmd/tsbs_run_queries_iotdb/main.go new file mode 100644 index 000000000..da1316142 --- /dev/null +++ b/cmd/tsbs_run_queries_iotdb/main.go @@ -0,0 +1,107 @@ +package main + +import ( + "fmt" + "log" + "time" + + "github.com/blagojts/viper" + "github.com/spf13/pflag" + "github.com/timescale/tsbs/internal/utils" + "github.com/timescale/tsbs/pkg/query" + + "github.com/apache/iotdb-client-go/client" +) + +// database option vars +var ( + clientConfig client.Config + timeoutInMs int64 // 0 for no timeout +) + +// Global vars: +var ( + runner *query.BenchmarkRunner +) + +// Parse args: +func init() { + var config query.BenchmarkRunnerConfig + config.AddToFlagSet(pflag.CommandLine) + + pflag.String("host", "localhost", "Hostname of IoTDB instance") + pflag.String("port", "6667", "Which port to connect to on the database host") + pflag.String("user", "root", "The user who connect to IoTDB") + pflag.String("password", "root", "The password for user connecting to IoTDB") + + pflag.Parse() + + err := utils.SetupConfigFile() + + if err != nil { + panic(fmt.Errorf("fatal error config file: %s", err)) + } + + if err := viper.Unmarshal(&config); err != nil { + panic(fmt.Errorf("unable to decode config: %s", err)) + } + + host := viper.GetString("host") + port := viper.GetString("port") + user := viper.GetString("user") + password := viper.GetString("password") + workers := viper.GetUint("workers") + timeoutInMs = 0 // 0 for no timeout + + log.Printf("tsbs_run_queries_iotdb target: %s:%s. Loading with %d workers.\n", host, port, workers) + if workers < 5 { + log.Println("Insertion throughput is strongly related to the number of threads. Use more workers for better performance.") + } + + clientConfig = client.Config{ + Host: host, + Port: port, + UserName: user, + Password: password, + } + + runner = query.NewBenchmarkRunner(config) +} + +func main() { + runner.Run(&query.IoTDBPool, newProcessor) +} + +type processor struct { + session client.Session +} + +func newProcessor() query.Processor { return &processor{} } + +func (p *processor) Init(workerNumber int) { + p.session = client.NewSession(&clientConfig) + if err := p.session.Open(false, int(timeoutInMs)); err != nil { + errMsg := fmt.Sprintf("query processor init error, session is not open: %v\n", err) + errMsg = errMsg + fmt.Sprintf("timeout setting: %d ms", timeoutInMs) + log.Fatal(errMsg) + } +} + +func (p *processor) ProcessQuery(q query.Query, _ bool) ([]*query.Stat, error) { + iotdbQ := q.(*query.IoTDB) + sql := string(iotdbQ.SqlQuery) + + start := time.Now().UnixNano() + _, err := p.session.ExecuteQueryStatement(sql, &timeoutInMs) // 0 for no timeout + + took := time.Now().UnixNano() - start + if err != nil { + // CRTODO 更换一个更合适的方式 + log.Printf("log! ERROR! %v", err) + return nil, err + } + lag := float64(took) / float64(time.Millisecond) // in milliseconds + stat := query.GetStat() + stat.Init(q.HumanLabelName(), lag) + return []*query.Stat{stat}, err +} diff --git a/pkg/query/factories/init_factories.go b/pkg/query/factories/init_factories.go index ff3faf47d..27292914f 100644 --- a/pkg/query/factories/init_factories.go +++ b/pkg/query/factories/init_factories.go @@ -6,6 +6,7 @@ import ( "github.com/timescale/tsbs/cmd/tsbs_generate_queries/databases/clickhouse" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/databases/cratedb" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/databases/influx" + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/databases/iotdb" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/databases/mongo" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/databases/questdb" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/databases/siridb" @@ -39,5 +40,9 @@ func InitQueryFactories(config *config.QueryGeneratorConfig) map[string]interfac DBName: config.DbName, } factories[constants.FormatQuestDB] = &questdb.BaseGenerator{} + factories[constants.FormatIoTDB] = &iotdb.BaseGenerator{ + BasicPath: "root", + BasicPathLevel: 0, + } return factories } diff --git a/pkg/query/iotdb.go b/pkg/query/iotdb.go index c96cf20fc..74aa8cd44 100644 --- a/pkg/query/iotdb.go +++ b/pkg/query/iotdb.go @@ -3,8 +3,8 @@ package query import ( "fmt" "sync" - - "github.com/apache/iotdb-client-go/client" + // CRTODO: 删掉不需要的内容 + // "github.com/apache/iotdb-client-go/client" ) // IoTDB encodes a IoTDB request. This will be serialized for use @@ -14,9 +14,9 @@ type IoTDB struct { HumanDescription []byte // CRTODO: 设计好这里的结构，这里可能还需要其他东西 - ClientSession client.Session - SqlQuery []byte - id uint64 + // ClientSession client.Session + SqlQuery []byte + id uint64 } // IoTDBPool is a sync.Pool of IoTDB Query types From c3850f4a4427b4a4e0c327e5378779b3819f2cff Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Wed, 9 Nov 2022 09:49:43 +0800 Subject: [PATCH 14/47] =?UTF-8?q?CRTODO=20removed=EF=BC=8Cusing=20back=20q?= =?UTF-8?q?uote(`)=20in=20IP=20address?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CRTODO removed using back quote(`) in IP address instead of using quotation marks(") --- .../databases/iotdb/common.go | 1 - .../databases/iotdb/devops.go | 4 ++-- .../databases/iotdb/devops_test.go | 18 +++++++++--------- cmd/tsbs_load_iotdb/creator.go | 2 +- cmd/tsbs_load_iotdb/main.go | 5 ++--- cmd/tsbs_load_iotdb/process.go | 2 +- cmd/tsbs_load_iotdb/scan.go | 3 +-- cmd/tsbs_run_queries_iotdb/main.go | 3 +-- pkg/query/iotdb.go | 5 ----- pkg/targets/iotdb/implemented_target.go | 4 ++-- pkg/targets/iotdb/serializer.go | 10 +++------- 11 files changed, 22 insertions(+), 35 deletions(-) diff --git a/cmd/tsbs_generate_queries/databases/iotdb/common.go b/cmd/tsbs_generate_queries/databases/iotdb/common.go index 35f0bd1ca..bf1a42536 100644 --- a/cmd/tsbs_generate_queries/databases/iotdb/common.go +++ b/cmd/tsbs_generate_queries/databases/iotdb/common.go @@ -27,7 +27,6 @@ func (g *BaseGenerator) fillInQuery(qi query.Query, humanLabel, humanDesc, sql s q.HumanLabel = []byte(humanLabel) q.HumanDescription = []byte(humanDesc) q.SqlQuery = []byte(sql) - // CRTODO: 在修改了结构之后，这里是否还需要更多的东西？ } // NewDevops creates a new devops use case query generator. diff --git a/cmd/tsbs_generate_queries/databases/iotdb/devops.go b/cmd/tsbs_generate_queries/databases/iotdb/devops.go index fb4fa4970..7c4b1bdaf 100644 --- a/cmd/tsbs_generate_queries/databases/iotdb/devops.go +++ b/cmd/tsbs_generate_queries/databases/iotdb/devops.go @@ -28,9 +28,9 @@ type Devops struct { func (d *Devops) modifyHostnames(hostnames []string) []string { for i, hostname := range hostnames { if strings.Contains(hostname, ".") { - if !(hostname[:1] == "\"" && hostname[len(hostname)-1:] == "\"") { + if !(hostname[:1] == "`" && hostname[len(hostname)-1:] == "`") { // not modified yet - hostnames[i] = "\"" + hostnames[i] + "\"" + hostnames[i] = "`" + hostnames[i] + "`" } } diff --git a/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go b/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go index e8ab4827c..daaa84194 100644 --- a/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go +++ b/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go @@ -17,24 +17,24 @@ func TestModifyHostnames(t *testing.T) { expected []string }{ { - description: "normal node names", + description: "normal node name", hostnames: []string{"hostname", "hello_world"}, expected: []string{"hostname", "hello_world"}, }, { description: "IP address or URL as hostnames", hostnames: []string{"192.168.1.1", "8.8.8.8", "iotdb.apache.org"}, - expected: []string{"\"192.168.1.1\"", "\"8.8.8.8\"", "\"iotdb.apache.org\""}, + expected: []string{"`192.168.1.1`", "`8.8.8.8`", "`iotdb.apache.org`"}, }, { - description: "already modified case", - hostnames: []string{"\"192.168.1.1\"", "\"8.8.8.8\"", "\"iotdb.apache.org\""}, - expected: []string{"\"192.168.1.1\"", "\"8.8.8.8\"", "\"iotdb.apache.org\""}, + description: "already modified cases", + hostnames: []string{"`192.168.1.1`", "`8.8.8.8`", "`iotdb.apache.org`"}, + expected: []string{"`192.168.1.1`", "`8.8.8.8`", "`iotdb.apache.org`"}, }, { description: "mixed host names", - hostnames: []string{"192.168.1.1", "hostname", "iotdb.apache.org", "\"8.8.8.8\""}, - expected: []string{"\"192.168.1.1\"", "hostname", "\"iotdb.apache.org\"", "\"8.8.8.8\""}, + hostnames: []string{"192.168.1.1", "hostname", "iotdb.apache.org", "`8.8.8.8`"}, + expected: []string{"`192.168.1.1`", "hostname", "`iotdb.apache.org`", "`8.8.8.8`"}, }, } @@ -84,8 +84,8 @@ func TestDevopsGetHostFromString(t *testing.T) { description: "multi host (3) with special node names", basicPath: "root", basicPathLevel: 0, - hostnames: []string{"host1", "192.168.1.1", "\"iotdb.apache.org\""}, - expected: "root.cpu.host1, root.cpu.\"192.168.1.1\", root.cpu.\"iotdb.apache.org\"", + hostnames: []string{"host1", "192.168.1.1", "`iotdb.apache.org`"}, + expected: "root.cpu.host1, root.cpu.`192.168.1.1`, root.cpu.`iotdb.apache.org`", }, } diff --git a/cmd/tsbs_load_iotdb/creator.go b/cmd/tsbs_load_iotdb/creator.go index 416899d5a..27d258000 100644 --- a/cmd/tsbs_load_iotdb/creator.go +++ b/cmd/tsbs_load_iotdb/creator.go @@ -16,7 +16,7 @@ type dbCreator struct { func (d *dbCreator) Init() { d.session = client.NewSession(&clientConfig) if err := d.session.Open(false, timeoutInMs); err != nil { - errMsg := fmt.Sprintf("dbCreator init error, session is not open: %v\n", err) + errMsg := fmt.Sprintf("IoTDB dbCreator init error, session is not open: %v\n", err) errMsg = errMsg + fmt.Sprintf("timeout setting: %d ms", timeoutInMs) fatal(errMsg) } diff --git a/cmd/tsbs_load_iotdb/main.go b/cmd/tsbs_load_iotdb/main.go index d80dd1446..214237a5a 100644 --- a/cmd/tsbs_load_iotdb/main.go +++ b/cmd/tsbs_load_iotdb/main.go @@ -28,14 +28,13 @@ var ( // Global vars var ( target targets.ImplementedTarget - // CRTODO 如果没用就移除它 + loaderConfig load.BenchmarkRunnerConfig loader load.BenchmarkRunner - // bufPool sync.Pool ) // allows for testing -var fatal = log.Fatalf // CRTODO 如果没用就移除它 +var fatal = log.Fatalf // Parse args: func init() { diff --git a/cmd/tsbs_load_iotdb/process.go b/cmd/tsbs_load_iotdb/process.go index 9dc43d668..933b998b9 100644 --- a/cmd/tsbs_load_iotdb/process.go +++ b/cmd/tsbs_load_iotdb/process.go @@ -17,7 +17,7 @@ func (p *processor) Init(_ int, doLoad, _ bool) { } p.session = client.NewSession(&clientConfig) if err := p.session.Open(false, timeoutInMs); err != nil { - errMsg := fmt.Sprintf("processor init error, session is not open: %v\n", err) + errMsg := fmt.Sprintf("IoTDB processor init error, session is not open: %v\n", err) errMsg = errMsg + fmt.Sprintf("timeout setting: %d ms", timeoutInMs) fatal(errMsg) } diff --git a/cmd/tsbs_load_iotdb/scan.go b/cmd/tsbs_load_iotdb/scan.go index 33de5f5be..ee494f7c6 100644 --- a/cmd/tsbs_load_iotdb/scan.go +++ b/cmd/tsbs_load_iotdb/scan.go @@ -64,7 +64,7 @@ func (d *fileDataSource) NextItem() data.LoadedPoint { if err == nil { // End of file return data.LoadedPoint{} } else { // Some error occurred - fatal("scan error: %v", err) + fatal("IoTDB scan error: %v", err) return data.LoadedPoint{} } } @@ -93,7 +93,6 @@ func (b *iotdbBatch) Len() uint { } func (b *iotdbBatch) Append(item data.LoadedPoint) { - // CRTODO 可能需要解析内容 b.rows++ b.points = append(b.points, item.Data.(*iotdbPoint)) b.metrics += item.Data.(*iotdbPoint).fieldsCnt diff --git a/cmd/tsbs_run_queries_iotdb/main.go b/cmd/tsbs_run_queries_iotdb/main.go index da1316142..272cdf115 100644 --- a/cmd/tsbs_run_queries_iotdb/main.go +++ b/cmd/tsbs_run_queries_iotdb/main.go @@ -96,8 +96,7 @@ func (p *processor) ProcessQuery(q query.Query, _ bool) ([]*query.Stat, error) { took := time.Now().UnixNano() - start if err != nil { - // CRTODO 更换一个更合适的方式 - log.Printf("log! ERROR! %v", err) + log.Printf("An error occurred while executing SQL: %s\n", sql) return nil, err } lag := float64(took) / float64(time.Millisecond) // in milliseconds diff --git a/pkg/query/iotdb.go b/pkg/query/iotdb.go index 74aa8cd44..e7adf03b8 100644 --- a/pkg/query/iotdb.go +++ b/pkg/query/iotdb.go @@ -3,8 +3,6 @@ package query import ( "fmt" "sync" - // CRTODO: 删掉不需要的内容 - // "github.com/apache/iotdb-client-go/client" ) // IoTDB encodes a IoTDB request. This will be serialized for use @@ -13,8 +11,6 @@ type IoTDB struct { HumanLabel []byte HumanDescription []byte - // CRTODO: 设计好这里的结构，这里可能还需要其他东西 - // ClientSession client.Session SqlQuery []byte id uint64 } @@ -66,7 +62,6 @@ func (q *IoTDB) HumanDescriptionName() []byte { // Release resets and returns this Query to its pool func (q *IoTDB) Release() { - // CRTODO: 弄清楚这里是否还需要其他的操作 q.HumanLabel = q.HumanLabel[:0] q.HumanDescription = q.HumanDescription[:0] q.id = 0 diff --git a/pkg/targets/iotdb/implemented_target.go b/pkg/targets/iotdb/implemented_target.go index a0e47d2bd..13cebb3e0 100644 --- a/pkg/targets/iotdb/implemented_target.go +++ b/pkg/targets/iotdb/implemented_target.go @@ -33,6 +33,6 @@ func (t *iotdbTarget) Serializer() serialize.PointSerializer { } func (t *iotdbTarget) Benchmark(string, *source.DataSourceConfig, *viper.Viper) (targets.Benchmark, error) { - // CRTODO: 搞清楚这个是干什么的，MongoDB和questdb都没做这个事情 - panic("not implemented") + // benchmark for tsbs_load_iotdb is implemented in "cmd/tsbs_load_iotdb/main.go/main()" + panic("Benchmark() not implemented! Benchmark for tsbs_load_iotdb is implemented in \"cmd/tsbs_load_iotdb/main.go/main()\"") } diff --git a/pkg/targets/iotdb/serializer.go b/pkg/targets/iotdb/serializer.go index 5c965390a..8e1f2dd65 100644 --- a/pkg/targets/iotdb/serializer.go +++ b/pkg/targets/iotdb/serializer.go @@ -44,11 +44,7 @@ func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { } } if !foundHostname { - // errMsg := "IoTDB Serialize Error, 'hostname' tag not found.\n Tags are:" - // for i, _ := range tagKeys { - // errMsg += fmt.Sprintf("%s, ", string(tagKeys[i])) - // } - // return fmt.Errorf("%s", errMsg) + // Unable to find hostname as part of device id hostname = "unknown" } buf2 := make([]byte, 0, 1024) @@ -78,9 +74,9 @@ func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { // In this case, quotes are recognized as part of the node name to avoid ambiguity. func modifyHostname(hostname string) string { if strings.Contains(hostname, ".") { - if !(hostname[:1] == "\"" && hostname[len(hostname)-1:] == "\"") { + if !(hostname[:1] == "`" && hostname[len(hostname)-1:] == "`") { // not modified yet - hostname = "\"" + hostname + "\"" + hostname = "`" + hostname + "`" } } From 9cc69bcf6fdc819652f0d25d2f729798e1abfce3 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Wed, 9 Nov 2022 19:14:10 +0800 Subject: [PATCH 15/47] using DATETIME-INPUT in data serializer using DATETIME-INPUT in data serializer timestamp in LONG INT is ambiguous --- cmd/tsbs_load_iotdb/creator.go | 6 ------ cmd/tsbs_load_iotdb/main.go | 5 ----- cmd/tsbs_load_iotdb/scan.go | 6 +++--- pkg/targets/iotdb/serializer.go | 8 +++++--- pkg/targets/iotdb/serializer_test.go | 8 ++++---- 5 files changed, 12 insertions(+), 21 deletions(-) diff --git a/cmd/tsbs_load_iotdb/creator.go b/cmd/tsbs_load_iotdb/creator.go index 27d258000..9df8506f3 100644 --- a/cmd/tsbs_load_iotdb/creator.go +++ b/cmd/tsbs_load_iotdb/creator.go @@ -46,9 +46,6 @@ func (d *dbCreator) getAllStorageGroup() ([]string, error) { } func (d *dbCreator) DBExists(dbName string) bool { - // d.session = client.NewSession(&clientConfig) - // defer d.session.Close() - sgList, err := d.getAllStorageGroup() if err != nil { fatal("DBExists error: %v", err) @@ -68,9 +65,6 @@ func (d *dbCreator) CreateDB(dbName string) error { } func (d *dbCreator) RemoveOldDB(dbName string) error { - // d.session = client.NewSession(&clientConfig) - // defer d.session.Close() - sg := fmt.Sprintf("root.%s", dbName) _, err := d.session.DeleteStorageGroup(sg) return err diff --git a/cmd/tsbs_load_iotdb/main.go b/cmd/tsbs_load_iotdb/main.go index 214237a5a..0ad6517f3 100644 --- a/cmd/tsbs_load_iotdb/main.go +++ b/cmd/tsbs_load_iotdb/main.go @@ -81,11 +81,6 @@ func init() { } func main() { - // bufPool = sync.Pool{ - // New: func() interface{} { - // return bytes.NewBuffer(make([]byte, 0, 4*1024*1024)) - // }, - // } benchmark := newBenchmark(clientConfig, loaderConfig) loader.RunBenchmark(benchmark) diff --git a/cmd/tsbs_load_iotdb/scan.go b/cmd/tsbs_load_iotdb/scan.go index ee494f7c6..9c91261f6 100644 --- a/cmd/tsbs_load_iotdb/scan.go +++ b/cmd/tsbs_load_iotdb/scan.go @@ -14,7 +14,7 @@ import ( type iotdbPoint struct { deviceID string // the deviceID(path) of this record, e.g. "root.cpu.host_0" fieldKeyStr string // the keys of fields, e.g. "timestamp,value,str" - fieldValueStr string // the values of fields in string, e.g. "1666281600000,44.0,'host_1'" + fieldValueStr string // the values of fields in string, e.g. "2016-01-01 00:00:00,44.0,'host_1'" fieldsCnt uint64 } @@ -32,8 +32,8 @@ type fileDataSource struct { // deviceID,timestamp,,,,... // ,,,,,... // -// deviceID,timestamp,hostname,tag2 -// root.cpu.host_1,1666281600000,'host_1',44.0 +// deviceID,timestamp,hostname,value +// root.cpu.host_1,2016-01-01 00:00:00,'host_1',44.0 // // return : bool -> true means got one point, else reaches EOF or error happens func (d *fileDataSource) nextTwoLines() (bool, string, string, error) { diff --git a/pkg/targets/iotdb/serializer.go b/pkg/targets/iotdb/serializer.go index 8e1f2dd65..00b8317de 100644 --- a/pkg/targets/iotdb/serializer.go +++ b/pkg/targets/iotdb/serializer.go @@ -12,6 +12,8 @@ import ( // Serializer writes a Point in a serialized form for MongoDB type Serializer struct{} +const iotdbTimeFmt = "2006-01-02 15:04:05" + // Serialize writes Point p to the given Writer w, so it can be // loaded by the IoTDB loader. The format is CSV with two lines per Point, // with the first row being the names of fields and the second row being the @@ -21,8 +23,8 @@ type Serializer struct{} // deviceID,timestamp,,,,... // ,,,,,... // -// deviceID,timestamp,hostname,tag2 -// root.cpu.host_1,1666281600000,'host_1',44.0 +// deviceID,timestamp,hostname,value +// root.cpu.host_1,2016-01-01 00:00:00,'host_1',44.0 func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { // Tag row first, prefixed with 'time,path' buf1 := make([]byte, 0, 1024) @@ -49,7 +51,7 @@ func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { } buf2 := make([]byte, 0, 1024) buf2 = append(buf2, []byte(fmt.Sprintf("root.%s.%s,", modifyHostname(string(p.MeasurementName())), hostname))...) - buf2 = append(buf2, []byte(fmt.Sprintf("%d", p.Timestamp().UTC().UnixMicro()))...) + buf2 = append(buf2, []byte(fmt.Sprintf("%s", p.Timestamp().UTC().Format(iotdbTimeFmt)))...) buf2 = append(buf2, tempBuf...) // Fields fieldKeys := p.FieldKeys() diff --git a/pkg/targets/iotdb/serializer_test.go b/pkg/targets/iotdb/serializer_test.go index d8e7610d7..e577ef71c 100644 --- a/pkg/targets/iotdb/serializer_test.go +++ b/pkg/targets/iotdb/serializer_test.go @@ -18,22 +18,22 @@ func TestSerialize_001(t *testing.T) { { description: "a regular point ", inputPoint: serialize.TestPointDefault(), - expected: "deviceID,timestamp,region,datacenter,usage_guest_nice\nroot.cpu.host_0,1451606400000000,'eu-west-1','eu-west-1b',38.24311829\n", + expected: "deviceID,timestamp,region,datacenter,usage_guest_nice\nroot.cpu.host_0,2016-01-01 00:00:00,'eu-west-1','eu-west-1b',38.24311829\n", }, { description: "a regular Point using int as value", inputPoint: serialize.TestPointInt(), - expected: "deviceID,timestamp,region,datacenter,usage_guest\nroot.cpu.host_0,1451606400000000,'eu-west-1','eu-west-1b',38\n", + expected: "deviceID,timestamp,region,datacenter,usage_guest\nroot.cpu.host_0,2016-01-01 00:00:00,'eu-west-1','eu-west-1b',38\n", }, { description: "a regular Point with multiple fields", inputPoint: serialize.TestPointMultiField(), - expected: "deviceID,timestamp,region,datacenter,big_usage_guest,usage_guest,usage_guest_nice\nroot.cpu.host_0,1451606400000000,'eu-west-1','eu-west-1b',5000000000,38,38.24311829\n", + expected: "deviceID,timestamp,region,datacenter,big_usage_guest,usage_guest,usage_guest_nice\nroot.cpu.host_0,2016-01-01 00:00:00,'eu-west-1','eu-west-1b',5000000000,38,38.24311829\n", }, { description: "a Point with no tags", inputPoint: serialize.TestPointNoTags(), - expected: "deviceID,timestamp,usage_guest_nice\nroot.cpu.unknown,1451606400000000,38.24311829\n", + expected: "deviceID,timestamp,usage_guest_nice\nroot.cpu.unknown,2016-01-01 00:00:00,38.24311829\n", }, } for _, c := range cases { From 1a831328eb2258ca9ae15c51fe515e0edff00226 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Wed, 9 Nov 2022 20:36:20 +0800 Subject: [PATCH 16/47] fix bug in GroupByTimeAndPrimaryTag fix bug in GroupByTimeAndPrimaryTag should use one hour in `group by` instead of one minute --- cmd/tsbs_generate_queries/databases/iotdb/devops.go | 2 +- cmd/tsbs_generate_queries/databases/iotdb/devops_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/tsbs_generate_queries/databases/iotdb/devops.go b/cmd/tsbs_generate_queries/databases/iotdb/devops.go index 7c4b1bdaf..da0b04660 100644 --- a/cmd/tsbs_generate_queries/databases/iotdb/devops.go +++ b/cmd/tsbs_generate_queries/databases/iotdb/devops.go @@ -124,7 +124,7 @@ func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) { sql = sql + fmt.Sprintf("SELECT %s", selectClause) sql = sql + fmt.Sprintf(" FROM %s.cpu.*", d.BasicPath) // sql = sql + fmt.Sprintf(" WHERE time >= %s AND time < %s", interval.StartString(), interval.EndString()) - sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1m), LEVEL = %d", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt), d.BasicPathLevel+2) + sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1h), LEVEL = %d", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt), d.BasicPathLevel+2) d.fillInQuery(qi, humanLabel, humanDesc, sql) } diff --git a/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go b/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go index daaa84194..eed20054c 100644 --- a/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go +++ b/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go @@ -182,7 +182,7 @@ func TestGroupByTimeAndPrimaryTag(t *testing.T) { baseGenerator: BaseGenerator{BasicPath: "root.sg", BasicPathLevel: 1}, expectedHumanLabel: "IoTDB mean of 1 metrics, all hosts, random 12h0m0s by 1h", expectedHumanDesc: "IoTDB mean of 1 metrics, all hosts, random 12h0m0s by 1h: 1970-01-01T00:16:22Z", - expectedSQLQuery: "SELECT AVG(usage_user) FROM root.sg.cpu.* GROUP BY ([1970-01-01 00:16:22, 1970-01-01 12:16:22), 1m), LEVEL = 3", + expectedSQLQuery: "SELECT AVG(usage_user) FROM root.sg.cpu.* GROUP BY ([1970-01-01 00:16:22, 1970-01-01 12:16:22), 1h), LEVEL = 3", }, { description: "5 metric with storage group 'root'", @@ -190,7 +190,7 @@ func TestGroupByTimeAndPrimaryTag(t *testing.T) { baseGenerator: BaseGenerator{BasicPath: "root", BasicPathLevel: 0}, expectedHumanLabel: "IoTDB mean of 5 metrics, all hosts, random 12h0m0s by 1h", expectedHumanDesc: "IoTDB mean of 5 metrics, all hosts, random 12h0m0s by 1h: 1970-01-01T00:16:22Z", - expectedSQLQuery: "SELECT AVG(usage_user), AVG(usage_system), AVG(usage_idle), AVG(usage_nice), AVG(usage_iowait) FROM root.cpu.* GROUP BY ([1970-01-01 00:16:22, 1970-01-01 12:16:22), 1m), LEVEL = 2", + expectedSQLQuery: "SELECT AVG(usage_user), AVG(usage_system), AVG(usage_idle), AVG(usage_nice), AVG(usage_iowait) FROM root.cpu.* GROUP BY ([1970-01-01 00:16:22, 1970-01-01 12:16:22), 1h), LEVEL = 2", }, } From c5a630c85460146b752334c3aa444147f15a3ed7 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Thu, 10 Nov 2022 18:37:04 +0800 Subject: [PATCH 17/47] added doc, readme for iotdb added documentation, readme for iotdb --- docs/iotdb.md | 95 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 docs/iotdb.md diff --git a/docs/iotdb.md b/docs/iotdb.md new file mode 100644 index 000000000..4c31cbbbf --- /dev/null +++ b/docs/iotdb.md @@ -0,0 +1,95 @@ +# TSBS Supplemental Guide: Apache IoTDB + +**This should be read *after* the main README.** + +Apache IoTDB (Database for Internet of Things) is an IoT native database with +high performance for data management and analysis, deployable on the edge and +the cloud. For more details about Apache IoTDB, please take a look at: +[https://iotdb.apache.org/](https://iotdb.apache.org/) + +This supplemental guide explains how the data generated for TSBS is stored, +additional flags available when using the data importer (`tsbs_load_iotdb`), +and additional flags available for the query runner (`tsbs_run_queries_iotdb`). + +## Data format + +Data generated by `tsbs_generate_data` for IoTDB is serialized in a "pseudo-CSV" +format, along with a custom header at the beginning: + +* one line composed of a comma-separated list of "deviceID" and fieldNames +* one line composed of a comma-separated list of deviceID and fieldValues + +For example: + +```text +deviceID,timestamp,,,,... +,,,,,... + +deviceID,timestamp,hostname,value +root.cpu.host_1,2016-01-01 00:00:00,'host_1',44.0 +``` + +`deviceID` describes the storage path which is composed of several nodes. +IoTDB uses storage group to manage data. For example, in test case `devops`, +if `measurementName` is `cpu` while `hostname` tag is `host_0`, then `deviceID` +is `root.cpu.host_0`. +IoTDB treat tags as fields, except primary key like `hostname` in `devops`. + +An example for the `cpu-only` use case: + +```text +deviceID,timestamp,region,datacenter,rack,os,arch,team,service,service_version,service_environment,usage_user,usage_system,usage_idle,usage_nice,usage_iowait,usage_irq,usage_softirq,usage_steal,usage_guest,usage_guest_nice +root.cpu.host_0,2016-01-01 00:00:00,'eu-west-1','eu-west-1c','87','Ubuntu16.04LTS','x64','NYC','18','1','production',58,2,24,61,22,63,6,44,80,38 + +``` + +--- + +## `tsbs_load_iotdb` Additional Flags + +### IoTDB related + +#### `-host` (type: `string`, default: `localhost`) + +Hostname of IoTDB instance. + +#### `-port` (type: `string`, default: `6667`) + +Which port to connect to on the database host. + +#### `-user` (type: `string`, default: `root`) + +The username of user who connect to IoTDB. + +#### `-password` (type: `string`, default: `root`) + +The password for user connecting to IoTDB. + +#### `-timeout` (type: `int`, default: `0`) + +Session timeout check in millisecond. After session config initialization, +client will try to get response from database server to make sure the connection +is established. This argument is the session timeout in millisecond. Use 0 for +no timeout. + +--- + +## `tsbs_run_queries_iotdb` Additional Flags + +### IoTDB related + +#### `-host` (type: `string`, default: `localhost`) + +Hostname of IoTDB instance. + +#### `-port` (type: `string`, default: `6667`) + +Which port to connect to on the database host. + +#### `-user` (type: `string`, default: `root`) + +The username of user who connect to IoTDB. + +#### `-password` (type: `string`, default: `root`) + +The password for user connecting to IoTDB. From e1439b550c55a5c0cf726413221f1e89ce9bbdde Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Sun, 13 Nov 2022 11:34:26 +0800 Subject: [PATCH 18/47] update main readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index bf78a77df..fad7c25d6 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Current databases supported: + ClickHouse [(supplemental docs)](docs/clickhouse.md) + CrateDB [(supplemental docs)](docs/cratedb.md) + InfluxDB [(supplemental docs)](docs/influx.md) ++ IoTDB [(supplemental docs)](docs/iotdb.md) + MongoDB [(supplemental docs)](docs/mongo.md) + QuestDB [(supplemental docs)](docs/questdb.md) + SiriDB [(supplemental docs)](docs/siridb.md) From b009f190202723c1faba10deeac577af53b68b60 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Sun, 13 Nov 2022 16:00:42 +0800 Subject: [PATCH 19/47] [Important] using InsertRecords 1.using InsertRecords 2.serilaizer string without '' 3.using unixNano as timestamp in serializer output file, but should be converted into millisecond before insert. 4.readme updated 5.there is 3 lines in serializer output for each data point --- cmd/tsbs_load_iotdb/benchmark.go | 14 ++- cmd/tsbs_load_iotdb/main.go | 8 +- cmd/tsbs_load_iotdb/process.go | 57 +++++++++++- cmd/tsbs_load_iotdb/scan.go | 117 ++++++++++++++++++------ cmd/tsbs_load_iotdb/scan_test.go | 50 ++++++---- docs/iotdb.md | 9 +- pkg/targets/iotdb/implemented_target.go | 1 + pkg/targets/iotdb/serializer.go | 67 +++++++++----- pkg/targets/iotdb/serializer_test.go | 69 +++++++++++++- 9 files changed, 305 insertions(+), 87 deletions(-) diff --git a/cmd/tsbs_load_iotdb/benchmark.go b/cmd/tsbs_load_iotdb/benchmark.go index cdee29173..c23291e75 100644 --- a/cmd/tsbs_load_iotdb/benchmark.go +++ b/cmd/tsbs_load_iotdb/benchmark.go @@ -10,14 +10,16 @@ import ( func newBenchmark(clientConfig client.Config, loaderConfig load.BenchmarkRunnerConfig) targets.Benchmark { return &iotdbBenchmark{ - cilentConfig: clientConfig, - loaderConfig: loaderConfig, + cilentConfig: clientConfig, + loaderConfig: loaderConfig, + recordsMaxRows: recordsMaxRows, } } type iotdbBenchmark struct { - cilentConfig client.Config - loaderConfig load.BenchmarkRunnerConfig + cilentConfig client.Config + loaderConfig load.BenchmarkRunnerConfig + recordsMaxRows int } func (b *iotdbBenchmark) GetDataSource() targets.DataSource { @@ -33,7 +35,9 @@ func (b *iotdbBenchmark) GetPointIndexer(maxPartitions uint) targets.PointIndexe } func (b *iotdbBenchmark) GetProcessor() targets.Processor { - return &processor{} + return &processor{ + recordsMaxRows: b.recordsMaxRows, + } } func (b *iotdbBenchmark) GetDBCreator() targets.DBCreator { diff --git a/cmd/tsbs_load_iotdb/main.go b/cmd/tsbs_load_iotdb/main.go index 0ad6517f3..a742ce5f3 100644 --- a/cmd/tsbs_load_iotdb/main.go +++ b/cmd/tsbs_load_iotdb/main.go @@ -21,8 +21,9 @@ import ( // database option vars var ( - clientConfig client.Config - timeoutInMs int // 0 for no timeout + clientConfig client.Config + timeoutInMs int // 0 for no timeout + recordsMaxRows int // max rows of records in 'InsertRecords' ) // Global vars @@ -58,11 +59,12 @@ func init() { user := viper.GetString("user") password := viper.GetString("password") workers := viper.GetUint("workers") + recordsMaxRows = viper.GetInt("records-max-rows") timeoutInMs = viper.GetInt("timeout") timeoutStr := fmt.Sprintf("timeout for session opening check: %d ms", timeoutInMs) if timeoutInMs <= 0 { - timeoutInMs = 0 // + timeoutInMs = 0 // 0 for no timeout. timeoutStr = "no timeout for session opening check" } log.Printf("tsbs_load_iotdb target: %s:%s, %s. Loading with %d workers.\n", host, port, timeoutStr, workers) diff --git a/cmd/tsbs_load_iotdb/process.go b/cmd/tsbs_load_iotdb/process.go index 933b998b9..3f1cdeced 100644 --- a/cmd/tsbs_load_iotdb/process.go +++ b/cmd/tsbs_load_iotdb/process.go @@ -8,7 +8,8 @@ import ( ) type processor struct { - session client.Session + session client.Session + recordsMaxRows int // max rows of records in 'InsertRecords' } func (p *processor) Init(_ int, doLoad, _ bool) { @@ -28,10 +29,58 @@ func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (metricCount, row // Write records if doLoad { - for _, row := range batch.points { - sql := row.generateInsertStatement() - p.session.ExecuteUpdateStatement(sql) + if p.recordsMaxRows > 0 { + for index := 0; index < len(batch.points); { + var ( + deviceId []string + measurements [][]string + dataTypes [][]client.TSDataType + values [][]interface{} + timestamps []int64 + ) + for thisRecordsCnt := 0; thisRecordsCnt < recordsMaxRows && index < len(batch.points); { + row := batch.points[index] + deviceId = append(deviceId, row.deviceID) + measurements = append(measurements, row.measurements) + dataTypes = append(dataTypes, row.dataTypes) + values = append(values, row.values) + timestamps = append(timestamps, row.timestamp) + thisRecordsCnt++ + index++ + } + _, err := p.session.InsertRecords( + deviceId, measurements, dataTypes, values, timestamps, + ) + if err != nil { + fatal("ProcessBatch error:%v", err) + } + } + } else { + var ( + deviceId []string + measurements [][]string + dataTypes [][]client.TSDataType + values [][]interface{} + timestamps []int64 + ) + for _, row := range batch.points { + deviceId = append(deviceId, row.deviceID) + measurements = append(measurements, row.measurements) + dataTypes = append(dataTypes, row.dataTypes) + values = append(values, row.values) + timestamps = append(timestamps, row.timestamp) + } + _, err := p.session.InsertRecords( + deviceId, measurements, dataTypes, values, timestamps, + ) + if err != nil { + fatal("ProcessBatch error:%v", err) + } } + // for _, row := range batch.points { + // sql := row.generateInsertStatement() + // p.session.ExecuteUpdateStatement(sql) + // } } metricCount = batch.metrics diff --git a/cmd/tsbs_load_iotdb/scan.go b/cmd/tsbs_load_iotdb/scan.go index 9c91261f6..a42c7d72d 100644 --- a/cmd/tsbs_load_iotdb/scan.go +++ b/cmd/tsbs_load_iotdb/scan.go @@ -3,8 +3,11 @@ package main import ( "bufio" "fmt" + "strconv" "strings" + "time" + "github.com/apache/iotdb-client-go/client" "github.com/timescale/tsbs/pkg/data" "github.com/timescale/tsbs/pkg/data/usecases/common" "github.com/timescale/tsbs/pkg/targets" @@ -12,54 +15,124 @@ import ( // iotdbPoint is a single record(row) of data type iotdbPoint struct { - deviceID string // the deviceID(path) of this record, e.g. "root.cpu.host_0" - fieldKeyStr string // the keys of fields, e.g. "timestamp,value,str" - fieldValueStr string // the values of fields in string, e.g. "2016-01-01 00:00:00,44.0,'host_1'" - fieldsCnt uint64 + deviceID string // the deviceID(path) of this record, e.g. "root.cpu.host_0" + timestamp int64 + measurements []string + values []interface{} + dataTypes []client.TSDataType + + fieldsCnt uint64 } -func (p *iotdbPoint) generateInsertStatement() string { - sql := fmt.Sprintf("INSERT INTO %s(%s) VALUES(%s)", p.deviceID, p.fieldKeyStr, p.fieldValueStr) - return sql +// parse datatype and convert string into interface +func parseDateToInterface(datatype client.TSDataType, str string) (interface{}, error) { + switch client.TSDataType(datatype) { + case client.BOOLEAN: + value, err := strconv.ParseBool(str) + return interface{}(value), err + case client.INT32: + value, err := strconv.ParseInt(str, 10, 32) + return interface{}(int32(value)), err + case client.INT64: + value, err := strconv.ParseInt(str, 10, 64) + return interface{}(int64(value)), err + case client.FLOAT: + value, err := strconv.ParseFloat(str, 32) + return interface{}(float32(value)), err + case client.DOUBLE: + value, err := strconv.ParseFloat(str, 64) + return interface{}(float64(value)), err + case client.TEXT: + return interface{}(str), nil + case client.UNKNOW: + return interface{}(nil), fmt.Errorf("datatype client.UNKNOW, value:%s", str) + default: + return interface{}(nil), fmt.Errorf("unknown datatype, value:%s", str) + } } type fileDataSource struct { scanner *bufio.Scanner } -// read new two line, which store one data point +// read new three line, which store one data point +// e.g., // e.g., // deviceID,timestamp,,,,... // ,,,,,... +// datatype,,,,... // // deviceID,timestamp,hostname,value -// root.cpu.host_1,2016-01-01 00:00:00,'host_1',44.0 +// root.cpu.host_1,1451606400000000000,'host_1',44.0 +// datatype,5,2 // // return : bool -> true means got one point, else reaches EOF or error happens -func (d *fileDataSource) nextTwoLines() (bool, string, string, error) { +func (d *fileDataSource) nextThreeLines() (bool, string, string, string, error) { ok := d.scanner.Scan() if !ok && d.scanner.Err() == nil { // nothing scanned & no error = EOF - return false, "", "", nil + return false, "", "", "", nil } else if !ok { - return false, "", "", fmt.Errorf("scan error: %v", d.scanner.Err()) + return false, "", "", "", fmt.Errorf("scan error: %v", d.scanner.Err()) } line1 := d.scanner.Text() line_ok := strings.HasPrefix(line1, "deviceID,timestamp,") if !line_ok { - return false, line1, "", fmt.Errorf("scan error, illegal line: %s", line1) + return false, line1, "", "", fmt.Errorf("scan error, illegal line: %s", line1) } ok = d.scanner.Scan() if !ok && d.scanner.Err() == nil { // nothing scanned & no error = EOF - return false, "", "", nil + return false, "", "", "", nil } else if !ok { - return false, "", "", fmt.Errorf("scan error: %v", d.scanner.Err()) + return false, "", "", "", fmt.Errorf("scan error: %v", d.scanner.Err()) } line2 := d.scanner.Text() - return true, line1, line2, nil + ok = d.scanner.Scan() + if !ok && d.scanner.Err() == nil { // nothing scanned & no error = EOF + return false, "", "", "", nil + } else if !ok { + return false, "", "", "", fmt.Errorf("scan error: %v", d.scanner.Err()) + } + line3 := d.scanner.Text() + return true, line1, line2, line3, nil +} + +func parseThreeLines(line1 string, line2 string, line3 string) data.LoadedPoint { + line1_parts := strings.Split(line1, ",") // 'deviceID' and rest keys of fields + line2_parts := strings.Split(line2, ",") // deviceID and rest values of fields + line3_parts := strings.Split(line3, ",") // deviceID and rest values of fields + timestamp, err := strconv.ParseInt(line2_parts[1], 10, 64) + if err != nil { + fatal("timestamp convert err: %v", err) + } + timestamp = int64(timestamp / int64(time.Millisecond)) + var measurements []string + var values []interface{} + var dataTypes []client.TSDataType + // handle measurements, datatype and values + measurements = append(measurements, line1_parts[2:]...) + for type_index := 1; type_index < len(line3_parts); type_index++ { + value_index := type_index + 1 + datatype, _ := strconv.ParseInt(line3_parts[type_index], 10, 8) + dataTypes = append(dataTypes, client.TSDataType(datatype)) + value, err := parseDateToInterface(client.TSDataType(datatype), line2_parts[value_index]) + if err != nil { + panic(fmt.Errorf("iotdb fileDataSource NextItem Parse error:%v", err)) + } + values = append(values, value) + } + return data.NewLoadedPoint( + &iotdbPoint{ + deviceID: line2_parts[0], + timestamp: timestamp, + measurements: measurements, + values: values, + dataTypes: dataTypes, + fieldsCnt: uint64(len(line1_parts) - 2), + }) } func (d *fileDataSource) NextItem() data.LoadedPoint { - scan_ok, line1, line2, err := d.nextTwoLines() + scan_ok, line1, line2, line3, err := d.nextThreeLines() if !scan_ok { if err == nil { // End of file return data.LoadedPoint{} @@ -68,15 +141,7 @@ func (d *fileDataSource) NextItem() data.LoadedPoint { return data.LoadedPoint{} } } - line1_parts := strings.SplitN(line1, ",", 2) // 'deviceID' and rest keys of fields - line2_parts := strings.SplitN(line2, ",", 2) // deviceID and rest values of fields - return data.NewLoadedPoint( - &iotdbPoint{ - deviceID: line2_parts[0], - fieldKeyStr: line1_parts[1], - fieldValueStr: line2_parts[1], - fieldsCnt: uint64(len(strings.Split(line1_parts[1], ","))), - }) + return parseThreeLines(line1, line2, line3) } func (d *fileDataSource) Headers() *common.GeneratedDataHeaders { return nil } diff --git a/cmd/tsbs_load_iotdb/scan_test.go b/cmd/tsbs_load_iotdb/scan_test.go index 935cd64e6..364ffe338 100644 --- a/cmd/tsbs_load_iotdb/scan_test.go +++ b/cmd/tsbs_load_iotdb/scan_test.go @@ -3,41 +3,55 @@ package main import ( "testing" + "github.com/apache/iotdb-client-go/client" "github.com/stretchr/testify/require" ) func TestGenerateInsertStatement(t *testing.T) { cases := []struct { description string - point iotdbPoint - expected string + lines []string + expected iotdbPoint }{ { - description: "one point(1)", - point: iotdbPoint{ - deviceID: "root.cpu.host_0", - fieldKeyStr: "timestamp,value,str", - fieldValueStr: "123456,999,'abc'", - fieldsCnt: 3, + description: "one point", + lines: []string{ + "deviceID,timestamp,value", + "root.cpu.host_9,1451606400000000000,3.1415926", + "datatype,4", + }, + expected: iotdbPoint{ + deviceID: "root.cpu.host_9", + timestamp: 1451606400000, + measurements: []string{"value"}, + values: []interface{}{float64(3.1415926)}, + dataTypes: []client.TSDataType{client.DOUBLE}, + fieldsCnt: 1, }, - expected: "INSERT INTO root.cpu.host_0(timestamp,value,str) VALUES(123456,999,'abc')", }, { - description: "one point(2)", - point: iotdbPoint{ - deviceID: "root.cpu.host_9", - fieldKeyStr: "timestamp,floatValue,str,intValue", - fieldValueStr: "123456,4321.9,'abc',45621", - fieldsCnt: 4, + description: "one point with different dataTypes", + lines: []string{ + "deviceID,timestamp,floatV,strV,int64V,int32V,boolV", + "root.cpu.host_0,1451606400000000000,3.1415926,hello,123,123,true", + "datatype,4,5,2,1,0", + }, + expected: iotdbPoint{ + deviceID: "root.cpu.host_0", + timestamp: 1451606400000, + measurements: []string{"floatV", "strV", "int64V", "int32V", "boolV"}, + values: []interface{}{float64(3.1415926), string("hello"), int64(123), int32(123), true}, + dataTypes: []client.TSDataType{client.DOUBLE, client.TEXT, client.INT64, client.INT32, client.BOOLEAN}, + fieldsCnt: 5, }, - expected: "INSERT INTO root.cpu.host_9(timestamp,floatValue,str,intValue) VALUES(123456,4321.9,'abc',45621)", }, } for _, c := range cases { t.Run(c.description, func(t *testing.T) { - actual := c.point.generateInsertStatement() - require.EqualValues(t, c.expected, actual) + require.True(t, len(c.lines) == 3) + actual := parseThreeLines(c.lines[0], c.lines[1], c.lines[2]) + require.EqualValues(t, &c.expected, actual.Data.(*iotdbPoint)) }) } } diff --git a/docs/iotdb.md b/docs/iotdb.md index 4c31cbbbf..d137c4a15 100644 --- a/docs/iotdb.md +++ b/docs/iotdb.md @@ -24,9 +24,11 @@ For example: ```text deviceID,timestamp,,,,... ,,,,,... +datatype,,,,... deviceID,timestamp,hostname,value -root.cpu.host_1,2016-01-01 00:00:00,'host_1',44.0 +root.cpu.host_1,1451606400000000000,'host_1',44.0 +datatype,5,2 ``` `deviceID` describes the storage path which is composed of several nodes. @@ -34,12 +36,15 @@ IoTDB uses storage group to manage data. For example, in test case `devops`, if `measurementName` is `cpu` while `hostname` tag is `host_0`, then `deviceID` is `root.cpu.host_0`. IoTDB treat tags as fields, except primary key like `hostname` in `devops`. +The unit of timestamp in generated data is nanosecond, but it will be converted +into millisecond before insert into database. An example for the `cpu-only` use case: ```text deviceID,timestamp,region,datacenter,rack,os,arch,team,service,service_version,service_environment,usage_user,usage_system,usage_idle,usage_nice,usage_iowait,usage_irq,usage_softirq,usage_steal,usage_guest,usage_guest_nice -root.cpu.host_0,2016-01-01 00:00:00,'eu-west-1','eu-west-1c','87','Ubuntu16.04LTS','x64','NYC','18','1','production',58,2,24,61,22,63,6,44,80,38 +root.cpu.host_0,1451606400000000000,eu-west-1,eu-west-1c,87,Ubuntu16.04LTS,x64,NYC,18,1,production,58,2,24,61,22,63,6,44,80,38 +datatype,5,5,5,5,5,5,5,5,5,2,2,2,2,2,2,2,2,2,2 ``` diff --git a/pkg/targets/iotdb/implemented_target.go b/pkg/targets/iotdb/implemented_target.go index 13cebb3e0..62ae03d4f 100644 --- a/pkg/targets/iotdb/implemented_target.go +++ b/pkg/targets/iotdb/implemented_target.go @@ -22,6 +22,7 @@ func (t *iotdbTarget) TargetSpecificFlags(flagPrefix string, flagSet *pflag.Flag flagSet.String(flagPrefix+"user", "root", "The user who connect to IoTDB") flagSet.String(flagPrefix+"password", "root", "The password for user connecting to IoTDB") flagSet.Int(flagPrefix+"timeout", 0, "Session timeout check in millisecond. Use 0 for no timeout.") + flagSet.Int(flagPrefix+"records-max-rows", 0, "Max rows of 'InsertRecords'. Use 0 for no limit.") } func (t *iotdbTarget) TargetName() string { diff --git a/pkg/targets/iotdb/serializer.go b/pkg/targets/iotdb/serializer.go index 00b8317de..5fd1911c1 100644 --- a/pkg/targets/iotdb/serializer.go +++ b/pkg/targets/iotdb/serializer.go @@ -6,13 +6,16 @@ import ( "strconv" "strings" + "github.com/apache/iotdb-client-go/client" "github.com/timescale/tsbs/pkg/data" ) // Serializer writes a Point in a serialized form for MongoDB type Serializer struct{} -const iotdbTimeFmt = "2006-01-02 15:04:05" +// const iotdbTimeFmt = "2006-01-02 15:04:05" + +const defaultBufSize = 4096 // Serialize writes Point p to the given Writer w, so it can be // loaded by the IoTDB loader. The format is CSV with two lines per Point, @@ -22,14 +25,18 @@ const iotdbTimeFmt = "2006-01-02 15:04:05" // e.g., // deviceID,timestamp,,,,... // ,,,,,... +// datatype,,,,... // // deviceID,timestamp,hostname,value -// root.cpu.host_1,2016-01-01 00:00:00,'host_1',44.0 +// root.cpu.host_1,1451606400000000000,'host_1',44.0 +// datatype,5,2 func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { // Tag row first, prefixed with 'time,path' - buf1 := make([]byte, 0, 1024) + buf1 := make([]byte, 0, defaultBufSize) buf1 = append(buf1, []byte("deviceID,timestamp")...) - tempBuf := make([]byte, 0, 1024) + datatype_buf := make([]byte, 0, defaultBufSize) + datatype_buf = append(datatype_buf, []byte("datatype")...) + tempBuf := make([]byte, 0, defaultBufSize) var hostname string foundHostname := false tagKeys := p.TagKeys() @@ -41,17 +48,20 @@ func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { } else { buf1 = append(buf1, ',') buf1 = append(buf1, tagKeys[i]...) + valueInStrByte, datatype := iotdbFormat(v) tempBuf = append(tempBuf, ',') - tempBuf = iotdbFormatAppend(v, tempBuf) + tempBuf = append(tempBuf, valueInStrByte...) + datatype_buf = append(datatype_buf, ',') + datatype_buf = append(datatype_buf, []byte(fmt.Sprintf("%d", datatype))...) } } if !foundHostname { // Unable to find hostname as part of device id hostname = "unknown" } - buf2 := make([]byte, 0, 1024) + buf2 := make([]byte, 0, defaultBufSize) buf2 = append(buf2, []byte(fmt.Sprintf("root.%s.%s,", modifyHostname(string(p.MeasurementName())), hostname))...) - buf2 = append(buf2, []byte(fmt.Sprintf("%s", p.Timestamp().UTC().Format(iotdbTimeFmt)))...) + buf2 = append(buf2, []byte(fmt.Sprintf("%d", p.Timestamp().UTC().UnixNano()))...) buf2 = append(buf2, tempBuf...) // Fields fieldKeys := p.FieldKeys() @@ -59,15 +69,23 @@ func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { for i, v := range fieldValues { buf1 = append(buf1, ',') buf1 = append(buf1, fieldKeys[i]...) + // buf2 = iotdbFormatAppend(v, buf2) + valueInStrByte, datatype := iotdbFormat(v) buf2 = append(buf2, ',') - buf2 = iotdbFormatAppend(v, buf2) + buf2 = append(buf2, valueInStrByte...) + datatype_buf = append(datatype_buf, ',') + datatype_buf = append(datatype_buf, []byte(fmt.Sprintf("%d", datatype))...) } buf1 = append(buf1, '\n') buf2 = append(buf2, '\n') + datatype_buf = append(datatype_buf, '\n') _, err := w.Write(buf1) if err == nil { _, err = w.Write(buf2) } + if err == nil { + _, err = w.Write(datatype_buf) + } return err } @@ -86,35 +104,34 @@ func modifyHostname(hostname string) string { } // Utility function for appending various data types to a byte string -func iotdbFormatAppend(v interface{}, buf []byte) []byte { +func iotdbFormat(v interface{}) ([]byte, client.TSDataType) { switch v.(type) { + case uint: + return []byte(strconv.FormatInt(int64(v.(uint)), 10)), client.INT64 + case uint32: + return []byte(strconv.FormatInt(int64(v.(uint32)), 10)), client.INT64 + case uint64: + return []byte(strconv.FormatInt(int64(v.(uint64)), 10)), client.INT64 case int: - return strconv.AppendInt(buf, int64(v.(int)), 10) + return []byte(strconv.FormatInt(int64(v.(int)), 10)), client.INT64 + case int32: + return []byte(strconv.FormatInt(int64(v.(int32)), 10)), client.INT32 case int64: - return strconv.AppendInt(buf, v.(int64), 10) + return []byte(strconv.FormatInt(int64(v.(int64)), 10)), client.INT64 case float64: // Why -1 ? // From Golang source on genericFtoa (called by AppendFloat): 'Negative precision means "only as much as needed to be exact."' // Using this instead of an exact number for precision ensures we preserve the precision passed in to the function, allowing us // to use different precision for different use cases. - return strconv.AppendFloat(buf, v.(float64), 'f', -1, 64) + return []byte(strconv.FormatFloat(float64(v.(float64)), 'f', -1, 64)), client.DOUBLE case float32: - return strconv.AppendFloat(buf, float64(v.(float32)), 'f', -1, 32) + return []byte(strconv.FormatFloat(float64(v.(float32)), 'f', -1, 32)), client.FLOAT case bool: - return strconv.AppendBool(buf, v.(bool)) - case []byte: - buf = append(buf, []byte("'")...) - buf = append(buf, v.([]byte)...) - buf = append(buf, []byte("'")...) - return buf + return []byte(strconv.FormatBool(v.(bool))), client.BOOLEAN case string: - // buf = append(buf, []byte(fmt.Sprintf("\"%s\"", v.(string)))...) - buf = append(buf, []byte("'")...) - buf = append(buf, v.(string)...) - buf = append(buf, []byte("'")...) - return buf + return []byte(v.(string)), client.TEXT case nil: - return buf + return []byte(v.(string)), client.UNKNOW default: panic(fmt.Sprintf("unknown field type for %#v", v)) } diff --git a/pkg/targets/iotdb/serializer_test.go b/pkg/targets/iotdb/serializer_test.go index e577ef71c..9b58a7f26 100644 --- a/pkg/targets/iotdb/serializer_test.go +++ b/pkg/targets/iotdb/serializer_test.go @@ -4,11 +4,72 @@ import ( "bytes" "testing" + "github.com/apache/iotdb-client-go/client" "github.com/stretchr/testify/require" "github.com/timescale/tsbs/pkg/data" "github.com/timescale/tsbs/pkg/data/serialize" ) +func TestIotdbFormat(t *testing.T) { + cases := []struct { + description string + input interface{} + expectedByte []byte + expectedType client.TSDataType + }{ + { + description: "boolean true", + input: interface{}(true), + expectedByte: []byte("true"), + expectedType: client.BOOLEAN, + }, + { + description: "boolean false", + input: interface{}(false), + expectedByte: []byte("false"), + expectedType: client.BOOLEAN, + }, + { + description: "int32 -1", + input: interface{}(int32(-1)), + expectedByte: []byte("-1"), + expectedType: client.INT32, + }, + { + description: "int64 2147483648", + input: interface{}(int64(2147483648)), + expectedByte: []byte("2147483648"), + expectedType: client.INT64, + }, + { + description: "int64 9223372036854775801", + input: interface{}(int64(9223372036854775801)), + expectedByte: []byte("9223372036854775801"), + expectedType: client.INT64, + }, + { + description: "float32 0.1", + input: interface{}(float32(0.1)), + expectedByte: []byte("0.1"), + expectedType: client.FLOAT, + }, + { + description: "float64 0.12345678901234567890123456", + input: interface{}(float64(0.12345678901234567890123456)), + expectedByte: []byte("0.12345678901234568"), + expectedType: client.DOUBLE, + }, + } + for _, c := range cases { + t.Run(c.description, func(t *testing.T) { + actualByte, actualType := iotdbFormat(c.input) + require.EqualValues(t, c.expectedByte, actualByte) + require.EqualValues(t, c.expectedType, actualType) + }) + } + +} + func TestSerialize_001(t *testing.T) { cases := []struct { description string @@ -18,22 +79,22 @@ func TestSerialize_001(t *testing.T) { { description: "a regular point ", inputPoint: serialize.TestPointDefault(), - expected: "deviceID,timestamp,region,datacenter,usage_guest_nice\nroot.cpu.host_0,2016-01-01 00:00:00,'eu-west-1','eu-west-1b',38.24311829\n", + expected: "deviceID,timestamp,region,datacenter,usage_guest_nice\nroot.cpu.host_0,1451606400000000000,'eu-west-1','eu-west-1b',38.24311829\ndatatype,5,5,4\n", }, { description: "a regular Point using int as value", inputPoint: serialize.TestPointInt(), - expected: "deviceID,timestamp,region,datacenter,usage_guest\nroot.cpu.host_0,2016-01-01 00:00:00,'eu-west-1','eu-west-1b',38\n", + expected: "deviceID,timestamp,region,datacenter,usage_guest\nroot.cpu.host_0,1451606400000000000,'eu-west-1','eu-west-1b',38\ndatatype,5,5,2\n", }, { description: "a regular Point with multiple fields", inputPoint: serialize.TestPointMultiField(), - expected: "deviceID,timestamp,region,datacenter,big_usage_guest,usage_guest,usage_guest_nice\nroot.cpu.host_0,2016-01-01 00:00:00,'eu-west-1','eu-west-1b',5000000000,38,38.24311829\n", + expected: "deviceID,timestamp,region,datacenter,big_usage_guest,usage_guest,usage_guest_nice\nroot.cpu.host_0,1451606400000000000,'eu-west-1','eu-west-1b',5000000000,38,38.24311829\ndatatype,5,5,2,2,4\n", }, { description: "a Point with no tags", inputPoint: serialize.TestPointNoTags(), - expected: "deviceID,timestamp,usage_guest_nice\nroot.cpu.unknown,2016-01-01 00:00:00,38.24311829\n", + expected: "deviceID,timestamp,usage_guest_nice\nroot.cpu.unknown,1451606400000000000,38.24311829\ndatatype,4\n", }, } for _, c := range cases { From 079fc737c8333d63ee887d133aae9035a6f29bc5 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Wed, 16 Nov 2022 19:50:22 +0800 Subject: [PATCH 20/47] using 4lines structure for serializer, not completed using 4lines structure for serializer, not completed can insert data without tags. can NOT handle tags yet in this commit. --- cmd/tsbs_load_iotdb/scan.go | 49 +++++++++++++++++++++----------- cmd/tsbs_load_iotdb/scan_test.go | 4 ++- pkg/targets/iotdb/serializer.go | 32 ++++++++++++++++----- 3 files changed, 61 insertions(+), 24 deletions(-) diff --git a/cmd/tsbs_load_iotdb/scan.go b/cmd/tsbs_load_iotdb/scan.go index a42c7d72d..de01c75e4 100644 --- a/cmd/tsbs_load_iotdb/scan.go +++ b/cmd/tsbs_load_iotdb/scan.go @@ -20,10 +20,18 @@ type iotdbPoint struct { measurements []string values []interface{} dataTypes []client.TSDataType + tagString string fieldsCnt uint64 } +// CRTODO:使用这个函数来生成创建语句。 +func (p *iotdbPoint) generateTagsAttributesSQL() string { + sql := "CREATE timeseries %s._tags with datatype=INT32, encoding=RLE, compression=SNAPPY attributes(%s)" + // sql2 := "ALTER timeseries %s._tags UPSERT attributes(%s)" + return fmt.Sprintf(sql, p.deviceID, p.tagString) +} + // parse datatype and convert string into interface func parseDateToInterface(datatype client.TSDataType, str string) (interface{}, error) { switch client.TSDataType(datatype) { @@ -55,7 +63,7 @@ type fileDataSource struct { scanner *bufio.Scanner } -// read new three line, which store one data point +// read new four line, which store one data point // e.g., // e.g., // deviceID,timestamp,,,,... @@ -67,39 +75,47 @@ type fileDataSource struct { // datatype,5,2 // // return : bool -> true means got one point, else reaches EOF or error happens -func (d *fileDataSource) nextThreeLines() (bool, string, string, string, error) { +func (d *fileDataSource) nextFourLines() (bool, string, string, string, string, error) { ok := d.scanner.Scan() if !ok && d.scanner.Err() == nil { // nothing scanned & no error = EOF - return false, "", "", "", nil + return false, "", "", "", "", nil } else if !ok { - return false, "", "", "", fmt.Errorf("scan error: %v", d.scanner.Err()) + return false, "", "", "", "", fmt.Errorf("scan error: %v", d.scanner.Err()) } line1 := d.scanner.Text() line_ok := strings.HasPrefix(line1, "deviceID,timestamp,") if !line_ok { - return false, line1, "", "", fmt.Errorf("scan error, illegal line: %s", line1) + return false, line1, "", "", "", fmt.Errorf("scan error, illegal line: %s", line1) } ok = d.scanner.Scan() if !ok && d.scanner.Err() == nil { // nothing scanned & no error = EOF - return false, "", "", "", nil + return false, "", "", "", "", nil } else if !ok { - return false, "", "", "", fmt.Errorf("scan error: %v", d.scanner.Err()) + return false, "", "", "", "", fmt.Errorf("scan error: %v", d.scanner.Err()) } line2 := d.scanner.Text() ok = d.scanner.Scan() if !ok && d.scanner.Err() == nil { // nothing scanned & no error = EOF - return false, "", "", "", nil + return false, "", "", "", "", nil } else if !ok { - return false, "", "", "", fmt.Errorf("scan error: %v", d.scanner.Err()) + return false, "", "", "", "", fmt.Errorf("scan error: %v", d.scanner.Err()) } line3 := d.scanner.Text() - return true, line1, line2, line3, nil + ok = d.scanner.Scan() + if !ok && d.scanner.Err() == nil { // nothing scanned & no error = EOF + return false, "", "", "", "", nil + } else if !ok { + return false, "", "", "", "", fmt.Errorf("scan error: %v", d.scanner.Err()) + } + line4 := d.scanner.Text() + return true, line1, line2, line3, line4, nil } -func parseThreeLines(line1 string, line2 string, line3 string) data.LoadedPoint { - line1_parts := strings.Split(line1, ",") // 'deviceID' and rest keys of fields - line2_parts := strings.Split(line2, ",") // deviceID and rest values of fields - line3_parts := strings.Split(line3, ",") // deviceID and rest values of fields +func parseFourLines(line1 string, line2 string, line3 string, line4 string) data.LoadedPoint { + line1_parts := strings.Split(line1, ",") // 'deviceID' and rest keys of fields + line2_parts := strings.Split(line2, ",") // deviceID and rest values of fields + line3_parts := strings.Split(line3, ",") // deviceID and rest values of fields + line4_parts := strings.SplitN(line4, ",", 2) // 'tags' and string of tags timestamp, err := strconv.ParseInt(line2_parts[1], 10, 64) if err != nil { fatal("timestamp convert err: %v", err) @@ -127,12 +143,13 @@ func parseThreeLines(line1 string, line2 string, line3 string) data.LoadedPoint measurements: measurements, values: values, dataTypes: dataTypes, + tagString: line4_parts[1], fieldsCnt: uint64(len(line1_parts) - 2), }) } func (d *fileDataSource) NextItem() data.LoadedPoint { - scan_ok, line1, line2, line3, err := d.nextThreeLines() + scan_ok, line1, line2, line3, line4, err := d.nextFourLines() if !scan_ok { if err == nil { // End of file return data.LoadedPoint{} @@ -141,7 +158,7 @@ func (d *fileDataSource) NextItem() data.LoadedPoint { return data.LoadedPoint{} } } - return parseThreeLines(line1, line2, line3) + return parseFourLines(line1, line2, line3, line4) } func (d *fileDataSource) Headers() *common.GeneratedDataHeaders { return nil } diff --git a/cmd/tsbs_load_iotdb/scan_test.go b/cmd/tsbs_load_iotdb/scan_test.go index 364ffe338..d9e81d659 100644 --- a/cmd/tsbs_load_iotdb/scan_test.go +++ b/cmd/tsbs_load_iotdb/scan_test.go @@ -19,6 +19,7 @@ func TestGenerateInsertStatement(t *testing.T) { "deviceID,timestamp,value", "root.cpu.host_9,1451606400000000000,3.1415926", "datatype,4", + "tags", }, expected: iotdbPoint{ deviceID: "root.cpu.host_9", @@ -35,6 +36,7 @@ func TestGenerateInsertStatement(t *testing.T) { "deviceID,timestamp,floatV,strV,int64V,int32V,boolV", "root.cpu.host_0,1451606400000000000,3.1415926,hello,123,123,true", "datatype,4,5,2,1,0", + "tags", }, expected: iotdbPoint{ deviceID: "root.cpu.host_0", @@ -50,7 +52,7 @@ func TestGenerateInsertStatement(t *testing.T) { for _, c := range cases { t.Run(c.description, func(t *testing.T) { require.True(t, len(c.lines) == 3) - actual := parseThreeLines(c.lines[0], c.lines[1], c.lines[2]) + actual := parseFourLines(c.lines[0], c.lines[1], c.lines[2], c.lines[3]) require.EqualValues(t, &c.expected, actual.Data.(*iotdbPoint)) }) } diff --git a/pkg/targets/iotdb/serializer.go b/pkg/targets/iotdb/serializer.go index 5fd1911c1..057a08c4a 100644 --- a/pkg/targets/iotdb/serializer.go +++ b/pkg/targets/iotdb/serializer.go @@ -26,16 +26,20 @@ const defaultBufSize = 4096 // deviceID,timestamp,,,,... // ,,,,,... // datatype,,,,... +// tags,=,=,... // // deviceID,timestamp,hostname,value // root.cpu.host_1,1451606400000000000,'host_1',44.0 // datatype,5,2 +// tags,region='eu-west-1',datacenter='eu-west-1c',rack=87, func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { // Tag row first, prefixed with 'time,path' buf1 := make([]byte, 0, defaultBufSize) buf1 = append(buf1, []byte("deviceID,timestamp")...) datatype_buf := make([]byte, 0, defaultBufSize) datatype_buf = append(datatype_buf, []byte("datatype")...) + tags_buf := make([]byte, 0, defaultBufSize) + tags_buf = append(tags_buf, []byte("tags")...) tempBuf := make([]byte, 0, defaultBufSize) var hostname string foundHostname := false @@ -46,13 +50,24 @@ func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { foundHostname = true hostname = v.(string) } else { - buf1 = append(buf1, ',') - buf1 = append(buf1, tagKeys[i]...) + // handle other tags + + // buf1 = append(buf1, ',') + // buf1 = append(buf1, tagKeys[i]...) + // valueInStrByte, datatype := iotdbFormat(v) + // tempBuf = append(tempBuf, ',') + // tempBuf = append(tempBuf, valueInStrByte...) + // datatype_buf = append(datatype_buf, ',') + // datatype_buf = append(datatype_buf, []byte(fmt.Sprintf("%d", datatype))...) valueInStrByte, datatype := iotdbFormat(v) - tempBuf = append(tempBuf, ',') - tempBuf = append(tempBuf, valueInStrByte...) - datatype_buf = append(datatype_buf, ',') - datatype_buf = append(datatype_buf, []byte(fmt.Sprintf("%d", datatype))...) + if datatype == client.TEXT { + tagStr := fmt.Sprintf(",%s='%s'", keyStr, string(valueInStrByte)) + tags_buf = append(tags_buf, []byte(tagStr)...) + } else { + tagStr := fmt.Sprintf(",%s=", keyStr) + tags_buf = append(tags_buf, []byte(tagStr)...) + tags_buf = append(tags_buf, valueInStrByte...) + } } } if !foundHostname { @@ -69,7 +84,6 @@ func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { for i, v := range fieldValues { buf1 = append(buf1, ',') buf1 = append(buf1, fieldKeys[i]...) - // buf2 = iotdbFormatAppend(v, buf2) valueInStrByte, datatype := iotdbFormat(v) buf2 = append(buf2, ',') buf2 = append(buf2, valueInStrByte...) @@ -79,6 +93,7 @@ func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { buf1 = append(buf1, '\n') buf2 = append(buf2, '\n') datatype_buf = append(datatype_buf, '\n') + tags_buf = append(tags_buf, '\n') _, err := w.Write(buf1) if err == nil { _, err = w.Write(buf2) @@ -86,6 +101,9 @@ func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { if err == nil { _, err = w.Write(datatype_buf) } + if err == nil { + _, err = w.Write(tags_buf) + } return err } From bcbf8b39eaaf7b9bb3d374e77d42f27fb7d936e5 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Wed, 16 Nov 2022 20:18:11 +0800 Subject: [PATCH 21/47] using 4lines, storage tags in attributes of node '_tags' using 4lines serializer storage tags in attributes of node '_tags' --- cmd/tsbs_load_iotdb/process.go | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/cmd/tsbs_load_iotdb/process.go b/cmd/tsbs_load_iotdb/process.go index 3f1cdeced..613c71434 100644 --- a/cmd/tsbs_load_iotdb/process.go +++ b/cmd/tsbs_load_iotdb/process.go @@ -8,11 +8,14 @@ import ( ) type processor struct { - session client.Session - recordsMaxRows int // max rows of records in 'InsertRecords' + numWorker int // the worker(like thread) ID of this processor + session client.Session + recordsMaxRows int // max rows of records in 'InsertRecords' + ProcessedTagsDeviceIDMap map[string]bool // already processed device ID } -func (p *processor) Init(_ int, doLoad, _ bool) { +func (p *processor) Init(numWorker int, doLoad, _ bool) { + p.numWorker = numWorker if !doLoad { return } @@ -22,6 +25,7 @@ func (p *processor) Init(_ int, doLoad, _ bool) { errMsg = errMsg + fmt.Sprintf("timeout setting: %d ms", timeoutInMs) fatal(errMsg) } + p.ProcessedTagsDeviceIDMap = make(map[string]bool, 1024) } func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (metricCount, rowCount uint64) { @@ -29,6 +33,7 @@ func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (metricCount, row // Write records if doLoad { + var sqlList []string if p.recordsMaxRows > 0 { for index := 0; index < len(batch.points); { var ( @@ -45,6 +50,11 @@ func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (metricCount, row dataTypes = append(dataTypes, row.dataTypes) values = append(values, row.values) timestamps = append(timestamps, row.timestamp) + _, exist := p.ProcessedTagsDeviceIDMap[row.deviceID] + if !exist { + sqlList = append(sqlList, row.generateTagsAttributesSQL()) + p.ProcessedTagsDeviceIDMap[row.deviceID] = true + } thisRecordsCnt++ index++ } @@ -69,6 +79,11 @@ func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (metricCount, row dataTypes = append(dataTypes, row.dataTypes) values = append(values, row.values) timestamps = append(timestamps, row.timestamp) + _, exist := p.ProcessedTagsDeviceIDMap[row.deviceID] + if !exist { + sqlList = append(sqlList, row.generateTagsAttributesSQL()) + p.ProcessedTagsDeviceIDMap[row.deviceID] = true + } } _, err := p.session.InsertRecords( deviceId, measurements, dataTypes, values, timestamps, @@ -77,10 +92,10 @@ func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (metricCount, row fatal("ProcessBatch error:%v", err) } } - // for _, row := range batch.points { - // sql := row.generateInsertStatement() - // p.session.ExecuteUpdateStatement(sql) - // } + // handle create timeseries SQL to insert tags + for _, sql := range sqlList { + p.session.ExecuteUpdateStatement(sql) + } } metricCount = batch.metrics From eb2606311a44aad7bdf477965bd7276e1b445d82 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Thu, 17 Nov 2022 11:33:42 +0800 Subject: [PATCH 22/47] update workersNum warning in insertion test update workersNum warning in insertion test less than five --> less than 2 --- cmd/tsbs_load_iotdb/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/tsbs_load_iotdb/main.go b/cmd/tsbs_load_iotdb/main.go index a742ce5f3..8296f4fbf 100644 --- a/cmd/tsbs_load_iotdb/main.go +++ b/cmd/tsbs_load_iotdb/main.go @@ -68,7 +68,7 @@ func init() { timeoutStr = "no timeout for session opening check" } log.Printf("tsbs_load_iotdb target: %s:%s, %s. Loading with %d workers.\n", host, port, timeoutStr, workers) - if workers < 5 { + if workers < 2 { log.Println("Insertion throughput is strongly related to the number of threads. Use more workers for better performance.") } From 9c99cd9215cb273736f372aca028185d01839871 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Thu, 17 Nov 2022 12:09:07 +0800 Subject: [PATCH 23/47] optimize ProcessBatch function optimize ProcessBatch function using records struct to simplify code --- cmd/tsbs_load_iotdb/process.go | 95 ++++++++++++++++------------------ 1 file changed, 44 insertions(+), 51 deletions(-) diff --git a/cmd/tsbs_load_iotdb/process.go b/cmd/tsbs_load_iotdb/process.go index 613c71434..18514b442 100644 --- a/cmd/tsbs_load_iotdb/process.go +++ b/cmd/tsbs_load_iotdb/process.go @@ -28,69 +28,62 @@ func (p *processor) Init(numWorker int, doLoad, _ bool) { p.ProcessedTagsDeviceIDMap = make(map[string]bool, 1024) } +type records struct { + deviceId []string + measurements [][]string + dataTypes [][]client.TSDataType + values [][]interface{} + timestamps []int64 +} + +func (p *processor) pointsToRecords(points []*iotdbPoint) (records, []string) { + var rcds records + var sqlList []string + for _, row := range points { + rcds.deviceId = append(rcds.deviceId, row.deviceID) + rcds.measurements = append(rcds.measurements, row.measurements) + rcds.dataTypes = append(rcds.dataTypes, row.dataTypes) + rcds.values = append(rcds.values, row.values) + rcds.timestamps = append(rcds.timestamps, row.timestamp) + _, exist := p.ProcessedTagsDeviceIDMap[row.deviceID] + if !exist { + sqlList = append(sqlList, row.generateTagsAttributesSQL()) + p.ProcessedTagsDeviceIDMap[row.deviceID] = true + } + } + return rcds, sqlList +} + +func minInt(x int, y int) int { + if x < y { + return x + } + return y +} + func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (metricCount, rowCount uint64) { batch := b.(*iotdbBatch) // Write records if doLoad { var sqlList []string - if p.recordsMaxRows > 0 { - for index := 0; index < len(batch.points); { - var ( - deviceId []string - measurements [][]string - dataTypes [][]client.TSDataType - values [][]interface{} - timestamps []int64 - ) - for thisRecordsCnt := 0; thisRecordsCnt < recordsMaxRows && index < len(batch.points); { - row := batch.points[index] - deviceId = append(deviceId, row.deviceID) - measurements = append(measurements, row.measurements) - dataTypes = append(dataTypes, row.dataTypes) - values = append(values, row.values) - timestamps = append(timestamps, row.timestamp) - _, exist := p.ProcessedTagsDeviceIDMap[row.deviceID] - if !exist { - sqlList = append(sqlList, row.generateTagsAttributesSQL()) - p.ProcessedTagsDeviceIDMap[row.deviceID] = true - } - thisRecordsCnt++ - index++ - } - _, err := p.session.InsertRecords( - deviceId, measurements, dataTypes, values, timestamps, - ) - if err != nil { - fatal("ProcessBatch error:%v", err) - } - } - } else { - var ( - deviceId []string - measurements [][]string - dataTypes [][]client.TSDataType - values [][]interface{} - timestamps []int64 - ) - for _, row := range batch.points { - deviceId = append(deviceId, row.deviceID) - measurements = append(measurements, row.measurements) - dataTypes = append(dataTypes, row.dataTypes) - values = append(values, row.values) - timestamps = append(timestamps, row.timestamp) - _, exist := p.ProcessedTagsDeviceIDMap[row.deviceID] - if !exist { - sqlList = append(sqlList, row.generateTagsAttributesSQL()) - p.ProcessedTagsDeviceIDMap[row.deviceID] = true - } + for index := 0; index < len(batch.points); { + startIndex := index + var endIndex int + if p.recordsMaxRows > 0 { + endIndex = minInt(len(batch.points), index+p.recordsMaxRows) + } else { + endIndex = len(batch.points) } + rcds, tempSqlList := p.pointsToRecords(batch.points[startIndex:endIndex]) + sqlList = append(sqlList, tempSqlList...) _, err := p.session.InsertRecords( - deviceId, measurements, dataTypes, values, timestamps, + rcds.deviceId, rcds.measurements, rcds.dataTypes, rcds.values, rcds.timestamps, ) if err != nil { fatal("ProcessBatch error:%v", err) } + index = endIndex } // handle create timeseries SQL to insert tags for _, sql := range sqlList { From 70948554a3538b042438a13011cc5a6377387ef0 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Thu, 17 Nov 2022 13:56:36 +0800 Subject: [PATCH 24/47] fix typo parseDateToInterface -> parseDataToInterface fix typo parseDateToInterface -> parseDataToInterface --- cmd/tsbs_load_iotdb/scan.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/tsbs_load_iotdb/scan.go b/cmd/tsbs_load_iotdb/scan.go index de01c75e4..3dd36283b 100644 --- a/cmd/tsbs_load_iotdb/scan.go +++ b/cmd/tsbs_load_iotdb/scan.go @@ -33,7 +33,7 @@ func (p *iotdbPoint) generateTagsAttributesSQL() string { } // parse datatype and convert string into interface -func parseDateToInterface(datatype client.TSDataType, str string) (interface{}, error) { +func parseDataToInterface(datatype client.TSDataType, str string) (interface{}, error) { switch client.TSDataType(datatype) { case client.BOOLEAN: value, err := strconv.ParseBool(str) @@ -130,7 +130,7 @@ func parseFourLines(line1 string, line2 string, line3 string, line4 string) data value_index := type_index + 1 datatype, _ := strconv.ParseInt(line3_parts[type_index], 10, 8) dataTypes = append(dataTypes, client.TSDataType(datatype)) - value, err := parseDateToInterface(client.TSDataType(datatype), line2_parts[value_index]) + value, err := parseDataToInterface(client.TSDataType(datatype), line2_parts[value_index]) if err != nil { panic(fmt.Errorf("iotdb fileDataSource NextItem Parse error:%v", err)) } From 146f139b1883c1371777c14aa8908a860c674e09 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Thu, 17 Nov 2022 15:29:02 +0800 Subject: [PATCH 25/47] [important] support csv export support csv export but readme is not updated --- cmd/tsbs_load_iotdb/benchmark.go | 4 +- cmd/tsbs_load_iotdb/main.go | 15 ++- cmd/tsbs_load_iotdb/process.go | 120 +++++++++++++++++++----- pkg/targets/iotdb/implemented_target.go | 2 + pkg/targets/iotdb/serializer.go | 6 +- pkg/targets/iotdb/serializer_test.go | 2 +- 6 files changed, 117 insertions(+), 32 deletions(-) diff --git a/cmd/tsbs_load_iotdb/benchmark.go b/cmd/tsbs_load_iotdb/benchmark.go index c23291e75..5387046ad 100644 --- a/cmd/tsbs_load_iotdb/benchmark.go +++ b/cmd/tsbs_load_iotdb/benchmark.go @@ -36,7 +36,9 @@ func (b *iotdbBenchmark) GetPointIndexer(maxPartitions uint) targets.PointIndexe func (b *iotdbBenchmark) GetProcessor() targets.Processor { return &processor{ - recordsMaxRows: b.recordsMaxRows, + recordsMaxRows: b.recordsMaxRows, + loadToSCV: loadToSCV, + csvFilepathPrefix: csvFilepathPrefix, } } diff --git a/cmd/tsbs_load_iotdb/main.go b/cmd/tsbs_load_iotdb/main.go index 8296f4fbf..af10b4035 100644 --- a/cmd/tsbs_load_iotdb/main.go +++ b/cmd/tsbs_load_iotdb/main.go @@ -21,9 +21,11 @@ import ( // database option vars var ( - clientConfig client.Config - timeoutInMs int // 0 for no timeout - recordsMaxRows int // max rows of records in 'InsertRecords' + clientConfig client.Config + timeoutInMs int // 0 for no timeout + recordsMaxRows int // max rows of records in 'InsertRecords' + loadToSCV bool // if true, do NOT insert into databases, but generate csv files instead. + csvFilepathPrefix string // Prefix of filepath for csv files. Specific a folder or a folder with filename prefix. ) // Global vars @@ -58,9 +60,12 @@ func init() { port := viper.GetString("port") user := viper.GetString("user") password := viper.GetString("password") - workers := viper.GetUint("workers") - recordsMaxRows = viper.GetInt("records-max-rows") timeoutInMs = viper.GetInt("timeout") + recordsMaxRows = viper.GetInt("records-max-rows") + loadToSCV = viper.GetBool("to-csv") + csvFilepathPrefix = viper.GetString("csv-prefix") + + workers := viper.GetUint("workers") timeoutStr := fmt.Sprintf("timeout for session opening check: %d ms", timeoutInMs) if timeoutInMs <= 0 { diff --git a/cmd/tsbs_load_iotdb/process.go b/cmd/tsbs_load_iotdb/process.go index 18514b442..4d236d97b 100644 --- a/cmd/tsbs_load_iotdb/process.go +++ b/cmd/tsbs_load_iotdb/process.go @@ -2,16 +2,23 @@ package main import ( "fmt" + "os" + "strconv" + "strings" "github.com/apache/iotdb-client-go/client" "github.com/timescale/tsbs/pkg/targets" + "github.com/timescale/tsbs/pkg/targets/iotdb" ) type processor struct { numWorker int // the worker(like thread) ID of this processor session client.Session - recordsMaxRows int // max rows of records in 'InsertRecords' - ProcessedTagsDeviceIDMap map[string]bool // already processed device ID + recordsMaxRows int // max rows of records in 'InsertRecords' + ProcessedTagsDeviceIDMap map[string]bool // already processed device ID + loadToSCV bool // if true, do NOT insert into databases, but generate csv files instead. + csvFilepathPrefix string // Prefix of filepath for csv files. Specific a folder or a folder with filename prefix. + filePtrMap map[string]*os.File // file pointer for each deviceID } func (p *processor) Init(numWorker int, doLoad, _ bool) { @@ -26,6 +33,7 @@ func (p *processor) Init(numWorker int, doLoad, _ bool) { fatal(errMsg) } p.ProcessedTagsDeviceIDMap = make(map[string]bool, 1024) + p.filePtrMap = make(map[string]*os.File) } type records struct { @@ -61,34 +69,102 @@ func minInt(x int, y int) int { return y } +func getStringOfDatatype(datatype client.TSDataType) string { + switch datatype { + case client.BOOLEAN: + return "BOOLEAN" + case client.DOUBLE: + return "DOUBLE" + case client.FLOAT: + return "FLOAT" + case client.INT32: + return "INT32" + case client.INT64: + return "INT64" + case client.TEXT: + return "TEXT" + case client.UNKNOW: + return "UNKNOW" + default: + return "UNKNOW" + } +} + +func generateCSVHeader(point *iotdbPoint) string { + header := "Time" + for index, dataType := range point.dataTypes { + meta := fmt.Sprintf(",%s.%s(%s)", point.deviceID, point.measurements[index], + getStringOfDatatype(dataType)) + header = header + meta + } + header = header + "\n" + return header +} + +func generateCSVContent(point *iotdbPoint) string { + var valueList []string + valueList = append(valueList, strconv.FormatInt(point.timestamp, 10)) + for _, value := range point.values { + valueInStrByte, _ := iotdb.IotdbFormat(value) + valueList = append(valueList, string(valueInStrByte)) + } + content := strings.Join(valueList, ",") + content += "\n" + return content +} + func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (metricCount, rowCount uint64) { batch := b.(*iotdbBatch) // Write records if doLoad { - var sqlList []string - for index := 0; index < len(batch.points); { - startIndex := index - var endIndex int - if p.recordsMaxRows > 0 { - endIndex = minInt(len(batch.points), index+p.recordsMaxRows) - } else { - endIndex = len(batch.points) + if !p.loadToSCV { + var sqlList []string + for index := 0; index < len(batch.points); { + startIndex := index + var endIndex int + if p.recordsMaxRows > 0 { + endIndex = minInt(len(batch.points), index+p.recordsMaxRows) + } else { + endIndex = len(batch.points) + } + rcds, tempSqlList := p.pointsToRecords(batch.points[startIndex:endIndex]) + sqlList = append(sqlList, tempSqlList...) + _, err := p.session.InsertRecords( + rcds.deviceId, rcds.measurements, rcds.dataTypes, rcds.values, rcds.timestamps, + ) + if err != nil { + fatal("ProcessBatch error:%v", err) + } + index = endIndex } - rcds, tempSqlList := p.pointsToRecords(batch.points[startIndex:endIndex]) - sqlList = append(sqlList, tempSqlList...) - _, err := p.session.InsertRecords( - rcds.deviceId, rcds.measurements, rcds.dataTypes, rcds.values, rcds.timestamps, - ) - if err != nil { - fatal("ProcessBatch error:%v", err) + // handle create timeseries SQL to insert tags + for _, sql := range sqlList { + p.session.ExecuteUpdateStatement(sql) + } + } else { + for index := 0; index < len(batch.points); index++ { + point := batch.points[index] + _, exist := p.filePtrMap[point.deviceID] + if !exist { + // create file pointer + filepath := fmt.Sprintf("%s%s.csv", p.csvFilepathPrefix, point.deviceID) + filePtr, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY, 0777) + if err != nil { + fatal(fmt.Sprintf("ERROR occurs while creating csv file for deviceID: %s, filepath: %s", point.deviceID, filepath)) + panic(err) + } + p.filePtrMap[point.deviceID] = filePtr + // write header of this csv file + header := generateCSVHeader(point) + filePtr.WriteString(header) + } + filePtr := p.filePtrMap[point.deviceID] + pointRowInCSV := generateCSVContent(point) + filePtr.WriteString(pointRowInCSV) } - index = endIndex - } - // handle create timeseries SQL to insert tags - for _, sql := range sqlList { - p.session.ExecuteUpdateStatement(sql) } + } metricCount = batch.metrics diff --git a/pkg/targets/iotdb/implemented_target.go b/pkg/targets/iotdb/implemented_target.go index 62ae03d4f..dd0475007 100644 --- a/pkg/targets/iotdb/implemented_target.go +++ b/pkg/targets/iotdb/implemented_target.go @@ -23,6 +23,8 @@ func (t *iotdbTarget) TargetSpecificFlags(flagPrefix string, flagSet *pflag.Flag flagSet.String(flagPrefix+"password", "root", "The password for user connecting to IoTDB") flagSet.Int(flagPrefix+"timeout", 0, "Session timeout check in millisecond. Use 0 for no timeout.") flagSet.Int(flagPrefix+"records-max-rows", 0, "Max rows of 'InsertRecords'. Use 0 for no limit.") + flagSet.Bool(flagPrefix+"to-csv", false, "Do not insert into database, but to some csv files.") + flagSet.String(flagPrefix+"csv-prefix", "./", "Prefix of filepath for csv files. Specific a folder or a folder with filename prefix.") } func (t *iotdbTarget) TargetName() string { diff --git a/pkg/targets/iotdb/serializer.go b/pkg/targets/iotdb/serializer.go index 057a08c4a..9ca8dc58a 100644 --- a/pkg/targets/iotdb/serializer.go +++ b/pkg/targets/iotdb/serializer.go @@ -59,7 +59,7 @@ func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { // tempBuf = append(tempBuf, valueInStrByte...) // datatype_buf = append(datatype_buf, ',') // datatype_buf = append(datatype_buf, []byte(fmt.Sprintf("%d", datatype))...) - valueInStrByte, datatype := iotdbFormat(v) + valueInStrByte, datatype := IotdbFormat(v) if datatype == client.TEXT { tagStr := fmt.Sprintf(",%s='%s'", keyStr, string(valueInStrByte)) tags_buf = append(tags_buf, []byte(tagStr)...) @@ -84,7 +84,7 @@ func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { for i, v := range fieldValues { buf1 = append(buf1, ',') buf1 = append(buf1, fieldKeys[i]...) - valueInStrByte, datatype := iotdbFormat(v) + valueInStrByte, datatype := IotdbFormat(v) buf2 = append(buf2, ',') buf2 = append(buf2, valueInStrByte...) datatype_buf = append(datatype_buf, ',') @@ -122,7 +122,7 @@ func modifyHostname(hostname string) string { } // Utility function for appending various data types to a byte string -func iotdbFormat(v interface{}) ([]byte, client.TSDataType) { +func IotdbFormat(v interface{}) ([]byte, client.TSDataType) { switch v.(type) { case uint: return []byte(strconv.FormatInt(int64(v.(uint)), 10)), client.INT64 diff --git a/pkg/targets/iotdb/serializer_test.go b/pkg/targets/iotdb/serializer_test.go index 9b58a7f26..35b5ebe93 100644 --- a/pkg/targets/iotdb/serializer_test.go +++ b/pkg/targets/iotdb/serializer_test.go @@ -62,7 +62,7 @@ func TestIotdbFormat(t *testing.T) { } for _, c := range cases { t.Run(c.description, func(t *testing.T) { - actualByte, actualType := iotdbFormat(c.input) + actualByte, actualType := IotdbFormat(c.input) require.EqualValues(t, c.expectedByte, actualByte) require.EqualValues(t, c.expectedType, actualType) }) From a3b10a13dc91ad43d5f196aa5294be018f45cbfc Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Thu, 17 Nov 2022 19:57:27 +0800 Subject: [PATCH 26/47] make sure csv export can be used offline make sure csv export can be used offline if to-csv == true, then there is no need to connect to any database, because user want to generate csv files locally --- cmd/tsbs_load_iotdb/benchmark.go | 4 ++- cmd/tsbs_load_iotdb/creator.go | 47 ++++++++++++++++++++------------ cmd/tsbs_load_iotdb/process.go | 19 ++++++++----- 3 files changed, 44 insertions(+), 26 deletions(-) diff --git a/cmd/tsbs_load_iotdb/benchmark.go b/cmd/tsbs_load_iotdb/benchmark.go index 5387046ad..fdb5d140b 100644 --- a/cmd/tsbs_load_iotdb/benchmark.go +++ b/cmd/tsbs_load_iotdb/benchmark.go @@ -43,5 +43,7 @@ func (b *iotdbBenchmark) GetProcessor() targets.Processor { } func (b *iotdbBenchmark) GetDBCreator() targets.DBCreator { - return &dbCreator{} + return &dbCreator{ + loadToSCV: loadToSCV, + } } diff --git a/cmd/tsbs_load_iotdb/creator.go b/cmd/tsbs_load_iotdb/creator.go index 9df8506f3..e3e09c1e2 100644 --- a/cmd/tsbs_load_iotdb/creator.go +++ b/cmd/tsbs_load_iotdb/creator.go @@ -10,15 +10,19 @@ import ( // in preparation for running a benchmark against it. type dbCreator struct { - session client.Session + session client.Session + loadToSCV bool } func (d *dbCreator) Init() { - d.session = client.NewSession(&clientConfig) - if err := d.session.Open(false, timeoutInMs); err != nil { - errMsg := fmt.Sprintf("IoTDB dbCreator init error, session is not open: %v\n", err) - errMsg = errMsg + fmt.Sprintf("timeout setting: %d ms", timeoutInMs) - fatal(errMsg) + if !d.loadToSCV { + // no need to connect to database, because user want to generate csv files + d.session = client.NewSession(&clientConfig) + if err := d.session.Open(false, timeoutInMs); err != nil { + errMsg := fmt.Sprintf("IoTDB dbCreator init error, session is not open: %v\n", err) + errMsg = errMsg + fmt.Sprintf("timeout setting: %d ms", timeoutInMs) + fatal(errMsg) + } } } @@ -46,15 +50,18 @@ func (d *dbCreator) getAllStorageGroup() ([]string, error) { } func (d *dbCreator) DBExists(dbName string) bool { - sgList, err := d.getAllStorageGroup() - if err != nil { - fatal("DBExists error: %v", err) - return false - } - sg := fmt.Sprintf("root.%s", dbName) - for _, thisSG := range sgList { - if thisSG == sg { - return true + if !d.loadToSCV { + // no need to connect to database, because user want to generate csv files + sgList, err := d.getAllStorageGroup() + if err != nil { + fatal("DBExists error: %v", err) + return false + } + sg := fmt.Sprintf("root.%s", dbName) + for _, thisSG := range sgList { + if thisSG == sg { + return true + } } } return false @@ -65,7 +72,11 @@ func (d *dbCreator) CreateDB(dbName string) error { } func (d *dbCreator) RemoveOldDB(dbName string) error { - sg := fmt.Sprintf("root.%s", dbName) - _, err := d.session.DeleteStorageGroup(sg) - return err + if !d.loadToSCV { + // no need to connect to database, because user want to generate csv files + sg := fmt.Sprintf("root.%s", dbName) + _, err := d.session.DeleteStorageGroup(sg) + return err + } + return nil } diff --git a/cmd/tsbs_load_iotdb/process.go b/cmd/tsbs_load_iotdb/process.go index 4d236d97b..0b1875a6f 100644 --- a/cmd/tsbs_load_iotdb/process.go +++ b/cmd/tsbs_load_iotdb/process.go @@ -26,14 +26,17 @@ func (p *processor) Init(numWorker int, doLoad, _ bool) { if !doLoad { return } - p.session = client.NewSession(&clientConfig) - if err := p.session.Open(false, timeoutInMs); err != nil { - errMsg := fmt.Sprintf("IoTDB processor init error, session is not open: %v\n", err) - errMsg = errMsg + fmt.Sprintf("timeout setting: %d ms", timeoutInMs) - fatal(errMsg) + if p.loadToSCV { + p.ProcessedTagsDeviceIDMap = make(map[string]bool, 1024) + p.filePtrMap = make(map[string]*os.File) + } else { + p.session = client.NewSession(&clientConfig) + if err := p.session.Open(false, timeoutInMs); err != nil { + errMsg := fmt.Sprintf("IoTDB processor init error, session is not open: %v, ", err) + errMsg = errMsg + fmt.Sprintf("timeout setting: %d ms\n", timeoutInMs) + fatal(errMsg) + } } - p.ProcessedTagsDeviceIDMap = make(map[string]bool, 1024) - p.filePtrMap = make(map[string]*os.File) } type records struct { @@ -119,6 +122,7 @@ func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (metricCount, row // Write records if doLoad { if !p.loadToSCV { + // insert records into the database var sqlList []string for index := 0; index < len(batch.points); { startIndex := index @@ -143,6 +147,7 @@ func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (metricCount, row p.session.ExecuteUpdateStatement(sql) } } else { + // generate csv files. There is no requirement to connect to any database for index := 0; index < len(batch.points); index++ { point := batch.points[index] _, exist := p.filePtrMap[point.deviceID] From bedfeba67536422e97fb91b5b5e6d06863406688 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Thu, 17 Nov 2022 21:11:30 +0800 Subject: [PATCH 27/47] updated docs and comments. added arguments check. init bug fixed updated docs and comments. added arguments check init bug fixed: forget to init ProcessedTagsDeviceIDMap in Init. --- cmd/tsbs_load_iotdb/main.go | 6 ++ cmd/tsbs_load_iotdb/process.go | 13 +-- cmd/tsbs_load_iotdb/scan.go | 3 +- docs/iotdb.md | 112 ++++++++++++++++++++++-- pkg/targets/iotdb/implemented_target.go | 4 +- pkg/targets/iotdb/serializer.go | 3 +- 6 files changed, 124 insertions(+), 17 deletions(-) diff --git a/cmd/tsbs_load_iotdb/main.go b/cmd/tsbs_load_iotdb/main.go index af10b4035..56eb75f92 100644 --- a/cmd/tsbs_load_iotdb/main.go +++ b/cmd/tsbs_load_iotdb/main.go @@ -76,6 +76,12 @@ func init() { if workers < 2 { log.Println("Insertion throughput is strongly related to the number of threads. Use more workers for better performance.") } + if loadToSCV && workers != 1 { + err_msg := "Arguments conflicts! When using csv export method, `workers` should NOT be set more than 1. " + err_msg += fmt.Sprintf("Current setting: `to-csv`=%v, `workers`=%d.", loadToSCV, workers) + log.Println(err_msg) + panic(err_msg) + } clientConfig = client.Config{ Host: host, diff --git a/cmd/tsbs_load_iotdb/process.go b/cmd/tsbs_load_iotdb/process.go index 0b1875a6f..a1ed7ed24 100644 --- a/cmd/tsbs_load_iotdb/process.go +++ b/cmd/tsbs_load_iotdb/process.go @@ -14,11 +14,12 @@ import ( type processor struct { numWorker int // the worker(like thread) ID of this processor session client.Session - recordsMaxRows int // max rows of records in 'InsertRecords' - ProcessedTagsDeviceIDMap map[string]bool // already processed device ID - loadToSCV bool // if true, do NOT insert into databases, but generate csv files instead. - csvFilepathPrefix string // Prefix of filepath for csv files. Specific a folder or a folder with filename prefix. - filePtrMap map[string]*os.File // file pointer for each deviceID + recordsMaxRows int // max rows of records in 'InsertRecords' + ProcessedTagsDeviceIDMap map[string]bool // already processed device ID + + loadToSCV bool // if true, do NOT insert into databases, but generate csv files instead. + csvFilepathPrefix string // Prefix of filepath for csv files. Specific a folder or a folder with filename prefix. + filePtrMap map[string]*os.File // file pointer for each deviceID } func (p *processor) Init(numWorker int, doLoad, _ bool) { @@ -27,9 +28,9 @@ func (p *processor) Init(numWorker int, doLoad, _ bool) { return } if p.loadToSCV { - p.ProcessedTagsDeviceIDMap = make(map[string]bool, 1024) p.filePtrMap = make(map[string]*os.File) } else { + p.ProcessedTagsDeviceIDMap = make(map[string]bool) p.session = client.NewSession(&clientConfig) if err := p.session.Open(false, timeoutInMs); err != nil { errMsg := fmt.Sprintf("IoTDB processor init error, session is not open: %v, ", err) diff --git a/cmd/tsbs_load_iotdb/scan.go b/cmd/tsbs_load_iotdb/scan.go index 3dd36283b..0744c5444 100644 --- a/cmd/tsbs_load_iotdb/scan.go +++ b/cmd/tsbs_load_iotdb/scan.go @@ -65,14 +65,15 @@ type fileDataSource struct { // read new four line, which store one data point // e.g., -// e.g., // deviceID,timestamp,,,,... // ,,,,,... // datatype,,,,... +// tags,=,=,... // // deviceID,timestamp,hostname,value // root.cpu.host_1,1451606400000000000,'host_1',44.0 // datatype,5,2 +// tags,region='eu-west-1',datacenter='eu-west-1c',rack='87', // // return : bool -> true means got one point, else reaches EOF or error happens func (d *fileDataSource) nextFourLines() (bool, string, string, string, string, error) { diff --git a/docs/iotdb.md b/docs/iotdb.md index d137c4a15..ac6c58d4a 100644 --- a/docs/iotdb.md +++ b/docs/iotdb.md @@ -16,8 +16,10 @@ and additional flags available for the query runner (`tsbs_run_queries_iotdb`). Data generated by `tsbs_generate_data` for IoTDB is serialized in a "pseudo-CSV" format, along with a custom header at the beginning: -* one line composed of a comma-separated list of "deviceID" and fieldNames -* one line composed of a comma-separated list of deviceID and fieldValues +* one line composed of a comma-separated list of "deviceID" and field names +* one line composed of a comma-separated list of deviceID and field values +* one line composed of a comma-separated list of "datatype" and data types +* one line composed of a comma-separated list of tags For example: @@ -25,31 +27,67 @@ For example: deviceID,timestamp,,,,... ,,,,,... datatype,,,,... +tags,=,=,... deviceID,timestamp,hostname,value root.cpu.host_1,1451606400000000000,'host_1',44.0 datatype,5,2 +tags,region='eu-west-1',datacenter='eu-west-1c',rack='87', ``` `deviceID` describes the storage path which is composed of several nodes. -IoTDB uses storage group to manage data. For example, in test case `devops`, +IoTDB uses storage groups to manage data. For example, in test case `devops`, if `measurementName` is `cpu` while `hostname` tag is `host_0`, then `deviceID` is `root.cpu.host_0`. -IoTDB treat tags as fields, except primary key like `hostname` in `devops`. +`hostname` in `devops` test cases is one of primary keys, and each `hostname` +specifies a unique device, so it's selected to be part of `deviceID`. But other +tags are not so important and their values do not change during whole testing, +so they are designed to be stored in a node named `_tags` as attributes. +For more detail about how IoTDB manage attributes of a node, please take a look +at [Timeseries Management](https://iotdb.apache.org/UserGuide/Master/Operate-Metadata/Timeseries.html#create-timeseries). + The unit of timestamp in generated data is nanosecond, but it will be converted into millisecond before insert into database. An example for the `cpu-only` use case: ```text -deviceID,timestamp,region,datacenter,rack,os,arch,team,service,service_version,service_environment,usage_user,usage_system,usage_idle,usage_nice,usage_iowait,usage_irq,usage_softirq,usage_steal,usage_guest,usage_guest_nice -root.cpu.host_0,1451606400000000000,eu-west-1,eu-west-1c,87,Ubuntu16.04LTS,x64,NYC,18,1,production,58,2,24,61,22,63,6,44,80,38 -datatype,5,5,5,5,5,5,5,5,5,2,2,2,2,2,2,2,2,2,2 +deviceID,timestamp,usage_user,usage_system,usage_idle,usage_nice,usage_iowait,usage_irq,usage_softirq,usage_steal,usage_guest,usage_guest_nice +root.cpu.host_0,1451606400000000000,58,2,24,61,22,63,6,44,80,38 +datatype,2,2,2,2,2,2,2,2,2,2 +tags,region='eu-west-1',datacenter='eu-west-1c',rack='87',os='Ubuntu16.04LTS',arch='x64',team='NYC',service='18',service_version='1',service_environment='production' ``` --- +## CSV Export + +`tsbs_load_iotdb` provides CSV export function. User can export any test cases +into CSV files. In this method, `tsbs_load_iotdb` will NOT open any session, +and will NOT connect to database. It will create some CSV files and store data +into them instead. The amount of CSV files is equal to the amount of storage +groups. For example, if `sacle==4000` in `devops` test cases, there will be +`4,000 * 9 = 36,000` storage groups and 36,000 CSV files. + +An example for a CSV file: + +```text +Time,root.cpu.host_0.usage_user(INT64),root.cpu.host_0.usage_system(INT64),root.cpu.host_0.usage_idle(INT64),root.cpu.host_0.usage_nice(INT64),root.cpu.host_0.usage_iowait(INT64),root.cpu.host_0.usage_irq(INT64),root.cpu.host_0.usage_softirq(INT64),root.cpu.host_0.usage_steal(INT64),root.cpu.host_0.usage_guest(INT64),root.cpu.host_0.usage_guest_nice(INT64) +1451606400000,58,2,24,61,22,63,6,44,80,38 +1451606460000,58,1,24,62,22,61,7,48,80,38 +1451606520000,59,0,24,63,21,61,6,48,79,38 +1451606580000,59,1,25,62,22,60,6,49,80,36 +1451606640000,58,1,24,62,22,60,8,49,80,35 +1451606700000,58,2,24,62,21,61,8,48,79,36 +``` + +In this CSV file, the data aligned by time, and headers with data type. For more +information about this format, please take a look at +[Sample CSV file to be imported](https://iotdb.apache.org/UserGuide/Master/Write-Data/CSV-Tool.html#sample-csv-file-to-be-imported). + +--- + ## `tsbs_load_iotdb` Additional Flags ### IoTDB related @@ -77,6 +115,66 @@ client will try to get response from database server to make sure the connection is established. This argument is the session timeout in millisecond. Use 0 for no timeout. +#### `-records-max-rows` (type: `int`, default: `0`) + +Session use `insertRecords` to insert data into database. This argument is the +max size (max rows) of records. Use 0 for no limit. If this argument is 0, +`tsbs_load_iotdb` will insert all data in a batch at one time, that means the +behavior is equal to setting this arguments as `batch-size`. + +**Warning!** If this value is set too small, the insertion performance will be +seriously reduced. + +#### `-to-csv` (type: `bool`, default: `false`) + +This argument is relative to CSV export function. If this argument is `false`, +then `tsbs_load_iotdb` will insert data into IoTDB database. Else it will +generate CSV files locally without any connect to database. Therefore, if this +argument is `true`, 6 arguments are meaningless. + +**Warning!** While this argument is `true`, do NOT set `worker` more than 1. +Because file writing is a sequential operation. + +#### `-csv-prefix` (type: `string`, default: `./`) + +This is prefix of filepath for CSV files. Specific a folder or a folder with +filename prefix. +For example, if it's set to `/home/`, all exported CSV files will be stored in +folder `/home/`, like this: + +```text +|-- home + |-- root.cpu.host_0.csv + |-- root.cpu.host_1.csv + |-- root.cpu.host_2.csv + |-- root.cpu.host_3.csv + ... + |-- root.disk.host_0.csv + |-- root.disk.host_1.csv + |-- root.disk.host_2.csv + ... +``` + +For another example, if it's set to `/home/iotdb-data-`, all exported CSV files +will be stored in folder `/home/` with prefix `iotdb-data-`, like this: + +```text +|-- home + |-- iotdb-data-root.cpu.host_0.csv + |-- iotdb-data-root.cpu.host_1.csv + |-- iotdb-data-root.cpu.host_2.csv + |-- iotdb-data-root.cpu.host_3.csv + ... + |-- iotdb-data-root.disk.host_0.csv + |-- iotdb-data-root.disk.host_1.csv + |-- iotdb-data-root.disk.host_2.csv + ... +``` + +**Warning!** `tsbs_load_iotdb` uses native string connection to complete above +works, so please make sure folders in those paths are exist. That means user +should create those folders manually. + --- ## `tsbs_run_queries_iotdb` Additional Flags diff --git a/pkg/targets/iotdb/implemented_target.go b/pkg/targets/iotdb/implemented_target.go index dd0475007..3d0453a4a 100644 --- a/pkg/targets/iotdb/implemented_target.go +++ b/pkg/targets/iotdb/implemented_target.go @@ -23,8 +23,8 @@ func (t *iotdbTarget) TargetSpecificFlags(flagPrefix string, flagSet *pflag.Flag flagSet.String(flagPrefix+"password", "root", "The password for user connecting to IoTDB") flagSet.Int(flagPrefix+"timeout", 0, "Session timeout check in millisecond. Use 0 for no timeout.") flagSet.Int(flagPrefix+"records-max-rows", 0, "Max rows of 'InsertRecords'. Use 0 for no limit.") - flagSet.Bool(flagPrefix+"to-csv", false, "Do not insert into database, but to some csv files.") - flagSet.String(flagPrefix+"csv-prefix", "./", "Prefix of filepath for csv files. Specific a folder or a folder with filename prefix.") + flagSet.Bool(flagPrefix+"to-csv", false, "Do not insert into database, but to some CSV files.") + flagSet.String(flagPrefix+"csv-prefix", "./", "Prefix of filepath for CSV files. Specific a folder or a folder with filename prefix.") } func (t *iotdbTarget) TargetName() string { diff --git a/pkg/targets/iotdb/serializer.go b/pkg/targets/iotdb/serializer.go index 9ca8dc58a..4495937dd 100644 --- a/pkg/targets/iotdb/serializer.go +++ b/pkg/targets/iotdb/serializer.go @@ -31,7 +31,8 @@ const defaultBufSize = 4096 // deviceID,timestamp,hostname,value // root.cpu.host_1,1451606400000000000,'host_1',44.0 // datatype,5,2 -// tags,region='eu-west-1',datacenter='eu-west-1c',rack=87, +// tags,region='eu-west-1',datacenter='eu-west-1c',rack='87' +// func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { // Tag row first, prefixed with 'time,path' buf1 := make([]byte, 0, defaultBufSize) From b4df7a6d308eb9160225a2b08a8f2bc7d9113233 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Mon, 21 Nov 2022 20:45:07 +0800 Subject: [PATCH 28/47] updated test cases for scan.go and serializer.go updated test cases for scan.go and serializer.go return empty string while there is no tags --- cmd/tsbs_load_iotdb/scan.go | 11 ++++- cmd/tsbs_load_iotdb/scan_test.go | 69 +++++++++++++++++++++++++++- pkg/targets/iotdb/serializer_test.go | 8 ++-- 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/cmd/tsbs_load_iotdb/scan.go b/cmd/tsbs_load_iotdb/scan.go index 0744c5444..7bbb034ac 100644 --- a/cmd/tsbs_load_iotdb/scan.go +++ b/cmd/tsbs_load_iotdb/scan.go @@ -25,8 +25,11 @@ type iotdbPoint struct { fieldsCnt uint64 } -// CRTODO:使用这个函数来生成创建语句。 func (p *iotdbPoint) generateTagsAttributesSQL() string { + if p.tagString == "" { + // no tags for this host. This is not a normal behavior in benchmark. + return fmt.Sprintf("CREATE timeseries %s._tags with datatype=INT32, encoding=RLE, compression=SNAPPY", p.deviceID) + } sql := "CREATE timeseries %s._tags with datatype=INT32, encoding=RLE, compression=SNAPPY attributes(%s)" // sql2 := "ALTER timeseries %s._tags UPSERT attributes(%s)" return fmt.Sprintf(sql, p.deviceID, p.tagString) @@ -137,6 +140,10 @@ func parseFourLines(line1 string, line2 string, line3 string, line4 string) data } values = append(values, value) } + tagString := "" + if len(line4_parts) > 1 { + tagString = line4_parts[1] + } return data.NewLoadedPoint( &iotdbPoint{ deviceID: line2_parts[0], @@ -144,7 +151,7 @@ func parseFourLines(line1 string, line2 string, line3 string, line4 string) data measurements: measurements, values: values, dataTypes: dataTypes, - tagString: line4_parts[1], + tagString: tagString, fieldsCnt: uint64(len(line1_parts) - 2), }) } diff --git a/cmd/tsbs_load_iotdb/scan_test.go b/cmd/tsbs_load_iotdb/scan_test.go index d9e81d659..ac996a06f 100644 --- a/cmd/tsbs_load_iotdb/scan_test.go +++ b/cmd/tsbs_load_iotdb/scan_test.go @@ -7,6 +7,53 @@ import ( "github.com/stretchr/testify/require" ) +func TestGenerateTagsAttributesSQL(t *testing.T) { + cases := []struct { + description string + point iotdbPoint + expectedSQL string + }{ + { + description: "no tags", + point: iotdbPoint{ + deviceID: "root.cpu.host_9", + tagString: "", + }, + expectedSQL: "CREATE timeseries root.cpu.host_9._tags with datatype=INT32, encoding=RLE, compression=SNAPPY", + }, + { + description: "one tag", + point: iotdbPoint{ + deviceID: "root.cpu.host_0", + tagString: "key1='value'", + }, + expectedSQL: "CREATE timeseries root.cpu.host_0._tags with datatype=INT32, encoding=RLE, compression=SNAPPY attributes(key1='value')", + }, + { + description: "one tag that type is int", + point: iotdbPoint{ + deviceID: "root.cpu.host_2", + tagString: "key1=123", + }, + expectedSQL: "CREATE timeseries root.cpu.host_2._tags with datatype=INT32, encoding=RLE, compression=SNAPPY attributes(key1=123)", + }, + { + description: "two tags", + point: iotdbPoint{ + deviceID: "root.cpu.host_5", + tagString: "region='eu-west-1',datacenter='eu-west-1b'", + }, + expectedSQL: "CREATE timeseries root.cpu.host_5._tags with datatype=INT32, encoding=RLE, compression=SNAPPY attributes(region='eu-west-1',datacenter='eu-west-1b')", + }, + } + for _, c := range cases { + t.Run(c.description, func(t *testing.T) { + actual := c.point.generateTagsAttributesSQL() + require.EqualValues(t, c.expectedSQL, actual) + }) + } +} + func TestGenerateInsertStatement(t *testing.T) { cases := []struct { description string @@ -27,6 +74,7 @@ func TestGenerateInsertStatement(t *testing.T) { measurements: []string{"value"}, values: []interface{}{float64(3.1415926)}, dataTypes: []client.TSDataType{client.DOUBLE}, + tagString: "", fieldsCnt: 1, }, }, @@ -44,6 +92,25 @@ func TestGenerateInsertStatement(t *testing.T) { measurements: []string{"floatV", "strV", "int64V", "int32V", "boolV"}, values: []interface{}{float64(3.1415926), string("hello"), int64(123), int32(123), true}, dataTypes: []client.TSDataType{client.DOUBLE, client.TEXT, client.INT64, client.INT32, client.BOOLEAN}, + tagString: "", + fieldsCnt: 5, + }, + }, + { + description: "one point with different dataTypes", + lines: []string{ + "deviceID,timestamp,floatV,strV,int64V,int32V,boolV", + "root.cpu.host_0,1451606400000000000,3.1415926,hello,123,123,true", + "datatype,4,5,2,1,0", + "tags,region='eu-west-1',datacenter='eu-west-1b'", + }, + expected: iotdbPoint{ + deviceID: "root.cpu.host_0", + timestamp: 1451606400000, + measurements: []string{"floatV", "strV", "int64V", "int32V", "boolV"}, + values: []interface{}{float64(3.1415926), string("hello"), int64(123), int32(123), true}, + dataTypes: []client.TSDataType{client.DOUBLE, client.TEXT, client.INT64, client.INT32, client.BOOLEAN}, + tagString: "region='eu-west-1',datacenter='eu-west-1b'", fieldsCnt: 5, }, }, @@ -51,7 +118,7 @@ func TestGenerateInsertStatement(t *testing.T) { for _, c := range cases { t.Run(c.description, func(t *testing.T) { - require.True(t, len(c.lines) == 3) + require.True(t, len(c.lines) == 4) actual := parseFourLines(c.lines[0], c.lines[1], c.lines[2], c.lines[3]) require.EqualValues(t, &c.expected, actual.Data.(*iotdbPoint)) }) diff --git a/pkg/targets/iotdb/serializer_test.go b/pkg/targets/iotdb/serializer_test.go index 35b5ebe93..0bbc13191 100644 --- a/pkg/targets/iotdb/serializer_test.go +++ b/pkg/targets/iotdb/serializer_test.go @@ -79,22 +79,22 @@ func TestSerialize_001(t *testing.T) { { description: "a regular point ", inputPoint: serialize.TestPointDefault(), - expected: "deviceID,timestamp,region,datacenter,usage_guest_nice\nroot.cpu.host_0,1451606400000000000,'eu-west-1','eu-west-1b',38.24311829\ndatatype,5,5,4\n", + expected: "deviceID,timestamp,usage_guest_nice\nroot.cpu.host_0,1451606400000000000,38.24311829\ndatatype,4\ntags,region='eu-west-1',datacenter='eu-west-1b'\n", }, { description: "a regular Point using int as value", inputPoint: serialize.TestPointInt(), - expected: "deviceID,timestamp,region,datacenter,usage_guest\nroot.cpu.host_0,1451606400000000000,'eu-west-1','eu-west-1b',38\ndatatype,5,5,2\n", + expected: "deviceID,timestamp,usage_guest\nroot.cpu.host_0,1451606400000000000,38\ndatatype,2\ntags,region='eu-west-1',datacenter='eu-west-1b'\n", }, { description: "a regular Point with multiple fields", inputPoint: serialize.TestPointMultiField(), - expected: "deviceID,timestamp,region,datacenter,big_usage_guest,usage_guest,usage_guest_nice\nroot.cpu.host_0,1451606400000000000,'eu-west-1','eu-west-1b',5000000000,38,38.24311829\ndatatype,5,5,2,2,4\n", + expected: "deviceID,timestamp,big_usage_guest,usage_guest,usage_guest_nice\nroot.cpu.host_0,1451606400000000000,5000000000,38,38.24311829\ndatatype,2,2,4\ntags,region='eu-west-1',datacenter='eu-west-1b'\n", }, { description: "a Point with no tags", inputPoint: serialize.TestPointNoTags(), - expected: "deviceID,timestamp,usage_guest_nice\nroot.cpu.unknown,1451606400000000000,38.24311829\ndatatype,4\n", + expected: "deviceID,timestamp,usage_guest_nice\nroot.cpu.unknown,1451606400000000000,38.24311829\ndatatype,4\ntags\n", }, } for _, c := range cases { From 9d00fa0c49bb792e3df77477fbbd614f1e135a4d Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Mon, 21 Nov 2022 21:59:27 +0800 Subject: [PATCH 29/47] correct SQL queries; support printing responses correct SQL queries. support printing queries responses. --- .../databases/iotdb/devops.go | 16 +++---- .../databases/iotdb/devops_test.go | 20 ++++----- cmd/tsbs_run_queries_iotdb/main.go | 43 ++++++++++++++++++- 3 files changed, 57 insertions(+), 22 deletions(-) diff --git a/cmd/tsbs_generate_queries/databases/iotdb/devops.go b/cmd/tsbs_generate_queries/databases/iotdb/devops.go index da0b04660..84ac4c0e8 100644 --- a/cmd/tsbs_generate_queries/databases/iotdb/devops.go +++ b/cmd/tsbs_generate_queries/databases/iotdb/devops.go @@ -94,13 +94,12 @@ func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange t selectClause := d.getSelectClausesAggMetricsString("MAX_VALUE", metrics) fromHosts := d.getHostFromString(nHosts) - humanLabel := fmt.Sprintf("IoTDB %d cpu metric(s), random %4d hosts, random %s by 1m", numMetrics, nHosts, timeRange) + humanLabel := fmt.Sprintf("IoTDB %d cpu metric(s), random %4d hosts, random %s by 5m", numMetrics, nHosts, timeRange) humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) sql := "" sql = sql + fmt.Sprintf("SELECT %s", selectClause) sql = sql + fmt.Sprintf(" FROM %s", fromHosts) - // sql = sql + fmt.Sprintf(" WHERE time >= %s AND time < %s", interval.StartString(), interval.EndString()) - sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1m)", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt)) + sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 5m)", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt)) d.fillInQuery(qi, humanLabel, humanDesc, sql) } @@ -123,8 +122,7 @@ func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) { sql := "" sql = sql + fmt.Sprintf("SELECT %s", selectClause) sql = sql + fmt.Sprintf(" FROM %s.cpu.*", d.BasicPath) - // sql = sql + fmt.Sprintf(" WHERE time >= %s AND time < %s", interval.StartString(), interval.EndString()) - sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1h), LEVEL = %d", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt), d.BasicPathLevel+2) + sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1h)", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt)) d.fillInQuery(qi, humanLabel, humanDesc, sql) } @@ -155,7 +153,6 @@ func (d *Devops) MaxAllCPU(qi query.Query, nHosts int, duration time.Duration) { sql := "" sql = sql + fmt.Sprintf("SELECT %s", selectClause) sql = sql + fmt.Sprintf(" FROM %s", fromHosts) - // sql = sql + fmt.Sprintf(" WHERE time >= %s AND time < %s", interval.StartString(), interval.EndString()) sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1h)", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt)) d.fillInQuery(qi, humanLabel, humanDesc, sql) @@ -173,9 +170,8 @@ func (d *Devops) GroupByOrderByLimit(qi query.Query) { sql := "" sql = sql + fmt.Sprintf("SELECT %s", selectClause) sql = sql + fmt.Sprintf(" FROM %s.cpu.*", d.BasicPath) - // sql = sql + fmt.Sprintf(" WHERE time < %s", interval.EndString()) - sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1m)", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt)) - sql = sql + " LIMIT 5" + sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1m), LEVEL = %d", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt), d.BasicPathLevel+1) + sql = sql + " ORDER BY TIME DESC LIMIT 5" humanLabel := "IoTDB max cpu over last 5 min-intervals (random end)" humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) @@ -207,7 +203,7 @@ func (d *Devops) HighCPUForHosts(qi query.Query, nHosts int) { sql := "SELECT *" sql = sql + fmt.Sprintf(" FROM %s", fromHosts) - sql = sql + fmt.Sprintf(" WHERE usage_user > 90.0 AND time >= %s AND time < %s", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt)) + sql = sql + fmt.Sprintf(" WHERE usage_user > 90 AND time >= %s AND time < %s", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt)) d.fillInQuery(qi, humanLabel, humanDesc, sql) } diff --git a/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go b/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go index eed20054c..3192ef56a 100644 --- a/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go +++ b/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go @@ -158,9 +158,9 @@ func TestGroupByTime(t *testing.T) { actual := dp.GenerateEmptyQuery() expected := dp.GenerateEmptyQuery() dp.fillInQuery(expected, - "IoTDB 1 cpu metric(s), random 1 hosts, random 1s by 1m", - "IoTDB 1 cpu metric(s), random 1 hosts, random 1s by 1m: 1970-01-01T00:05:58Z", - "SELECT MAX_VALUE(usage_user) FROM root.sg.cpu.host_9 GROUP BY ([1970-01-01 00:05:58, 1970-01-01 00:05:59), 1m)", + "IoTDB 1 cpu metric(s), random 1 hosts, random 1s by 5m", + "IoTDB 1 cpu metric(s), random 1 hosts, random 1s by 5m: 1970-01-01T00:05:58Z", + "SELECT MAX_VALUE(usage_user) FROM root.sg.cpu.host_9 GROUP BY ([1970-01-01 00:05:58, 1970-01-01 00:05:59), 5m)", ) dp.GroupByTime(actual, nHosts, metrics, duration) @@ -182,7 +182,7 @@ func TestGroupByTimeAndPrimaryTag(t *testing.T) { baseGenerator: BaseGenerator{BasicPath: "root.sg", BasicPathLevel: 1}, expectedHumanLabel: "IoTDB mean of 1 metrics, all hosts, random 12h0m0s by 1h", expectedHumanDesc: "IoTDB mean of 1 metrics, all hosts, random 12h0m0s by 1h: 1970-01-01T00:16:22Z", - expectedSQLQuery: "SELECT AVG(usage_user) FROM root.sg.cpu.* GROUP BY ([1970-01-01 00:16:22, 1970-01-01 12:16:22), 1h), LEVEL = 3", + expectedSQLQuery: "SELECT AVG(usage_user) FROM root.sg.cpu.* GROUP BY ([1970-01-01 00:16:22, 1970-01-01 12:16:22), 1h)", }, { description: "5 metric with storage group 'root'", @@ -190,7 +190,7 @@ func TestGroupByTimeAndPrimaryTag(t *testing.T) { baseGenerator: BaseGenerator{BasicPath: "root", BasicPathLevel: 0}, expectedHumanLabel: "IoTDB mean of 5 metrics, all hosts, random 12h0m0s by 1h", expectedHumanDesc: "IoTDB mean of 5 metrics, all hosts, random 12h0m0s by 1h: 1970-01-01T00:16:22Z", - expectedSQLQuery: "SELECT AVG(usage_user), AVG(usage_system), AVG(usage_idle), AVG(usage_nice), AVG(usage_iowait) FROM root.cpu.* GROUP BY ([1970-01-01 00:16:22, 1970-01-01 12:16:22), 1h), LEVEL = 2", + expectedSQLQuery: "SELECT AVG(usage_user), AVG(usage_system), AVG(usage_idle), AVG(usage_nice), AVG(usage_iowait) FROM root.cpu.* GROUP BY ([1970-01-01 00:16:22, 1970-01-01 12:16:22), 1h)", }, } @@ -292,7 +292,7 @@ func TestMaxAllCPU(t *testing.T) { func TestGroupByOrderByLimit(t *testing.T) { rand.Seed(123) // Setting seed for testing purposes. - base := BaseGenerator{BasicPath: "root.ln", BasicPathLevel: 1} + base := BaseGenerator{BasicPath: "root", BasicPathLevel: 0} start := time.Unix(0, 0) end := start.Add(2 * time.Hour) queryGenerator, err := base.NewDevops(start, end, 10) @@ -304,7 +304,7 @@ func TestGroupByOrderByLimit(t *testing.T) { dp.fillInQuery(expected, "IoTDB max cpu over last 5 min-intervals (random end)", "IoTDB max cpu over last 5 min-intervals (random end): 1970-01-01T00:16:22Z", - "SELECT MAX_VALUE(usage_user) FROM root.ln.cpu.* GROUP BY ([1970-01-01 00:16:22, 1970-01-01 01:16:22), 1m) LIMIT 5", + "SELECT MAX_VALUE(usage_user) FROM root.cpu.* GROUP BY ([1970-01-01 00:16:22, 1970-01-01 01:16:22), 1m), LEVEL = 1 ORDER BY TIME DESC LIMIT 5", ) dp.GroupByOrderByLimit(actual) @@ -326,7 +326,7 @@ func TestHighCPUForHosts(t *testing.T) { baseGenerator: BaseGenerator{BasicPath: "root", BasicPathLevel: 0}, expectedHumanLabel: "IoTDB CPU over threshold, all hosts", expectedHumanDesc: "IoTDB CPU over threshold, all hosts: 1970-01-01T00:16:22Z", - expectedSQLQuery: "SELECT * FROM root.cpu.* WHERE usage_user > 90.0 AND time >= 1970-01-01 00:16:22 AND time < 1970-01-01 12:16:22", + expectedSQLQuery: "SELECT * FROM root.cpu.* WHERE usage_user > 90 AND time >= 1970-01-01 00:16:22 AND time < 1970-01-01 12:16:22", }, { description: "1 host with storage group 'root.sg.abc'", @@ -334,7 +334,7 @@ func TestHighCPUForHosts(t *testing.T) { baseGenerator: BaseGenerator{BasicPath: "root.sg.abc", BasicPathLevel: 2}, expectedHumanLabel: "IoTDB CPU over threshold, 1 host(s)", expectedHumanDesc: "IoTDB CPU over threshold, 1 host(s): 1970-01-01T00:16:22Z", - expectedSQLQuery: "SELECT * FROM root.sg.abc.cpu.host_9 WHERE usage_user > 90.0 AND time >= 1970-01-01 00:16:22 AND time < 1970-01-01 12:16:22", + expectedSQLQuery: "SELECT * FROM root.sg.abc.cpu.host_9 WHERE usage_user > 90 AND time >= 1970-01-01 00:16:22 AND time < 1970-01-01 12:16:22", }, { description: "5 host2 with storage group 'root.ln'", @@ -342,7 +342,7 @@ func TestHighCPUForHosts(t *testing.T) { baseGenerator: BaseGenerator{BasicPath: "root.ln", BasicPathLevel: 1}, expectedHumanLabel: "IoTDB CPU over threshold, 5 host(s)", expectedHumanDesc: "IoTDB CPU over threshold, 5 host(s): 1970-01-01T00:16:22Z", - expectedSQLQuery: "SELECT * FROM root.ln.cpu.host_9, root.ln.cpu.host_3, root.ln.cpu.host_5, root.ln.cpu.host_1, root.ln.cpu.host_7 WHERE usage_user > 90.0 AND time >= 1970-01-01 00:16:22 AND time < 1970-01-01 12:16:22", + expectedSQLQuery: "SELECT * FROM root.ln.cpu.host_9, root.ln.cpu.host_3, root.ln.cpu.host_5, root.ln.cpu.host_1, root.ln.cpu.host_7 WHERE usage_user > 90 AND time >= 1970-01-01 00:16:22 AND time < 1970-01-01 12:16:22", }, } diff --git a/cmd/tsbs_run_queries_iotdb/main.go b/cmd/tsbs_run_queries_iotdb/main.go index 272cdf115..778542d51 100644 --- a/cmd/tsbs_run_queries_iotdb/main.go +++ b/cmd/tsbs_run_queries_iotdb/main.go @@ -73,13 +73,15 @@ func main() { } type processor struct { - session client.Session + session client.Session + printResponses bool } func newProcessor() query.Processor { return &processor{} } func (p *processor) Init(workerNumber int) { p.session = client.NewSession(&clientConfig) + p.printResponses = runner.DoPrintResponses() if err := p.session.Open(false, int(timeoutInMs)); err != nil { errMsg := fmt.Sprintf("query processor init error, session is not open: %v\n", err) errMsg = errMsg + fmt.Sprintf("timeout setting: %d ms", timeoutInMs) @@ -92,7 +94,11 @@ func (p *processor) ProcessQuery(q query.Query, _ bool) ([]*query.Stat, error) { sql := string(iotdbQ.SqlQuery) start := time.Now().UnixNano() - _, err := p.session.ExecuteQueryStatement(sql, &timeoutInMs) // 0 for no timeout + dataSet, err := p.session.ExecuteQueryStatement(sql, &timeoutInMs) // 0 for no timeout + if err == nil && p.printResponses { + printDataSet(sql, dataSet) + dataSet.Close() + } took := time.Now().UnixNano() - start if err != nil { @@ -104,3 +110,36 @@ func (p *processor) ProcessQuery(q query.Query, _ bool) ([]*query.Stat, error) { stat.Init(q.HumanLabelName(), lag) return []*query.Stat{stat}, err } + +func printDataSet(sql string, sds *client.SessionDataSet) { + fmt.Printf("\nResponse for query '%s':\n", sql) + showTimestamp := !sds.IsIgnoreTimeStamp() + if showTimestamp { + fmt.Print("Time\t\t\t\t") + } + + for i := 0; i < sds.GetColumnCount(); i++ { + fmt.Printf("%s\t", sds.GetColumnName(i)) + } + fmt.Println() + + printedColsCount := 0 + for next, err := sds.Next(); err == nil && next; next, err = sds.Next() { + if showTimestamp { + fmt.Printf("%s\t", sds.GetText(client.TimestampColumnName)) + } + for i := 0; i < sds.GetColumnCount(); i++ { + columnName := sds.GetColumnName(i) + v := sds.GetValue(columnName) + if v == nil { + v = "null" + } + fmt.Printf("%v\t\t", v) + } + fmt.Println() + printedColsCount++ + } + if printedColsCount == 0 { + fmt.Println("Empty Set.") + } +} From ffe4bab0f6d28e357f00b975102bf5d4e22a81b7 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Tue, 22 Nov 2022 15:39:29 +0800 Subject: [PATCH 30/47] [important] Printing is not counted into time; do NOT send close. [important] Printing is not counted into query time. Do NOT send close signal after each query. --- cmd/tsbs_run_queries_iotdb/main.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cmd/tsbs_run_queries_iotdb/main.go b/cmd/tsbs_run_queries_iotdb/main.go index 778542d51..fbe5bd2c9 100644 --- a/cmd/tsbs_run_queries_iotdb/main.go +++ b/cmd/tsbs_run_queries_iotdb/main.go @@ -95,16 +95,18 @@ func (p *processor) ProcessQuery(q query.Query, _ bool) ([]*query.Stat, error) { start := time.Now().UnixNano() dataSet, err := p.session.ExecuteQueryStatement(sql, &timeoutInMs) // 0 for no timeout - if err == nil && p.printResponses { - printDataSet(sql, dataSet) - dataSet.Close() - } - took := time.Now().UnixNano() - start - if err != nil { + + if err == nil { + if p.printResponses { + printDataSet(sql, dataSet) + } + dataSet.Close() + } else { log.Printf("An error occurred while executing SQL: %s\n", sql) return nil, err } + lag := float64(took) / float64(time.Millisecond) // in milliseconds stat := query.GetStat() stat.Init(q.HumanLabelName(), lag) From eecb4cb457b225a6983ac246ba335b722a9d614d Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Tue, 22 Nov 2022 19:11:04 +0800 Subject: [PATCH 31/47] updated main readme updated main readme --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fad7c25d6..45adb954f 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ cases are implemented for each database: |ClickHouse|X|| |CrateDB|X|| |InfluxDB|X|X| +|IoTDB|X|| |MongoDB|X| |QuestDB|X|X |SiriDB|X| @@ -93,8 +94,8 @@ query execution performance. (It currently does not measure concurrent insert and query performance, which is a future priority.) To accomplish this in a fair way, the data to be inserted and the queries to run are pre-generated and native Go clients are used -wherever possible to connect to each database (e.g., `mgo` for MongoDB, -`aws sdk` for Timestream). +wherever possible to connect to each database (e.g., `mgo` for MongoDB, +`aws sdk` for Timestream, `iotdb-client-go` for IoTDB). Although the data is randomly generated, TSBS data and queries are entirely deterministic. By supplying the same PRNG (pseudo-random number @@ -135,8 +136,8 @@ Variables needed: 1. an end time. E.g., `2016-01-04T00:00:00Z` 1. how much time should be between each reading per device, in seconds. E.g., `10s` 1. and which database(s) you want to generate for. E.g., `timescaledb` - (choose from `cassandra`, `clickhouse`, `cratedb`, `influx`, `mongo`, `questdb`, `siridb`, - `timescaledb` or `victoriametrics`) + (choose from `cassandra`, `clickhouse`, `cratedb`, `influx`, `iotdb`, `mongo`, + `questdb`, `siridb`, `timescaledb` or `victoriametrics`) Given the above steps you can now generate a dataset (or multiple datasets, if you chose to generate for multiple databases) that can From 61c84f4b818946ef1da209a17b1244bd3d35087b Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Wed, 23 Nov 2022 09:55:09 +0800 Subject: [PATCH 32/47] fix bug: query dataset closing fix bug: query dataset did not close in all cases. Now dataset will always close after query, and will be traversed in query --- cmd/tsbs_run_queries_iotdb/main.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cmd/tsbs_run_queries_iotdb/main.go b/cmd/tsbs_run_queries_iotdb/main.go index fbe5bd2c9..661e6ec8f 100644 --- a/cmd/tsbs_run_queries_iotdb/main.go +++ b/cmd/tsbs_run_queries_iotdb/main.go @@ -95,15 +95,22 @@ func (p *processor) ProcessQuery(q query.Query, _ bool) ([]*query.Stat, error) { start := time.Now().UnixNano() dataSet, err := p.session.ExecuteQueryStatement(sql, &timeoutInMs) // 0 for no timeout - took := time.Now().UnixNano() - start + defer dataSet.Close() if err == nil { if p.printResponses { printDataSet(sql, dataSet) + } else { + var next bool + for next, err = dataSet.Next(); err == nil && next; next, err = dataSet.Next() { + // Traverse query results + } } - dataSet.Close() - } else { - log.Printf("An error occurred while executing SQL: %s\n", sql) + } + took := time.Now().UnixNano() - start + + if err != nil { + log.Printf("An error occurred while executing query SQL: %s\n", sql) return nil, err } From 8f8bdf12381617208ab93218eee19efc6ed9225a Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Wed, 23 Nov 2022 09:56:57 +0800 Subject: [PATCH 33/47] check dataset close error before return check dataset close error before return --- cmd/tsbs_run_queries_iotdb/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/tsbs_run_queries_iotdb/main.go b/cmd/tsbs_run_queries_iotdb/main.go index 661e6ec8f..3bdd37a1c 100644 --- a/cmd/tsbs_run_queries_iotdb/main.go +++ b/cmd/tsbs_run_queries_iotdb/main.go @@ -96,7 +96,6 @@ func (p *processor) ProcessQuery(q query.Query, _ bool) ([]*query.Stat, error) { start := time.Now().UnixNano() dataSet, err := p.session.ExecuteQueryStatement(sql, &timeoutInMs) // 0 for no timeout - defer dataSet.Close() if err == nil { if p.printResponses { printDataSet(sql, dataSet) @@ -109,6 +108,7 @@ func (p *processor) ProcessQuery(q query.Query, _ bool) ([]*query.Stat, error) { } took := time.Now().UnixNano() - start + err = dataSet.Close() // close error should also be checked if err != nil { log.Printf("An error occurred while executing query SQL: %s\n", sql) return nil, err From 596389620d3fd86de85c0a89deb14bcfde41d841 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Wed, 23 Nov 2022 10:06:20 +0800 Subject: [PATCH 34/47] better error catch for dataset closing better error catch for dataset closing --- cmd/tsbs_run_queries_iotdb/main.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cmd/tsbs_run_queries_iotdb/main.go b/cmd/tsbs_run_queries_iotdb/main.go index 3bdd37a1c..fd3a3ea36 100644 --- a/cmd/tsbs_run_queries_iotdb/main.go +++ b/cmd/tsbs_run_queries_iotdb/main.go @@ -89,13 +89,19 @@ func (p *processor) Init(workerNumber int) { } } +func (p *processor) closeDataSet(ds *client.SessionDataSet) { + err := ds.Close() + if err != nil { + log.Printf("[session:%d]An error occurred while closing sessionDataSet: %v\n", err, p.session.GetSessionId()) + } +} + func (p *processor) ProcessQuery(q query.Query, _ bool) ([]*query.Stat, error) { iotdbQ := q.(*query.IoTDB) sql := string(iotdbQ.SqlQuery) start := time.Now().UnixNano() dataSet, err := p.session.ExecuteQueryStatement(sql, &timeoutInMs) // 0 for no timeout - if err == nil { if p.printResponses { printDataSet(sql, dataSet) @@ -108,7 +114,7 @@ func (p *processor) ProcessQuery(q query.Query, _ bool) ([]*query.Stat, error) { } took := time.Now().UnixNano() - start - err = dataSet.Close() // close error should also be checked + defer p.closeDataSet(dataSet) // close error should also be checked if err != nil { log.Printf("An error occurred while executing query SQL: %s\n", sql) return nil, err From abcb39bc3357c616f4d5f622c58b4bddc290154a Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Wed, 23 Nov 2022 15:40:41 +0800 Subject: [PATCH 35/47] updated SQL query updated SQL query --- .../databases/iotdb/devops.go | 14 ++++++++++---- .../databases/iotdb/devops_test.go | 16 ++++++++-------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/cmd/tsbs_generate_queries/databases/iotdb/devops.go b/cmd/tsbs_generate_queries/databases/iotdb/devops.go index 84ac4c0e8..3d4adb991 100644 --- a/cmd/tsbs_generate_queries/databases/iotdb/devops.go +++ b/cmd/tsbs_generate_queries/databases/iotdb/devops.go @@ -99,7 +99,7 @@ func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange t sql := "" sql = sql + fmt.Sprintf("SELECT %s", selectClause) sql = sql + fmt.Sprintf(" FROM %s", fromHosts) - sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 5m)", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt)) + sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 5m), LEVEL = %d", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt), d.BasicPathLevel+1) d.fillInQuery(qi, humanLabel, humanDesc, sql) } @@ -153,7 +153,7 @@ func (d *Devops) MaxAllCPU(qi query.Query, nHosts int, duration time.Duration) { sql := "" sql = sql + fmt.Sprintf("SELECT %s", selectClause) sql = sql + fmt.Sprintf(" FROM %s", fromHosts) - sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1h)", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt)) + sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1h), LEVEL=%d", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt), d.BasicPathLevel+1) d.fillInQuery(qi, humanLabel, humanDesc, sql) } @@ -170,7 +170,13 @@ func (d *Devops) GroupByOrderByLimit(qi query.Query) { sql := "" sql = sql + fmt.Sprintf("SELECT %s", selectClause) sql = sql + fmt.Sprintf(" FROM %s.cpu.*", d.BasicPath) - sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1m), LEVEL = %d", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt), d.BasicPathLevel+1) + startTime := interval.Start() + endTime := interval.End() + optimizedStartTime := endTime.Add(time.Second * -300) // 5 mins ago + if optimizedStartTime.After(startTime) { + startTime = optimizedStartTime + } + sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1m), LEVEL = %d", startTime.Format(iotdbTimeFmt), endTime.Format(iotdbTimeFmt), d.BasicPathLevel+1) sql = sql + " ORDER BY TIME DESC LIMIT 5" humanLabel := "IoTDB max cpu over last 5 min-intervals (random end)" @@ -203,7 +209,7 @@ func (d *Devops) HighCPUForHosts(qi query.Query, nHosts int) { sql := "SELECT *" sql = sql + fmt.Sprintf(" FROM %s", fromHosts) - sql = sql + fmt.Sprintf(" WHERE usage_user > 90 AND time >= %s AND time < %s", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt)) + sql = sql + fmt.Sprintf(" WHERE usage_user > 90 AND time >= %s AND time < %s ALIGN BY DEVICE", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt)) d.fillInQuery(qi, humanLabel, humanDesc, sql) } diff --git a/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go b/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go index 3192ef56a..8d0fc51c8 100644 --- a/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go +++ b/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go @@ -146,7 +146,7 @@ func TestGroupByTime(t *testing.T) { rand.Seed(123) // Setting seed for testing purposes. start := time.Unix(0, 0) end := start.Add(time.Hour) - base := BaseGenerator{BasicPath: "root.sg", BasicPathLevel: 1} + base := BaseGenerator{BasicPath: "root", BasicPathLevel: 0} queryGenerator, err := base.NewDevops(start, end, 10) require.NoError(t, err, "Error while creating devops generator") dp := queryGenerator.(*Devops) @@ -160,7 +160,7 @@ func TestGroupByTime(t *testing.T) { dp.fillInQuery(expected, "IoTDB 1 cpu metric(s), random 1 hosts, random 1s by 5m", "IoTDB 1 cpu metric(s), random 1 hosts, random 1s by 5m: 1970-01-01T00:05:58Z", - "SELECT MAX_VALUE(usage_user) FROM root.sg.cpu.host_9 GROUP BY ([1970-01-01 00:05:58, 1970-01-01 00:05:59), 5m)", + "SELECT MAX_VALUE(usage_user) FROM root.cpu.host_9 GROUP BY ([1970-01-01 00:05:58, 1970-01-01 00:05:59), 5m), LEVEL = 1", ) dp.GroupByTime(actual, nHosts, metrics, duration) @@ -253,7 +253,7 @@ func TestMaxAllCPU(t *testing.T) { expectedSQLQuery: "SELECT MAX_VALUE(usage_user), MAX_VALUE(usage_system), MAX_VALUE(usage_idle), " + "MAX_VALUE(usage_nice), MAX_VALUE(usage_iowait), MAX_VALUE(usage_irq), MAX_VALUE(usage_softirq), " + "MAX_VALUE(usage_steal), MAX_VALUE(usage_guest), MAX_VALUE(usage_guest_nice) " + - "FROM root.cpu.host_9 GROUP BY ([1970-01-01 02:16:22, 1970-01-01 10:16:22), 1h)", + "FROM root.cpu.host_9 GROUP BY ([1970-01-01 02:16:22, 1970-01-01 10:16:22), 1h), LEVEL=1", }, { description: "3 hosts with storage group 'root'", @@ -264,7 +264,7 @@ func TestMaxAllCPU(t *testing.T) { expectedSQLQuery: "SELECT MAX_VALUE(usage_user), MAX_VALUE(usage_system), MAX_VALUE(usage_idle), " + "MAX_VALUE(usage_nice), MAX_VALUE(usage_iowait), MAX_VALUE(usage_irq), MAX_VALUE(usage_softirq), " + "MAX_VALUE(usage_steal), MAX_VALUE(usage_guest), MAX_VALUE(usage_guest_nice) " + - "FROM root.cpu.host_9, root.cpu.host_3, root.cpu.host_5 GROUP BY ([1970-01-01 02:16:22, 1970-01-01 10:16:22), 1h)", + "FROM root.cpu.host_9, root.cpu.host_3, root.cpu.host_5 GROUP BY ([1970-01-01 02:16:22, 1970-01-01 10:16:22), 1h), LEVEL=1", }, } @@ -304,7 +304,7 @@ func TestGroupByOrderByLimit(t *testing.T) { dp.fillInQuery(expected, "IoTDB max cpu over last 5 min-intervals (random end)", "IoTDB max cpu over last 5 min-intervals (random end): 1970-01-01T00:16:22Z", - "SELECT MAX_VALUE(usage_user) FROM root.cpu.* GROUP BY ([1970-01-01 00:16:22, 1970-01-01 01:16:22), 1m), LEVEL = 1 ORDER BY TIME DESC LIMIT 5", + "SELECT MAX_VALUE(usage_user) FROM root.cpu.* GROUP BY ([1970-01-01 01:11:22, 1970-01-01 01:16:22), 1m), LEVEL = 1 ORDER BY TIME DESC LIMIT 5", ) dp.GroupByOrderByLimit(actual) @@ -326,7 +326,7 @@ func TestHighCPUForHosts(t *testing.T) { baseGenerator: BaseGenerator{BasicPath: "root", BasicPathLevel: 0}, expectedHumanLabel: "IoTDB CPU over threshold, all hosts", expectedHumanDesc: "IoTDB CPU over threshold, all hosts: 1970-01-01T00:16:22Z", - expectedSQLQuery: "SELECT * FROM root.cpu.* WHERE usage_user > 90 AND time >= 1970-01-01 00:16:22 AND time < 1970-01-01 12:16:22", + expectedSQLQuery: "SELECT * FROM root.cpu.* WHERE usage_user > 90 AND time >= 1970-01-01 00:16:22 AND time < 1970-01-01 12:16:22 ALIGN BY DEVICE", }, { description: "1 host with storage group 'root.sg.abc'", @@ -334,7 +334,7 @@ func TestHighCPUForHosts(t *testing.T) { baseGenerator: BaseGenerator{BasicPath: "root.sg.abc", BasicPathLevel: 2}, expectedHumanLabel: "IoTDB CPU over threshold, 1 host(s)", expectedHumanDesc: "IoTDB CPU over threshold, 1 host(s): 1970-01-01T00:16:22Z", - expectedSQLQuery: "SELECT * FROM root.sg.abc.cpu.host_9 WHERE usage_user > 90 AND time >= 1970-01-01 00:16:22 AND time < 1970-01-01 12:16:22", + expectedSQLQuery: "SELECT * FROM root.sg.abc.cpu.host_9 WHERE usage_user > 90 AND time >= 1970-01-01 00:16:22 AND time < 1970-01-01 12:16:22 ALIGN BY DEVICE", }, { description: "5 host2 with storage group 'root.ln'", @@ -342,7 +342,7 @@ func TestHighCPUForHosts(t *testing.T) { baseGenerator: BaseGenerator{BasicPath: "root.ln", BasicPathLevel: 1}, expectedHumanLabel: "IoTDB CPU over threshold, 5 host(s)", expectedHumanDesc: "IoTDB CPU over threshold, 5 host(s): 1970-01-01T00:16:22Z", - expectedSQLQuery: "SELECT * FROM root.ln.cpu.host_9, root.ln.cpu.host_3, root.ln.cpu.host_5, root.ln.cpu.host_1, root.ln.cpu.host_7 WHERE usage_user > 90 AND time >= 1970-01-01 00:16:22 AND time < 1970-01-01 12:16:22", + expectedSQLQuery: "SELECT * FROM root.ln.cpu.host_9, root.ln.cpu.host_3, root.ln.cpu.host_5, root.ln.cpu.host_1, root.ln.cpu.host_7 WHERE usage_user > 90 AND time >= 1970-01-01 00:16:22 AND time < 1970-01-01 12:16:22 ALIGN BY DEVICE", }, } From 8234919395b5370fc074b29bb9fcf7e9e8ee2cd0 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Fri, 25 Nov 2022 09:35:33 +0800 Subject: [PATCH 36/47] close sql dataset in insertion close sql dataset in insertion --- cmd/tsbs_load_iotdb/process.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/tsbs_load_iotdb/process.go b/cmd/tsbs_load_iotdb/process.go index a1ed7ed24..68c3a2f8e 100644 --- a/cmd/tsbs_load_iotdb/process.go +++ b/cmd/tsbs_load_iotdb/process.go @@ -145,7 +145,11 @@ func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (metricCount, row } // handle create timeseries SQL to insert tags for _, sql := range sqlList { - p.session.ExecuteUpdateStatement(sql) + ds, err := p.session.ExecuteUpdateStatement(sql) + ds.Close() + if err != nil { + fatal("ProcessBatch SQL Execution error:%v", err) + } } } else { // generate csv files. There is no requirement to connect to any database From e5e25808db27f57f0d941c703ecf05fdf52df320 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Fri, 25 Nov 2022 09:48:07 +0800 Subject: [PATCH 37/47] Traverse query results in insertion Traverse query results in insertion --- cmd/tsbs_load_iotdb/process.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/tsbs_load_iotdb/process.go b/cmd/tsbs_load_iotdb/process.go index 68c3a2f8e..80af836dc 100644 --- a/cmd/tsbs_load_iotdb/process.go +++ b/cmd/tsbs_load_iotdb/process.go @@ -146,7 +146,14 @@ func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (metricCount, row // handle create timeseries SQL to insert tags for _, sql := range sqlList { ds, err := p.session.ExecuteUpdateStatement(sql) - ds.Close() + if err != nil { + fatal("ProcessBatch SQL Execution error:%v", err) + } + var next bool + for next, err = ds.Next(); err == nil && next; next, err = ds.Next() { + // Traverse query results + } + defer ds.Close() if err != nil { fatal("ProcessBatch SQL Execution error:%v", err) } From a16db5b3a3addabedc5b0c7a89c2a55689f626fd Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Fri, 25 Nov 2022 09:57:27 +0800 Subject: [PATCH 38/47] fix defer ds.Close() in insertion fix defer ds.Close() in insertion --- cmd/tsbs_load_iotdb/process.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/tsbs_load_iotdb/process.go b/cmd/tsbs_load_iotdb/process.go index 80af836dc..40e8a017d 100644 --- a/cmd/tsbs_load_iotdb/process.go +++ b/cmd/tsbs_load_iotdb/process.go @@ -146,6 +146,7 @@ func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (metricCount, row // handle create timeseries SQL to insert tags for _, sql := range sqlList { ds, err := p.session.ExecuteUpdateStatement(sql) + defer ds.Close() if err != nil { fatal("ProcessBatch SQL Execution error:%v", err) } @@ -153,7 +154,6 @@ func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (metricCount, row for next, err = ds.Next(); err == nil && next; next, err = ds.Next() { // Traverse query results } - defer ds.Close() if err != nil { fatal("ProcessBatch SQL Execution error:%v", err) } From 9b5389e34ca555f5e4418ef82ad9d00f4afa4945 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Fri, 25 Nov 2022 11:55:13 +0800 Subject: [PATCH 39/47] use ExecuteNonQueryStatement for inserting tags use ExecuteNonQueryStatement for inserting tags --- cmd/tsbs_load_iotdb/process.go | 10 +--------- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/cmd/tsbs_load_iotdb/process.go b/cmd/tsbs_load_iotdb/process.go index 40e8a017d..548078d0c 100644 --- a/cmd/tsbs_load_iotdb/process.go +++ b/cmd/tsbs_load_iotdb/process.go @@ -145,15 +145,7 @@ func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (metricCount, row } // handle create timeseries SQL to insert tags for _, sql := range sqlList { - ds, err := p.session.ExecuteUpdateStatement(sql) - defer ds.Close() - if err != nil { - fatal("ProcessBatch SQL Execution error:%v", err) - } - var next bool - for next, err = ds.Next(); err == nil && next; next, err = ds.Next() { - // Traverse query results - } + _, err := p.session.ExecuteNonQueryStatement(sql) if err != nil { fatal("ProcessBatch SQL Execution error:%v", err) } diff --git a/go.mod b/go.mod index 0ec959d0a..9f0b9d2fb 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/SiriDB/go-siridb-connector v0.0.0-20190110105621-86b34c44c921 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 - github.com/apache/iotdb-client-go v0.12.2-0.20221102040630-2731a9e19fc9 + github.com/apache/iotdb-client-go v0.14.0-preview1.0.20221125032227-e7dcc092f533 github.com/aws/aws-sdk-go v1.35.13 github.com/blagojts/viper v1.6.3-0.20200313094124-068f44cf5e69 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 diff --git a/go.sum b/go.sum index 4d4f7ad12..0451762a7 100644 --- a/go.sum +++ b/go.sum @@ -93,8 +93,8 @@ github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDa github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= -github.com/apache/iotdb-client-go v0.12.2-0.20221102040630-2731a9e19fc9 h1:0djW7kEV/yYbjj+DYKqNQnx5P9RRFAuvCYJjFDeYLy8= -github.com/apache/iotdb-client-go v0.12.2-0.20221102040630-2731a9e19fc9/go.mod h1:4Az8byJmeA6Nolvl0tYTvuFVP+6AJcF++SZ+gNSeaIA= +github.com/apache/iotdb-client-go v0.14.0-preview1.0.20221125032227-e7dcc092f533 h1:BpGjGsy1s2xsnw0ML9E1Agqn2bmwdSMr9L+fG7WTsq8= +github.com/apache/iotdb-client-go v0.14.0-preview1.0.20221125032227-e7dcc092f533/go.mod h1:4Az8byJmeA6Nolvl0tYTvuFVP+6AJcF++SZ+gNSeaIA= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.15.0 h1:aGvdaR0v1t9XLgjtBYwxcBvBOTMqClzwE26CHOgjW1Y= From 3a8975abd48ac29edf51de698e083fd87d720bff Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Fri, 25 Nov 2022 16:47:57 +0800 Subject: [PATCH 40/47] close dataSet in queries anyway close dataSet in queries anyway --- cmd/tsbs_run_queries_iotdb/main.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/cmd/tsbs_run_queries_iotdb/main.go b/cmd/tsbs_run_queries_iotdb/main.go index fd3a3ea36..e19d3ed5b 100644 --- a/cmd/tsbs_run_queries_iotdb/main.go +++ b/cmd/tsbs_run_queries_iotdb/main.go @@ -89,13 +89,6 @@ func (p *processor) Init(workerNumber int) { } } -func (p *processor) closeDataSet(ds *client.SessionDataSet) { - err := ds.Close() - if err != nil { - log.Printf("[session:%d]An error occurred while closing sessionDataSet: %v\n", err, p.session.GetSessionId()) - } -} - func (p *processor) ProcessQuery(q query.Query, _ bool) ([]*query.Stat, error) { iotdbQ := q.(*query.IoTDB) sql := string(iotdbQ.SqlQuery) @@ -114,7 +107,7 @@ func (p *processor) ProcessQuery(q query.Query, _ bool) ([]*query.Stat, error) { } took := time.Now().UnixNano() - start - defer p.closeDataSet(dataSet) // close error should also be checked + defer dataSet.Close() if err != nil { log.Printf("An error occurred while executing query SQL: %s\n", sql) return nil, err From 08712e494553a972b5d1336c54b8086d9a568887 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Sun, 27 Nov 2022 17:24:04 +0800 Subject: [PATCH 41/47] for GroupByTime use 1min MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit for GroupByTime use 1min， fix issues https://github.com/timescale/tsbs/issues/214 IoTDB is same with InfluxDB and TimescaleDB now。 And that meas the behavior of IoTDB single-group-by queries is NOT same with README, but is same with other databases. --- cmd/tsbs_generate_queries/databases/iotdb/devops.go | 4 ++-- cmd/tsbs_generate_queries/databases/iotdb/devops_test.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/tsbs_generate_queries/databases/iotdb/devops.go b/cmd/tsbs_generate_queries/databases/iotdb/devops.go index 3d4adb991..717bbabec 100644 --- a/cmd/tsbs_generate_queries/databases/iotdb/devops.go +++ b/cmd/tsbs_generate_queries/databases/iotdb/devops.go @@ -94,12 +94,12 @@ func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange t selectClause := d.getSelectClausesAggMetricsString("MAX_VALUE", metrics) fromHosts := d.getHostFromString(nHosts) - humanLabel := fmt.Sprintf("IoTDB %d cpu metric(s), random %4d hosts, random %s by 5m", numMetrics, nHosts, timeRange) + humanLabel := fmt.Sprintf("IoTDB %d cpu metric(s), random %4d hosts, random %s by 1m", numMetrics, nHosts, timeRange) humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) sql := "" sql = sql + fmt.Sprintf("SELECT %s", selectClause) sql = sql + fmt.Sprintf(" FROM %s", fromHosts) - sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 5m), LEVEL = %d", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt), d.BasicPathLevel+1) + sql = sql + fmt.Sprintf(" GROUP BY ([%s, %s), 1m), LEVEL = %d", interval.Start().Format(iotdbTimeFmt), interval.End().Format(iotdbTimeFmt), d.BasicPathLevel+1) d.fillInQuery(qi, humanLabel, humanDesc, sql) } diff --git a/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go b/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go index 8d0fc51c8..e405f1bd7 100644 --- a/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go +++ b/cmd/tsbs_generate_queries/databases/iotdb/devops_test.go @@ -158,9 +158,9 @@ func TestGroupByTime(t *testing.T) { actual := dp.GenerateEmptyQuery() expected := dp.GenerateEmptyQuery() dp.fillInQuery(expected, - "IoTDB 1 cpu metric(s), random 1 hosts, random 1s by 5m", - "IoTDB 1 cpu metric(s), random 1 hosts, random 1s by 5m: 1970-01-01T00:05:58Z", - "SELECT MAX_VALUE(usage_user) FROM root.cpu.host_9 GROUP BY ([1970-01-01 00:05:58, 1970-01-01 00:05:59), 5m), LEVEL = 1", + "IoTDB 1 cpu metric(s), random 1 hosts, random 1s by 1m", + "IoTDB 1 cpu metric(s), random 1 hosts, random 1s by 1m: 1970-01-01T00:05:58Z", + "SELECT MAX_VALUE(usage_user) FROM root.cpu.host_9 GROUP BY ([1970-01-01 00:05:58, 1970-01-01 00:05:59), 1m), LEVEL = 1", ) dp.GroupByTime(actual, nHosts, metrics, duration) From 2f14fa6bdb9d363897885206fe0293d9bf787b34 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Wed, 30 Nov 2022 16:35:40 +0800 Subject: [PATCH 42/47] user can choose TS type and whether to store tags User can choose TS type and whether to store tags now. Added flag `aligned-timeseries` and `store-tags` for tsbs_load_iotdb --- cmd/tsbs_load_iotdb/benchmark.go | 8 ++++--- cmd/tsbs_load_iotdb/main.go | 25 +++++++++++++++------- cmd/tsbs_load_iotdb/process.go | 28 ++++++++++++++++++------- docs/iotdb.md | 11 ++++++++++ pkg/targets/iotdb/implemented_target.go | 2 ++ 5 files changed, 56 insertions(+), 18 deletions(-) diff --git a/cmd/tsbs_load_iotdb/benchmark.go b/cmd/tsbs_load_iotdb/benchmark.go index fdb5d140b..c9b3ff645 100644 --- a/cmd/tsbs_load_iotdb/benchmark.go +++ b/cmd/tsbs_load_iotdb/benchmark.go @@ -36,9 +36,11 @@ func (b *iotdbBenchmark) GetPointIndexer(maxPartitions uint) targets.PointIndexe func (b *iotdbBenchmark) GetProcessor() targets.Processor { return &processor{ - recordsMaxRows: b.recordsMaxRows, - loadToSCV: loadToSCV, - csvFilepathPrefix: csvFilepathPrefix, + recordsMaxRows: b.recordsMaxRows, + loadToSCV: loadToSCV, + csvFilepathPrefix: csvFilepathPrefix, + useAlignedTimeseries: useAlignedTimeseries, + storeTags: storeTags, } } diff --git a/cmd/tsbs_load_iotdb/main.go b/cmd/tsbs_load_iotdb/main.go index 56eb75f92..b3a7131b5 100644 --- a/cmd/tsbs_load_iotdb/main.go +++ b/cmd/tsbs_load_iotdb/main.go @@ -21,11 +21,13 @@ import ( // database option vars var ( - clientConfig client.Config - timeoutInMs int // 0 for no timeout - recordsMaxRows int // max rows of records in 'InsertRecords' - loadToSCV bool // if true, do NOT insert into databases, but generate csv files instead. - csvFilepathPrefix string // Prefix of filepath for csv files. Specific a folder or a folder with filename prefix. + clientConfig client.Config + timeoutInMs int // 0 for no timeout + recordsMaxRows int // max rows of records in 'InsertRecords' + loadToSCV bool // if true, do NOT insert into databases, but generate csv files instead. + csvFilepathPrefix string // Prefix of filepath for csv files. Specific a folder or a folder with filename prefix. + useAlignedTimeseries bool // using aligned timeseries if set true. + storeTags bool // store tags if set true. Can NOT be used if useAlignedTimeseries is set true. ) // Global vars @@ -64,6 +66,8 @@ func init() { recordsMaxRows = viper.GetInt("records-max-rows") loadToSCV = viper.GetBool("to-csv") csvFilepathPrefix = viper.GetString("csv-prefix") + useAlignedTimeseries = viper.GetBool("aligned-timeseries") + storeTags = viper.GetBool("store-tags") workers := viper.GetUint("workers") @@ -73,15 +77,20 @@ func init() { timeoutStr = "no timeout for session opening check" } log.Printf("tsbs_load_iotdb target: %s:%s, %s. Loading with %d workers.\n", host, port, timeoutStr, workers) - if workers < 2 { - log.Println("Insertion throughput is strongly related to the number of threads. Use more workers for better performance.") - } if loadToSCV && workers != 1 { err_msg := "Arguments conflicts! When using csv export method, `workers` should NOT be set more than 1. " err_msg += fmt.Sprintf("Current setting: `to-csv`=%v, `workers`=%d.", loadToSCV, workers) log.Println(err_msg) panic(err_msg) } + if useAlignedTimeseries && storeTags { + warn_msg := "[Waring] Can NOT store tags while using aligned timeseries!" + warn_msg += " Because IoTDB do NOT support 'attributes' and 'tags' for aligned timeseries yet." + log.Println(warn_msg) + warn_msg = "Automatic parameter correction: 'store-tags' is set to false." + log.Println(warn_msg) + storeTags = false + } clientConfig = client.Config{ Host: host, diff --git a/cmd/tsbs_load_iotdb/process.go b/cmd/tsbs_load_iotdb/process.go index 548078d0c..ac38b046a 100644 --- a/cmd/tsbs_load_iotdb/process.go +++ b/cmd/tsbs_load_iotdb/process.go @@ -20,6 +20,9 @@ type processor struct { loadToSCV bool // if true, do NOT insert into databases, but generate csv files instead. csvFilepathPrefix string // Prefix of filepath for csv files. Specific a folder or a folder with filename prefix. filePtrMap map[string]*os.File // file pointer for each deviceID + + useAlignedTimeseries bool // using aligned timeseries if set true. + storeTags bool // store tags if set true. Can NOT be used if useAlignedTimeseries is set true. } func (p *processor) Init(numWorker int, doLoad, _ bool) { @@ -57,10 +60,13 @@ func (p *processor) pointsToRecords(points []*iotdbPoint) (records, []string) { rcds.dataTypes = append(rcds.dataTypes, row.dataTypes) rcds.values = append(rcds.values, row.values) rcds.timestamps = append(rcds.timestamps, row.timestamp) - _, exist := p.ProcessedTagsDeviceIDMap[row.deviceID] - if !exist { - sqlList = append(sqlList, row.generateTagsAttributesSQL()) - p.ProcessedTagsDeviceIDMap[row.deviceID] = true + // append tags if "storeTags" is set true + if p.storeTags { + _, exist := p.ProcessedTagsDeviceIDMap[row.deviceID] + if !exist { + sqlList = append(sqlList, row.generateTagsAttributesSQL()) + p.ProcessedTagsDeviceIDMap[row.deviceID] = true + } } } return rcds, sqlList @@ -135,9 +141,17 @@ func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (metricCount, row } rcds, tempSqlList := p.pointsToRecords(batch.points[startIndex:endIndex]) sqlList = append(sqlList, tempSqlList...) - _, err := p.session.InsertRecords( - rcds.deviceId, rcds.measurements, rcds.dataTypes, rcds.values, rcds.timestamps, - ) + // using relative API according to "aligned-timeseries" setting + var err error + if p.useAlignedTimeseries { + _, err = p.session.InsertAlignedRecords( + rcds.deviceId, rcds.measurements, rcds.dataTypes, rcds.values, rcds.timestamps, + ) + } else { + _, err = p.session.InsertRecords( + rcds.deviceId, rcds.measurements, rcds.dataTypes, rcds.values, rcds.timestamps, + ) + } if err != nil { fatal("ProcessBatch error:%v", err) } diff --git a/docs/iotdb.md b/docs/iotdb.md index ac6c58d4a..64ff0586f 100644 --- a/docs/iotdb.md +++ b/docs/iotdb.md @@ -175,6 +175,17 @@ will be stored in folder `/home/` with prefix `iotdb-data-`, like this: works, so please make sure folders in those paths are exist. That means user should create those folders manually. +#### `-aligned-timeseries` (type: `bool`, default: `true`) + +Using aligned timeseries for all metrics if set true. That means using +InsertAlignedRecords, which is faster than InsertRecords. + +#### `-store-tags` (type: `bool`, default: `false`) + +Store tags if set true. Can NOT be used if `-aligned-timeseries` is set true. +That's because IoTDB do NOT support 'attributes' and 'tags' for aligned +timeseries yet. + --- ## `tsbs_run_queries_iotdb` Additional Flags diff --git a/pkg/targets/iotdb/implemented_target.go b/pkg/targets/iotdb/implemented_target.go index 3d0453a4a..5559b4a99 100644 --- a/pkg/targets/iotdb/implemented_target.go +++ b/pkg/targets/iotdb/implemented_target.go @@ -25,6 +25,8 @@ func (t *iotdbTarget) TargetSpecificFlags(flagPrefix string, flagSet *pflag.Flag flagSet.Int(flagPrefix+"records-max-rows", 0, "Max rows of 'InsertRecords'. Use 0 for no limit.") flagSet.Bool(flagPrefix+"to-csv", false, "Do not insert into database, but to some CSV files.") flagSet.String(flagPrefix+"csv-prefix", "./", "Prefix of filepath for CSV files. Specific a folder or a folder with filename prefix.") + flagSet.Bool(flagPrefix+"aligned-timeseries", true, "Using aligned timeseries for all metrics if set true.") + flagSet.Bool(flagPrefix+"store-tags", false, "Store tags if set true. Can NOT be used if aligned-timeseries is set true.") } func (t *iotdbTarget) TargetName() string { From fa950bb85a92dc6ca80ca857dcb6c6cc71de13ee Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Thu, 15 Dec 2022 13:41:24 +0800 Subject: [PATCH 43/47] do NOT traverse query results after queries, not traverse query results --- cmd/tsbs_run_queries_iotdb/main.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/tsbs_run_queries_iotdb/main.go b/cmd/tsbs_run_queries_iotdb/main.go index e19d3ed5b..f9675eb55 100644 --- a/cmd/tsbs_run_queries_iotdb/main.go +++ b/cmd/tsbs_run_queries_iotdb/main.go @@ -99,10 +99,10 @@ func (p *processor) ProcessQuery(q query.Query, _ bool) ([]*query.Stat, error) { if p.printResponses { printDataSet(sql, dataSet) } else { - var next bool - for next, err = dataSet.Next(); err == nil && next; next, err = dataSet.Next() { - // Traverse query results - } + // var next bool + // for next, err = dataSet.Next(); err == nil && next; next, err = dataSet.Next() { + // // Traverse query results + // } } } took := time.Now().UnixNano() - start From 82902f320fef4ed07cf0f94c2b5905de00c6418b Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Fri, 23 Dec 2022 17:05:54 +0800 Subject: [PATCH 44/47] treat all integer as int32 treat all integer as int32 --- pkg/targets/iotdb/serializer.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/pkg/targets/iotdb/serializer.go b/pkg/targets/iotdb/serializer.go index 4495937dd..37f5d1d85 100644 --- a/pkg/targets/iotdb/serializer.go +++ b/pkg/targets/iotdb/serializer.go @@ -124,19 +124,26 @@ func modifyHostname(hostname string) string { // Utility function for appending various data types to a byte string func IotdbFormat(v interface{}) ([]byte, client.TSDataType) { + // treat all integer as int32 switch v.(type) { case uint: - return []byte(strconv.FormatInt(int64(v.(uint)), 10)), client.INT64 + return []byte(strconv.FormatInt(int64(v.(uint)), 10)), client.INT32 + // return []byte(strconv.FormatInt(int64(v.(uint)), 10)), client.INT64 case uint32: - return []byte(strconv.FormatInt(int64(v.(uint32)), 10)), client.INT64 + return []byte(strconv.FormatInt(int64(v.(uint32)), 10)), client.INT32 + // return []byte(strconv.FormatInt(int64(v.(uint32)), 10)), client.INT64 case uint64: - return []byte(strconv.FormatInt(int64(v.(uint64)), 10)), client.INT64 + return []byte(strconv.FormatInt(int64(v.(uint64)), 10)), client.INT32 + // return []byte(strconv.FormatInt(int64(v.(uint64)), 10)), client.INT64 case int: - return []byte(strconv.FormatInt(int64(v.(int)), 10)), client.INT64 + return []byte(strconv.FormatInt(int64(v.(int)), 10)), client.INT32 + // return []byte(strconv.FormatInt(int64(v.(int)), 10)), client.INT64 case int32: return []byte(strconv.FormatInt(int64(v.(int32)), 10)), client.INT32 + // return []byte(strconv.FormatInt(int64(v.(int32)), 10)), client.INT32 case int64: - return []byte(strconv.FormatInt(int64(v.(int64)), 10)), client.INT64 + return []byte(strconv.FormatInt(int64(v.(int64)), 10)), client.INT32 + // return []byte(strconv.FormatInt(int64(v.(int64)), 10)), client.INT64 case float64: // Why -1 ? // From Golang source on genericFtoa (called by AppendFloat): 'Negative precision means "only as much as needed to be exact."' From 96ce1442f3f648eaef7808291e8e4ff8dbec5935 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Fri, 6 Jan 2023 13:26:17 +0800 Subject: [PATCH 45/47] do NOT treat all integer as int32 do NOT treat all integer as int32 --- pkg/targets/iotdb/serializer.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/targets/iotdb/serializer.go b/pkg/targets/iotdb/serializer.go index 37f5d1d85..678ee39fb 100644 --- a/pkg/targets/iotdb/serializer.go +++ b/pkg/targets/iotdb/serializer.go @@ -127,23 +127,23 @@ func IotdbFormat(v interface{}) ([]byte, client.TSDataType) { // treat all integer as int32 switch v.(type) { case uint: - return []byte(strconv.FormatInt(int64(v.(uint)), 10)), client.INT32 - // return []byte(strconv.FormatInt(int64(v.(uint)), 10)), client.INT64 + // return []byte(strconv.FormatInt(int64(v.(uint)), 10)), client.INT32 + return []byte(strconv.FormatInt(int64(v.(uint)), 10)), client.INT64 case uint32: - return []byte(strconv.FormatInt(int64(v.(uint32)), 10)), client.INT32 - // return []byte(strconv.FormatInt(int64(v.(uint32)), 10)), client.INT64 + // return []byte(strconv.FormatInt(int64(v.(uint32)), 10)), client.INT32 + return []byte(strconv.FormatInt(int64(v.(uint32)), 10)), client.INT64 case uint64: - return []byte(strconv.FormatInt(int64(v.(uint64)), 10)), client.INT32 - // return []byte(strconv.FormatInt(int64(v.(uint64)), 10)), client.INT64 + // return []byte(strconv.FormatInt(int64(v.(uint64)), 10)), client.INT32 + return []byte(strconv.FormatInt(int64(v.(uint64)), 10)), client.INT64 case int: - return []byte(strconv.FormatInt(int64(v.(int)), 10)), client.INT32 - // return []byte(strconv.FormatInt(int64(v.(int)), 10)), client.INT64 + // return []byte(strconv.FormatInt(int64(v.(int)), 10)), client.INT32 + return []byte(strconv.FormatInt(int64(v.(int)), 10)), client.INT64 case int32: - return []byte(strconv.FormatInt(int64(v.(int32)), 10)), client.INT32 // return []byte(strconv.FormatInt(int64(v.(int32)), 10)), client.INT32 + return []byte(strconv.FormatInt(int64(v.(int32)), 10)), client.INT32 case int64: - return []byte(strconv.FormatInt(int64(v.(int64)), 10)), client.INT32 - // return []byte(strconv.FormatInt(int64(v.(int64)), 10)), client.INT64 + // return []byte(strconv.FormatInt(int64(v.(int64)), 10)), client.INT32 + return []byte(strconv.FormatInt(int64(v.(int64)), 10)), client.INT64 case float64: // Why -1 ? // From Golang source on genericFtoa (called by AppendFloat): 'Negative precision means "only as much as needed to be exact."' From 559ebd60050a1c20eba6481d9830dbf0e6f86363 Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Mon, 3 Apr 2023 18:58:09 +0800 Subject: [PATCH 46/47] Fixed Bug, supported using BasicPath for data generation Fixed Bug, supported using BasicPath for data generation --- pkg/targets/iotdb/implemented_target.go | 12 +++++- pkg/targets/iotdb/serializer.go | 9 ++-- pkg/targets/iotdb/serializer_test.go | 56 ++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 6 deletions(-) diff --git a/pkg/targets/iotdb/implemented_target.go b/pkg/targets/iotdb/implemented_target.go index 5559b4a99..25ca5a0b4 100644 --- a/pkg/targets/iotdb/implemented_target.go +++ b/pkg/targets/iotdb/implemented_target.go @@ -10,10 +10,15 @@ import ( ) func NewTarget() targets.ImplementedTarget { - return &iotdbTarget{} + return &iotdbTarget{ + BasicPath: "root", + BasicPathLevel: 0, + } } type iotdbTarget struct { + BasicPath string // e.g. "root.sg" is basic path of "root.sg.device". default : "root" + BasicPathLevel int32 // e.g. 0 for "root", 1 for "root.device" } func (t *iotdbTarget) TargetSpecificFlags(flagPrefix string, flagSet *pflag.FlagSet) { @@ -34,7 +39,10 @@ func (t *iotdbTarget) TargetName() string { } func (t *iotdbTarget) Serializer() serialize.PointSerializer { - return &Serializer{} + return &Serializer{ + BasicPath: t.BasicPath, + BasicPathLevel: t.BasicPathLevel, + } } func (t *iotdbTarget) Benchmark(string, *source.DataSourceConfig, *viper.Viper) (targets.Benchmark, error) { diff --git a/pkg/targets/iotdb/serializer.go b/pkg/targets/iotdb/serializer.go index 678ee39fb..143aa5471 100644 --- a/pkg/targets/iotdb/serializer.go +++ b/pkg/targets/iotdb/serializer.go @@ -10,8 +10,11 @@ import ( "github.com/timescale/tsbs/pkg/data" ) -// Serializer writes a Point in a serialized form for MongoDB -type Serializer struct{} +// Serializer writes a Point in a serialized form for IoTDB +type Serializer struct { + BasicPath string // e.g. "root.sg" is basic path of "root.sg.device". default : "root" + BasicPathLevel int32 // e.g. 0 for "root", 1 for "root.device" +} // const iotdbTimeFmt = "2006-01-02 15:04:05" @@ -76,7 +79,7 @@ func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { hostname = "unknown" } buf2 := make([]byte, 0, defaultBufSize) - buf2 = append(buf2, []byte(fmt.Sprintf("root.%s.%s,", modifyHostname(string(p.MeasurementName())), hostname))...) + buf2 = append(buf2, []byte(fmt.Sprintf("%s.%s.%s,", s.BasicPath, modifyHostname(string(p.MeasurementName())), hostname))...) buf2 = append(buf2, []byte(fmt.Sprintf("%d", p.Timestamp().UTC().UnixNano()))...) buf2 = append(buf2, tempBuf...) // Fields diff --git a/pkg/targets/iotdb/serializer_test.go b/pkg/targets/iotdb/serializer_test.go index 0bbc13191..752d781a5 100644 --- a/pkg/targets/iotdb/serializer_test.go +++ b/pkg/targets/iotdb/serializer_test.go @@ -70,7 +70,7 @@ func TestIotdbFormat(t *testing.T) { } -func TestSerialize_001(t *testing.T) { +func TestSerialize_normal(t *testing.T) { cases := []struct { description string inputPoint *data.Point @@ -110,3 +110,57 @@ func TestSerialize_001(t *testing.T) { } } + +func TestSerialize_nonDefaultBasicPath(t *testing.T) { + cases := []struct { + description string + inputPoint *data.Point + BasicPath string + BasicPathLevel int32 + expected string + }{ + { + description: "a regular point ", + inputPoint: serialize.TestPointDefault(), + BasicPath: "root.sg", + BasicPathLevel: 1, + expected: "deviceID,timestamp,usage_guest_nice\nroot.sg.cpu.host_0,1451606400000000000,38.24311829\ndatatype,4\ntags,region='eu-west-1',datacenter='eu-west-1b'\n", + }, + { + description: "a regular Point using int as value", + inputPoint: serialize.TestPointInt(), + BasicPath: "root.ln", + BasicPathLevel: 1, + expected: "deviceID,timestamp,usage_guest\nroot.ln.cpu.host_0,1451606400000000000,38\ndatatype,2\ntags,region='eu-west-1',datacenter='eu-west-1b'\n", + }, + { + description: "a regular Point with multiple fields", + inputPoint: serialize.TestPointMultiField(), + BasicPath: "root.db.abc", + BasicPathLevel: 2, + expected: "deviceID,timestamp,big_usage_guest,usage_guest,usage_guest_nice\nroot.db.abc.cpu.host_0,1451606400000000000,5000000000,38,38.24311829\ndatatype,2,2,4\ntags,region='eu-west-1',datacenter='eu-west-1b'\n", + }, + { + description: "a Point with no tags", + inputPoint: serialize.TestPointNoTags(), + BasicPath: "root", + BasicPathLevel: 0, + expected: "deviceID,timestamp,usage_guest_nice\nroot.cpu.unknown,1451606400000000000,38.24311829\ndatatype,4\ntags\n", + }, + } + for _, c := range cases { + t.Run(c.description, func(t *testing.T) { + ps := &Serializer{ + BasicPath: c.BasicPath, + BasicPathLevel: c.BasicPathLevel, + } + b := new(bytes.Buffer) + err := ps.Serialize(c.inputPoint, b) + require.NoError(t, err) + actual := b.String() + + require.EqualValues(t, c.expected, actual) + }) + } + +} From 38c9e1b2f9ce8e5b3f7df93c5bdc6b4941cb300b Mon Sep 17 00:00:00 2001 From: citrusreticulata <1372722289@qq.com> Date: Mon, 3 Apr 2023 19:09:36 +0800 Subject: [PATCH 47/47] Fix test cases TestSerialize_normal Fix test cases TestSerialize_normal --- pkg/targets/iotdb/serializer_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/targets/iotdb/serializer_test.go b/pkg/targets/iotdb/serializer_test.go index 752d781a5..e438c2bac 100644 --- a/pkg/targets/iotdb/serializer_test.go +++ b/pkg/targets/iotdb/serializer_test.go @@ -99,7 +99,10 @@ func TestSerialize_normal(t *testing.T) { } for _, c := range cases { t.Run(c.description, func(t *testing.T) { - ps := &Serializer{} + ps := &Serializer{ + BasicPath: "root", + BasicPathLevel: 0, + } b := new(bytes.Buffer) err := ps.Serialize(c.inputPoint, b) require.NoError(t, err)