From d5dd5e6e928f115a7010083873cdbaaaea288afe Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 12 Jun 2024 13:48:37 +0800 Subject: [PATCH 01/10] fix: change files --- src/benchUtil.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/benchUtil.c b/src/benchUtil.c index 50524289..13648262 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -1052,6 +1052,7 @@ int convertStringToDatatype(char *type, int length) { } } + int compare(const void *a, const void *b) { return *(int64_t *)a - *(int64_t *)b; } From 2f2aa168fefab094444f9dfc8ab2b8ce68e9ce55 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 13 Jun 2024 17:53:24 +0800 Subject: [PATCH 02/10] fix: add geometry and varbinary --- inc/bench.h | 1 + src/benchData.c | 119 ++++++++++++++++++++++++++++++++++++++++++++- src/benchDataMix.c | 3 ++ src/benchInsert.c | 5 ++ src/benchJsonOpt.c | 2 + src/benchUtil.c | 4 ++ 6 files changed, 132 insertions(+), 2 deletions(-) diff --git a/inc/bench.h b/inc/bench.h index 801dfcfa..248a2e8f 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -1180,5 +1180,6 @@ uint64_t tmpUint64Impl(Field *field, int32_t angle, int64_t k); float tmpFloatImpl(Field *field, int i, int32_t angle, int32_t k); double tmpDoubleImpl(Field *field, int32_t angle, int32_t k); int tmpStr(char *tmp, int iface, Field *field, int64_t k); +int tmpGeometry(char *tmp, int iface, Field *field, int64_t k); #endif // INC_BENCH_H_ diff --git a/src/benchData.c b/src/benchData.c index ff522704..2788a56b 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -391,7 +391,6 @@ uint32_t accumulateRowLen(BArray *fields, int iface) { case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_GEOMETRY: case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_GEOMETRY: len += field->length + 3; break; case TSDB_DATA_TYPE_INT: @@ -483,6 +482,41 @@ int tmpStr(char *tmp, int iface, Field *field, int64_t k) { return 0; } +int tmpGeometry(char *tmp, int iface, Field *field, int64_t k) { + // values + if (field->values) { + int arraySize = tools_cJSON_GetArraySize(field->values); + if (arraySize) { + tools_cJSON *buf = tools_cJSON_GetArrayItem( + field->values, + taosRandom() % arraySize); + snprintf(tmp, field->length, + "%s", buf->valuestring); + } else { + errorPrint("%s() cannot read correct value " + "from json file. array size: %d\n", + __func__, arraySize); + return -1; + } + return 0; + } + + // gen point count + int32_t cnt = field->length / 24; + if(cnt == 0) { + snprintf(tmp, field->length, "POINT(%d %d)", tmpUint16(field), tmpUint16(field)); + return 0; + } + + int32_t pos = snprintf(tmp, field->length, "LINESTRING("); + for(int32_t i = 0; i < cnt; i++) { + pos = snprintf(tmp + pos, field->length - pos, "%d %d", tmpUint16(field), tmpUint16(field)); + } + strcat(tmp, ")"); + + return 0; +} + bool tmpBool(Field *field) { bool boolTmp; if (field->min == field->max) { @@ -1001,6 +1035,28 @@ static int fillStmt( tmfree(tmp); break; } + case TSDB_DATA_TYPE_GEOMETRY: { + char *tmp = benchCalloc(1, field->length + 1, false); + if (0 != tmpGeometry(tmp, stbInfo->iface, field, k)) { + tmfree(tmp); + return -1; + } + if (childCol) { + snprintf((char *)childCol->stmtData.data + + k * field->length, + field->length, + "%s", tmp); + } else { + snprintf((char *)field->stmtData.data + + k * field->length, + field->length, + "%s", tmp); + } + n = snprintf(sampleDataBuf + pos, bufLen - pos, + "'%s',", tmp); + tmfree(tmp); + break; + } case TSDB_DATA_TYPE_JSON: { pos += tmpJson(sampleDataBuf, bufLen, pos, fieldsSize, field); @@ -1231,7 +1287,7 @@ static int generateRandDataSmlTelnet(SSuperTable *stbInfo, char *sampleDataBuf, case TSDB_DATA_TYPE_NCHAR: { char *tmp = benchCalloc(1, field->length + 1, false); if (0 != tmpStr(tmp, stbInfo->iface, field, k)) { - free(tmp); + tmfree(tmp); return -1; } if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY) { @@ -1272,6 +1328,31 @@ static int generateRandDataSmlTelnet(SSuperTable *stbInfo, char *sampleDataBuf, tmfree(tmp); break; } + case TSDB_DATA_TYPE_GEOMETRY: { + char *tmp = benchCalloc(1, field->length + 1, false); + if (0 != tmpGeometry(tmp, stbInfo->iface, field, k)) { + tmfree(tmp); + return -1; + } + if (tag) { + n = snprintf(sampleDataBuf + pos, bufLen - pos, + "%s=L\"%s\" ", + field->name, tmp); + } else { + n = snprintf(sampleDataBuf + pos, bufLen - pos, + "\"%s\" ", tmp); + } + if (n < 0 || n >= bufLen - pos) { + errorPrint("%s() LN%d snprintf overflow\n", + __func__, __LINE__); + tmfree(tmp); + return -1; + } else { + pos += n; + } + tmfree(tmp); + break; + } case TSDB_DATA_TYPE_JSON: { pos += tmpJson(sampleDataBuf, bufLen, pos, fieldsSize, field); @@ -1534,6 +1615,18 @@ static int generateRandDataSmlLine(SSuperTable *stbInfo, char *sampleDataBuf, tmfree(tmp); break; } + case TSDB_DATA_TYPE_GEOMETRY: { + char *tmp = benchCalloc(1, field->length + 1, false); + if (0 != tmpGeometry(tmp, stbInfo->iface, field, k)) { + tmfree(tmp); + return -1; + } + n = snprintf(sampleDataBuf + pos, bufLen - pos, + "%s=\"%s\",", + field->name, tmp); + tmfree(tmp); + break; + } case TSDB_DATA_TYPE_JSON: { n = tmpJson(sampleDataBuf, bufLen, pos, fieldsSize, field); @@ -2075,6 +2168,15 @@ void generateSmlJsonValues( tmfree(buf); break; } + case TSDB_DATA_TYPE_GEOMETRY: { + char *buf = (char *)benchCalloc(col->length + 1, 1, false); + tmpGeometry(buf, stbInfo->iface, col, 0); + value_buf = benchCalloc(len_key + col->length + 3, 1, true); + snprintf(value_buf, len_key + col->length + 3, + "\"value\":\"%s\",", buf); + tmfree(buf); + break; + } default: { value_buf = benchCalloc(len_key + 20, 1, true); double doubleTmp = tmpDouble(col); @@ -2120,6 +2222,13 @@ void generateSmlJsonCols(tools_cJSON *array, tools_cJSON *tag, tmfree(buf); break; } + case TSDB_DATA_TYPE_GEOMETRY: { + char *buf = (char *)benchCalloc(col->length + 1, 1, false); + tmpGeometry(buf, stbInfo->iface, col, 0); + tools_cJSON_AddStringToObject(record, "value", buf); + tmfree(buf); + break; + } default: { double doubleTmp = tmpDouble(col); tools_cJSON_AddNumberToObject(record, "value", doubleTmp); @@ -2180,6 +2289,12 @@ void generateSmlTaosJsonCols(tools_cJSON *array, tools_cJSON *tag, tmfree(buf); break; } + case TSDB_DATA_TYPE_GEOMETRY: + char *buf = (char *)benchCalloc(col->length + 1, 1, false); + tmpGeometry(buf, stbInfo->iface, col, 0); + tools_cJSON_AddStringToObject(value, "value", buf); + tools_cJSON_AddStringToObject(value, "type", "geometry"); + tmfree(buf); default: { double dblTmp = (double)col->min; if (col->max != col->min) { diff --git a/src/benchDataMix.c b/src/benchDataMix.c index 117b2c48..3692e57f 100644 --- a/src/benchDataMix.c +++ b/src/benchDataMix.c @@ -157,6 +157,9 @@ uint32_t dataGenByField(Field* fd, char* pstr, uint32_t len, char* prefix, int64 format = ",\'%s\'"; tmpStr(val, 0, fd, *k); break; + case TSDB_DATA_TYPE_GEOMETRY: + tmpGeometry(val, 0, fd, 0); + break; default: break; } diff --git a/src/benchInsert.c b/src/benchInsert.c index e7246dcb..83e91f38 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -247,6 +247,7 @@ static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) { int n; if (col->type == TSDB_DATA_TYPE_BINARY || col->type == TSDB_DATA_TYPE_NCHAR || + col->type == TSDB_DATA_TYPE_VARBINARY || col->type == TSDB_DATA_TYPE_GEOMETRY) { n = snprintf(colsBuf + len, col_buffer_len - len, ",%s %s(%d)", col->name, @@ -318,6 +319,7 @@ static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) { Field *tag = benchArrayGet(stbInfo->tags, tagIndex); if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_NCHAR || + tag->type == TSDB_DATA_TYPE_VARBINARY || tag->type == TSDB_DATA_TYPE_GEOMETRY) { n = snprintf(tagsBuf + len, tag_buffer_len - len, "%s %s(%d),", tag->name, @@ -2834,6 +2836,8 @@ static int initStmtDataValue(SSuperTable *stbInfo, SChildTable *childTbl) { break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_GEOMETRY: { size_t tmpLen = strlen(tmpStr); debugPrint("%s() LN%d, index: %d, " @@ -2926,6 +2930,7 @@ static void initStmtData(char dataType, void **data, uint32_t length) { case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_GEOMETRY: tmpP = calloc(1, g_arguments->prepared_rand * length); assert(tmpP); tmfree(*data); diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 6481e40e..162db5a2 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -458,6 +458,8 @@ static int getColumnAndTagTypeFromInsertJsonFile( } else { if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_JSON + || type == TSDB_DATA_TYPE_VARBINARY + || type == TSDB_DATA_TYPE_GEOMETRY || type == TSDB_DATA_TYPE_NCHAR) { length = g_arguments->binwidth; } else { diff --git a/src/benchUtil.c b/src/benchUtil.c index 13648262..e860ef55 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -1006,6 +1006,8 @@ int convertStringToDatatype(char *type, int length) { return TSDB_DATA_TYPE_BINARY; } else if (0 == strcasecmp(type, "varbinary")) { return TSDB_DATA_TYPE_VARBINARY; + } else if (0 == strcasecmp(type, "geometry")) { + return TSDB_DATA_TYPE_GEOMETRY; } else { errorPrint("unknown data type: %s\n", type); exit(EXIT_FAILURE); @@ -1045,6 +1047,8 @@ int convertStringToDatatype(char *type, int length) { return TSDB_DATA_TYPE_BINARY; } else if (0 == strncasecmp(type, "varbinary", length)) { return TSDB_DATA_TYPE_VARBINARY; + } else if (0 == strncasecmp(type, "geometry", length)) { + return TSDB_DATA_TYPE_GEOMETRY; } else { errorPrint("unknown data type: %s\n", type); exit(EXIT_FAILURE); From d912320fb1a22e9674b4391b5c8e713d501b891a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 13 Jun 2024 18:15:30 +0800 Subject: [PATCH 03/10] fix: fix csv read line --- src/benchCommandOpt.c | 5 +---- src/benchData.c | 15 ++++++++++++--- src/taosdump.c | 6 +----- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/benchCommandOpt.c b/src/benchCommandOpt.c index c6273f39..86d8a837 100644 --- a/src/benchCommandOpt.c +++ b/src/benchCommandOpt.c @@ -28,9 +28,6 @@ extern char g_configDir[MAX_PATH_LEN]; #define TAOSBENCHMARK_STATUS "unknown" #endif -#ifndef TD_PRODUCT_NAME -#define TD_PRODUCT_NAME "TDengine" -#endif // libtaos.so extern char buildinfo[]; @@ -52,7 +49,7 @@ void printVersion() { char taosBenchmark_status[] = TAOSBENCHMARK_STATUS; // version - printf("%s\ntaosBenchmark version: %s\ngit: %s\n", TD_PRODUCT_NAME, taosBenchmark_ver, taosBenchmark_commit); + printf("taosBenchmark version: %s\ngit: %s\n", taosBenchmark_ver, taosBenchmark_commit); #ifdef LINUX printf("build: %s\n ", buildinfo); #endif diff --git a/src/benchData.c b/src/benchData.c index 2788a56b..4c79d92a 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -323,8 +323,17 @@ static int generateSampleFromCsv(char *buffer, char* file, FILE* fp, int32_t len continue; } - if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { - line[--readLen] = 0; + int32_t pos = (int32_t)(readLen - 1); + if(pos > 0) { + if (('\r' == line[pos]) || ('\n' == line[pos])) { + line[pos] = 0; + } + } + pos = (int32_t)(readLen - 2); + if(pos > 0) { + if (('\r' == line[pos]) || ('\n' == line[pos])) { + line[pos] = 0; + } } if (readLen == 0) { @@ -339,7 +348,7 @@ static int generateSampleFromCsv(char *buffer, char* file, FILE* fp, int32_t len continue; } - memcpy(buffer + getRows * length, line, readLen); + memcpy(buffer + getRows * length, line, readLen + 1); getRows++; if (getRows == size) { diff --git a/src/taosdump.c b/src/taosdump.c index 334f76d7..c53fc576 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -82,10 +82,6 @@ #define TAOSDUMP_STATUS "unknown" #endif -#ifndef TD_PRODUCT_NAME -#define TD_PRODUCT_NAME "TDengine" -#endif - // use 256 as normal buffer length #define BUFFER_LEN 256 @@ -673,7 +669,7 @@ static void printVersion(FILE *file) { char taosdump_commit[] = TAOSDUMP_COMMIT_SHA1; - fprintf(file,"%s\ntaosdump version: %s\ngit: %s\n", TD_PRODUCT_NAME, taostools_ver, taosdump_commit); + fprintf(file,"ntaosdump version: %s\ngit: %s\n", taostools_ver, taosdump_commit); #ifdef LINUX printf("build: %s\n ", buildinfo); #endif From 5ef449b1a7722aac1ea0ca54243e78a3e30c704d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 13 Jun 2024 18:20:12 +0800 Subject: [PATCH 04/10] fix: free command twice --- src/taosdump.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/taosdump.c b/src/taosdump.c index c53fc576..e9475da9 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -2177,7 +2177,6 @@ static int64_t getTbCountOfStbNative(const char *dbName, const char *stbName) { int32_t code = taos_errno(res); if (code != 0) { cleanIfQueryFailed(__func__, __LINE__, command, res); - free(command); taos_close(taos); return -1; } @@ -9976,7 +9975,7 @@ int readNextTableDesWS(void* ws_res, TableDes* tbDes, int *idx, int *cnt) { // read next table tags to tbDes int readNextTableDesNative(void* res, TableDes* tbDes) { // tbname, tagName , tagValue - TAOS_ROW row; + TAOS_ROW row = NULL; int index = 0; while( index < tbDes->tags && NULL != (row = taos_fetch_row(res))) { // tbname changed check From 134f11169796914c04bc6c4f4fa1ac25c1f10247 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 13 Jun 2024 18:29:33 +0800 Subject: [PATCH 05/10] fix: build error --- deps/avro/lang/c/examples/quickstop.c | 2 +- src/benchData.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deps/avro/lang/c/examples/quickstop.c b/deps/avro/lang/c/examples/quickstop.c index ff9e9700..a1e5a0f1 100644 --- a/deps/avro/lang/c/examples/quickstop.c +++ b/deps/avro/lang/c/examples/quickstop.c @@ -107,7 +107,7 @@ int print_person(avro_file_reader_t db, avro_schema_t reader_schema) if (rval == 0) { int64_t id; int32_t age; - int32_t *p; + char *p = NULL; size_t size; avro_value_t id_value; avro_value_t first_value; diff --git a/src/benchData.c b/src/benchData.c index 4c79d92a..d4144a28 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -843,7 +843,7 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_GEOMETRY: { - int bufferSize = geoCalcBufferSize(field->length); + int bufferSize = field->length + 1; char *tmp = benchCalloc(1, bufferSize, false); if (0 != tmpGeometry(tmp, stbInfo->iface, field, i)) { free(tmp); From 5ebe168633a3b8a5a4d606ccae810182cd99d250 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 13 Jun 2024 19:16:24 +0800 Subject: [PATCH 06/10] remove -V blank --- src/benchCommandOpt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchCommandOpt.c b/src/benchCommandOpt.c index 86d8a837..fa2fd435 100644 --- a/src/benchCommandOpt.c +++ b/src/benchCommandOpt.c @@ -51,7 +51,7 @@ void printVersion() { // version printf("taosBenchmark version: %s\ngit: %s\n", taosBenchmark_ver, taosBenchmark_commit); #ifdef LINUX - printf("build: %s\n ", buildinfo); + printf("build: %s\n", buildinfo); #endif if (strlen(taosBenchmark_status) > 0) { printf("status: %s\n", taosBenchmark_status); From 5e2929a29af9295f7f5ac9b2e486050fccde53a1 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 13 Jun 2024 19:29:34 +0800 Subject: [PATCH 07/10] fix: tmpGeometry format --- src/benchData.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchData.c b/src/benchData.c index d4144a28..3cc906d2 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -519,7 +519,7 @@ int tmpGeometry(char *tmp, int iface, Field *field, int64_t k) { int32_t pos = snprintf(tmp, field->length, "LINESTRING("); for(int32_t i = 0; i < cnt; i++) { - pos = snprintf(tmp + pos, field->length - pos, "%d %d", tmpUint16(field), tmpUint16(field)); + pos += snprintf(tmp + pos, field->length - pos, "%d %d", tmpUint16(field), tmpUint16(field)); } strcat(tmp, ")"); From 6521354b5306afdf170f19db76ca9cffed77ac83 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 13 Jun 2024 19:33:43 +0800 Subject: [PATCH 08/10] fix: tmpGeometry format1 --- src/benchData.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/benchData.c b/src/benchData.c index 3cc906d2..dfb5acd8 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -518,8 +518,12 @@ int tmpGeometry(char *tmp, int iface, Field *field, int64_t k) { } int32_t pos = snprintf(tmp, field->length, "LINESTRING("); + char * format = ",%d %d"; for(int32_t i = 0; i < cnt; i++) { - pos += snprintf(tmp + pos, field->length - pos, "%d %d", tmpUint16(field), tmpUint16(field)); + if (i == 0) { + format = "%d %d"; + } + pos += snprintf(tmp + pos, field->length - pos, format, tmpUint16(field), tmpUint16(field)); } strcat(tmp, ")"); From 1310afc362ce41f6e1dbad4d921d7273c6434ba2 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 13 Jun 2024 19:37:07 +0800 Subject: [PATCH 09/10] fix: tmpGeometry format2 --- src/benchData.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/benchData.c b/src/benchData.c index dfb5acd8..ef8f0b75 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -518,11 +518,11 @@ int tmpGeometry(char *tmp, int iface, Field *field, int64_t k) { } int32_t pos = snprintf(tmp, field->length, "LINESTRING("); - char * format = ",%d %d"; + char * format = "%d %d,"; for(int32_t i = 0; i < cnt; i++) { - if (i == 0) { + if (i == cnt - 1) { format = "%d %d"; - } + } pos += snprintf(tmp + pos, field->length - pos, format, tmpUint16(field), tmpUint16(field)); } strcat(tmp, ")"); From 1278ac0035f8348f35f8fbf7eaf6956b999b0922 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 14 Jun 2024 15:09:30 +0800 Subject: [PATCH 10/10] fix: build error --- src/benchData.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/benchData.c b/src/benchData.c index ef8f0b75..b3fd211b 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -2302,12 +2302,13 @@ void generateSmlTaosJsonCols(tools_cJSON *array, tools_cJSON *tag, tmfree(buf); break; } - case TSDB_DATA_TYPE_GEOMETRY: + case TSDB_DATA_TYPE_GEOMETRY: { char *buf = (char *)benchCalloc(col->length + 1, 1, false); tmpGeometry(buf, stbInfo->iface, col, 0); tools_cJSON_AddStringToObject(value, "value", buf); tools_cJSON_AddStringToObject(value, "type", "geometry"); tmfree(buf); + } default: { double dblTmp = (double)col->min; if (col->max != col->min) {