From 8fa2b58a660f6b6fb0d7af4ab49aa1f3fb3d1fd1 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Fri, 15 Sep 2023 10:06:19 +0800 Subject: [PATCH 01/57] feat: support geometry (not include schemaless) --- src/benchData.c | 3 ++- src/benchInsert.c | 8 +++++--- src/benchJsonOpt.c | 5 +++-- src/benchUtil.c | 10 ++++++++-- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/benchData.c b/src/benchData.c index a196ada3..9393cac9 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -697,7 +697,8 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_GEOMETRY: { char *tmp = benchCalloc(1, field->length + 1, false); if (0 != tmpStr(tmp, stbInfo->iface, field, i)) { free(tmp); diff --git a/src/benchInsert.c b/src/benchInsert.c index 5c3f0f24..1590708a 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -231,7 +231,8 @@ static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) { Field * col = benchArrayGet(stbInfo->cols, colIndex); int n; if (col->type == TSDB_DATA_TYPE_BINARY || - col->type == TSDB_DATA_TYPE_NCHAR) { + col->type == TSDB_DATA_TYPE_NCHAR || + col->type == TSDB_DATA_TYPE_GEOMETRY) { n = snprintf(colsBuf + len, col_buffer_len - len, ",%s %s(%d)", col->name, convertDatatypeToString(col->type), col->length); @@ -282,7 +283,8 @@ static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) { for (tagIndex = 0; tagIndex < stbInfo->tags->size; tagIndex++) { Field *tag = benchArrayGet(stbInfo->tags, tagIndex); if (tag->type == TSDB_DATA_TYPE_BINARY || - tag->type == TSDB_DATA_TYPE_NCHAR) { + tag->type == TSDB_DATA_TYPE_NCHAR || + tag->type == TSDB_DATA_TYPE_GEOMETRY) { n = snprintf(tagsBuf + len, tag_buffer_len - len, "%s %s(%d),", tag->name, convertDatatypeToString(tag->type), tag->length); @@ -1738,7 +1740,7 @@ static void *syncWriteInterlace(void *sarg) { infoPrint( "thread[%d] has currently inserted rows: %" PRIu64 ", peroid insert rate: %.3f rows/s \n", - pThreadInfo->threadID, pThreadInfo->totalInsertRows, + pThreadInfo->threadID, pThreadInfo->totalInsertRows, (double)(pThreadInfo->totalInsertRows - lastTotalInsertRows) * 1000.0/(currentPrintTime - lastPrintTime)); lastPrintTime = currentPrintTime; lastTotalInsertRows = pThreadInfo->totalInsertRows; diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 535d960b..8c83bf4a 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -171,7 +171,8 @@ static int getColumnAndTagTypeFromInsertJsonFile( } else { if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_JSON - || type == TSDB_DATA_TYPE_NCHAR) { + || type == TSDB_DATA_TYPE_NCHAR + || type == TSDB_DATA_TYPE_GEOMETRY) { length = g_arguments->binwidth; } else { length = convertTypeToLength(type); @@ -1840,7 +1841,7 @@ static int getMetaFromTmqJsonFile(tools_cJSON *json) { if (tools_cJSON_IsString(groupMode)) { g_tmqInfo.consumerInfo.groupMode = groupMode->valuestring; } - + tools_cJSON *pollDelay = tools_cJSON_GetObjectItem(tmqInfo, "poll_delay"); if (tools_cJSON_IsNumber(pollDelay)) { diff --git a/src/benchUtil.c b/src/benchUtil.c index 6eb97972..2b9fb69f 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -320,7 +320,7 @@ void closeBenchConn(SBenchConn* conn) { if(conn->taos) { taos_close(conn->taos); conn->taos = NULL; - } + } if (conn->ctaos) { taos_close(conn->ctaos); conn->ctaos = NULL; @@ -840,6 +840,8 @@ char *convertDatatypeToString(int type) { return "double"; case TSDB_DATA_TYPE_JSON: return "json"; + case TSDB_DATA_TYPE_GEOMETRY: + return "geometry"; default: break; } @@ -972,6 +974,8 @@ int convertStringToDatatype(char *type, int length) { return TSDB_DATA_TYPE_JSON; } else if (0 == strcasecmp(type, "varchar")) { return TSDB_DATA_TYPE_BINARY; + } else if (0 == strcasecmp(type, "geometry")) { + return TSDB_DATA_TYPE_GEOMETRY; } else { errorPrint("unknown data type: %s\n", type); exit(EXIT_FAILURE); @@ -1009,6 +1013,8 @@ int convertStringToDatatype(char *type, int length) { return TSDB_DATA_TYPE_JSON; } else if (0 == strncasecmp(type, "varchar", length)) { return TSDB_DATA_TYPE_BINARY; + } else if (0 == strcasecmp(type, "geometry")) { + return TSDB_DATA_TYPE_GEOMETRY; } else { errorPrint("unknown data type: %s\n", type); exit(EXIT_FAILURE); @@ -1191,7 +1197,7 @@ void destroySockFd(int sockfd) { if (sockfd < 0) { return; } - + // shutdown the connection since no more data will be sent int result; result = shutdown(sockfd, SHUT_WR); From c9094f4f465923600a2d287e97f3b800e5686a01 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Mon, 18 Sep 2023 16:02:21 +0800 Subject: [PATCH 02/57] feat: support for random geometry --- inc/benchData.h | 6 +- src/CMakeLists.txt | 12 +-- src/benchData.c | 37 +++++++++- src/benchDataGeometry.c | 158 ++++++++++++++++++++++++++++++++++++++++ src/benchInsert.c | 10 +++ src/benchUtil.c | 2 +- 6 files changed, 214 insertions(+), 11 deletions(-) create mode 100644 src/benchDataGeometry.c diff --git a/inc/benchData.h b/inc/benchData.h index 01dc5425..cf42916a 100644 --- a/inc/benchData.h +++ b/inc/benchData.h @@ -20,8 +20,10 @@ /***** Global variables ******/ /***** Declare functions *****/ void rand_string(char *str, int size, bool chinese); -int64_t getTSRandTail(int64_t timeStampStep, int32_t seq, int disorderRatio, - int disorderRange); +void rand_geometry(char *str, int fieldLen, int maxType); +int geoCalcBufferSize(int fieldLen); +int getGeoMaxType(int fieldLen); +int64_t getTSRandTail(int64_t timeStampStep, int32_t seq, int disorderRatio, int disorderRange); int generateRandData(SSuperTable *stbInfo, char *sampleDataBuf, int64_t bufLen, int lenOfOneRow, BArray * fields, int64_t loop, diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 06fda650..cb2a20f1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -222,9 +222,9 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin ADD_DEPENDENCIES(taosdump deps-jansson) ADD_DEPENDENCIES(taosdump deps-snappy) IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0") - ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) + ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchDataGeometry.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) ELSE() - ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) + ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchDataGeometry.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) ENDIF() ELSE () INCLUDE_DIRECTORIES(/usr/local/include) @@ -233,9 +233,9 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin SET(OS_ID "Darwin") IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0") - ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) + ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchDataGeometry.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) ELSE() - ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) + ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchDataGeometry.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) ENDIF() ENDIF () @@ -427,9 +427,9 @@ ELSE () SET(CMAKE_C_STANDARD 11) SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /utf-8") IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0") - ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsString.c toolsSys.c toolsString.c) + ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchDataGeometry.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsString.c toolsSys.c toolsString.c) ELSE () - ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) + ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchDataGeometry.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) ENDIF () ADD_EXECUTABLE(taosdump taosdump.c toolsSys.c toolstime.c toolsDir.c toolsString.c) diff --git a/src/benchData.c b/src/benchData.c index 9393cac9..a01e0cd8 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -300,6 +300,7 @@ uint32_t accumulateRowLen(BArray *fields, int iface) { switch (field->type) { case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_GEOMETRY: len += field->length + 3; break; case TSDB_DATA_TYPE_INT: @@ -387,6 +388,28 @@ static int tmpStr(char *tmp, int iface, Field *field, int i) { return 0; } +static int tmpGeometry(char *tmp, int iface, Field *field, int i) { + if (g_arguments->demo_mode) { + // TODO + } else if (field->values) { + int arraySize = tools_cJSON_GetArraySize(field->values); + if (arraySize) { + tools_cJSON *buf = tools_cJSON_GetArrayItem(field->values, taosRandom() % arraySize); + snprintf(tmp, field->length, "%s", buf->valuestring); + } else { + errorPrint( + "%s() cannot read correct value " + "from json file. array size: %d\n", + __func__, arraySize); + return -1; + } + } else { + int maxType = getGeoMaxType(field->length); + rand_geometry(tmp, field->length, maxType); + } + return 0; +} + FORCE_INLINE double tmpDoubleImpl(Field *field, int32_t angle) { double doubleTmp = (double)(field->min); @@ -697,8 +720,7 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_GEOMETRY: { + case TSDB_DATA_TYPE_NCHAR: { char *tmp = benchCalloc(1, field->length + 1, false); if (0 != tmpStr(tmp, stbInfo->iface, field, i)) { free(tmp); @@ -709,6 +731,17 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf, tmfree(tmp); break; } + case TSDB_DATA_TYPE_GEOMETRY: { + int bufferSize = geoCalcBufferSize(field->length); + char *tmp = benchCalloc(1, bufferSize, false); + if (0 != tmpGeometry(tmp, stbInfo->iface, field, i)) { + free(tmp); + return -1; + } + n = snprintf(sampleDataBuf + pos, bufLen - pos, "'%s',", tmp); + tmfree(tmp); + break; + } case TSDB_DATA_TYPE_JSON: { pos += tmpJson(sampleDataBuf, bufLen, pos, fieldsSize, field); diff --git a/src/benchDataGeometry.c b/src/benchDataGeometry.c new file mode 100644 index 00000000..a13189a1 --- /dev/null +++ b/src/benchDataGeometry.c @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the MIT license as published by the Free Software + * Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + */ + +#include +#include + +typedef struct { + double x; + double y; +} SGeoCoord2D; + +typedef enum { GEO_SUB_TYPE_POINT = 0, GEO_SUB_TYPE_LINESTRING, GEO_SUB_TYPE_POLYGON, GEO_SUB_TYPE_COUNT } EGeoSubType; + +const int CCoordSize = 16; + +typedef struct { + const char *preifx; + const char *suffix; + int baseLen; + int minCoordNum; + int maxCoordNum; + bool isClosed; // if true, first coord and last coord must be the same +} SGeoSubTypeInfo; + +// should be ordered by (baseLen + minCoordNum * CCoordSize), ASC +const SGeoSubTypeInfo GeoInfo[] = { + {"POINT(", ")", 5, 1, 1, false}, {"LINESTRING(", ")", 9, 2, 4094, false}, {"POLYGON((", "))", 13, 3, 4094, true}}; + +typedef struct { + EGeoSubType type; + BArray *coordArray; // element: SGeoCoord2D +} SGeoObj2D; + +static SGeoObj2D *geoObject2DInit(EGeoSubType subType, BArray *coordArray); +static SGeoObj2D *geoObject2DRandInit(EGeoSubType subType, int coordNum); +static void geoObject2DDestory(SGeoObj2D *pObj); + +static int geoCoord2DToStr(char *buffer, SGeoCoord2D *pCoord); +static int geoCoord2DArrayToStr(char *buffer, BArray *coordArray); +static int geoObject2DToStr(char *buffer, SGeoObj2D *object); + +static BArray *randCoordArray(int count); + +/*--- init & destory ---*/ +static SGeoObj2D *geoObject2DInit(EGeoSubType subType, BArray *coordArray) { + SGeoObj2D *pObj = (SGeoObj2D *)benchCalloc(1, sizeof(SGeoObj2D), true); + pObj->type = subType; + pObj->coordArray = coordArray; + return pObj; +} + +static SGeoObj2D *geoObject2DRandInit(EGeoSubType subType, int coordNum) { + const SGeoSubTypeInfo *info = &GeoInfo[subType]; + + if (info->isClosed) { + coordNum = coordNum - 1; + } + BArray *array = randCoordArray(coordNum); + if (info->isClosed) { + SGeoCoord2D *pCoord = (SGeoCoord2D *)benchCalloc(1, sizeof(SGeoCoord2D), true); + SGeoCoord2D *pFirstCoord= benchArrayGet(array, 0); + memcpy(pCoord, pFirstCoord, sizeof(SGeoCoord2D)); + benchArrayPush(array, pCoord); + } + return geoObject2DInit(subType, array); +} + +static void geoObject2DDestory(SGeoObj2D *pObj) { + if (!pObj) return; + benchArrayDestroy(pObj->coordArray); + tmfree(pObj); +} + +/*--- string formatters ---*/ +static int geoCoord2DToStr(char *buffer, SGeoCoord2D *pCoord) { return sprintf(buffer, "%10.6lf %10.6lf", pCoord->x, pCoord->y); } + +static int geoCoord2DArrayToStr(char *buffer, BArray *coordArray) { + int pos = 0; + bool firstCoord = true; + for (int i = 0; i < coordArray->size; i++) { + int size = 0; + if (firstCoord) { + firstCoord = false; + } else { + size = sprintf(buffer + pos, "%s", ", "); + pos += size; + } + size = geoCoord2DToStr(buffer + pos, benchArrayGet(coordArray, i)); + pos += size; + } + return pos; +} + +static int geoObject2DToStr(char *buffer, SGeoObj2D *object) { + int pos = sprintf(buffer, "%s", GeoInfo[object->type].preifx); + pos += geoCoord2DArrayToStr(buffer + pos, object->coordArray); + pos += sprintf(buffer + pos, "%s", GeoInfo[object->type].suffix); + return pos; +} + +static BArray *randCoordArray(int count) { + BArray *array = benchArrayInit(8, sizeof(SGeoCoord2D)); + int minVal = -1000, maxVal = 1000; + for (int i = 0; i < count; i++) { + double x = minVal + 1.0 * taosRandom() / RAND_MAX * (maxVal - minVal); + double y = minVal + 1.0 * taosRandom() / RAND_MAX * (maxVal - minVal); + SGeoCoord2D *pCoord = (SGeoCoord2D *)benchCalloc(1, sizeof(SGeoCoord2D), true); + pCoord->x = x; + pCoord->y = y; + benchArrayPush(array, pCoord); + } + return array; +} + +int geoCalcBufferSize(int fieldLen) { + // not accurate, but enough + return fieldLen + 20; +} + +int getGeoMaxType(int fieldLen) { + int maxType = -1; + for (int type = GEO_SUB_TYPE_COUNT - 1; type >= 0; type--) { + const SGeoSubTypeInfo *info = &GeoInfo[type]; + + int minLen = info->baseLen + info->minCoordNum * CCoordSize; + if (fieldLen >= minLen) { + maxType = type; + break; + } + } + return maxType; +} + +void rand_geometry(char *str, int fieldLen, int maxType) { + EGeoSubType type = taosRandom() % (maxType + 1); + const SGeoSubTypeInfo *info = &GeoInfo[type]; + + int maxCoordNum = (fieldLen - info->baseLen) / CCoordSize; + maxCoordNum = min(maxCoordNum, info->maxCoordNum); + + int coordNum = info->minCoordNum; + if (maxCoordNum > info->minCoordNum) { + coordNum = info->minCoordNum + taosRandom() % (maxCoordNum - info->minCoordNum); + } + + SGeoObj2D *pObj = geoObject2DRandInit(type, coordNum); + geoObject2DToStr(str, pObj); + geoObject2DDestory(pObj); +} diff --git a/src/benchInsert.c b/src/benchInsert.c index 1590708a..e7e9bab6 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -236,6 +236,11 @@ static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) { n = snprintf(colsBuf + len, col_buffer_len - len, ",%s %s(%d)", col->name, convertDatatypeToString(col->type), col->length); + if (col->type == TSDB_DATA_TYPE_GEOMETRY && col->length < 21) { + errorPrint("%s() LN%d, geometry filed len must be greater than 21 on %d\n", __func__, __LINE__, + colIndex); + return -1; + } } else { n = snprintf(colsBuf + len, col_buffer_len - len, ",%s %s", col->name, @@ -288,6 +293,11 @@ static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) { n = snprintf(tagsBuf + len, tag_buffer_len - len, "%s %s(%d),", tag->name, convertDatatypeToString(tag->type), tag->length); + if (tag->type == TSDB_DATA_TYPE_GEOMETRY && tag->length < 21) { + errorPrint("%s() LN%d, geometry filed len must be greater than 21 on %d\n", __func__, __LINE__, + tagIndex); + return -1; + } } else if (tag->type == TSDB_DATA_TYPE_JSON) { n = snprintf(tagsBuf + len, tag_buffer_len - len, "%s json", tag->name); diff --git a/src/benchUtil.c b/src/benchUtil.c index 2b9fb69f..4b54854d 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -1071,7 +1071,7 @@ void* benchArrayAddBatch(BArray* pArray, void* pData, int32_t elems) { void* dst = BARRAY_GET_ELEM(pArray, pArray->size); memcpy(dst, pData, pArray->elemSize * elems); - tmfree(pData); + tmfree(pData); // TODO remove this pArray->size += elems; return dst; } From 958b2bbae550026dc46b5d49b9497c8a82df5957 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 21 May 2024 10:37:45 +0800 Subject: [PATCH 03/57] feat: init csv moudle --- case/exportCsv.json | 44 ++++++++ inc/bench.h | 2 + inc/benchCsv.h | 35 +++++++ src/benchCsv.c | 250 ++++++++++++++++++++++++++++++++++++++++++++ src/benchJsonOpt.c | 17 ++- src/benchMain.c | 6 ++ 6 files changed, 353 insertions(+), 1 deletion(-) create mode 100644 case/exportCsv.json create mode 100644 inc/benchCsv.h create mode 100644 src/benchCsv.c diff --git a/case/exportCsv.json b/case/exportCsv.json new file mode 100644 index 00000000..1155b9b6 --- /dev/null +++ b/case/exportCsv.json @@ -0,0 +1,44 @@ +{ + "filetype": "csv", + "csvDir": "/root/csv/", + "databases": [ + { + "dbinfo": { + "name": "exportCsv" + }, + "super_tables": [ + { + "name": "meters", + "childtable_count": 100, + "insert_rows": 100000, + "childtable_prefix": "d", + "timestamp_step": 10, + "start_timestamp":1600000000000, + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "min": 1}, + { "type": "double", "name": "dc", "min":10}, + { "type": "tinyint", "name": "ti"}, + { "type": "smallint", "name": "si"}, + { "type": "int", "name": "ic", "fillNull":"false"}, + { "type": "bigint", "name": "bi"}, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi"}, + { "type": "uint", "name": "ui"}, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 32}, + { "type": "nchar", "name": "nch", "len": 64} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"type": "binary", "name": "location", "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/inc/bench.h b/inc/bench.h index 10b8bb9a..65139afa 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -470,6 +470,7 @@ enum TEST_MODE { INSERT_TEST, // 0 QUERY_TEST, // 1 SUBSCRIBE_TEST, // 2 + CSVFILE_TEST // 3 }; enum enumSYNC_MODE { SYNC_MODE, ASYNC_MODE, MODE_BUT }; @@ -965,6 +966,7 @@ typedef struct SArguments_S { bool mistMode; bool escape_character; bool pre_load_tb_meta; + char csvPath[MAX_FILE_NAME_LEN]; } SArguments; typedef struct SBenchConn { diff --git a/inc/benchCsv.h b/inc/benchCsv.h new file mode 100644 index 00000000..51325ae1 --- /dev/null +++ b/inc/benchCsv.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the MIT license as published by the Free Software + * Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef INC_BENCHCSV_H_ +#define INC_BENCHCSV_H_ + +#include + +int csvTestProcess(); + +int genWithSTable(SDataBase* db, SSuperTable* stb, char* outDir); + +void csvWriteThread(void* param); + +char * genTagData(char* buf, SSuperTable* stb, int64_t i, int64_t *k); + +char * genColumnData(char* buf, SSuperTable* stb, int64_t i, int precision, int64_t *k) + +int32_t genRowByField(char* buf, int32_t pos1, BArray* fields, int16_t fieldCnt, char* binanryPrefix, char* ncharPrefix, int64_t *k); + +void obtainCsvFile(char * outFile, SDataBase* db, SSuperTable* stb, char* outDir); + +#endif // INC_BENCHCSV_H_ diff --git a/src/benchCsv.c b/src/benchCsv.c new file mode 100644 index 00000000..c0ac1e2a --- /dev/null +++ b/src/benchCsv.c @@ -0,0 +1,250 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the MIT license as published by the Free Software + * Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + */ + +#include +#include +#include + + +// +// main etry +// +int csvTestProcess() { + pthread_t handle = NULL; + int ret = pthread_create(handle, NULL, csvWriteThread, NULL); + if (ret != 0) { + errorPrint("pthread_create failed. error code =%d", ret); + return -1; + } + pthread_join(handle); + return 0; +} + +void csvWriteThread(void* param) { + // write thread + for (int i = 0; i < g_arguments->databases->size; i++) { + // database + SDataBase * db = benchArrayGet(g_arguments->databases, i); + for (int j=0; i< db->superTbls->size; j++) { + // stb + SSuperTable* stb = benchArrayGet(database->superTbls, j); + // gen csv + int ret = genWithSTable(db, stb, g_arguments->csvPath); + if(ret != 0) { + errorPrint("failed generate to csv. db=%s stb=%s error code=%d", db->dbName, stb->stbName, ret); + return ret; + } + } + } +} + +int genWithSTable(SDataBase* db, SSuperTable* stb, char* outDir) { + // filename + int ret = 0; + char outFile[MAX_FILE_NAME_LEN] = {0}; + obtainCsvFile(outFile, db, stb, outDir); + FILE * fs = fopen(outFile, "w"); + if(fs == NULL) { + errorPrint("failed create csv file. file=%s, last errno=%d strerror=%s", outFile, errno, strerror(errno)); + return -1; + } + + int rowLen = TSDB_TABLE_NAME_LEN + stb->lenOfTags + stb->lenOfCols + stb->tags->size + stb->cols->size; + int bufLen = rowLen * g_arguments->reqPerReq; + char* buf = benchCalloc(1, bufLen, true); + + if (stb->interlaceRows > 0) { + // interlace mode + ret = interlaceWriteCsv(db, stb, fs, buf, bufLen, rowLen * 2); + } else { + // batch mode + ret = batchWriteCsv(db, stb, fs, buf, bufLen, rowLen * 2); + } + + tmfree(buf); + close(fs); + + return ret; +} + + +void obtainCsvFile(char * outFile, SDataBase* db, SSuperTable* stb, char* outDir) { + sprintf(outFile, "%s//%s-%s.csv", outDir, db->dbName, stb->stbName); +} + + + +int32_t writeCsvFile(FILE* f, char * buf, int32_t len) { + size_t size = fwrite(buf, 1, len, f); + if(size ! = len) { + errorPrint("failed to write csv file. expect write length:%d real write length:%d", len, size); + return -1; + } + return 0; +} + +int batchWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufLen, int minRemain) { + int ret = 0; + int cnt = 0; + int pos = 0; + int64_t n = 0; // already inserted rows for one child table + + int tagDataLen = stb->lenOfTags + stb->tags->size + 256; + char * tagData = (char *) benchCalloc(1, tagDataLen, true); + int colDataLen = stb->lenOfCols + stb->cols->size + 256; + char * colData = (char *) benchCalloc(1, colDataLen, true); + + // gen child name + for (int64_t i = 0; i < stb->childTblCount; i++) { + int64_t ts = stb->startTimestamp; + // tags + genTagData(tagData, stb, i); + // insert child column data + for(int64_t j=0; j< stb->insertRows; j++) { + genColumnData(colData, ts, i, j); + // combine + pos += sprintf(buf + pos, "%s,%s\n", tagData, colData); + if (bufLen - pos < minRemain ) { + // submit + ret = writeCsvFile(fs, buf, pos); + if (ret != 0) { + goto END; + } + + pos = 0; + } + + // ts move next + ts += stb->timestamp_step; + } + } + +END: + // free + tmfree(tagData); + tmfree(colData); + return ret; +} + +int interlaceWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufLen, int minRemain) { + int ret = 0; + int cnt = 0; + int pos = 0; + int64_t n = 0; // already inserted rows for one child table + int64_t tk = 0; + + char **tagDatas = (char **)benchCalloc(stb->childTblCount, sizeof(char *), true); + int colDataLen = stb->lenOfCols + stb->cols->size + 256; + char * colData = (char *) benchCalloc(1, colDataLen, true); + + while (n < stb->insertRows ) { + for (int64_t i = 0; i < stb->childTblCount; i++) { + // start one table + int64_t ts = stb->startTimestamp; + int64_t ck = 0; + // tags + tagDatas[i] = genTagData(NULL, stb, i, &tk); + // calc need insert rows + int64_t needInserts = stb->interlaceRows; + if(needInserts > stb->insertRows - n) { + needInserts = stb->insertRows - n; + } + + for (int64_t j = 0; j < needInserts; j++) { + genColumnData(colData, ts, i, j, &ck, db->precision); + // combine tags,cols + pos += sprintf(buf + pos, "%s,%s\n", tagDatas[i], colData); + if (bufLen - pos < ) { + // submit + ret = writeToFile(fs, buf, pos); + if (ret != 0) { + goto END: + } + pos = 0; + } + + // ts move next + ts += stb->timestamp_step; + } + + if (i == 0) { + n += needInserts; + } + } + } + +END: + // free + for(int64_t m = 0 ; m < stb->childTblCount; m ++) { + if (tagDatas[m]) { + tmfree(tagDatas[m]); + } + } + tmfree(colData); + return ret; +} + +// gen tag data +char * genTagData(char* buf, SSuperTable* stb, int64_t i, int64_t *k) { + // malloc + char* tagData; + if (buf == NULL) { + int tagDataLen = TSDB_TABLE_NAME_LEN + stb->lenOfTags + stb->tags->size + 32; + tagData = benchCalloc(1, tagDataLen, true); + } else { + tagData = buf; + } + + int pos = 0; + // tbname + pos += sprintf(tagData + pos, "%s%d", stb->childTblPrefix); + // tags + pos += genRowByField(tagData + pos, stb->tags, stb->tags->size, stb->binaryPrefex, stb->ncharPrefex, k); + + return tagData; +} + +// gen column data +char * genColumnData(char* colData, SSuperTable* stb, int64_t ts, int64_t i, int32_t precision, int64_t *k) { + int pos = 0; + char str[128] = ""; + toolsFormatTimestamp(colData, ts, precision); + pos = strlen(colData); + + // columns + genRowByField(colData + pos, stb->cols, stb->cols->size, stb->binaryPrefex, stb->ncharPrefex, precision, k); + return colData; +} + + +int32_t genRowByField(char* buf, int32_t pos1, BArray* fields, int16_t fieldCnt, char* binanryPrefix, char* ncharPrefix, int64_t *k) { + + // other cols data + int32_t pos2 = 0; + for(uint16_t i = 0; i < fieldCnt; i++) { + Field* fd = benchArrayGet(fields, GET_IDX(i)); + char* prefix = ""; + if(fd->type == TSDB_DATA_TYPE_BINARY || fd->type == TSDB_DATA_TYPE_VARBINARY) { + if(stb->binaryPrefex) { + prefix = stb->binaryPrefex; + } + } else if(fd->type == TSDB_DATA_TYPE_NCHAR) { + if(stb->ncharPrefex) { + prefix = stb->ncharPrefex; + } + } + + pos2 += dataGenByField(fd, buf + pos1 + pos2, prefix, k); + } + + return pos2; +} diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 4f7da1c2..735810de 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -1425,6 +1425,19 @@ static int getMetaFromCommonJsonFile(tools_cJSON *json) { } } + g_arguments->csvPath[0] = 0; + tools_cJSON *csv = tools_cJSON_GetObjectItem(json, "csvPath"); + if (csvPath && (csv->type == tools_cJSON_String) + && (csv->valuestring != NULL)) { + tstrncpy(g_arguments->csvPath, csv->valuestring, MAX_FILE_NAME_LEN); + } + + if(g_arguments->csvPath[0] == 0) { + // set default with current path + strcpy(g_arguments->csvPath, "./output"); + mkdir(_arguments->csvPath, 0775); + } + code = 0; return code; } @@ -2230,6 +2243,8 @@ int getInfoFromJsonFile() { g_arguments->test_mode = QUERY_TEST; } else if (0 == strcasecmp("subscribe", filetype->valuestring)) { g_arguments->test_mode = SUBSCRIBE_TEST; + } else if (0 == strcasecmp("csvfile", filetype->valuestring)) { + g_arguments->test_mode = CSVFILE_TEST; } else { errorPrint("%s", "failed to read json, filetype not support\n"); @@ -2241,7 +2256,7 @@ int getInfoFromJsonFile() { // read common item code = getMetaFromCommonJsonFile(root); - if (INSERT_TEST == g_arguments->test_mode) { + if (INSERT_TEST == g_arguments->test_mode || CSVFILE_TEST == g_arguments->test_mode) { code = getMetaFromInsertJsonFile(root); #ifdef TD_VER_COMPATIBLE_3_0_0_0 } else if (QUERY_TEST == g_arguments->test_mode) { diff --git a/src/benchMain.c b/src/benchMain.c index 82e45ddb..f790c2cf 100644 --- a/src/benchMain.c +++ b/src/benchMain.c @@ -11,6 +11,7 @@ */ #include +#include #include SArguments* g_arguments; @@ -117,6 +118,11 @@ int main(int argc, char* argv[]) { errorPrint("%s", "insert test process failed\n"); ret = -1; } + } else if (g_arguments->test_mode == CSVFILE_TEST) { + if (csvTestProcess()) { + errorPrint("%s", "query test process failed\n"); + ret = -1; + } } else if (g_arguments->test_mode == QUERY_TEST) { if (queryTestProcess(g_arguments)) { errorPrint("%s", "query test process failed\n"); From 14d2d6867839041b6d72c07f7749899dd69dd5f4 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 21 May 2024 10:41:00 +0800 Subject: [PATCH 04/57] fix: read json file error --- src/benchJsonOpt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 735810de..cb9cabae 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -1427,7 +1427,7 @@ static int getMetaFromCommonJsonFile(tools_cJSON *json) { g_arguments->csvPath[0] = 0; tools_cJSON *csv = tools_cJSON_GetObjectItem(json, "csvPath"); - if (csvPath && (csv->type == tools_cJSON_String) + if (csv && (csv->type == tools_cJSON_String) && (csv->valuestring != NULL)) { tstrncpy(g_arguments->csvPath, csv->valuestring, MAX_FILE_NAME_LEN); } From bb8eea8c0d3c53ad6b64d93695b87f4c3c0851cb Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 21 May 2024 11:09:48 +0800 Subject: [PATCH 05/57] fix: timestamp now with toolsFormatTimestamp replace --- inc/benchCsv.h | 2 +- src/benchCsv.c | 16 +++++++++------- src/benchDataMix.c | 3 ++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/inc/benchCsv.h b/inc/benchCsv.h index 51325ae1..37b3ea62 100644 --- a/inc/benchCsv.h +++ b/inc/benchCsv.h @@ -26,7 +26,7 @@ void csvWriteThread(void* param); char * genTagData(char* buf, SSuperTable* stb, int64_t i, int64_t *k); -char * genColumnData(char* buf, SSuperTable* stb, int64_t i, int precision, int64_t *k) +char * genColumnData(char* colData, SSuperTable* stb, int64_t ts, int32_t precision, int64_t *k); int32_t genRowByField(char* buf, int32_t pos1, BArray* fields, int16_t fieldCnt, char* binanryPrefix, char* ncharPrefix, int64_t *k); diff --git a/src/benchCsv.c b/src/benchCsv.c index c0ac1e2a..65bc5220 100644 --- a/src/benchCsv.c +++ b/src/benchCsv.c @@ -71,7 +71,7 @@ int genWithSTable(SDataBase* db, SSuperTable* stb, char* outDir) { } tmfree(buf); - close(fs); + fclose(fs); return ret; } @@ -97,6 +97,7 @@ int batchWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufL int cnt = 0; int pos = 0; int64_t n = 0; // already inserted rows for one child table + int64_t tk = 0; int tagDataLen = stb->lenOfTags + stb->tags->size + 256; char * tagData = (char *) benchCalloc(1, tagDataLen, true); @@ -106,11 +107,12 @@ int batchWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufL // gen child name for (int64_t i = 0; i < stb->childTblCount; i++) { int64_t ts = stb->startTimestamp; + int64_t ck = 0; // tags - genTagData(tagData, stb, i); + genTagData(tagData, stb, i, &tk); // insert child column data - for(int64_t j=0; j< stb->insertRows; j++) { - genColumnData(colData, ts, i, j); + for(int64_t j = 0; j < stb->insertRows; j++) { + genColumnData(colData, ts, db->precision, &ck); // combine pos += sprintf(buf + pos, "%s,%s\n", tagData, colData); if (bufLen - pos < minRemain ) { @@ -160,7 +162,7 @@ int interlaceWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int } for (int64_t j = 0; j < needInserts; j++) { - genColumnData(colData, ts, i, j, &ck, db->precision); + genColumnData(colData, ts, db->precision, &ck); // combine tags,cols pos += sprintf(buf + pos, "%s,%s\n", tagDatas[i], colData); if (bufLen - pos < ) { @@ -206,7 +208,7 @@ char * genTagData(char* buf, SSuperTable* stb, int64_t i, int64_t *k) { int pos = 0; // tbname - pos += sprintf(tagData + pos, "%s%d", stb->childTblPrefix); + pos += sprintf(tagData, "%s%"PRId64, stb->childTblPrefix, i); // tags pos += genRowByField(tagData + pos, stb->tags, stb->tags->size, stb->binaryPrefex, stb->ncharPrefex, k); @@ -214,7 +216,7 @@ char * genTagData(char* buf, SSuperTable* stb, int64_t i, int64_t *k) { } // gen column data -char * genColumnData(char* colData, SSuperTable* stb, int64_t ts, int64_t i, int32_t precision, int64_t *k) { +char * genColumnData(char* colData, SSuperTable* stb, int64_t ts, int32_t precision, int64_t *k) { int pos = 0; char str[128] = ""; toolsFormatTimestamp(colData, ts, precision); diff --git a/src/benchDataMix.c b/src/benchDataMix.c index 1521f263..dec69104 100644 --- a/src/benchDataMix.c +++ b/src/benchDataMix.c @@ -111,7 +111,8 @@ uint32_t dataGenByField(Field* fd, char* pstr, uint32_t len, char* prefix, int64 break; // timestamp case TSDB_DATA_TYPE_TIMESTAMP: - strcpy(val, "now"); + int64_t nowts = toolsGetTimestampMs(); + toolsFormatTimestamp(val, nowts, TSDB_TIME_PRECISION_MILLI); break; // signed case TSDB_DATA_TYPE_TINYINT: From feac5b1fe25ccdd6d1eeb5430fd8660ae557b25d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 21 May 2024 11:24:09 +0800 Subject: [PATCH 06/57] fix: add source benchCsv.c --- src/CMakeLists.txt | 8 ++++---- src/benchJsonOpt.c | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6a177a5e..969f5a25 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -235,9 +235,9 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin ADD_DEPENDENCIES(taosdump deps-jansson) ADD_DEPENDENCIES(taosdump deps-snappy) IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0") - ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) + ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) ELSE() - ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) + ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) ENDIF() ELSE () INCLUDE_DIRECTORIES(/usr/local/include) @@ -246,9 +246,9 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin SET(OS_ID "Darwin") IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0") - ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) + ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) ELSE() - ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) + ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) ENDIF() ENDIF () diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index cb9cabae..0da21ed8 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -11,6 +11,7 @@ */ #include +#include #include extern char g_configDir[MAX_PATH_LEN]; @@ -1435,7 +1436,7 @@ static int getMetaFromCommonJsonFile(tools_cJSON *json) { if(g_arguments->csvPath[0] == 0) { // set default with current path strcpy(g_arguments->csvPath, "./output"); - mkdir(_arguments->csvPath, 0775); + mkdir(g_arguments->csvPath, 0775); } code = 0; From 589991ff112e55711912d7b4b2237d3999867026 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 21 May 2024 16:37:55 +0800 Subject: [PATCH 07/57] feat : basic function finished --- case/exportCsv.json | 4 +- inc/benchCsv.h | 7 +-- src/benchCsv.c | 104 ++++++++++++++++++++++++-------------------- 3 files changed, 64 insertions(+), 51 deletions(-) diff --git a/case/exportCsv.json b/case/exportCsv.json index 1155b9b6..0b8ba753 100644 --- a/case/exportCsv.json +++ b/case/exportCsv.json @@ -1,6 +1,6 @@ { - "filetype": "csv", - "csvDir": "/root/csv/", + "filetype": "csvfile", + "csvPath": "/root/csv/", "databases": [ { "dbinfo": { diff --git a/inc/benchCsv.h b/inc/benchCsv.h index 37b3ea62..25d0c55e 100644 --- a/inc/benchCsv.h +++ b/inc/benchCsv.h @@ -22,14 +22,15 @@ int csvTestProcess(); int genWithSTable(SDataBase* db, SSuperTable* stb, char* outDir); -void csvWriteThread(void* param); - char * genTagData(char* buf, SSuperTable* stb, int64_t i, int64_t *k); char * genColumnData(char* colData, SSuperTable* stb, int64_t ts, int32_t precision, int64_t *k); -int32_t genRowByField(char* buf, int32_t pos1, BArray* fields, int16_t fieldCnt, char* binanryPrefix, char* ncharPrefix, int64_t *k); +int32_t genRowByField(char* buf, BArray* fields, int16_t fieldCnt, char* binanryPrefix, char* ncharPrefix, int64_t *k); void obtainCsvFile(char * outFile, SDataBase* db, SSuperTable* stb, char* outDir); +int interlaceWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufLen, int minRemain); +int batchWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufLen, int minRemain); + #endif // INC_BENCHCSV_H_ diff --git a/src/benchCsv.c b/src/benchCsv.c index 65bc5220..c75d9120 100644 --- a/src/benchCsv.c +++ b/src/benchCsv.c @@ -18,33 +18,35 @@ // // main etry // -int csvTestProcess() { - pthread_t handle = NULL; - int ret = pthread_create(handle, NULL, csvWriteThread, NULL); - if (ret != 0) { - errorPrint("pthread_create failed. error code =%d", ret); - return -1; - } - pthread_join(handle); - return 0; -} -void csvWriteThread(void* param) { +static void *csvWriteThread(void *param) { // write thread for (int i = 0; i < g_arguments->databases->size; i++) { // database SDataBase * db = benchArrayGet(g_arguments->databases, i); - for (int j=0; i< db->superTbls->size; j++) { + for (int j=0; j < db->superTbls->size; j++) { // stb - SSuperTable* stb = benchArrayGet(database->superTbls, j); + SSuperTable* stb = benchArrayGet(db->superTbls, j); // gen csv int ret = genWithSTable(db, stb, g_arguments->csvPath); if(ret != 0) { errorPrint("failed generate to csv. db=%s stb=%s error code=%d", db->dbName, stb->stbName, ret); - return ret; + return NULL; } } } + return NULL; +} + +int csvTestProcess() { + pthread_t handle; + int ret = pthread_create(&handle, NULL, csvWriteThread, NULL); + if (ret != 0) { + errorPrint("pthread_create failed. error code =%d", ret); + return -1; + } + pthread_join(handle, NULL); + return 0; } int genWithSTable(SDataBase* db, SSuperTable* stb, char* outDir) { @@ -85,8 +87,8 @@ void obtainCsvFile(char * outFile, SDataBase* db, SSuperTable* stb, char* outDir int32_t writeCsvFile(FILE* f, char * buf, int32_t len) { size_t size = fwrite(buf, 1, len, f); - if(size ! = len) { - errorPrint("failed to write csv file. expect write length:%d real write length:%d", len, size); + if(size != len) { + errorPrint("failed to write csv file. expect write length:%d real write length:%d", len, (int32_t)size); return -1; } return 0; @@ -94,9 +96,7 @@ int32_t writeCsvFile(FILE* f, char * buf, int32_t len) { int batchWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufLen, int minRemain) { int ret = 0; - int cnt = 0; int pos = 0; - int64_t n = 0; // already inserted rows for one child table int64_t tk = 0; int tagDataLen = stb->lenOfTags + stb->tags->size + 256; @@ -112,10 +112,10 @@ int batchWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufL genTagData(tagData, stb, i, &tk); // insert child column data for(int64_t j = 0; j < stb->insertRows; j++) { - genColumnData(colData, ts, db->precision, &ck); + genColumnData(colData, stb, ts, db->precision, &ck); // combine pos += sprintf(buf + pos, "%s,%s\n", tagData, colData); - if (bufLen - pos < minRemain ) { + if (bufLen - pos < minRemain) { // submit ret = writeCsvFile(fs, buf, pos); if (ret != 0) { @@ -130,6 +130,11 @@ int batchWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufL } } + if (pos > 0) { + ret = writeCsvFile(fs, buf, pos); + pos = 0; + } + END: // free tmfree(tagData); @@ -139,7 +144,6 @@ int batchWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufL int interlaceWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufLen, int minRemain) { int ret = 0; - int cnt = 0; int pos = 0; int64_t n = 0; // already inserted rows for one child table int64_t tk = 0; @@ -147,14 +151,18 @@ int interlaceWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int char **tagDatas = (char **)benchCalloc(stb->childTblCount, sizeof(char *), true); int colDataLen = stb->lenOfCols + stb->cols->size + 256; char * colData = (char *) benchCalloc(1, colDataLen, true); + int64_t last_ts = stb->startTimestamp; while (n < stb->insertRows ) { for (int64_t i = 0; i < stb->childTblCount; i++) { // start one table - int64_t ts = stb->startTimestamp; + int64_t ts = last_ts; int64_t ck = 0; // tags - tagDatas[i] = genTagData(NULL, stb, i, &tk); + if (tagDatas[i] == NULL) { + tagDatas[i] = genTagData(NULL, stb, i, &tk); + } + // calc need insert rows int64_t needInserts = stb->interlaceRows; if(needInserts > stb->insertRows - n) { @@ -162,14 +170,14 @@ int interlaceWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int } for (int64_t j = 0; j < needInserts; j++) { - genColumnData(colData, ts, db->precision, &ck); + genColumnData(colData, stb, ts, db->precision, &ck); // combine tags,cols pos += sprintf(buf + pos, "%s,%s\n", tagDatas[i], colData); - if (bufLen - pos < ) { + if (bufLen - pos < minRemain) { // submit - ret = writeToFile(fs, buf, pos); + ret = writeCsvFile(fs, buf, pos); if (ret != 0) { - goto END: + goto END; } pos = 0; } @@ -178,18 +186,23 @@ int interlaceWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int ts += stb->timestamp_step; } - if (i == 0) { + // if last child table + if (i + 1 == stb->childTblCount ) { n += needInserts; + last_ts = ts; } } } + if (pos > 0) { + ret = writeCsvFile(fs, buf, pos); + pos = 0; + } + END: // free - for(int64_t m = 0 ; m < stb->childTblCount; m ++) { - if (tagDatas[m]) { + for(int64_t m = 0 ; m < stb->childTblCount; m ++) { tmfree(tagDatas[m]); - } } tmfree(colData); return ret; @@ -208,7 +221,7 @@ char * genTagData(char* buf, SSuperTable* stb, int64_t i, int64_t *k) { int pos = 0; // tbname - pos += sprintf(tagData, "%s%"PRId64, stb->childTblPrefix, i); + pos += sprintf(tagData, "\'%s%"PRId64"\'", stb->childTblPrefix, i); // tags pos += genRowByField(tagData + pos, stb->tags, stb->tags->size, stb->binaryPrefex, stb->ncharPrefex, k); @@ -217,36 +230,35 @@ char * genTagData(char* buf, SSuperTable* stb, int64_t i, int64_t *k) { // gen column data char * genColumnData(char* colData, SSuperTable* stb, int64_t ts, int32_t precision, int64_t *k) { - int pos = 0; - char str[128] = ""; - toolsFormatTimestamp(colData, ts, precision); - pos = strlen(colData); + char szTime[128] = {0}; + toolsFormatTimestamp(szTime, ts, precision); + int pos = sprintf(colData, "\'%s\'", szTime); // columns - genRowByField(colData + pos, stb->cols, stb->cols->size, stb->binaryPrefex, stb->ncharPrefex, precision, k); + genRowByField(colData + pos, stb->cols, stb->cols->size, stb->binaryPrefex, stb->ncharPrefex, k); return colData; } -int32_t genRowByField(char* buf, int32_t pos1, BArray* fields, int16_t fieldCnt, char* binanryPrefix, char* ncharPrefix, int64_t *k) { +int32_t genRowByField(char* buf, BArray* fields, int16_t fieldCnt, char* binanryPrefix, char* ncharPrefix, int64_t *k) { // other cols data - int32_t pos2 = 0; + int32_t pos1 = 0; for(uint16_t i = 0; i < fieldCnt; i++) { - Field* fd = benchArrayGet(fields, GET_IDX(i)); + Field* fd = benchArrayGet(fields, i); char* prefix = ""; if(fd->type == TSDB_DATA_TYPE_BINARY || fd->type == TSDB_DATA_TYPE_VARBINARY) { - if(stb->binaryPrefex) { - prefix = stb->binaryPrefex; + if(binanryPrefix) { + prefix = binanryPrefix; } } else if(fd->type == TSDB_DATA_TYPE_NCHAR) { - if(stb->ncharPrefex) { - prefix = stb->ncharPrefex; + if(ncharPrefix) { + prefix = ncharPrefix; } } - pos2 += dataGenByField(fd, buf + pos1 + pos2, prefix, k); + pos1 += dataGenByField(fd, buf, pos1, prefix, k); } - return pos2; + return pos1; } From 28a3ddcbd3d51e5f85f12557bcd5289c445ca975 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 21 May 2024 17:32:53 +0800 Subject: [PATCH 08/57] fix: test passed version --- case/exportCsv.json | 42 +++++++++++++++++++++++++++++++++++++----- src/benchData.c | 28 ++++++++++++++-------------- src/benchDataMix.c | 13 +++++++------ 3 files changed, 58 insertions(+), 25 deletions(-) diff --git a/case/exportCsv.json b/case/exportCsv.json index 0b8ba753..cf9572dd 100644 --- a/case/exportCsv.json +++ b/case/exportCsv.json @@ -4,20 +4,52 @@ "databases": [ { "dbinfo": { - "name": "exportCsv" + "name": "csvdb" }, "super_tables": [ { - "name": "meters", - "childtable_count": 100, - "insert_rows": 100000, + "name": "batchTable", + "childtable_count": 10, + "insert_rows": 50, "childtable_prefix": "d", "timestamp_step": 10, "start_timestamp":1600000000000, "columns": [ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc", "min": 1}, - { "type": "double", "name": "dc", "min":10}, + { "type": "double", "name": "dc", "min":10, "max":10}, + { "type": "tinyint", "name": "ti"}, + { "type": "smallint", "name": "si"}, + { "type": "int", "name": "ic", "fillNull":"false"}, + { "type": "bigint", "name": "bi"}, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi", "min":100, "max":120}, + { "type": "uint", "name": "ui"}, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 16}, + { "type": "nchar", "name": "nch", "len": 16} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"type": "binary", "name": "location", "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + }, + { + "name": "interlaceTable", + "childtable_count": 5, + "insert_rows": 100, + "interlace_rows": 10, + "childtable_prefix": "e", + "timestamp_step": 1000, + "start_timestamp":1700000000000, + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "min":16}, + { "type": "double", "name": "dc", "min":16}, { "type": "tinyint", "name": "ti"}, { "type": "smallint", "name": "si"}, { "type": "int", "name": "ic", "fillNull":"false"}, diff --git a/src/benchData.c b/src/benchData.c index ebd3aedc..31ca2245 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -440,19 +440,7 @@ uint32_t accumulateRowLen(BArray *fields, int iface) { int tmpStr(char *tmp, int iface, Field *field, int64_t k) { - if (g_arguments->demo_mode) { - unsigned int tmpRand = taosRandom(); - if (g_arguments->chinese) { - snprintf(tmp, field->length, "%s", - locations_chinese[tmpRand % 10]); - } else if (SML_IFACE == iface) { - snprintf(tmp, field->length, "%s", - locations_sml[tmpRand % 10]); - } else { - snprintf(tmp, field->length, "%s", - locations[tmpRand % 10]); - } - } else if (field->values) { + if (field->values) { int arraySize = tools_cJSON_GetArraySize(field->values); if (arraySize) { tools_cJSON *buf = tools_cJSON_GetArrayItem( @@ -466,11 +454,23 @@ int tmpStr(char *tmp, int iface, Field *field, int64_t k) { __func__, arraySize); return -1; } + } else if (g_arguments->demo_mode) { + unsigned int tmpRand = taosRandom(); + if (g_arguments->chinese) { + snprintf(tmp, field->length, "%s", + locations_chinese[tmpRand % 10]); + } else if (SML_IFACE == iface) { + snprintf(tmp, field->length, "%s", + locations_sml[tmpRand % 10]); + } else { + snprintf(tmp, field->length, "%s", + locations[tmpRand % 10]); + } } else { if(field->gen == GEN_ORDER) { snprintf(tmp, field->length, "%"PRId64, k); } else { - rand_string(tmp, field->length, g_arguments->chinese); + rand_string(tmp, taosRandom() % field->length, g_arguments->chinese); } } return 0; diff --git a/src/benchDataMix.c b/src/benchDataMix.c index dec69104..d819ec9f 100644 --- a/src/benchDataMix.c +++ b/src/benchDataMix.c @@ -105,6 +105,8 @@ uint32_t dataGenByField(Field* fd, char* pstr, uint32_t len, char* prefix, int64 return size; } + const char * format = ",%s"; + switch (fd->type) { case TSDB_DATA_TYPE_BOOL: sprintf(val, "%d", tmpBool(fd)); @@ -112,7 +114,9 @@ uint32_t dataGenByField(Field* fd, char* pstr, uint32_t len, char* prefix, int64 // timestamp case TSDB_DATA_TYPE_TIMESTAMP: int64_t nowts = toolsGetTimestampMs(); + strcpy(val, "\'"); toolsFormatTimestamp(val, nowts, TSDB_TIME_PRECISION_MILLI); + strcat(val, "\'"); break; // signed case TSDB_DATA_TYPE_TINYINT: @@ -150,17 +154,14 @@ uint32_t dataGenByField(Field* fd, char* pstr, uint32_t len, char* prefix, int64 // binary nchar case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_BINARY: - if(fd->gen == GEN_ORDER) { - tmpStr(val, 0, fd, *k); - } else { - genRadomString(val, fd->length > sizeof(val) ? sizeof(val) : fd->length, prefix); - } + format = ",\'%s\'"; + tmpStr(val, 0, fd, *k); break; default: break; } - size += sprintf(pstr + len, ",%s", val); + size += sprintf(pstr + len, format, val); return size; } From 14379459d22ef8d4880596066800dfa1b7bb24e1 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 21 May 2024 17:42:23 +0800 Subject: [PATCH 09/57] fix: add batch size --- case/exportCsv.json | 1 + 1 file changed, 1 insertion(+) diff --git a/case/exportCsv.json b/case/exportCsv.json index cf9572dd..f031266c 100644 --- a/case/exportCsv.json +++ b/case/exportCsv.json @@ -1,6 +1,7 @@ { "filetype": "csvfile", "csvPath": "/root/csv/", + "num_of_records_per_req": 10000, "databases": [ { "dbinfo": { From d4b9557f02200e00ce129b1d2e1c678f01340cb1 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 21 May 2024 18:36:12 +0800 Subject: [PATCH 10/57] feat: add progress tips --- src/benchCsv.c | 48 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/src/benchCsv.c b/src/benchCsv.c index c75d9120..11cf3a8c 100644 --- a/src/benchCsv.c +++ b/src/benchCsv.c @@ -19,6 +19,8 @@ // main etry // +#define SHOW_CNT 100000 + static void *csvWriteThread(void *param) { // write thread for (int i = 0; i < g_arguments->databases->size; i++) { @@ -30,9 +32,9 @@ static void *csvWriteThread(void *param) { // gen csv int ret = genWithSTable(db, stb, g_arguments->csvPath); if(ret != 0) { - errorPrint("failed generate to csv. db=%s stb=%s error code=%d", db->dbName, stb->stbName, ret); + errorPrint("failed generate to csv. db=%s stb=%s error code=%d \n", db->dbName, stb->stbName, ret); return NULL; - } + } } } return NULL; @@ -42,10 +44,16 @@ int csvTestProcess() { pthread_t handle; int ret = pthread_create(&handle, NULL, csvWriteThread, NULL); if (ret != 0) { - errorPrint("pthread_create failed. error code =%d", ret); + errorPrint("pthread_create failed. error code =%d \n", ret); return -1; } + + infoPrint("start output to csv %s ...\n", g_arguments->csvPath); + int64_t start = toolsGetTimestampMs(); pthread_join(handle, NULL); + int64_t delay = toolsGetTimestampMs() - start; + infoPrint("output to csv %s finished. delay:%"PRId64"s \n", g_arguments->csvPath, delay/1000); + return 0; } @@ -56,7 +64,7 @@ int genWithSTable(SDataBase* db, SSuperTable* stb, char* outDir) { obtainCsvFile(outFile, db, stb, outDir); FILE * fs = fopen(outFile, "w"); if(fs == NULL) { - errorPrint("failed create csv file. file=%s, last errno=%d strerror=%s", outFile, errno, strerror(errno)); + errorPrint("failed create csv file. file=%s, last errno=%d strerror=%s \n", outFile, errno, strerror(errno)); return -1; } @@ -64,6 +72,8 @@ int genWithSTable(SDataBase* db, SSuperTable* stb, char* outDir) { int bufLen = rowLen * g_arguments->reqPerReq; char* buf = benchCalloc(1, bufLen, true); + infoPrint("start write csv file: %s \n", outFile); + if (stb->interlaceRows > 0) { // interlace mode ret = interlaceWriteCsv(db, stb, fs, buf, bufLen, rowLen * 2); @@ -75,6 +85,7 @@ int genWithSTable(SDataBase* db, SSuperTable* stb, char* outDir) { tmfree(buf); fclose(fs); + succPrint("end write csv file: %s \n", outFile); return ret; } @@ -88,7 +99,7 @@ void obtainCsvFile(char * outFile, SDataBase* db, SSuperTable* stb, char* outDir int32_t writeCsvFile(FILE* f, char * buf, int32_t len) { size_t size = fwrite(buf, 1, len, f); if(size != len) { - errorPrint("failed to write csv file. expect write length:%d real write length:%d", len, (int32_t)size); + errorPrint("failed to write csv file. expect write length:%d real write length:%d \n", len, (int32_t)size); return -1; } return 0; @@ -98,6 +109,7 @@ int batchWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufL int ret = 0; int pos = 0; int64_t tk = 0; + int64_t show = 0; int tagDataLen = stb->lenOfTags + stb->tags->size + 256; char * tagData = (char *) benchCalloc(1, tagDataLen, true); @@ -127,6 +139,19 @@ int batchWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufL // ts move next ts += stb->timestamp_step; + + // check cancel + if(g_arguments->terminate) { + infoPrint("%s", "You are cancel, exiting ...\n"); + ret = -1; + goto END; + } + + // print show + if (++show % SHOW_CNT == 0) { + infoPrint("batch write child table cnt = %"PRId64 " all rows = %" PRId64 "\n", i, show); + } + } } @@ -147,6 +172,7 @@ int interlaceWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int int pos = 0; int64_t n = 0; // already inserted rows for one child table int64_t tk = 0; + int64_t show = 0; char **tagDatas = (char **)benchCalloc(stb->childTblCount, sizeof(char *), true); int colDataLen = stb->lenOfCols + stb->cols->size + 256; @@ -184,6 +210,18 @@ int interlaceWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int // ts move next ts += stb->timestamp_step; + + // check cancel + if(g_arguments->terminate) { + infoPrint("%s", "You are cancel, exiting ... \n"); + ret = -1; + goto END; + } + + // print show + if (++show % SHOW_CNT == 0) { + infoPrint("interlace write child table index = %"PRId64 " all rows = %"PRId64 "\n", i, show); + } } // if last child table From a176e1f93c9e78e0346f8c76dd92ee8e5037d151 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 21 May 2024 18:36:59 +0800 Subject: [PATCH 11/57] feat: add progress tips --- src/benchCsv.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/benchCsv.c b/src/benchCsv.c index 11cf3a8c..6f7c3d2b 100644 --- a/src/benchCsv.c +++ b/src/benchCsv.c @@ -149,7 +149,7 @@ int batchWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufL // print show if (++show % SHOW_CNT == 0) { - infoPrint("batch write child table cnt = %"PRId64 " all rows = %" PRId64 "\n", i, show); + infoPrint("batch write child table cnt = %"PRId64 " all rows = %" PRId64 "\n", i+1, show); } } @@ -220,7 +220,7 @@ int interlaceWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int // print show if (++show % SHOW_CNT == 0) { - infoPrint("interlace write child table index = %"PRId64 " all rows = %"PRId64 "\n", i, show); + infoPrint("interlace write child table index = %"PRId64 " all rows = %"PRId64 "\n", i+1, show); } } From 3faa6b1ee7949df80ef75f1ecb62964f4a9bb986 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 22 May 2024 09:27:27 +0800 Subject: [PATCH 12/57] fix: move addbatch to exec --- inc/bench.h | 2 +- inc/benchData.h | 2 +- src/benchData.c | 10 +--------- src/benchInsert.c | 24 ++++++++++++++++++------ src/benchInsertMix.c | 4 ++-- 5 files changed, 23 insertions(+), 19 deletions(-) diff --git a/inc/bench.h b/inc/bench.h index 10b8bb9a..7eb4b424 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -1157,7 +1157,7 @@ void printErrCmdCodeStr(char *cmd, int32_t code, TAOS_RES *res); int32_t benchParseArgsNoArgp(int argc, char* argv[]); #endif -int32_t execInsert(threadInfo *pThreadInfo, uint32_t k); +int32_t execInsert(threadInfo *pThreadInfo, uint32_t k, int64_t* delay3); // if return true, timestmap must add timestap_step, else timestamp no need changed bool needChangeTs(SSuperTable * stbInfo, int32_t *pkCur, int32_t *pkCnt); diff --git a/inc/benchData.h b/inc/benchData.h index a88271b1..e4bb74f5 100644 --- a/inc/benchData.h +++ b/inc/benchData.h @@ -29,7 +29,7 @@ int generateRandData(SSuperTable *stbInfo, char *sampleDataBuf, int prepareStmt(SSuperTable *stbInfo, TAOS_STMT *stmt, char* tagData, uint64_t tableSeq); uint32_t bindParamBatch(threadInfo *pThreadInfo, uint32_t batch, int64_t startTime, - SChildTable *childTbl, int32_t *pkCur, int32_t *pkCnt, int32_t *n, int64_t *delay2, int64_t *delay3); + SChildTable *childTbl, int32_t *pkCur, int32_t *pkCnt, int32_t *n, int64_t *delay2); int prepareSampleData(SDataBase* database, SSuperTable* stbInfo); void generateSmlJsonTags(tools_cJSON *tagsList, char **sml_tags_json_array, diff --git a/src/benchData.c b/src/benchData.c index ebd3aedc..79934c90 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -1798,7 +1798,7 @@ int64_t getTSRandTail(int64_t timeStampStep, int32_t seq, int disorderRatio, uint32_t bindParamBatch(threadInfo *pThreadInfo, uint32_t batch, int64_t startTime, - SChildTable *childTbl, int32_t *pkCur, int32_t *pkCnt, int32_t *n, int64_t *delay2, int64_t *delay3) { + SChildTable *childTbl, int32_t *pkCur, int32_t *pkCnt, int32_t *n, int64_t *delay2) { TAOS_STMT *stmt = pThreadInfo->conn->stmt; SSuperTable *stbInfo = pThreadInfo->stbInfo; uint32_t columnCount = stbInfo->cols->size; @@ -1874,14 +1874,6 @@ uint32_t bindParamBatch(threadInfo *pThreadInfo, tmfree(param->length); } - // if msg > 3MB, break - start = toolsGetTimestampUs(); - if (taos_stmt_add_batch(stmt)) { - errorPrint("taos_stmt_add_batch() failed! reason: %s\n", - taos_stmt_errstr(stmt)); - return 0; - } - *delay3 += toolsGetTimestampUs() - start; return batch; } diff --git a/src/benchInsert.c b/src/benchInsert.c index fcb8f99f..61adb68c 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -1250,7 +1250,7 @@ void postFreeResource() { tools_cJSON_Delete(root); } -int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) { +int32_t execInsert(threadInfo *pThreadInfo, uint32_t k, int64_t *delay3) { SDataBase * database = pThreadInfo->dbInfo; SSuperTable *stbInfo = pThreadInfo->stbInfo; TAOS_RES * res = NULL; @@ -1309,6 +1309,18 @@ int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) { break; case STMT_IFACE: + // add batch + int64_t start = toolsGetTimestampUs(); + if (taos_stmt_add_batch(pThreadInfo->conn->stmt) != 0) { + errorPrint("taos_stmt_add_batch() failed! reason: %s\n", + taos_stmt_errstr(pThreadInfo->conn->stmt)); + return -1; + } + if(delay3) { + *delay3 += toolsGetTimestampUs() - start; + } + + // execute code = taos_stmt_execute(pThreadInfo->conn->stmt); if (code) { errorPrint( @@ -1743,7 +1755,7 @@ static void *syncWriteInterlace(void *sarg) { int32_t n = 0; generated = bindParamBatch(pThreadInfo, interlaceRows, - childTbl->ts, childTbl, &childTbl->pkCur, &childTbl->pkCnt, &n, &delay2, &delay3); + childTbl->ts, childTbl, &childTbl->pkCur, &childTbl->pkCnt, &n, &delay2); childTbl->ts += stbInfo->timestamp_step * n; break; @@ -1838,7 +1850,7 @@ static void *syncWriteInterlace(void *sarg) { } startTs = toolsGetTimestampUs(); - if (execInsert(pThreadInfo, generated)) { + if (execInsert(pThreadInfo, generated, &delay3)) { g_fail = true; goto free_of_interlace; } @@ -1962,7 +1974,7 @@ static int32_t prepareProgressDataStmt( (g_arguments->reqPerReq > (stbInfo->insertRows - i)) ? (stbInfo->insertRows - i) : g_arguments->reqPerReq, - *timestamp, childTbl, pkCur, pkCnt, &n, delay2, delay3); + *timestamp, childTbl, pkCur, pkCnt, &n, delay2); *timestamp += n * stbInfo->timestamp_step; return generated; } @@ -2531,7 +2543,7 @@ void *syncWriteProgressive(void *sarg) { } // only measure insert startTs = toolsGetTimestampUs(); - int code = execInsert(pThreadInfo, generated); + int code = execInsert(pThreadInfo, generated, &delay3); if (code) { if (NO_IF_FAILED == stbInfo->continueIfFail) { warnPrint("The super table parameter " @@ -2572,7 +2584,7 @@ void *syncWriteProgressive(void *sarg) { w = 0; } - code = execInsert(pThreadInfo, generated); + code = execInsert(pThreadInfo, generated, &delay3); if (code) { g_fail = true; goto free_of_progressive; diff --git a/src/benchInsertMix.c b/src/benchInsertMix.c index 3e62143e..87618954 100644 --- a/src/benchInsertMix.c +++ b/src/benchInsertMix.c @@ -843,7 +843,7 @@ bool insertDataMix(threadInfo* info, SDataBase* db, SSuperTable* stb) { int64_t startTs = toolsGetTimestampUs(); //g_arguments->debug_print = false; - if(execInsert(info, batchRows) != 0) { + if(execInsert(info, batchRows, NULL) != 0) { FAILED_BREAK() } //g_arguments->debug_print = true; @@ -893,7 +893,7 @@ bool insertDataMix(threadInfo* info, SDataBase* db, SSuperTable* stb) { batTotal.delRows = genBatchDelSql(stb, &mixRatio, batStartTime, info->conn->taos, tbName, info->buffer, len, querySql); if (batTotal.delRows > 0) { // g_arguments->debug_print = false; - if (execInsert(info, batTotal.delRows) != 0) { + if (execInsert(info, batTotal.delRows, NULL) != 0) { FAILED_BREAK() } From 221fb7b1cb2ae7e3bb8593749cf42965580929b3 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 22 May 2024 09:30:57 +0800 Subject: [PATCH 13/57] fix: build error --- src/benchInsert.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index 61adb68c..1e3f8aad 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -1256,6 +1256,7 @@ int32_t execInsert(threadInfo *pThreadInfo, uint32_t k, int64_t *delay3) { TAOS_RES * res = NULL; int32_t code = 0; uint16_t iface = stbInfo->iface; + int64_t start = 0; int32_t trying = (stbInfo->keep_trying)? stbInfo->keep_trying:g_arguments->keep_trying; @@ -1310,7 +1311,7 @@ int32_t execInsert(threadInfo *pThreadInfo, uint32_t k, int64_t *delay3) { case STMT_IFACE: // add batch - int64_t start = toolsGetTimestampUs(); + start = toolsGetTimestampUs(); if (taos_stmt_add_batch(pThreadInfo->conn->stmt) != 0) { errorPrint("taos_stmt_add_batch() failed! reason: %s\n", taos_stmt_errstr(pThreadInfo->conn->stmt)); From 4f2bc5f21f6d770f5f9a4f06f5d48a7334100bbd Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 22 May 2024 11:38:31 +0800 Subject: [PATCH 14/57] fix: output path --- src/benchCsv.c | 4 +--- src/benchJsonOpt.c | 12 ++++++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/benchCsv.c b/src/benchCsv.c index 6f7c3d2b..ac2e7cb9 100644 --- a/src/benchCsv.c +++ b/src/benchCsv.c @@ -91,11 +91,9 @@ int genWithSTable(SDataBase* db, SSuperTable* stb, char* outDir) { void obtainCsvFile(char * outFile, SDataBase* db, SSuperTable* stb, char* outDir) { - sprintf(outFile, "%s//%s-%s.csv", outDir, db->dbName, stb->stbName); + sprintf(outFile, "%s%s-%s.csv", outDir, db->dbName, stb->stbName); } - - int32_t writeCsvFile(FILE* f, char * buf, int32_t len) { size_t size = fwrite(buf, 1, len, f); if(size != len) { diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 0da21ed8..3c9121f4 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -1433,9 +1433,17 @@ static int getMetaFromCommonJsonFile(tools_cJSON *json) { tstrncpy(g_arguments->csvPath, csv->valuestring, MAX_FILE_NAME_LEN); } - if(g_arguments->csvPath[0] == 0) { + size_t len = strlen(g_arguments->csvPath); + + if(len == 0) { // set default with current path - strcpy(g_arguments->csvPath, "./output"); + strcpy(g_arguments->csvPath, "./output/"); + mkdir(g_arguments->csvPath, 0775); + } else { + // append end + if (g_arguments->csvPath[len-1] != '/' ) { + strcat(g_arguments->csvPath, "/"); + } mkdir(g_arguments->csvPath, 0775); } From 44137d8b61e57e9768256d77ed1d27f00e0ec751 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 22 May 2024 14:17:35 +0800 Subject: [PATCH 15/57] fix: replace null with blank --- src/benchCsv.c | 2 +- src/benchDataMix.c | 8 +++----- src/benchInsertMix.c | 4 +++- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/benchCsv.c b/src/benchCsv.c index ac2e7cb9..34e0ba7b 100644 --- a/src/benchCsv.c +++ b/src/benchCsv.c @@ -293,7 +293,7 @@ int32_t genRowByField(char* buf, BArray* fields, int16_t fieldCnt, char* binanry } } - pos1 += dataGenByField(fd, buf, pos1, prefix, k); + pos1 += dataGenByField(fd, buf, pos1, prefix, k, ""); } return pos1; diff --git a/src/benchDataMix.c b/src/benchDataMix.c index d819ec9f..d2f59e7b 100644 --- a/src/benchDataMix.c +++ b/src/benchDataMix.c @@ -14,8 +14,6 @@ #include "benchDataMix.h" #include -#define VAL_NULL "NULL" - #define VBOOL_CNT 3 int32_t inul = 20; // interval null count @@ -97,11 +95,11 @@ uint32_t genRadomString(char* val, uint32_t len, char* prefix) { // data row generate by randowm -uint32_t dataGenByField(Field* fd, char* pstr, uint32_t len, char* prefix, int64_t *k) { +uint32_t dataGenByField(Field* fd, char* pstr, uint32_t len, char* prefix, int64_t *k, char* nullVal) { uint32_t size = 0; - char val[512] = VAL_NULL; + char val[512] = {0}; if( fd->fillNull && RD(inul) == 0 ) { - size = sprintf(pstr + len, ",%s", VAL_NULL); + size = sprintf(pstr + len, ",%s", nullVal); return size; } diff --git a/src/benchInsertMix.c b/src/benchInsertMix.c index 3e62143e..87ac2670 100644 --- a/src/benchInsertMix.c +++ b/src/benchInsertMix.c @@ -285,6 +285,8 @@ uint32_t appendRowRuleOld(SSuperTable* stb, char* pstr, uint32_t len, int64_t ti } #define GET_IDX(i) info->batCols[i] +#define VAL_NULL "NULL" + uint32_t genRowMixAll(threadInfo* info, SSuperTable* stb, char* pstr, uint32_t len, int64_t ts, int64_t* k) { uint32_t size = 0; // first col is ts @@ -317,7 +319,7 @@ uint32_t genRowMixAll(threadInfo* info, SSuperTable* stb, char* pstr, uint32_t l } } - size += dataGenByField(fd, pstr, len + size, prefix, k); + size += dataGenByField(fd, pstr, len + size, prefix, k, VAL_NULL); } // end From 9bf9d780057000db558a40d25d344e37d615a6fa Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 22 May 2024 14:22:36 +0800 Subject: [PATCH 16/57] fix: build error --- inc/benchDataMix.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inc/benchDataMix.h b/inc/benchDataMix.h index 4beeb8ee..fe352015 100644 --- a/inc/benchDataMix.h +++ b/inc/benchDataMix.h @@ -17,7 +17,7 @@ #define __BENCHDATAMIX_H_ -uint32_t dataGenByField(Field* fd, char* pstr, uint32_t len, char* prefix, int64_t *k); +uint32_t dataGenByField(Field* fd, char* pstr, uint32_t len, char* prefix, int64_t *k, char* nullVal); // data generate by calc ts uint32_t dataGenByCalcTs(Field* fd, char* pstr, uint32_t len, int64_t ts); From d7927569e9d0178fe9eeaa431f93003570bb0ede Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 22 May 2024 14:24:31 +0800 Subject: [PATCH 17/57] fix: build error --- inc/bench.h | 2 ++ src/benchInsertMix.c | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/inc/bench.h b/inc/bench.h index 65139afa..7b92c07f 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -647,6 +647,8 @@ typedef struct STSMA { #define SUIT_DATAPOS_MUL_FILE 4 #define SUIT_DATAPOS_MIX 5 +#define VAL_NULL "NULL" + enum CONTINUE_IF_FAIL_MODE { NO_IF_FAILED, // 0 YES_IF_FAILED, // 1 diff --git a/src/benchInsertMix.c b/src/benchInsertMix.c index 87ac2670..795efab7 100644 --- a/src/benchInsertMix.c +++ b/src/benchInsertMix.c @@ -285,8 +285,6 @@ uint32_t appendRowRuleOld(SSuperTable* stb, char* pstr, uint32_t len, int64_t ti } #define GET_IDX(i) info->batCols[i] -#define VAL_NULL "NULL" - uint32_t genRowMixAll(threadInfo* info, SSuperTable* stb, char* pstr, uint32_t len, int64_t ts, int64_t* k) { uint32_t size = 0; // first col is ts From 864ccef4cb684d50c1b9c57c1b8c31cb1709a862 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 25 May 2024 09:59:51 +0800 Subject: [PATCH 18/57] fix: build error for nowts --- src/benchDataMix.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/benchDataMix.c b/src/benchDataMix.c index d2f59e7b..7e8e4396 100644 --- a/src/benchDataMix.c +++ b/src/benchDataMix.c @@ -97,6 +97,7 @@ uint32_t genRadomString(char* val, uint32_t len, char* prefix) { // data row generate by randowm uint32_t dataGenByField(Field* fd, char* pstr, uint32_t len, char* prefix, int64_t *k, char* nullVal) { uint32_t size = 0; + int64_t nowts= 0; char val[512] = {0}; if( fd->fillNull && RD(inul) == 0 ) { size = sprintf(pstr + len, ",%s", nullVal); @@ -111,7 +112,7 @@ uint32_t dataGenByField(Field* fd, char* pstr, uint32_t len, char* prefix, int64 break; // timestamp case TSDB_DATA_TYPE_TIMESTAMP: - int64_t nowts = toolsGetTimestampMs(); + nowts = toolsGetTimestampMs(); strcpy(val, "\'"); toolsFormatTimestamp(val, nowts, TSDB_TIME_PRECISION_MILLI); strcat(val, "\'"); From 166b32a91ae069e2bd2a1258f6184ea9b312e1c1 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 25 May 2024 11:04:05 +0800 Subject: [PATCH 19/57] fix: case passed --- tests/taosbenchmark/sml_telnet_alltypes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/taosbenchmark/sml_telnet_alltypes.py b/tests/taosbenchmark/sml_telnet_alltypes.py index 2536763c..cea2fa1f 100644 --- a/tests/taosbenchmark/sml_telnet_alltypes.py +++ b/tests/taosbenchmark/sml_telnet_alltypes.py @@ -93,7 +93,7 @@ def run(self): if major_ver == "3": tdSql.checkData(1, 1, "VARCHAR") tdSql.checkData( - 1, 2, 16 + 1, 2, 8 ) # 3.0 will use a bit more space for schemaless create table else: tdSql.checkData(1, 1, "BINARY") @@ -102,7 +102,7 @@ def run(self): tdSql.query("describe db.stb13") tdSql.checkData(1, 1, "NCHAR") if major_ver == "3": - tdSql.checkData(1, 2, 16) + tdSql.checkData(1, 2, 8) else: tdSql.checkData(1, 2, 8) From 50a107431c8d07bdc20fd6e395ba7f20e863fc4e Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 25 May 2024 11:15:48 +0800 Subject: [PATCH 20/57] feat: add newtype varbinary --- src/benchData.c | 20 ++++++++++++++++---- src/benchDataMix.c | 2 ++ src/benchUtil.c | 4 ++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/benchData.c b/src/benchData.c index 31ca2245..65eadd35 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -384,6 +384,8 @@ uint32_t accumulateRowLen(BArray *fields, int iface) { Field *field = benchArrayGet(fields, i); switch (field->type) { case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_GEOMETRY: case TSDB_DATA_TYPE_NCHAR: len += field->length + 3; break; @@ -780,6 +782,7 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: { char *tmp = benchCalloc(1, field->length + 1, false); if (0 != tmpStr(tmp, stbInfo->iface, field, k)) { @@ -959,6 +962,7 @@ static int fillStmt( break; } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: { char *tmp = benchCalloc(1, field->length + 1, false); if (0 != tmpStr(tmp, stbInfo->iface, field, k)) { @@ -1207,13 +1211,14 @@ static int generateRandDataSmlTelnet(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: { char *tmp = benchCalloc(1, field->length + 1, false); if (0 != tmpStr(tmp, stbInfo->iface, field, k)) { free(tmp); return -1; } - if (field->type == TSDB_DATA_TYPE_BINARY) { + if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY) { if (tag) { n = snprintf(sampleDataBuf + pos, bufLen - pos, "%s=L\"%s\" ", @@ -1359,6 +1364,7 @@ static int generateRandDataSmlJson(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: { char *tmp = benchCalloc(1, field->length + 1, false); if (0 != tmpStr(tmp, stbInfo->iface, field, k)) { @@ -1493,6 +1499,7 @@ static int generateRandDataSmlLine(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: { char *tmp = benchCalloc(1, field->length + 1, false); if (0 != tmpStr(tmp, stbInfo->iface, field, k)) { @@ -1915,10 +1922,11 @@ void generateSmlJsonTags(tools_cJSON *tagsList, } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: { char *buf = (char *)benchCalloc(tag->length + 1, 1, false); rand_string(buf, tag->length, g_arguments->chinese); - if (tag->type == TSDB_DATA_TYPE_BINARY) { + if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_VARBINARY) { tools_cJSON_AddStringToObject(tags, tagName, buf); } else { tools_cJSON_AddStringToObject(tags, tagName, buf); @@ -1981,10 +1989,11 @@ void generateSmlTaosJsonTags(tools_cJSON *tagsList, SSuperTable *stbInfo, } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: { char *buf = (char *)benchCalloc(tag->length + 1, 1, false); rand_string(buf, tag->length, g_arguments->chinese); - if (tag->type == TSDB_DATA_TYPE_BINARY) { + if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_VARBINARY) { tools_cJSON_AddStringToObject(tagObj, "value", buf); tools_cJSON_AddStringToObject(tagObj, "type", "binary"); } else { @@ -2040,6 +2049,7 @@ void generateSmlJsonValues( break; } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: { char *buf = (char *)benchCalloc(col->length + 1, 1, false); rand_string(buf, col->length, g_arguments->chinese); @@ -2082,6 +2092,7 @@ void generateSmlJsonCols(tools_cJSON *array, tools_cJSON *tag, break; } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: { char *buf = (char *)benchCalloc(col->length + 1, 1, false); rand_string(buf, col->length, g_arguments->chinese); @@ -2139,10 +2150,11 @@ void generateSmlTaosJsonCols(tools_cJSON *array, tools_cJSON *tag, break; } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: { char *buf = (char *)benchCalloc(col->length + 1, 1, false); rand_string(buf, col->length, g_arguments->chinese); - if (col->type == TSDB_DATA_TYPE_BINARY) { + if (col->type == TSDB_DATA_TYPE_BINARY || col->type == TSDB_DATA_TYPE_VARBINARY) { tools_cJSON_AddStringToObject(value, "value", buf); tools_cJSON_AddStringToObject(value, "type", "binary"); } else { diff --git a/src/benchDataMix.c b/src/benchDataMix.c index 7e8e4396..117b2c48 100644 --- a/src/benchDataMix.c +++ b/src/benchDataMix.c @@ -153,6 +153,7 @@ uint32_t dataGenByField(Field* fd, char* pstr, uint32_t len, char* prefix, int64 // binary nchar case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: format = ",\'%s\'"; tmpStr(val, 0, fd, *k); break; @@ -203,6 +204,7 @@ uint32_t dataGenByCalcTs(Field* fd, char* pstr, uint32_t len, int64_t ts) { break; // binary nchar case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: sprintf(val, "%" PRId64, ts); break; diff --git a/src/benchUtil.c b/src/benchUtil.c index 156e6a7f..2c3f7646 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -1002,6 +1002,8 @@ int convertStringToDatatype(char *type, int length) { return TSDB_DATA_TYPE_JSON; } else if (0 == strcasecmp(type, "varchar")) { return TSDB_DATA_TYPE_BINARY; + } else if (0 == strcasecmp(type, "varbinary")) { + return TSDB_DATA_TYPE_VARBINARY; } else { errorPrint("unknown data type: %s\n", type); exit(EXIT_FAILURE); @@ -1039,6 +1041,8 @@ int convertStringToDatatype(char *type, int length) { return TSDB_DATA_TYPE_JSON; } else if (0 == strncasecmp(type, "varchar", length)) { return TSDB_DATA_TYPE_BINARY; + } else if (0 == strcnasecmp(type, "varbinary"), length) { + return TSDB_DATA_TYPE_VARBINARY; } else { errorPrint("unknown data type: %s\n", type); exit(EXIT_FAILURE); From 34f700e9d1ed85b892be97b2156b9528f9b8f560 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 25 May 2024 11:34:42 +0800 Subject: [PATCH 21/57] fix: build error --- src/benchUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchUtil.c b/src/benchUtil.c index 2c3f7646..3318f2ee 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -1041,7 +1041,7 @@ int convertStringToDatatype(char *type, int length) { return TSDB_DATA_TYPE_JSON; } else if (0 == strncasecmp(type, "varchar", length)) { return TSDB_DATA_TYPE_BINARY; - } else if (0 == strcnasecmp(type, "varbinary"), length) { + } else if (0 == strcnasecmp(type, "varbinary", length) { return TSDB_DATA_TYPE_VARBINARY; } else { errorPrint("unknown data type: %s\n", type); From 9bb1cf5e33950a285c5d5751d4432c766dfcbff8 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 27 May 2024 15:30:45 +0800 Subject: [PATCH 22/57] feat: support thread bind vgroup --- inc/bench.h | 4 +- src/benchInsert.c | 612 +++++++++++++++++++++------------------------ src/benchJsonOpt.c | 7 + 3 files changed, 299 insertions(+), 324 deletions(-) diff --git a/inc/bench.h b/inc/bench.h index 7b92c07f..1c63553f 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -791,14 +791,12 @@ typedef struct SSTREAM_S { bool drop; } SSTREAM; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 typedef struct SVGroup_S { int32_t vgId; uint64_t tbCountPerVgId; SChildTable **childTblArray; uint64_t tbOffset; // internal use } SVGroup; -#endif // TD_VER_COMPATIBLE_3_0_0_0 // typedef struct SDataBase_S { char * dbName; @@ -969,6 +967,8 @@ typedef struct SArguments_S { bool escape_character; bool pre_load_tb_meta; char csvPath[MAX_FILE_NAME_LEN]; + + bool bind_vgroup; } SArguments; typedef struct SBenchConn { diff --git a/src/benchInsert.c b/src/benchInsert.c index fcb8f99f..4aff9b8e 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -428,7 +428,7 @@ static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) { return ret; } -#ifdef TD_VER_COMPATIBLE_3_0_0_0 + int32_t getVgroupsOfDb(SBenchConn *conn, SDataBase *database) { int vgroups = 0; char cmd[SHORT_1K_SQL_BUFF_LEN] = "\0"; @@ -491,13 +491,11 @@ int32_t getVgroupsOfDb(SBenchConn *conn, SDataBase *database) { return vgroups; } -#endif // TD_VER_COMPATIBLE_3_0_0_0 int geneDbCreateCmd(SDataBase *database, char *command, int remainVnodes) { int dataLen = 0; int n; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (g_arguments->nthreads_auto || (-1 != g_arguments->inputted_vgroups)) { + if (-1 != g_arguments->inputted_vgroups) { n = snprintf(command + dataLen, SHORT_1K_SQL_BUFF_LEN - dataLen, g_arguments->escape_character ? "CREATE DATABASE IF NOT EXISTS `%s` VGROUPS %d" @@ -513,12 +511,7 @@ int geneDbCreateCmd(SDataBase *database, char *command, int remainVnodes) { : "CREATE DATABASE IF NOT EXISTS %s", database->dbName); } -#else - n = snprintf(command + dataLen, SHORT_1K_SQL_BUFF_LEN - dataLen, - g_arguments->escape_character - ? "CREATE DATABASE IF NOT EXISTS `%s`" - : "CREATE DATABASE IF NOT EXISTS %s", database->dbName); -#endif // TD_VER_COMPATIBLE_3_0_0_0 + if (n < 0 || n >= SHORT_1K_SQL_BUFF_LEN - dataLen) { errorPrint("%s() LN%d snprintf overflow\n", __func__, __LINE__); @@ -651,27 +644,29 @@ int32_t getRemainVnodes(SBenchConn *conn) { int createDatabaseTaosc(SDataBase* database) { char command[SHORT_1K_SQL_BUFF_LEN] = "\0"; + // conn SBenchConn* conn = initBenchConn(); if (NULL == conn) { return -1; } - if (g_arguments->taosc_version == 3) { - for (int i = 0; i < g_arguments->streams->size; i++) { - SSTREAM* stream = benchArrayGet(g_arguments->streams, i); - if (stream->drop) { - snprintf(command, SHORT_1K_SQL_BUFF_LEN, - "DROP STREAM IF EXISTS %s;", - stream->stream_name); - if (queryDbExecCall(conn, command)) { - closeBenchConn(conn); - return -1; - } - infoPrint("%s\n", command); - memset(command, 0, SHORT_1K_SQL_BUFF_LEN); + + // drop stream in old database + for (int i = 0; i < g_arguments->streams->size; i++) { + SSTREAM* stream = benchArrayGet(g_arguments->streams, i); + if (stream->drop) { + snprintf(command, SHORT_1K_SQL_BUFF_LEN, + "DROP STREAM IF EXISTS %s;", + stream->stream_name); + if (queryDbExecCall(conn, command)) { + closeBenchConn(conn); + return -1; } + infoPrint("%s\n", command); + memset(command, 0, SHORT_1K_SQL_BUFF_LEN); } } + // drop old database snprintf(command, SHORT_1K_SQL_BUFF_LEN, g_arguments->escape_character ? "DROP DATABASE IF EXISTS `%s`;": @@ -691,9 +686,9 @@ int createDatabaseTaosc(SDataBase* database) { #endif } + // get remain vgroups int remainVnodes = INT_MAX; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (g_arguments->nthreads_auto) { + if (g_arguments->bind_vgroup) { remainVnodes = getRemainVnodes(conn); if (0 >= remainVnodes) { errorPrint("Remain vnodes %d, failed to create database\n", @@ -701,9 +696,9 @@ int createDatabaseTaosc(SDataBase* database) { return -1; } } -#endif - geneDbCreateCmd(database, command, remainVnodes); + // generate and execute create database sql + geneDbCreateCmd(database, command, remainVnodes); int32_t code = queryDbExecCall(conn, command); int32_t trying = g_arguments->keep_trying; while (code && trying) { @@ -736,19 +731,17 @@ int createDatabaseTaosc(SDataBase* database) { } infoPrint("command to create database: <%s>\n", command); -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (database->superTbls) { - if (g_arguments->nthreads_auto) { - int32_t vgroups = getVgroupsOfDb(conn, database); - if (vgroups <=0) { - closeBenchConn(conn); - errorPrint("Database %s's vgroups is %d\n", - database->dbName, vgroups); - return -1; - } + + // malloc and get vgroup + if (g_arguments->bind_vgroup) { + int32_t vgroups = getVgroupsOfDb(conn, database); + if (vgroups <= 0) { + closeBenchConn(conn); + errorPrint("Database %s's vgroups is %d\n", + database->dbName, vgroups); + return -1; } } -#endif // TD_VER_COMPATIBLE_3_0_0_0 closeBenchConn(conn); return 0; @@ -1226,22 +1219,17 @@ void postFreeResource() { tmfree(stbInfo->sqls); } - -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if ((0 == stbInfo->interlaceRows) - && (g_arguments->nthreads_auto)) { + // thread_bind + if (database->vgArray) { for (int32_t v = 0; v < database->vgroups; v++) { SVGroup *vg = benchArrayGet(database->vgArray, v); tmfree(vg->childTblArray); vg->childTblArray = NULL; } + benchArrayDestroy(database->vgArray); + database->vgArray = NULL; } -#endif // TD_VER_COMPATIBLE_3_0_0_0 } -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (database->vgArray) - benchArrayDestroy(database->vgArray); -#endif // TD_VER_COMPATIBLE_3_0_0_0 benchArrayDestroy(database->superTbls); } } @@ -1585,7 +1573,18 @@ static void *syncWriteInterlace(void *sarg) { goto free_of_interlace; } int64_t pos = pThreadInfo->pos; - SChildTable *childTbl = stbInfo->childTblArray[tableSeq]; + + // get childTable + SChildTable *childTbl; + if (g_arguments->bind_vgroup) { + childTbl = pThreadInfo->vg->childTblArray[tableSeq]; + } else { + childTbl = stbInfo->childTblArray[ + stbInfo->childTblExists? + tableSeq: + stbInfo->childTblFrom + tableSeq]; + } + char * tableName = childTbl->name; char *sampleDataBuf = childTbl->useOwnSample? childTbl->sampleDataBuf: @@ -1924,6 +1923,7 @@ static void *syncWriteInterlace(void *sarg) { lastTotalInsertRows = pThreadInfo->totalInsertRows; } } + free_of_interlace: cleanupAndPrint(pThreadInfo, "interlace"); if(csvFile) { @@ -2421,8 +2421,8 @@ void *syncWriteProgressive(void *sarg) { tableSeq <= pThreadInfo->end_table_to; tableSeq++) { char *sampleDataBuf; SChildTable *childTbl; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (g_arguments->nthreads_auto) { + + if (g_arguments->bind_vgroup) { childTbl = pThreadInfo->vg->childTblArray[tableSeq]; } else { childTbl = stbInfo->childTblArray[ @@ -2430,12 +2430,7 @@ void *syncWriteProgressive(void *sarg) { tableSeq: stbInfo->childTblFrom + tableSeq]; } -#else - childTbl = stbInfo->childTblArray[ - stbInfo->childTblExists? - tableSeq: - stbInfo->childTblFrom + tableSeq]; -#endif + if (childTbl->useOwnSample) { sampleDataBuf = childTbl->sampleDataBuf; } else { @@ -3216,16 +3211,8 @@ static bool calcExprFromServer(SDataBase *database, SSuperTable *stbInfo) { return true; } -static int startMultiThreadInsertData(SDataBase* database, - SSuperTable* stbInfo) { - if ((stbInfo->iface == SML_IFACE || stbInfo->iface == SML_REST_IFACE) - && !stbInfo->use_metric) { - errorPrint("%s", "schemaless cannot work without stable\n"); - return -1; - } - - preProcessArgument(stbInfo); - +int64_t obtainTableCount(SDataBase* database, SSuperTable* stbInfo) { + // ntable calc int64_t ntables; if (stbInfo->childTblTo > 0) { ntables = stbInfo->childTblTo - stbInfo->childTblFrom; @@ -3234,222 +3221,174 @@ static int startMultiThreadInsertData(SDataBase* database, } else { ntables = stbInfo->childTblCount; } - if (ntables == 0) { + + return ntables; +} + +// assign table to thread with vgroups, return assign thread count +int32_t assignTableToThread(SDataBase* database, SSuperTable* stbInfo) { + SBenchConn* conn = initBenchConn(); + if (NULL == conn) { return 0; } + int32_t threads = 0; - uint64_t tableFrom = 0; - int32_t threads = g_arguments->nthreads; - int64_t a = 0, b = 0; - -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if ((0 == stbInfo->interlaceRows) - && (g_arguments->nthreads_auto)) { - SBenchConn* conn = initBenchConn(); - if (NULL == conn) { + // calc table count per vgroup + for (int64_t i = 0; i < stbInfo->childTblCount; i++) { + int vgId; + int ret = taos_get_table_vgId( + conn->taos, database->dbName, + stbInfo->childTblArray[i]->name, &vgId); + if (ret < 0) { + errorPrint("Failed to get %s db's %s table's vgId\n", + database->dbName, + stbInfo->childTblArray[i]->name); + closeBenchConn(conn); return -1; } - - for (int64_t i = 0; i < stbInfo->childTblCount; i++) { - int vgId; - int ret = taos_get_table_vgId( - conn->taos, database->dbName, - stbInfo->childTblArray[i]->name, &vgId); - if (ret < 0) { - errorPrint("Failed to get %s db's %s table's vgId\n", - database->dbName, - stbInfo->childTblArray[i]->name); - closeBenchConn(conn); - return -1; - } - debugPrint("Db %s\'s table\'s %s vgId is: %d\n", - database->dbName, - stbInfo->childTblArray[i]->name, vgId); - for (int32_t v = 0; v < database->vgroups; v++) { - SVGroup *vg = benchArrayGet(database->vgArray, v); - if (vgId == vg->vgId) { - vg->tbCountPerVgId++; - } - } - } - - threads = 0; - for (int v = 0; v < database->vgroups; v++) { + debugPrint("Db %s\'s table\'s %s vgId is: %d\n", + database->dbName, + stbInfo->childTblArray[i]->name, vgId); + for (int32_t v = 0; v < database->vgroups; v++) { SVGroup *vg = benchArrayGet(database->vgArray, v); - infoPrint("Total %"PRId64" tables on bb %s's vgroup %d (id: %d)\n", - vg->tbCountPerVgId, database->dbName, v, vg->vgId); - if (vg->tbCountPerVgId) { - threads++; - } else { - continue; + if (vgId == vg->vgId) { + vg->tbCountPerVgId++; } - vg->childTblArray = benchCalloc( - vg->tbCountPerVgId, sizeof(SChildTable*), true); - vg->tbOffset = 0; } - for (int64_t i = 0; i < stbInfo->childTblCount; i++) { - int vgId; - int ret = taos_get_table_vgId( - conn->taos, database->dbName, - stbInfo->childTblArray[i]->name, &vgId); - if (ret < 0) { - errorPrint("Failed to get %s db's %s table's vgId\n", - database->dbName, - stbInfo->childTblArray[i]->name); + } - closeBenchConn(conn); - return -1; - } - debugPrint("Db %s\'s table\'s %s vgId is: %d\n", - database->dbName, - stbInfo->childTblArray[i]->name, vgId); - for (int32_t v = 0; v < database->vgroups; v++) { - SVGroup *vg = benchArrayGet(database->vgArray, v); - if (vgId == vg->vgId) { - vg->childTblArray[vg->tbOffset] = - stbInfo->childTblArray[i]; - vg->tbOffset++; - } - } - } - closeBenchConn(conn); - } else { - a = ntables / threads; - if (a < 1) { - threads = (int32_t)ntables; - a = 1; - } - b = 0; - if (threads != 0) { - b = ntables % threads; + // malloc vg->childTblArray memory with table count + for (int v = 0; v < database->vgroups; v++) { + SVGroup *vg = benchArrayGet(database->vgArray, v); + infoPrint("Total %"PRId64" tables on %s's vgroup %d (id: %d)\n", + vg->tbCountPerVgId, database->dbName, v, vg->vgId); + if (vg->tbCountPerVgId) { + threads++; + } else { + continue; } + vg->childTblArray = benchCalloc(vg->tbCountPerVgId, sizeof(SChildTable*), true); + vg->tbOffset = 0; } + + // set vg->childTblArray data + for (int64_t i = 0; i < stbInfo->childTblCount; i++) { + int vgId; + int ret = taos_get_table_vgId( + conn->taos, database->dbName, + stbInfo->childTblArray[i]->name, &vgId); + if (ret < 0) { + errorPrint("Failed to get %s db's %s table's vgId\n", + database->dbName, + stbInfo->childTblArray[i]->name); - // valid check - if(threads <= 0) { - errorPrint("db: %s threads num is invalid. threads=%d\n", + closeBenchConn(conn); + return 0; + } + debugPrint("Db %s\'s table\'s %s vgId is: %d\n", database->dbName, - threads); - return -1; + stbInfo->childTblArray[i]->name, vgId); + for (int32_t v = 0; v < database->vgroups; v++) { + SVGroup *vg = benchArrayGet(database->vgArray, v); + if (vgId == vg->vgId) { + vg->childTblArray[vg->tbOffset] = stbInfo->childTblArray[i]; + vg->tbOffset++; + } + } } + closeBenchConn(conn); - int32_t vgFrom = 0; -#else - a = ntables / threads; - if (a < 1) { - threads = (int32_t)ntables; - a = 1; - } - b = 0; - if (threads != 0) { - b = ntables % threads; - } -#endif // TD_VER_COMPATIBLE_3_0_0_0 + return threads; +} + +// init insert thread +int32_t initInsertThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthreads, threadInfo *infos, int64_t div, int64_t mod) { + int32_t ret = -1; + uint64_t tbNext = stbInfo->childTblFrom; + int32_t vgNext = 0; + FILE* csvFile = NULL; + char* tagData = NULL; + bool stmtN = stbInfo->iface == STMT_IFACE && stbInfo->autoTblCreating == false; + int w = 0; - FILE* csvFile = NULL; - char* tagData = NULL; - bool stmtN = (stbInfo->iface == STMT_IFACE && stbInfo->autoTblCreating == false); - int w = 0; if (stmtN) { csvFile = openTagCsv(stbInfo); tagData = benchCalloc(TAG_BATCH_COUNT, stbInfo->lenOfTags, false); } - - pthread_t *pids = benchCalloc(1, threads * sizeof(pthread_t), true); - threadInfo *infos = benchCalloc(1, threads * sizeof(threadInfo), true); - - for (int32_t i = 0; i < threads; i++) { + + for (int32_t i = 0; i < nthreads; i++) { + // set table threadInfo *pThreadInfo = infos + i; - pThreadInfo->threadID = i; - pThreadInfo->dbInfo = database; - pThreadInfo->stbInfo = stbInfo; + pThreadInfo->threadID = i; + pThreadInfo->dbInfo = database; + pThreadInfo->stbInfo = stbInfo; pThreadInfo->start_time = stbInfo->startTimestamp; - pThreadInfo->pos = 0; + pThreadInfo->pos = 0; + pThreadInfo->samplePos = 0; pThreadInfo->totalInsertRows = 0; - pThreadInfo->samplePos = 0; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if ((0 == stbInfo->interlaceRows) - && (g_arguments->nthreads_auto)) { - int32_t j; - for (j = vgFrom; i < database->vgroups; j++) { + + if (g_arguments->bind_vgroup) { + for (int32_t j = vgNext; j < database->vgroups; j++) { SVGroup *vg = benchArrayGet(database->vgArray, j); if (0 == vg->tbCountPerVgId) { continue; } - pThreadInfo->vg = vg; + pThreadInfo->vg = vg; + pThreadInfo->ntables = vg->tbCountPerVgId; pThreadInfo->start_table_from = 0; - pThreadInfo->ntables = vg->tbCountPerVgId; - pThreadInfo->end_table_to = vg->tbCountPerVgId-1; + pThreadInfo->end_table_to = vg->tbCountPerVgId - 1; + vgNext = j + 1; break; - } - vgFrom = j + 1; + } } else { - pThreadInfo->start_table_from = tableFrom; - pThreadInfo->ntables = i < b ? a + 1 : a; - pThreadInfo->end_table_to = (i < b)?(tableFrom+a):(tableFrom+a-1); - tableFrom = pThreadInfo->end_table_to + 1; + pThreadInfo->start_table_from = tbNext; + pThreadInfo->ntables = i < mod ? div + 1 : div; + pThreadInfo->end_table_to = i < mod ? tbNext + div : tbNext + div - 1; + tbNext = pThreadInfo->end_table_to + 1; } -#else - pThreadInfo->start_table_from = tableFrom; - pThreadInfo->ntables = i < b ? a + 1 : a; - pThreadInfo->end_table_to = (i < b)?(tableFrom+a):(tableFrom+a-1); - tableFrom = pThreadInfo->end_table_to + 1; -#endif // TD_VER_COMPATIBLE_3_0_0_0 + + // init conn pThreadInfo->delayList = benchArrayInit(1, sizeof(int64_t)); switch (stbInfo->iface) { + // rest case REST_IFACE: { if (stbInfo->interlaceRows > 0) { pThreadInfo->buffer = new_ds(0); } else { - pThreadInfo->buffer = - benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); + pThreadInfo->buffer = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); } int sockfd = createSockFd(); if (sockfd < 0) { - FREE_PIDS_INFOS_RETURN_MINUS_1(); + goto END; } pThreadInfo->sockfd = sockfd; break; } + // stmt case STMT_IFACE: { pThreadInfo->conn = initBenchConn(); if (NULL == pThreadInfo->conn) { - FREE_PIDS_INFOS_RETURN_MINUS_1(); + goto END; } - pThreadInfo->conn->stmt = - taos_stmt_init(pThreadInfo->conn->taos); + pThreadInfo->conn->stmt = taos_stmt_init(pThreadInfo->conn->taos); if (NULL == pThreadInfo->conn->stmt) { - errorPrint("taos_stmt_init() failed, reason: %s\n", - taos_errstr(NULL)); - FREE_RESOURCE(); - return -1; + errorPrint("taos_stmt_init() failed, reason: %s\n", taos_errstr(NULL)); + goto END; } if (taos_select_db(pThreadInfo->conn->taos, database->dbName)) { - errorPrint("taos select database(%s) failed\n", - database->dbName); - FREE_RESOURCE(); - return -1; + errorPrint("taos select database(%s) failed\n", database->dbName); + goto END; } if (stmtN) { // generator if (w == 0) { if(!generateTagData(stbInfo, tagData, TAG_BATCH_COUNT, csvFile)) { - if(csvFile){ - fclose(csvFile); - } - tmfree(tagData); - FREE_RESOURCE(); - return -1; + goto END; } } if (prepareStmt(stbInfo, pThreadInfo->conn->stmt, tagData, w)) { - if(csvFile){ - fclose(csvFile); - } - tmfree(tagData); - FREE_RESOURCE(); - return -1; + goto END; } // move next @@ -3459,18 +3398,14 @@ static int startMultiThreadInsertData(SDataBase* database, } } - pThreadInfo->bind_ts = benchCalloc(1, sizeof(int64_t), true); - pThreadInfo->bind_ts_array = - benchCalloc(1, sizeof(int64_t)*g_arguments->reqPerReq, - true); - pThreadInfo->bindParams = benchCalloc( - 1, sizeof(TAOS_MULTI_BIND)*(stbInfo->cols->size + 1), - true); - pThreadInfo->is_null = benchCalloc(1, g_arguments->reqPerReq, - true); + // malloc bind + pThreadInfo->bind_ts = benchCalloc(1, sizeof(int64_t), true); + pThreadInfo->bind_ts_array = benchCalloc(1, sizeof(int64_t)*g_arguments->reqPerReq, true); + pThreadInfo->bindParams = benchCalloc(1, sizeof(TAOS_MULTI_BIND)*(stbInfo->cols->size + 1), true); + pThreadInfo->is_null = benchCalloc(1, g_arguments->reqPerReq, true); + parseBufferToStmtBatch(stbInfo); - for (int64_t child = 0; - child < stbInfo->childTblCount; child++) { + for (int64_t child = 0; child < stbInfo->childTblCount; child++) { SChildTable *childTbl = stbInfo->childTblArray[child]; if (childTbl->useOwnSample) { parseBufferToStmtBatchChildTbl(stbInfo, childTbl); @@ -3479,44 +3414,34 @@ static int startMultiThreadInsertData(SDataBase* database, break; } + // sml rest case SML_REST_IFACE: { int sockfd = createSockFd(); if (sockfd < 0) { - free(pids); - free(infos); - return -1; + goto END; } pThreadInfo->sockfd = sockfd; } + // sml case SML_IFACE: { pThreadInfo->conn = initBenchConn(); if (pThreadInfo->conn == NULL) { errorPrint("%s() init connection failed\n", __func__); - FREE_RESOURCE(); - return -1; + goto END; } if (taos_select_db(pThreadInfo->conn->taos, database->dbName)) { - errorPrint("taos select database(%s) failed\n", - database->dbName); - FREE_RESOURCE(); - return -1; + errorPrint("taos select database(%s) failed\n", database->dbName); + goto END; } - pThreadInfo->max_sql_len = - stbInfo->lenOfCols + stbInfo->lenOfTags; + pThreadInfo->max_sql_len = stbInfo->lenOfCols + stbInfo->lenOfTags; if (stbInfo->iface == SML_REST_IFACE) { - pThreadInfo->buffer = - benchCalloc(1, g_arguments->reqPerReq * - (1 + pThreadInfo->max_sql_len), true); + pThreadInfo->buffer = benchCalloc(1, g_arguments->reqPerReq * (1 + pThreadInfo->max_sql_len), true); } int protocol = stbInfo->lineProtocol; - if (TSDB_SML_JSON_PROTOCOL != protocol - && SML_JSON_TAOS_FORMAT != protocol) { - pThreadInfo->sml_tags = - (char **)benchCalloc(pThreadInfo->ntables, - sizeof(char *), true); + if (TSDB_SML_JSON_PROTOCOL != protocol && SML_JSON_TAOS_FORMAT != protocol) { + pThreadInfo->sml_tags = (char **)benchCalloc(pThreadInfo->ntables, sizeof(char *), true); for (int t = 0; t < pThreadInfo->ntables; t++) { - pThreadInfo->sml_tags[t] = - benchCalloc(1, stbInfo->lenOfTags, true); + pThreadInfo->sml_tags[t] = benchCalloc(1, stbInfo->lenOfTags, true); } for (int t = 0; t < pThreadInfo->ntables; t++) { @@ -3530,20 +3455,14 @@ static int startMultiThreadInsertData(SDataBase* database, debugPrint("pThreadInfo->sml_tags[%d]: %s\n", t, pThreadInfo->sml_tags[t]); } - pThreadInfo->lines = - benchCalloc(g_arguments->reqPerReq, - sizeof(char *), true); - - for (int j = 0; (j < g_arguments->reqPerReq - && !g_arguments->terminate); j++) { - pThreadInfo->lines[j] = - benchCalloc(1, pThreadInfo->max_sql_len, true); + pThreadInfo->lines = benchCalloc(g_arguments->reqPerReq, sizeof(char *), true); + for (int j = 0; (j < g_arguments->reqPerReq && !g_arguments->terminate); j++) { + pThreadInfo->lines[j] = benchCalloc(1, pThreadInfo->max_sql_len, true); } } else { - pThreadInfo->json_array = tools_cJSON_CreateArray(); - pThreadInfo->sml_json_tags = tools_cJSON_CreateArray(); - pThreadInfo->sml_tags_json_array = (char **)benchCalloc( - pThreadInfo->ntables, sizeof(char *), true); + pThreadInfo->json_array = tools_cJSON_CreateArray(); + pThreadInfo->sml_json_tags = tools_cJSON_CreateArray(); + pThreadInfo->sml_tags_json_array = (char **)benchCalloc( pThreadInfo->ntables, sizeof(char *), true); for (int t = 0; t < pThreadInfo->ntables; t++) { if (stbInfo->lineProtocol == TSDB_SML_JSON_PROTOCOL) { generateSmlJsonTags( @@ -3557,48 +3476,33 @@ static int startMultiThreadInsertData(SDataBase* database, pThreadInfo->start_table_from, t); } } - pThreadInfo->lines = (char **)benchCalloc( - 1, sizeof(char *), true); - if ((0 == stbInfo->interlaceRows) - && (TSDB_SML_JSON_PROTOCOL == protocol)) { - pThreadInfo->line_buf_len = - g_arguments->reqPerReq * - accumulateRowLen(pThreadInfo->stbInfo->tags, - pThreadInfo->stbInfo->iface); - debugPrint("%s() LN%d, line_buf_len=%d\n", - __func__, __LINE__, pThreadInfo->line_buf_len); - pThreadInfo->lines[0] = benchCalloc( - 1, pThreadInfo->line_buf_len, true); - pThreadInfo->sml_json_value_array = - (char **)benchCalloc( - pThreadInfo->ntables, sizeof(char *), true); + pThreadInfo->lines = (char **)benchCalloc(1, sizeof(char *), true); + if (0 == stbInfo->interlaceRows && TSDB_SML_JSON_PROTOCOL == protocol) { + pThreadInfo->line_buf_len = g_arguments->reqPerReq * accumulateRowLen(pThreadInfo->stbInfo->tags, pThreadInfo->stbInfo->iface); + debugPrint("%s() LN%d, line_buf_len=%d\n", __func__, __LINE__, pThreadInfo->line_buf_len); + pThreadInfo->lines[0] = benchCalloc(1, pThreadInfo->line_buf_len, true); + pThreadInfo->sml_json_value_array = (char **)benchCalloc(pThreadInfo->ntables, sizeof(char *), true); for (int t = 0; t < pThreadInfo->ntables; t++) { - generateSmlJsonValues( - pThreadInfo->sml_json_value_array, stbInfo, t); + generateSmlJsonValues(pThreadInfo->sml_json_value_array, stbInfo, t); } } } break; } + // taos case TAOSC_IFACE: { pThreadInfo->conn = initBenchConn(); if (pThreadInfo->conn == NULL) { errorPrint("%s() failed to connect\n", __func__); - FREE_RESOURCE(); - return -1; + goto END; } char* command = benchCalloc(1, SHORT_1K_SQL_BUFF_LEN, false); snprintf(command, SHORT_1K_SQL_BUFF_LEN, - g_arguments->escape_character - ? "USE `%s`" - : "USE %s", + g_arguments->escape_character ? "USE `%s`" : "USE %s", database->dbName); if (queryDbExecCall(pThreadInfo->conn, command)) { - errorPrint("taos select database(%s) failed\n", - database->dbName); - FREE_RESOURCE(); - tmfree(command); - return -1; + errorPrint("taos select database(%s) failed\n", database->dbName); + goto END; } tmfree(command); command = NULL; @@ -3606,11 +3510,9 @@ static int startMultiThreadInsertData(SDataBase* database, if (stbInfo->interlaceRows > 0) { pThreadInfo->buffer = new_ds(0); } else { - pThreadInfo->buffer = - benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); + pThreadInfo->buffer = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); if (g_arguments->check_sql) { - pThreadInfo->csql = - benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); + pThreadInfo->csql = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); memset(pThreadInfo->csql, 0, TSDB_MAX_ALLOWED_SQL_LEN); } } @@ -3622,29 +3524,20 @@ static int startMultiThreadInsertData(SDataBase* database, } } + // success + ret = 0; + +END: if (csvFile) { fclose(csvFile); } tmfree(tagData); + return ret; +} - infoPrint("Estimate memory usage: %.2fMB\n", - (double)g_memoryUsage / 1048576); - prompt(0); - - // create threads - int threadCnt = 0; - for (int i = 0; (i < threads && !g_arguments->terminate); i++) { - threadInfo *pThreadInfo = infos + i; - if (stbInfo->interlaceRows > 0) { - pthread_create(pids + i, NULL, - syncWriteInterlace, pThreadInfo); - } else { - pthread_create(pids + i, NULL, - syncWriteProgressive, pThreadInfo); - } - threadCnt ++; - } +// free resource +int32_t waitInsertThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthreads, int32_t threadCnt, threadInfo *infos, pthread_t *pids) { int64_t start = toolsGetTimestampUs(); // wait threads @@ -3653,6 +3546,7 @@ static int startMultiThreadInsertData(SDataBase* database, pthread_join(pids[i], NULL); } + // thread end int64_t end = toolsGetTimestampUs()+1; if (g_arguments->terminate) toolsMsleep(100); @@ -3665,7 +3559,7 @@ static int startMultiThreadInsertData(SDataBase* database, uint64_t totalInsertRows = 0; // free threads resource - for (int i = 0; i < threads; i++) { + for (int i = 0; i < nthreads; i++) { threadInfo *pThreadInfo = infos + i; // free check sql if (pThreadInfo->csql) { @@ -3673,6 +3567,7 @@ static int startMultiThreadInsertData(SDataBase* database, pThreadInfo->csql = NULL; } + // close conn int protocol = stbInfo->lineProtocol; switch (stbInfo->iface) { case REST_IFACE: @@ -3773,18 +3668,93 @@ static int startMultiThreadInsertData(SDataBase* database, if (g_arguments->terminate) toolsMsleep(100); - free(pids); - free(infos); + tmfree(pids); + tmfree(infos); + // print result int ret = printTotalDelay(database, totalDelay, totalDelay1, totalDelay2, totalDelay3, - total_delay_list, threads, totalInsertRows, start, end); + total_delay_list, nthreads, totalInsertRows, start, end); benchArrayDestroy(total_delay_list); - if (g_fail || ret) { + if (g_fail || ret != 0) { return -1; } return 0; } +static int startMultiThreadInsertData(SDataBase* database, SSuperTable* stbInfo) { + if ((stbInfo->iface == SML_IFACE || stbInfo->iface == SML_REST_IFACE) + && !stbInfo->use_metric) { + errorPrint("%s", "schemaless cannot work without stable\n"); + return -1; + } + + // check argument valid + preProcessArgument(stbInfo); + + // ntable + int64_t ntables = obtainTableCount(database, stbInfo); + if (ntables == 0) { + errorPrint("insert table count is zero. %s.%s\n", database->dbName, stbInfo->stbName); + return -1; + } + + // assign table to thread + int32_t nthreads = g_arguments->nthreads; + int64_t div = 0; // ntable / nthread division + int64_t mod = 0; // ntable % nthread + + if (g_arguments->bind_vgroup) { + nthreads = assignTableToThread(database, stbInfo); + if(nthreads == 0) { + errorPrint("bind vgroup assign theads count is zero. %s.%s\n", database->dbName, stbInfo->stbName); + return -1; + } + } else { + if(nthreads == 0) { + errorPrint("argument thread_count can not be zero. %s.%s\n", database->dbName, stbInfo->stbName); + return -1; + } + div = ntables / nthreads; + if (div < 1) { + nthreads = (int32_t)ntables; + div = 1; + } + mod = ntables % nthreads; + } + + + // init each thread information + pthread_t *pids = benchCalloc(1, nthreads * sizeof(pthread_t), true); + threadInfo *infos = benchCalloc(1, nthreads * sizeof(threadInfo), true); + int32_t ret = initInsertThread(database, stbInfo, nthreads, infos, div, mod); + if( ret != 0) { + errorPrint("init insert thread failed. %s.%s\n", database->dbName, stbInfo->stbName); + FREE_RESOURCE(); + return ret; + } + + infoPrint("Estimate memory usage: %.2fMB\n", (double)g_memoryUsage / 1048576); + prompt(0); + + // create threads + int threadCnt = 0; + for (int i = 0; (i < nthreads && !g_arguments->terminate); i++) { + threadInfo *pThreadInfo = infos + i; + if (stbInfo->interlaceRows > 0) { + pthread_create(pids + i, NULL, + syncWriteInterlace, pThreadInfo); + } else { + pthread_create(pids + i, NULL, + syncWriteProgressive, pThreadInfo); + } + threadCnt ++; + } + + // wait insert end + ret = waitInsertThread(database, stbInfo, nthreads, threadCnt, infos, pids); + return ret; +} + static int getStbInsertedRows(char* dbName, char* stbName, TAOS* taos) { int rows = 0; char command[SHORT_1K_SQL_BUFF_LEN]; @@ -3977,7 +3947,6 @@ int insertTestProcess() { succPrint("created database (%s)\n", database->dbName); } else { // database already exist, get vgroups from server - #ifdef TD_VER_COMPATIBLE_3_0_0_0 if (database->superTbls) { SBenchConn* conn = initBenchConn(); if (conn) { @@ -3991,7 +3960,6 @@ int insertTestProcess() { succPrint("Database (%s) get vgroups num is %d from server.\n", database->dbName, vgroups); } } - #endif // TD_VER_COMPATIBLE_3_0_0_0 } } @@ -4045,7 +4013,7 @@ int insertTestProcess() { } } - // create threads + // tsma if (g_arguments->taosc_version == 3) { for (int i = 0; i < g_arguments->databases->size; i++) { SDataBase* database = benchArrayGet(g_arguments->databases, i); diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 3c9121f4..4001889a 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -1489,6 +1489,13 @@ static int getMetaFromInsertJsonFile(tools_cJSON *json) { g_arguments->nthreads = (uint32_t)threads->valueint; } + tools_cJSON *bindVGroup = tools_cJSON_GetObjectItem(json, "thread_bind_vgroup"); + if (tools_cJSON_IsString(bindVGroup)) { + if (0 == strcasecmp(bindVGroup->valuestring, "yes")) { + g_arguments->bind_vgroup = true; + } + } + tools_cJSON *keepTrying = tools_cJSON_GetObjectItem(json, "keep_trying"); if (keepTrying && keepTrying->type == tools_cJSON_Number) { g_arguments->keep_trying = (int32_t)keepTrying->valueint; From 28ad4c527ad4e7eeaed4e3c2ab68caa455a81db5 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 27 May 2024 20:53:36 +0800 Subject: [PATCH 23/57] feat: add bind vgroup new function --- case/insertBindVGroup.json | 58 ++++++++++++++++++++++++++++++++++++++ src/benchInsert.c | 3 +- src/benchUtil.c | 2 +- 3 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 case/insertBindVGroup.json diff --git a/case/insertBindVGroup.json b/case/insertBindVGroup.json new file mode 100644 index 00000000..beaadfb5 --- /dev/null +++ b/case/insertBindVGroup.json @@ -0,0 +1,58 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "num_of_records_per_req": 200, + "thread_count": 20, + "thread_bind_vgroup": "yes", + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "binddb", + "drop": "yes", + "vgroups": 2 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 4, + "insert_rows": 100, + "interlace_rows": 10, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 1000, + "start_timestamp":1500000000000, + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "max": 1, "min": 0 }, + { "type": "double", "name": "dc", "max": 1, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": 0 }, + { "type": "smallint", "name": "si", "max": 100, "min": 0 }, + { "type": "int", "name": "ic", "max": 100, "min": 0 }, + { "type": "bigint", "name": "bi", "max": 100, "min": 0 }, + { "type": "utinyint", "name": "uti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "usi", "max": 100, "min": 0 }, + { "type": "uint", "name": "ui", "max": 100, "min": 0 }, + { "type": "ubigint", "name": "ubi", "max": 100, "min": 0 }, + { "type": "binary", "name": "bin", "len": 32}, + { "type": "nchar", "name": "nch", "len": 64} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"type": "binary", "name": "location", "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/src/benchInsert.c b/src/benchInsert.c index 4aff9b8e..1ce7e8ce 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -3729,7 +3729,8 @@ static int startMultiThreadInsertData(SDataBase* database, SSuperTable* stbInfo) int32_t ret = initInsertThread(database, stbInfo, nthreads, infos, div, mod); if( ret != 0) { errorPrint("init insert thread failed. %s.%s\n", database->dbName, stbInfo->stbName); - FREE_RESOURCE(); + tmfree(pids); + tmfree(infos); return ret; } diff --git a/src/benchUtil.c b/src/benchUtil.c index 3318f2ee..5c8d19e5 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -1041,7 +1041,7 @@ int convertStringToDatatype(char *type, int length) { return TSDB_DATA_TYPE_JSON; } else if (0 == strncasecmp(type, "varchar", length)) { return TSDB_DATA_TYPE_BINARY; - } else if (0 == strcnasecmp(type, "varbinary", length) { + } else if (0 == strncasecmp(type, "varbinary", length)) { return TSDB_DATA_TYPE_VARBINARY; } else { errorPrint("unknown data type: %s\n", type); From f3ee2a124eb8dfb83651afa7f603c5737354485d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 29 May 2024 15:22:10 +0800 Subject: [PATCH 24/57] feat: create table add speed --- src/benchInsert.c | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index b4212636..c359d4bf 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -820,28 +820,30 @@ static int getIntervalOfTblCreating(threadInfo *pThreadInfo, return 0; } +// table create thread static void *createTable(void *sarg) { if (g_arguments->supplementInsert) { return NULL; } - threadInfo * pThreadInfo = (threadInfo *)sarg; - SDataBase * database = pThreadInfo->dbInfo; - SSuperTable *stbInfo = pThreadInfo->stbInfo; + threadInfo *pThreadInfo = (threadInfo *)sarg; + SDataBase *database = pThreadInfo->dbInfo; + SSuperTable *stbInfo = pThreadInfo->stbInfo; + uint64_t lastTotalCreate = 0; + uint64_t lastPrintTime = toolsGetTimestampMs(); + int32_t len = 0; + int32_t batchNum = 0; + char ttl[SMALL_BUFF_LEN] = ""; + #ifdef LINUX prctl(PR_SET_NAME, "createTable"); #endif - uint64_t lastPrintTime = toolsGetTimestampMs(); pThreadInfo->buffer = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); - int len = 0; - int batchNum = 0; infoPrint( "thread[%d] start creating table from %" PRIu64 " to %" PRIu64 "\n", pThreadInfo->threadID, pThreadInfo->start_table_from, pThreadInfo->end_table_to); - - char ttl[SMALL_BUFF_LEN] = ""; if (stbInfo->ttl != 0) { snprintf(ttl, SMALL_BUFF_LEN, "TTL %d", stbInfo->ttl); } @@ -954,10 +956,11 @@ static void *createTable(void *sarg) { batchNum = 0; uint64_t currentPrintTime = toolsGetTimestampMs(); if (currentPrintTime - lastPrintTime > PRINT_STAT_INTERVAL) { - infoPrint( - "thread[%d] already created %" PRId64 " tables\n", - pThreadInfo->threadID, pThreadInfo->tables_created); - lastPrintTime = currentPrintTime; + float rate = (pThreadInfo->tables_created - lastTotalCreate) * 1000 / (currentPrintTime - lastPrintTime); + infoPrint("thread[%d] already created %" PRId64 " tables, peroid rate: %.0f tables/s\n", + pThreadInfo->threadID, pThreadInfo->tables_created, rate); + lastPrintTime = currentPrintTime; + lastTotalCreate = pThreadInfo->tables_created; } } @@ -1120,11 +1123,14 @@ static int createChildTables() { double end = (double)toolsGetTimestampMs(); succPrint( "Spent %.4f seconds to create %" PRId64 - " table(s) with %d thread(s), already exist %" PRId64 + " table(s) with %d thread(s) rate: %.0f tables/s, already exist %" PRId64 " table(s), actual %" PRId64 " table(s) pre created, %" PRId64 " table(s) will be auto created\n", - (end - start) / 1000.0, g_arguments->totalChildTables, - g_arguments->table_threads, g_arguments->existedChildTables, + (end - start) / 1000.0, + g_arguments->totalChildTables, + g_arguments->table_threads, + g_arguments->actualChildTables * 1000 / (end - start), + g_arguments->existedChildTables, g_arguments->actualChildTables, g_arguments->autoCreatedChildTables); return 0; @@ -1530,7 +1536,9 @@ void loadChildTableInfo(threadInfo* pThreadInfo) { pos = 0; } } - infoPrint("end load child tables info. delay=%.2fs\n", (toolsGetTimestampUs() - start)/1E6); + int64_t delay = toolsGetTimestampUs() - start; + infoPrint("end load child tables info. delay=%.2fs\n", delay/1E6); + pThreadInfo->totalDelay += delay; tmfree(buf); } From b105aa37ea7477d7f1dc420a54f23d5569b56fa9 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 29 May 2024 15:22:23 +0800 Subject: [PATCH 25/57] feat: create table add speed --- src/benchInsert.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index c359d4bf..ad1d234c 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -956,9 +956,9 @@ static void *createTable(void *sarg) { batchNum = 0; uint64_t currentPrintTime = toolsGetTimestampMs(); if (currentPrintTime - lastPrintTime > PRINT_STAT_INTERVAL) { - float rate = (pThreadInfo->tables_created - lastTotalCreate) * 1000 / (currentPrintTime - lastPrintTime); - infoPrint("thread[%d] already created %" PRId64 " tables, peroid rate: %.0f tables/s\n", - pThreadInfo->threadID, pThreadInfo->tables_created, rate); + float speed = (pThreadInfo->tables_created - lastTotalCreate) * 1000 / (currentPrintTime - lastPrintTime); + infoPrint("thread[%d] already created %" PRId64 " tables, peroid speed: %.0f tables/s\n", + pThreadInfo->threadID, pThreadInfo->tables_created, speed); lastPrintTime = currentPrintTime; lastTotalCreate = pThreadInfo->tables_created; } @@ -1123,7 +1123,7 @@ static int createChildTables() { double end = (double)toolsGetTimestampMs(); succPrint( "Spent %.4f seconds to create %" PRId64 - " table(s) with %d thread(s) rate: %.0f tables/s, already exist %" PRId64 + " table(s) with %d thread(s) speed: %.0f tables/s, already exist %" PRId64 " table(s), actual %" PRId64 " table(s) pre created, %" PRId64 " table(s) will be auto created\n", (end - start) / 1000.0, From dc8f52e0210fd73432e909d00d35ca9634e042b5 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 31 May 2024 14:25:58 +0800 Subject: [PATCH 26/57] fix: with limit thread frame build ok --- src/benchInsert.c | 74 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 22 deletions(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index ad1d234c..f93cde97 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -3070,9 +3070,9 @@ static int printTotalDelay(SDataBase *database, BArray *total_delay_list, int threads, int64_t totalInsertRows, - int64_t start, int64_t end) { + int64_t spend) { // zero check - if (total_delay_list->size == 0 || (end - start) == 0 || threads == 0) { + if (total_delay_list->size == 0 || spend == 0 || threads == 0) { return -1; } @@ -3086,9 +3086,9 @@ static int printTotalDelay(SDataBase *database, succPrint("Spent %.6f (real %.6f) seconds to insert rows: %" PRIu64 " with %d thread(s) into %s %.2f (real %.2f) records/second%s\n", - (end - start)/1E6, totalDelay/threads/1E6, totalInsertRows, threads, + spend/1E6, totalDelay/threads/1E6, totalInsertRows, threads, database->dbName, - (double)(totalInsertRows / ((end - start)/1E6)), + (double)(totalInsertRows / (spend/1E6)), (double)(totalInsertRows / (totalDelay/threads/1E6)), subDelay); if (!total_delay_list->size) { return -1; @@ -3556,9 +3556,29 @@ int32_t initInsertThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthr return ret; } +// run with limit thread +int32_t runInsertLimitThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthreads, int32_t limitThread, threadInfo *infos, pthread_t *pids, int64_t *spend) { + infoPrint("run with limit insert thread. limit threads=%d nthread=%d\n", limitThread, nthreads); + return 0; +} + +// run +int32_t runInsertThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthreads, threadInfo *infos, pthread_t *pids, int64_t *spend) { + infoPrint("run insert thread. real nthread=%d\n", nthreads); + // create threads + int threadCnt = 0; + for (int i = 0; (i < nthreads && !g_arguments->terminate); i++) { + threadInfo *pThreadInfo = infos + i; + if (stbInfo->interlaceRows > 0) { + pthread_create(pids + i, NULL, + syncWriteInterlace, pThreadInfo); + } else { + pthread_create(pids + i, NULL, + syncWriteProgressive, pThreadInfo); + } + threadCnt ++; + } -// free resource -int32_t waitInsertThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthreads, int32_t threadCnt, threadInfo *infos, pthread_t *pids) { int64_t start = toolsGetTimestampUs(); // wait threads @@ -3568,7 +3588,19 @@ int32_t waitInsertThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthr } // thread end - int64_t end = toolsGetTimestampUs()+1; + int64_t end = toolsGetTimestampUs(); + if(end == start) { + *spend = 1; + } else { + *spend = end - start; + } + + return 0; +} + + +// exit and free resource +int32_t exitInsertThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthreads, threadInfo *infos, pthread_t *pids, int64_t spend) { if (g_arguments->terminate) toolsMsleep(100); @@ -3694,7 +3726,7 @@ int32_t waitInsertThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthr // print result int ret = printTotalDelay(database, totalDelay, totalDelay1, totalDelay2, totalDelay3, - total_delay_list, nthreads, totalInsertRows, start, end); + total_delay_list, nthreads, totalInsertRows, spend); benchArrayDestroy(total_delay_list); if (g_fail || ret != 0) { return -1; @@ -3723,6 +3755,7 @@ static int startMultiThreadInsertData(SDataBase* database, SSuperTable* stbInfo) int32_t nthreads = g_arguments->nthreads; int64_t div = 0; // ntable / nthread division int64_t mod = 0; // ntable % nthread + int64_t spend = 0; if (g_arguments->bind_vgroup) { nthreads = assignTableToThread(database, stbInfo); @@ -3747,6 +3780,8 @@ static int startMultiThreadInsertData(SDataBase* database, SSuperTable* stbInfo) // init each thread information pthread_t *pids = benchCalloc(1, nthreads * sizeof(pthread_t), true); threadInfo *infos = benchCalloc(1, nthreads * sizeof(threadInfo), true); + + // init int32_t ret = initInsertThread(database, stbInfo, nthreads, infos, div, mod); if( ret != 0) { errorPrint("init insert thread failed. %s.%s\n", database->dbName, stbInfo->stbName); @@ -3758,22 +3793,17 @@ static int startMultiThreadInsertData(SDataBase* database, SSuperTable* stbInfo) infoPrint("Estimate memory usage: %.2fMB\n", (double)g_memoryUsage / 1048576); prompt(0); - // create threads - int threadCnt = 0; - for (int i = 0; (i < nthreads && !g_arguments->terminate); i++) { - threadInfo *pThreadInfo = infos + i; - if (stbInfo->interlaceRows > 0) { - pthread_create(pids + i, NULL, - syncWriteInterlace, pThreadInfo); - } else { - pthread_create(pids + i, NULL, - syncWriteProgressive, pThreadInfo); - } - threadCnt ++; + // run + if(g_arguments->bind_vgroup && g_arguments->nthreads < nthreads ) { + // need many batch execute all threads + ret = runInsertLimitThread(database, stbInfo, nthreads, g_arguments->nthreads, infos, pids, &spend); + } else { + // only one batch execute all threads + ret = runInsertThread(database, stbInfo, nthreads, infos, pids, &spend); } - // wait insert end - ret = waitInsertThread(database, stbInfo, nthreads, threadCnt, infos, pids); + // exit + ret = exitInsertThread(database, stbInfo, nthreads, infos, pids, spend); return ret; } From 24dbb7871abcda6f0e4b0a7ba79ed3956acf0a82 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 31 May 2024 18:38:12 +0800 Subject: [PATCH 27/57] fix: limit thread is finish --- src/benchInsert.c | 94 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 73 insertions(+), 21 deletions(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index f93cde97..27c368e1 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -3556,30 +3556,81 @@ int32_t initInsertThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthr return ret; } +#define EMPTY_SLOT -1 // run with limit thread -int32_t runInsertLimitThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthreads, int32_t limitThread, threadInfo *infos, pthread_t *pids, int64_t *spend) { - infoPrint("run with limit insert thread. limit threads=%d nthread=%d\n", limitThread, nthreads); +int32_t runInsertLimitThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthreads, int32_t limitThread, threadInfo *infos, pthread_t *pids) { + infoPrint("run with bind vgroups limit thread. limit threads=%d nthread=%d\n", limitThread, nthreads); + + // slots save threadInfo array index + int32_t* slot = benchCalloc(limitThread, sizeof(int32_t), false); + int32_t t = 0; // thread index + for (int32_t i = 0; i < limitThread; i++) { + slot[i] = EMPTY_SLOT; + } + + while (!g_arguments->terminate) { + int32_t emptySlot = 0; + for (int32_t i = 0; i < limitThread; i++) { + int32_t idx = slot[i]; + // check slot thread end + if(idx != EMPTY_SLOT) { + if (pthread_tryjoin_np(pids[idx], NULL) == EBUSY ) { + // thread is running + toolsMsleep(2000); + } else { + // thread is end , set slot empty + infoPrint("slot[%d] finished tidx=%d. completed thread count=%d\n", i, slot[i], t); + slot[i] = EMPTY_SLOT; + } + } + + if (slot[i] == EMPTY_SLOT && t < nthreads) { + // slot is empty , set new thread to running + threadInfo *pThreadInfo = infos + t; + if (stbInfo->interlaceRows > 0) { + pthread_create(pids + t, NULL, syncWriteInterlace, pThreadInfo); + } else { + pthread_create(pids + t, NULL, syncWriteProgressive, pThreadInfo); + } + + // save current and move next + slot[i] = t; + t++; + infoPrint("slot[%d] start new thread tidx=%d. \n", i, slot[i]); + } + + // check slot empty + if(slot[i] == EMPTY_SLOT) { + emptySlot++; + } + } + + // check all thread end + if(emptySlot == limitThread) { + debugPrint("all threads(%d) is run finished.\n", nthreads); + break; + } else { + debugPrint("current thread index=%d all thread=%d\n", t, nthreads); + } + } + return 0; } // run -int32_t runInsertThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthreads, threadInfo *infos, pthread_t *pids, int64_t *spend) { +int32_t runInsertThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthreads, threadInfo *infos, pthread_t *pids) { infoPrint("run insert thread. real nthread=%d\n", nthreads); // create threads int threadCnt = 0; - for (int i = 0; (i < nthreads && !g_arguments->terminate); i++) { + for (int i = 0; i < nthreads && !g_arguments->terminate; i++) { threadInfo *pThreadInfo = infos + i; if (stbInfo->interlaceRows > 0) { - pthread_create(pids + i, NULL, - syncWriteInterlace, pThreadInfo); + pthread_create(pids + i, NULL, syncWriteInterlace, pThreadInfo); } else { - pthread_create(pids + i, NULL, - syncWriteProgressive, pThreadInfo); + pthread_create(pids + i, NULL, syncWriteProgressive, pThreadInfo); } threadCnt ++; - } - - int64_t start = toolsGetTimestampUs(); + } // wait threads for (int i = 0; i < threadCnt; i++) { @@ -3587,14 +3638,6 @@ int32_t runInsertThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthre pthread_join(pids[i], NULL); } - // thread end - int64_t end = toolsGetTimestampUs(); - if(end == start) { - *spend = 1; - } else { - *spend = end - start; - } - return 0; } @@ -3793,13 +3836,22 @@ static int startMultiThreadInsertData(SDataBase* database, SSuperTable* stbInfo) infoPrint("Estimate memory usage: %.2fMB\n", (double)g_memoryUsage / 1048576); prompt(0); + // run + int64_t start = toolsGetTimestampUs(); if(g_arguments->bind_vgroup && g_arguments->nthreads < nthreads ) { // need many batch execute all threads - ret = runInsertLimitThread(database, stbInfo, nthreads, g_arguments->nthreads, infos, pids, &spend); + ret = runInsertLimitThread(database, stbInfo, nthreads, g_arguments->nthreads, infos, pids); } else { // only one batch execute all threads - ret = runInsertThread(database, stbInfo, nthreads, infos, pids, &spend); + ret = runInsertThread(database, stbInfo, nthreads, infos, pids); + } + + int64_t end = toolsGetTimestampUs(); + if(end == start) { + spend = 1; + } else { + spend = end - start; } // exit From 8c7dee239c3ec5fa48b8b75acdc82696d0bc10b7 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 1 Jun 2024 11:00:05 +0800 Subject: [PATCH 28/57] fix: remove old auto_thread variant --- inc/bench.h | 1 - src/benchCommandOpt.c | 1 - src/benchInsert.c | 23 +++++------------------ src/benchMain.c | 1 - src/benchSys.c | 12 ------------ 5 files changed, 5 insertions(+), 33 deletions(-) diff --git a/inc/bench.h b/inc/bench.h index de88c8a6..003edc3f 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -924,7 +924,6 @@ typedef struct SArguments_S { uint32_t binwidth; uint32_t intColumnCount; uint32_t nthreads; - bool nthreads_auto; uint32_t table_threads; uint64_t prepared_rand; uint32_t reqPerReq; diff --git a/src/benchCommandOpt.c b/src/benchCommandOpt.c index faa2e407..3a18540d 100644 --- a/src/benchCommandOpt.c +++ b/src/benchCommandOpt.c @@ -242,7 +242,6 @@ void initArgument() { g_arguments->performance_print = 0; g_arguments->output_file = DEFAULT_OUTPUT; g_arguments->nthreads = DEFAULT_NTHREADS; - g_arguments->nthreads_auto = true; g_arguments->table_threads = DEFAULT_NTHREADS; g_arguments->prepared_rand = DEFAULT_PREPARED_RAND; g_arguments->reqPerReq = DEFAULT_REQ_PER_REQ; diff --git a/src/benchInsert.c b/src/benchInsert.c index 27c368e1..dbc56957 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -2403,25 +2403,12 @@ void *syncWriteProgressive(void *sarg) { return NULL; } -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (g_arguments->nthreads_auto) { - if (0 == pThreadInfo->vg->tbCountPerVgId) { - return NULL; - } - } else { - infoPrint( - "thread[%d] start progressive inserting into table from " - "%" PRIu64 " to %" PRIu64 "\n", - pThreadInfo->threadID, pThreadInfo->start_table_from, - pThreadInfo->end_table_to + 1); - } -#else infoPrint( - "thread[%d] start progressive inserting into table from " - "%" PRIu64 " to %" PRIu64 "\n", - pThreadInfo->threadID, pThreadInfo->start_table_from, - pThreadInfo->end_table_to + 1); -#endif + "thread[%d] start progressive inserting into table from " + "%" PRIu64 " to %" PRIu64 "\n", + pThreadInfo->threadID, pThreadInfo->start_table_from, + pThreadInfo->end_table_to + 1); + uint64_t lastPrintTime = toolsGetTimestampMs(); uint64_t lastTotalInsertRows = 0; int64_t startTs = toolsGetTimestampUs(); diff --git a/src/benchMain.c b/src/benchMain.c index f790c2cf..04b21191 100644 --- a/src/benchMain.c +++ b/src/benchMain.c @@ -87,7 +87,6 @@ int main(int argc, char* argv[]) { if (dsn != NULL) { g_arguments->dsn = dsn; g_arguments->websocket = true; - g_arguments->nthreads_auto = false; } else { g_arguments->dsn = false; } diff --git a/src/benchSys.c b/src/benchSys.c index 62183db0..4b6875fb 100644 --- a/src/benchSys.c +++ b/src/benchSys.c @@ -254,13 +254,11 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { case 'f': g_arguments->demo_mode = false; g_arguments->metaFile = arg; - g_arguments->nthreads_auto = false; break; case 'h': g_arguments->host = arg; g_arguments->host_auto = false; - g_arguments->nthreads_auto = false; break; case 'P': @@ -290,7 +288,6 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { stbInfo->iface = STMT_IFACE; } else if (0 == strcasecmp(arg, "rest")) { stbInfo->iface = REST_IFACE; - g_arguments->nthreads_auto = false; if (false == g_arguments->port_inputted) { g_arguments->port = DEFAULT_REST_PORT; } @@ -311,19 +308,15 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { || (0 == strcasecmp(arg, "sml-rest-line"))) { stbInfo->iface = SML_REST_IFACE; stbInfo->lineProtocol = TSDB_SML_LINE_PROTOCOL; - g_arguments->nthreads_auto = false; } else if (0 == strcasecmp(arg, "sml-rest-telnet")) { stbInfo->iface = SML_REST_IFACE; stbInfo->lineProtocol = TSDB_SML_TELNET_PROTOCOL; - g_arguments->nthreads_auto = false; } else if (0 == strcasecmp(arg, "sml-rest-json")) { stbInfo->iface = SML_REST_IFACE; stbInfo->lineProtocol = TSDB_SML_JSON_PROTOCOL; - g_arguments->nthreads_auto = false; } else if (0 == strcasecmp(arg, "sml-rest-taosjson")) { stbInfo->iface = SML_REST_IFACE; stbInfo->lineProtocol = SML_JSON_TAOS_FORMAT; - g_arguments->nthreads_auto = false; } else { errorPrint( "Invalid -I: %s, will auto set to default (taosc)\n", @@ -361,8 +354,6 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { "Invalid -T: %s, will auto set to default(8)\n", arg); g_arguments->nthreads = DEFAULT_NTHREADS; - } else { - g_arguments->nthreads_auto = false; } break; @@ -448,7 +439,6 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { case 'U': g_arguments->supplementInsert = true; - g_arguments->nthreads_auto = false; break; case 't': @@ -652,7 +642,6 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { #ifdef WEBSOCKET case 'W': - g_arguments->nthreads_auto = false; g_arguments->dsn = arg; break; @@ -669,7 +658,6 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { if (!toolsIsStringNumber(arg)) { errorPrintReqArg2(CUS_PROMPT"Benchmark", "v"); } - g_arguments->nthreads_auto = false; g_arguments->inputted_vgroups = atoi(arg); break; #endif From 64e57a702dd0e3cb2826670f7a2abf1e99b116a0 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 1 Jun 2024 12:00:31 +0800 Subject: [PATCH 29/57] fix: ci case pass --- src/benchInsertMix.c | 10 +++++++++- tests/taosbenchmark/commandline-sml.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/benchInsertMix.c b/src/benchInsertMix.c index 15f777ac..905237fb 100644 --- a/src/benchInsertMix.c +++ b/src/benchInsertMix.c @@ -766,6 +766,7 @@ bool checkCorrect(threadInfo* info, SDataBase* db, SSuperTable* stb, char* tbNam // bool insertDataMix(threadInfo* info, SDataBase* db, SSuperTable* stb) { int64_t lastPrintTime = 0; + infoPrint("insert mode is mix. generate_row_rule=%d\n", stb->genRowRule); // check interface if (stb->iface != TAOSC_IFACE) { @@ -797,7 +798,14 @@ bool insertDataMix(threadInfo* info, SDataBase* db, SSuperTable* stb) { // loop insert child tables for (uint64_t tbIdx = info->start_table_from; tbIdx <= info->end_table_to; ++tbIdx) { - char* tbName = stb->childTblArray[tbIdx]->name; + // get child table + SChildTable *childTbl; + if (g_arguments->bind_vgroup) { + childTbl = pThreadInfo->vg->childTblArray[tbIdx]; + } else { + childTbl = stb->childTblArray[tbIdx]; + } + char* tbName = childTbl->name; SMixRatio mixRatio; mixRatioInit(&mixRatio, stb); diff --git a/tests/taosbenchmark/commandline-sml.py b/tests/taosbenchmark/commandline-sml.py index 436720a4..e8faa490 100644 --- a/tests/taosbenchmark/commandline-sml.py +++ b/tests/taosbenchmark/commandline-sml.py @@ -88,7 +88,7 @@ def run(self): tdSql.query("select count(*) from test.meters") tdSql.checkData(0, 0, 1) - cmd = "%s -N -I sml -y" % binPath + cmd = "%s -I sml -y" % binPath tdLog.info("%s" % cmd) assert os.system("%s" % cmd) != 0 From 4f8ef781c3b9df064da8aa479d0ded7a1ced5b6c Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 1 Jun 2024 12:09:23 +0800 Subject: [PATCH 30/57] fix: modify check methon --- tests/taosbenchmark/commandline-sml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/taosbenchmark/commandline-sml.py b/tests/taosbenchmark/commandline-sml.py index e8faa490..aebe88cb 100644 --- a/tests/taosbenchmark/commandline-sml.py +++ b/tests/taosbenchmark/commandline-sml.py @@ -88,9 +88,9 @@ def run(self): tdSql.query("select count(*) from test.meters") tdSql.checkData(0, 0, 1) - cmd = "%s -I sml -y" % binPath + cmd = "%s -I sml -t 10 -n 10000 -y" % binPath tdLog.info("%s" % cmd) - assert os.system("%s" % cmd) != 0 + tdSql.checkData(0, 0, 10*10000) def stop(self): tdSql.close() From 1398ccccbf11a1629d6074204707daf696a08c00 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 1 Jun 2024 12:36:40 +0800 Subject: [PATCH 31/57] fix: case error for commandline.py --- src/benchInsertMix.c | 2 +- tests/taosbenchmark/commandline.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/benchInsertMix.c b/src/benchInsertMix.c index 905237fb..73b45464 100644 --- a/src/benchInsertMix.c +++ b/src/benchInsertMix.c @@ -801,7 +801,7 @@ bool insertDataMix(threadInfo* info, SDataBase* db, SSuperTable* stb) { // get child table SChildTable *childTbl; if (g_arguments->bind_vgroup) { - childTbl = pThreadInfo->vg->childTblArray[tbIdx]; + childTbl = info->vg->childTblArray[tbIdx]; } else { childTbl = stb->childTblArray[tbIdx]; } diff --git a/tests/taosbenchmark/commandline.py b/tests/taosbenchmark/commandline.py index 7e03b0c9..fe7b5cae 100644 --- a/tests/taosbenchmark/commandline.py +++ b/tests/taosbenchmark/commandline.py @@ -338,7 +338,7 @@ def run(self): tdLog.info("%s" % cmd) assert os.system("%s" % cmd) != 0 - cmd = "%s -n 1 -t 1 -y -A int,json" % binPath + cmd = "%s -n 1 -t 1 -y -A json" % binPath tdLog.info("%s" % cmd) assert os.system("%s" % cmd) != 0 From 4eb2937e5e4770f96f9494ce09bbaae918478d73 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 1 Jun 2024 14:36:58 +0800 Subject: [PATCH 32/57] fix: zero div --- src/benchInsert.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index dbc56957..809b3a70 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -1089,7 +1089,7 @@ static int createChildTables() { "start creating %" PRId64 " table(s) with %d thread(s)\n", g_arguments->totalChildTables, g_arguments->table_threads); } - double start = (double)toolsGetTimestampMs(); + int64_t start = (double)toolsGetTimestampMs(); for (int i = 0; (i < g_arguments->databases->size && !g_arguments->terminate); i++) { @@ -1120,16 +1120,19 @@ static int createChildTables() { } } - double end = (double)toolsGetTimestampMs(); + int64_t end = toolsGetTimestampMs(); + if(end == start) { + end += 1; + } succPrint( "Spent %.4f seconds to create %" PRId64 " table(s) with %d thread(s) speed: %.0f tables/s, already exist %" PRId64 " table(s), actual %" PRId64 " table(s) pre created, %" PRId64 " table(s) will be auto created\n", - (end - start) / 1000.0, + (float)(end - start) / 1000.0, g_arguments->totalChildTables, g_arguments->table_threads, - g_arguments->actualChildTables * 1000 / (end - start), + g_arguments->actualChildTables * 1000 / (float)(end - start), g_arguments->existedChildTables, g_arguments->actualChildTables, g_arguments->autoCreatedChildTables); From 16f0cdb2e667e3a59a2759bdb70984bce78d1e99 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 1 Jun 2024 16:07:52 +0800 Subject: [PATCH 33/57] fix: window build add benchCsv.c --- src/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 969f5a25..ff36f7b7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -441,9 +441,9 @@ ELSE () SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /utf-8") SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /utf-8") IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0") - ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsString.c toolsSys.c toolsString.c) + ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsString.c toolsSys.c toolsString.c) ELSE () - ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) + ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) ENDIF () ADD_EXECUTABLE(taosdump taosdump.c toolsSys.c toolstime.c toolsDir.c toolsString.c) From c0d63d9bcc0c34a39f5c918c25bc753eba940d46 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 1 Jun 2024 17:14:14 +0800 Subject: [PATCH 34/57] fix: ci case default_json.py --- src/benchInsert.c | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index 809b3a70..c8114ced 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -855,9 +855,9 @@ static void *createTable(void *sarg) { int w = 0; // record tagData int smallBatchCount = 0; - for (uint64_t i = pThreadInfo->start_table_from + stbInfo->childTblFrom; - (i <= (pThreadInfo->end_table_to + stbInfo->childTblFrom) - && !g_arguments->terminate); i++) { + for (uint64_t i = pThreadInfo->start_table_from; + i <= pThreadInfo->end_table_to && !g_arguments->terminate; + i++) { if (g_arguments->terminate) { goto create_table_end; } @@ -998,35 +998,34 @@ static void *createTable(void *sarg) { return NULL; } -static int startMultiThreadCreateChildTable( - SDataBase* database, SSuperTable* stbInfo) { - int code = -1; - int threads = g_arguments->table_threads; - int64_t ntables; +static int startMultiThreadCreateChildTable(SDataBase* database, SSuperTable* stbInfo) { + int32_t code = -1; + int32_t threads = g_arguments->table_threads; + int64_t ntables; if (stbInfo->childTblTo > 0) { - ntables = stbInfo->childTblTo - stbInfo->childTblFrom; + ntables = stbInfo->childTblTo - stbInfo->childTblFrom + 1; + } else if(stbInfo->childTblFrom > 0) { + ntables = stbInfo->childTblCount - stbInfo->childTblFrom; } else { ntables = stbInfo->childTblCount; } pthread_t *pids = benchCalloc(1, threads * sizeof(pthread_t), false); threadInfo *infos = benchCalloc(1, threads * sizeof(threadInfo), false); - uint64_t tableFrom = 0; + uint64_t tableFrom = stbInfo->childTblFrom; if (threads < 1) { threads = 1; } - - int64_t a = ntables / threads; - if (a < 1) { - threads = (int)ntables; - a = 1; - } - if (ntables == 0) { - errorPrint("failed to create child table, childTblCount: %"PRId64"\n", - ntables); + errorPrint("failed to create child table, childTblCount: %"PRId64"\n", ntables); goto over; } - int64_t b = ntables % threads; + + int64_t div = ntables / threads; + if (div < 1) { + threads = (int)ntables; + div = 1; + } + int64_t mod = ntables % threads; int threadCnt = 0; for (uint32_t i = 0; (i < threads && !g_arguments->terminate); i++) { @@ -1047,10 +1046,12 @@ static int startMultiThreadCreateChildTable( } } pThreadInfo->start_table_from = tableFrom; - pThreadInfo->ntables = i < b ? a + 1 : a; - pThreadInfo->end_table_to = i < b ? tableFrom + a : tableFrom + a - 1; + pThreadInfo->ntables = i < mod ? div + 1 : div; + pThreadInfo->end_table_to = i < mod ? tableFrom + div : tableFrom + div - 1; tableFrom = pThreadInfo->end_table_to + 1; pThreadInfo->tables_created = 0; + debugPrint("div table by thread. i=%d from=%"PRId64" to=%"PRId64" ntable=%"PRId64"\n", i, pThreadInfo->start_table_from, + pThreadInfo->end_table_to, pThreadInfo->ntables); pthread_create(pids + i, NULL, createTable, pThreadInfo); threadCnt ++; } From 91f1286d221be066b84fa3b68d938f3de1234472 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 1 Jun 2024 17:45:08 +0800 Subject: [PATCH 35/57] fix: from and to argument valid --- src/benchInsert.c | 13 ++++--------- src/benchInsertMix.c | 4 ++-- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index c8114ced..0978b813 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -1003,7 +1003,7 @@ static int startMultiThreadCreateChildTable(SDataBase* database, SSuperTable* st int32_t threads = g_arguments->table_threads; int64_t ntables; if (stbInfo->childTblTo > 0) { - ntables = stbInfo->childTblTo - stbInfo->childTblFrom + 1; + ntables = stbInfo->childTblTo - stbInfo->childTblFrom; } else if(stbInfo->childTblFrom > 0) { ntables = stbInfo->childTblCount - stbInfo->childTblFrom; } else { @@ -1604,10 +1604,7 @@ static void *syncWriteInterlace(void *sarg) { if (g_arguments->bind_vgroup) { childTbl = pThreadInfo->vg->childTblArray[tableSeq]; } else { - childTbl = stbInfo->childTblArray[ - stbInfo->childTblExists? - tableSeq: - stbInfo->childTblFrom + tableSeq]; + childTbl = stbInfo->childTblArray[tableSeq]; } char * tableName = childTbl->name; @@ -2437,11 +2434,9 @@ void *syncWriteProgressive(void *sarg) { if (g_arguments->bind_vgroup) { childTbl = pThreadInfo->vg->childTblArray[tableSeq]; } else { - childTbl = stbInfo->childTblArray[ - stbInfo->childTblExists? - tableSeq: - stbInfo->childTblFrom + tableSeq]; + childTbl = stbInfo->childTblArray[tableSeq]; } + debugPrint("tableSeq=%"PRId64" childTbl->name=%s\n", tableSeq, childTbl->name); if (childTbl->useOwnSample) { sampleDataBuf = childTbl->sampleDataBuf; diff --git a/src/benchInsertMix.c b/src/benchInsertMix.c index 73b45464..a137f90a 100644 --- a/src/benchInsertMix.c +++ b/src/benchInsertMix.c @@ -766,8 +766,6 @@ bool checkCorrect(threadInfo* info, SDataBase* db, SSuperTable* stb, char* tbNam // bool insertDataMix(threadInfo* info, SDataBase* db, SSuperTable* stb) { int64_t lastPrintTime = 0; - infoPrint("insert mode is mix. generate_row_rule=%d\n", stb->genRowRule); - // check interface if (stb->iface != TAOSC_IFACE) { return false; @@ -778,6 +776,8 @@ bool insertDataMix(threadInfo* info, SDataBase* db, SSuperTable* stb) { return false; } + infoPrint("insert mode is mix. generate_row_rule=%d\n", stb->genRowRule); + FILE* csvFile = NULL; char* tagData = NULL; bool acreate = (stb->genRowRule == RULE_OLD || stb->genRowRule == RULE_MIX_RANDOM) && stb->autoTblCreating; From e8c1536e7699a5388afc6f2769e720db3279a5df Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 5 Jun 2024 20:20:52 +0800 Subject: [PATCH 36/57] fix: stmt set columns data only once --- inc/bench.h | 1 + src/benchData.c | 75 +++++++++++++++++++++++------------------------ src/benchInsert.c | 6 ++++ 3 files changed, 43 insertions(+), 39 deletions(-) diff --git a/inc/bench.h b/inc/bench.h index 003edc3f..53480d9c 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -1036,6 +1036,7 @@ typedef struct SThreadInfo_S { // check sql result char *csql; int32_t clen; // csql current write position + bool stmtBind; } threadInfo; typedef struct SQueryThreadInfo_S { diff --git a/src/benchData.c b/src/benchData.c index 95234c94..7af436a3 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -1809,42 +1809,46 @@ uint32_t bindParamBatch(threadInfo *pThreadInfo, TAOS_STMT *stmt = pThreadInfo->conn->stmt; SSuperTable *stbInfo = pThreadInfo->stbInfo; uint32_t columnCount = stbInfo->cols->size; - memset(pThreadInfo->bindParams, 0, - (sizeof(TAOS_MULTI_BIND) * (columnCount + 1))); - memset(pThreadInfo->is_null, 0, batch); - - for (int c = 0; c < columnCount + 1; c++) { - TAOS_MULTI_BIND *param = - (TAOS_MULTI_BIND *)(pThreadInfo->bindParams + - sizeof(TAOS_MULTI_BIND) * c); - char data_type; - if (c == 0) { - data_type = TSDB_DATA_TYPE_TIMESTAMP; - param->buffer_length = sizeof(int64_t); - param->buffer = pThreadInfo->bind_ts_array; - } else { - Field *col = benchArrayGet(stbInfo->cols, c - 1); - data_type = col->type; - if (childTbl->useOwnSample) { - ChildField *childCol = benchArrayGet(childTbl->childCols, c-1); - param->buffer = childCol->stmtData.data; - param->is_null = childCol->stmtData.is_null; + + if (!pThreadInfo->stmtBind) { + pThreadInfo->stmtBind = true; + memset(pThreadInfo->bindParams, 0, + (sizeof(TAOS_MULTI_BIND) * (columnCount + 1))); + memset(pThreadInfo->is_null, 0, batch); + + for (int c = 0; c < columnCount + 1; c++) { + TAOS_MULTI_BIND *param = + (TAOS_MULTI_BIND *)(pThreadInfo->bindParams + + sizeof(TAOS_MULTI_BIND) * c); + char data_type; + if (c == 0) { + data_type = TSDB_DATA_TYPE_TIMESTAMP; + param->buffer_length = sizeof(int64_t); + param->buffer = pThreadInfo->bind_ts_array; } else { - param->buffer = col->stmtData.data; - param->is_null = col->stmtData.is_null; + Field *col = benchArrayGet(stbInfo->cols, c - 1); + data_type = col->type; + if (childTbl->useOwnSample) { + ChildField *childCol = benchArrayGet(childTbl->childCols, c-1); + param->buffer = childCol->stmtData.data; + param->is_null = childCol->stmtData.is_null; + } else { + param->buffer = col->stmtData.data; + param->is_null = col->stmtData.is_null; + } + param->buffer_length = col->length; + debugPrint("col[%d]: type: %s, len: %d\n", c, + convertDatatypeToString(data_type), + col->length); } - param->buffer_length = col->length; - debugPrint("col[%d]: type: %s, len: %d\n", c, - convertDatatypeToString(data_type), - col->length); - } - param->buffer_type = data_type; - param->length = benchCalloc(batch, sizeof(int32_t), true); + param->buffer_type = data_type; + param->length = benchCalloc(batch, sizeof(int32_t), true); - for (int b = 0; b < batch; b++) { - param->length[b] = (int32_t)param->buffer_length; + for (int b = 0; b < batch; b++) { + param->length[b] = (int32_t)param->buffer_length; + } + param->num = batch; } - param->num = batch; } // set ts array values @@ -1874,13 +1878,6 @@ uint32_t bindParamBatch(threadInfo *pThreadInfo, } *delay2 += toolsGetTimestampUs() - start; - for (int c = 0; c < stbInfo->cols->size + 1; c++) { - TAOS_MULTI_BIND *param = - (TAOS_MULTI_BIND *)(pThreadInfo->bindParams + - sizeof(TAOS_MULTI_BIND) * c); - tmfree(param->length); - } - return batch; } diff --git a/src/benchInsert.c b/src/benchInsert.c index 0978b813..dff6e88f 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -3710,6 +3710,12 @@ int32_t exitInsertThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthr case STMT_IFACE: taos_stmt_close(pThreadInfo->conn->stmt); + + // free length + for (int c = 0; c < stbInfo->cols->size + 1; c++) { + TAOS_MULTI_BIND *param = (TAOS_MULTI_BIND *)(pThreadInfo->bindParams + sizeof(TAOS_MULTI_BIND) * c); + tmfree(param->length); + } tmfree(pThreadInfo->bind_ts); tmfree(pThreadInfo->bind_ts_array); tmfree(pThreadInfo->bindParams); From dbfa467c26e8c1cf671418404f4693e76ad71b60 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 7 Jun 2024 09:28:06 +0800 Subject: [PATCH 37/57] fix: add stbInsert support --- src/benchInsert.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/benchInsert.c b/src/benchInsert.c index dff6e88f..8b13489f 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -3378,6 +3378,11 @@ int32_t initInsertThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthr goto END; } pThreadInfo->conn->stmt = taos_stmt_init(pThreadInfo->conn->taos); + TAOS_STMT_OPTIONS op; + op.reqId = 0; + op.singleStbInsert = true; + op.singleTableBindOnce = true; + pThreadInfo->conn->stmt = taos_stmt_init_with_options(pThreadInfo->conn->taos, &op); if (NULL == pThreadInfo->conn->stmt) { errorPrint("taos_stmt_init() failed, reason: %s\n", taos_errstr(NULL)); goto END; From 28f95e9d13e1e4d393118cd459b97b201d3df08d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 7 Jun 2024 14:33:36 +0800 Subject: [PATCH 38/57] fix: taos_stmt_init_with_options --- src/benchInsert.c | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index dff6e88f..bebd59e1 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -2454,7 +2454,16 @@ void *syncWriteProgressive(void *sarg) { int64_t delay3 = 0; if (stmt) { taos_stmt_close(pThreadInfo->conn->stmt); - pThreadInfo->conn->stmt = taos_stmt_init(pThreadInfo->conn->taos); + if(stbInfo->autoTblCreating) { + pThreadInfo->conn->stmt = taos_stmt_init(pThreadInfo->conn->taos); + } else { + TAOS_STMT_OPTIONS op; + op.reqId = 0; + op.singleStbInsert = true; + op.singleTableBindOnce = true; + pThreadInfo->conn->stmt = taos_stmt_init_with_options(pThreadInfo->conn->taos, &op); + } + if (NULL == pThreadInfo->conn->stmt) { errorPrint("taos_stmt_init() failed, reason: %s\n", taos_errstr(NULL)); @@ -3377,7 +3386,18 @@ int32_t initInsertThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthr if (NULL == pThreadInfo->conn) { goto END; } - pThreadInfo->conn->stmt = taos_stmt_init(pThreadInfo->conn->taos); + + taos_stmt_close(pThreadInfo->conn->stmt); + if(stbInfo->autoTblCreating) { + pThreadInfo->conn->stmt = taos_stmt_init(pThreadInfo->conn->taos); + } else { + TAOS_STMT_OPTIONS op; + op.reqId = 0; + op.singleStbInsert = true; + op.singleTableBindOnce = true; + pThreadInfo->conn->stmt = taos_stmt_init_with_options(pThreadInfo->conn->taos, &op); + } + if (NULL == pThreadInfo->conn->stmt) { errorPrint("taos_stmt_init() failed, reason: %s\n", taos_errstr(NULL)); goto END; From d5dd5e6e928f115a7010083873cdbaaaea288afe Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 12 Jun 2024 13:48:37 +0800 Subject: [PATCH 39/57] fix: change files --- src/benchUtil.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/benchUtil.c b/src/benchUtil.c index 50524289..13648262 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -1052,6 +1052,7 @@ int convertStringToDatatype(char *type, int length) { } } + int compare(const void *a, const void *b) { return *(int64_t *)a - *(int64_t *)b; } From 2f2aa168fefab094444f9dfc8ab2b8ce68e9ce55 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 13 Jun 2024 17:53:24 +0800 Subject: [PATCH 40/57] fix: add geometry and varbinary --- inc/bench.h | 1 + src/benchData.c | 119 ++++++++++++++++++++++++++++++++++++++++++++- src/benchDataMix.c | 3 ++ src/benchInsert.c | 5 ++ src/benchJsonOpt.c | 2 + src/benchUtil.c | 4 ++ 6 files changed, 132 insertions(+), 2 deletions(-) diff --git a/inc/bench.h b/inc/bench.h index 801dfcfa..248a2e8f 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -1180,5 +1180,6 @@ uint64_t tmpUint64Impl(Field *field, int32_t angle, int64_t k); float tmpFloatImpl(Field *field, int i, int32_t angle, int32_t k); double tmpDoubleImpl(Field *field, int32_t angle, int32_t k); int tmpStr(char *tmp, int iface, Field *field, int64_t k); +int tmpGeometry(char *tmp, int iface, Field *field, int64_t k); #endif // INC_BENCH_H_ diff --git a/src/benchData.c b/src/benchData.c index ff522704..2788a56b 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -391,7 +391,6 @@ uint32_t accumulateRowLen(BArray *fields, int iface) { case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_GEOMETRY: case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_GEOMETRY: len += field->length + 3; break; case TSDB_DATA_TYPE_INT: @@ -483,6 +482,41 @@ int tmpStr(char *tmp, int iface, Field *field, int64_t k) { return 0; } +int tmpGeometry(char *tmp, int iface, Field *field, int64_t k) { + // values + if (field->values) { + int arraySize = tools_cJSON_GetArraySize(field->values); + if (arraySize) { + tools_cJSON *buf = tools_cJSON_GetArrayItem( + field->values, + taosRandom() % arraySize); + snprintf(tmp, field->length, + "%s", buf->valuestring); + } else { + errorPrint("%s() cannot read correct value " + "from json file. array size: %d\n", + __func__, arraySize); + return -1; + } + return 0; + } + + // gen point count + int32_t cnt = field->length / 24; + if(cnt == 0) { + snprintf(tmp, field->length, "POINT(%d %d)", tmpUint16(field), tmpUint16(field)); + return 0; + } + + int32_t pos = snprintf(tmp, field->length, "LINESTRING("); + for(int32_t i = 0; i < cnt; i++) { + pos = snprintf(tmp + pos, field->length - pos, "%d %d", tmpUint16(field), tmpUint16(field)); + } + strcat(tmp, ")"); + + return 0; +} + bool tmpBool(Field *field) { bool boolTmp; if (field->min == field->max) { @@ -1001,6 +1035,28 @@ static int fillStmt( tmfree(tmp); break; } + case TSDB_DATA_TYPE_GEOMETRY: { + char *tmp = benchCalloc(1, field->length + 1, false); + if (0 != tmpGeometry(tmp, stbInfo->iface, field, k)) { + tmfree(tmp); + return -1; + } + if (childCol) { + snprintf((char *)childCol->stmtData.data + + k * field->length, + field->length, + "%s", tmp); + } else { + snprintf((char *)field->stmtData.data + + k * field->length, + field->length, + "%s", tmp); + } + n = snprintf(sampleDataBuf + pos, bufLen - pos, + "'%s',", tmp); + tmfree(tmp); + break; + } case TSDB_DATA_TYPE_JSON: { pos += tmpJson(sampleDataBuf, bufLen, pos, fieldsSize, field); @@ -1231,7 +1287,7 @@ static int generateRandDataSmlTelnet(SSuperTable *stbInfo, char *sampleDataBuf, case TSDB_DATA_TYPE_NCHAR: { char *tmp = benchCalloc(1, field->length + 1, false); if (0 != tmpStr(tmp, stbInfo->iface, field, k)) { - free(tmp); + tmfree(tmp); return -1; } if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY) { @@ -1272,6 +1328,31 @@ static int generateRandDataSmlTelnet(SSuperTable *stbInfo, char *sampleDataBuf, tmfree(tmp); break; } + case TSDB_DATA_TYPE_GEOMETRY: { + char *tmp = benchCalloc(1, field->length + 1, false); + if (0 != tmpGeometry(tmp, stbInfo->iface, field, k)) { + tmfree(tmp); + return -1; + } + if (tag) { + n = snprintf(sampleDataBuf + pos, bufLen - pos, + "%s=L\"%s\" ", + field->name, tmp); + } else { + n = snprintf(sampleDataBuf + pos, bufLen - pos, + "\"%s\" ", tmp); + } + if (n < 0 || n >= bufLen - pos) { + errorPrint("%s() LN%d snprintf overflow\n", + __func__, __LINE__); + tmfree(tmp); + return -1; + } else { + pos += n; + } + tmfree(tmp); + break; + } case TSDB_DATA_TYPE_JSON: { pos += tmpJson(sampleDataBuf, bufLen, pos, fieldsSize, field); @@ -1534,6 +1615,18 @@ static int generateRandDataSmlLine(SSuperTable *stbInfo, char *sampleDataBuf, tmfree(tmp); break; } + case TSDB_DATA_TYPE_GEOMETRY: { + char *tmp = benchCalloc(1, field->length + 1, false); + if (0 != tmpGeometry(tmp, stbInfo->iface, field, k)) { + tmfree(tmp); + return -1; + } + n = snprintf(sampleDataBuf + pos, bufLen - pos, + "%s=\"%s\",", + field->name, tmp); + tmfree(tmp); + break; + } case TSDB_DATA_TYPE_JSON: { n = tmpJson(sampleDataBuf, bufLen, pos, fieldsSize, field); @@ -2075,6 +2168,15 @@ void generateSmlJsonValues( tmfree(buf); break; } + case TSDB_DATA_TYPE_GEOMETRY: { + char *buf = (char *)benchCalloc(col->length + 1, 1, false); + tmpGeometry(buf, stbInfo->iface, col, 0); + value_buf = benchCalloc(len_key + col->length + 3, 1, true); + snprintf(value_buf, len_key + col->length + 3, + "\"value\":\"%s\",", buf); + tmfree(buf); + break; + } default: { value_buf = benchCalloc(len_key + 20, 1, true); double doubleTmp = tmpDouble(col); @@ -2120,6 +2222,13 @@ void generateSmlJsonCols(tools_cJSON *array, tools_cJSON *tag, tmfree(buf); break; } + case TSDB_DATA_TYPE_GEOMETRY: { + char *buf = (char *)benchCalloc(col->length + 1, 1, false); + tmpGeometry(buf, stbInfo->iface, col, 0); + tools_cJSON_AddStringToObject(record, "value", buf); + tmfree(buf); + break; + } default: { double doubleTmp = tmpDouble(col); tools_cJSON_AddNumberToObject(record, "value", doubleTmp); @@ -2180,6 +2289,12 @@ void generateSmlTaosJsonCols(tools_cJSON *array, tools_cJSON *tag, tmfree(buf); break; } + case TSDB_DATA_TYPE_GEOMETRY: + char *buf = (char *)benchCalloc(col->length + 1, 1, false); + tmpGeometry(buf, stbInfo->iface, col, 0); + tools_cJSON_AddStringToObject(value, "value", buf); + tools_cJSON_AddStringToObject(value, "type", "geometry"); + tmfree(buf); default: { double dblTmp = (double)col->min; if (col->max != col->min) { diff --git a/src/benchDataMix.c b/src/benchDataMix.c index 117b2c48..3692e57f 100644 --- a/src/benchDataMix.c +++ b/src/benchDataMix.c @@ -157,6 +157,9 @@ uint32_t dataGenByField(Field* fd, char* pstr, uint32_t len, char* prefix, int64 format = ",\'%s\'"; tmpStr(val, 0, fd, *k); break; + case TSDB_DATA_TYPE_GEOMETRY: + tmpGeometry(val, 0, fd, 0); + break; default: break; } diff --git a/src/benchInsert.c b/src/benchInsert.c index e7246dcb..83e91f38 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -247,6 +247,7 @@ static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) { int n; if (col->type == TSDB_DATA_TYPE_BINARY || col->type == TSDB_DATA_TYPE_NCHAR || + col->type == TSDB_DATA_TYPE_VARBINARY || col->type == TSDB_DATA_TYPE_GEOMETRY) { n = snprintf(colsBuf + len, col_buffer_len - len, ",%s %s(%d)", col->name, @@ -318,6 +319,7 @@ static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) { Field *tag = benchArrayGet(stbInfo->tags, tagIndex); if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_NCHAR || + tag->type == TSDB_DATA_TYPE_VARBINARY || tag->type == TSDB_DATA_TYPE_GEOMETRY) { n = snprintf(tagsBuf + len, tag_buffer_len - len, "%s %s(%d),", tag->name, @@ -2834,6 +2836,8 @@ static int initStmtDataValue(SSuperTable *stbInfo, SChildTable *childTbl) { break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_GEOMETRY: { size_t tmpLen = strlen(tmpStr); debugPrint("%s() LN%d, index: %d, " @@ -2926,6 +2930,7 @@ static void initStmtData(char dataType, void **data, uint32_t length) { case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_GEOMETRY: tmpP = calloc(1, g_arguments->prepared_rand * length); assert(tmpP); tmfree(*data); diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 6481e40e..162db5a2 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -458,6 +458,8 @@ static int getColumnAndTagTypeFromInsertJsonFile( } else { if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_JSON + || type == TSDB_DATA_TYPE_VARBINARY + || type == TSDB_DATA_TYPE_GEOMETRY || type == TSDB_DATA_TYPE_NCHAR) { length = g_arguments->binwidth; } else { diff --git a/src/benchUtil.c b/src/benchUtil.c index 13648262..e860ef55 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -1006,6 +1006,8 @@ int convertStringToDatatype(char *type, int length) { return TSDB_DATA_TYPE_BINARY; } else if (0 == strcasecmp(type, "varbinary")) { return TSDB_DATA_TYPE_VARBINARY; + } else if (0 == strcasecmp(type, "geometry")) { + return TSDB_DATA_TYPE_GEOMETRY; } else { errorPrint("unknown data type: %s\n", type); exit(EXIT_FAILURE); @@ -1045,6 +1047,8 @@ int convertStringToDatatype(char *type, int length) { return TSDB_DATA_TYPE_BINARY; } else if (0 == strncasecmp(type, "varbinary", length)) { return TSDB_DATA_TYPE_VARBINARY; + } else if (0 == strncasecmp(type, "geometry", length)) { + return TSDB_DATA_TYPE_GEOMETRY; } else { errorPrint("unknown data type: %s\n", type); exit(EXIT_FAILURE); From d912320fb1a22e9674b4391b5c8e713d501b891a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 13 Jun 2024 18:15:30 +0800 Subject: [PATCH 41/57] fix: fix csv read line --- src/benchCommandOpt.c | 5 +---- src/benchData.c | 15 ++++++++++++--- src/taosdump.c | 6 +----- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/benchCommandOpt.c b/src/benchCommandOpt.c index c6273f39..86d8a837 100644 --- a/src/benchCommandOpt.c +++ b/src/benchCommandOpt.c @@ -28,9 +28,6 @@ extern char g_configDir[MAX_PATH_LEN]; #define TAOSBENCHMARK_STATUS "unknown" #endif -#ifndef TD_PRODUCT_NAME -#define TD_PRODUCT_NAME "TDengine" -#endif // libtaos.so extern char buildinfo[]; @@ -52,7 +49,7 @@ void printVersion() { char taosBenchmark_status[] = TAOSBENCHMARK_STATUS; // version - printf("%s\ntaosBenchmark version: %s\ngit: %s\n", TD_PRODUCT_NAME, taosBenchmark_ver, taosBenchmark_commit); + printf("taosBenchmark version: %s\ngit: %s\n", taosBenchmark_ver, taosBenchmark_commit); #ifdef LINUX printf("build: %s\n ", buildinfo); #endif diff --git a/src/benchData.c b/src/benchData.c index 2788a56b..4c79d92a 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -323,8 +323,17 @@ static int generateSampleFromCsv(char *buffer, char* file, FILE* fp, int32_t len continue; } - if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { - line[--readLen] = 0; + int32_t pos = (int32_t)(readLen - 1); + if(pos > 0) { + if (('\r' == line[pos]) || ('\n' == line[pos])) { + line[pos] = 0; + } + } + pos = (int32_t)(readLen - 2); + if(pos > 0) { + if (('\r' == line[pos]) || ('\n' == line[pos])) { + line[pos] = 0; + } } if (readLen == 0) { @@ -339,7 +348,7 @@ static int generateSampleFromCsv(char *buffer, char* file, FILE* fp, int32_t len continue; } - memcpy(buffer + getRows * length, line, readLen); + memcpy(buffer + getRows * length, line, readLen + 1); getRows++; if (getRows == size) { diff --git a/src/taosdump.c b/src/taosdump.c index 334f76d7..c53fc576 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -82,10 +82,6 @@ #define TAOSDUMP_STATUS "unknown" #endif -#ifndef TD_PRODUCT_NAME -#define TD_PRODUCT_NAME "TDengine" -#endif - // use 256 as normal buffer length #define BUFFER_LEN 256 @@ -673,7 +669,7 @@ static void printVersion(FILE *file) { char taosdump_commit[] = TAOSDUMP_COMMIT_SHA1; - fprintf(file,"%s\ntaosdump version: %s\ngit: %s\n", TD_PRODUCT_NAME, taostools_ver, taosdump_commit); + fprintf(file,"ntaosdump version: %s\ngit: %s\n", taostools_ver, taosdump_commit); #ifdef LINUX printf("build: %s\n ", buildinfo); #endif From 5ef449b1a7722aac1ea0ca54243e78a3e30c704d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 13 Jun 2024 18:20:12 +0800 Subject: [PATCH 42/57] fix: free command twice --- src/taosdump.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/taosdump.c b/src/taosdump.c index c53fc576..e9475da9 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -2177,7 +2177,6 @@ static int64_t getTbCountOfStbNative(const char *dbName, const char *stbName) { int32_t code = taos_errno(res); if (code != 0) { cleanIfQueryFailed(__func__, __LINE__, command, res); - free(command); taos_close(taos); return -1; } @@ -9976,7 +9975,7 @@ int readNextTableDesWS(void* ws_res, TableDes* tbDes, int *idx, int *cnt) { // read next table tags to tbDes int readNextTableDesNative(void* res, TableDes* tbDes) { // tbname, tagName , tagValue - TAOS_ROW row; + TAOS_ROW row = NULL; int index = 0; while( index < tbDes->tags && NULL != (row = taos_fetch_row(res))) { // tbname changed check From 134f11169796914c04bc6c4f4fa1ac25c1f10247 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 13 Jun 2024 18:29:33 +0800 Subject: [PATCH 43/57] fix: build error --- deps/avro/lang/c/examples/quickstop.c | 2 +- src/benchData.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deps/avro/lang/c/examples/quickstop.c b/deps/avro/lang/c/examples/quickstop.c index ff9e9700..a1e5a0f1 100644 --- a/deps/avro/lang/c/examples/quickstop.c +++ b/deps/avro/lang/c/examples/quickstop.c @@ -107,7 +107,7 @@ int print_person(avro_file_reader_t db, avro_schema_t reader_schema) if (rval == 0) { int64_t id; int32_t age; - int32_t *p; + char *p = NULL; size_t size; avro_value_t id_value; avro_value_t first_value; diff --git a/src/benchData.c b/src/benchData.c index 4c79d92a..d4144a28 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -843,7 +843,7 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_GEOMETRY: { - int bufferSize = geoCalcBufferSize(field->length); + int bufferSize = field->length + 1; char *tmp = benchCalloc(1, bufferSize, false); if (0 != tmpGeometry(tmp, stbInfo->iface, field, i)) { free(tmp); From 5ebe168633a3b8a5a4d606ccae810182cd99d250 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 13 Jun 2024 19:16:24 +0800 Subject: [PATCH 44/57] remove -V blank --- src/benchCommandOpt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchCommandOpt.c b/src/benchCommandOpt.c index 86d8a837..fa2fd435 100644 --- a/src/benchCommandOpt.c +++ b/src/benchCommandOpt.c @@ -51,7 +51,7 @@ void printVersion() { // version printf("taosBenchmark version: %s\ngit: %s\n", taosBenchmark_ver, taosBenchmark_commit); #ifdef LINUX - printf("build: %s\n ", buildinfo); + printf("build: %s\n", buildinfo); #endif if (strlen(taosBenchmark_status) > 0) { printf("status: %s\n", taosBenchmark_status); From 5e2929a29af9295f7f5ac9b2e486050fccde53a1 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 13 Jun 2024 19:29:34 +0800 Subject: [PATCH 45/57] fix: tmpGeometry format --- src/benchData.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchData.c b/src/benchData.c index d4144a28..3cc906d2 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -519,7 +519,7 @@ int tmpGeometry(char *tmp, int iface, Field *field, int64_t k) { int32_t pos = snprintf(tmp, field->length, "LINESTRING("); for(int32_t i = 0; i < cnt; i++) { - pos = snprintf(tmp + pos, field->length - pos, "%d %d", tmpUint16(field), tmpUint16(field)); + pos += snprintf(tmp + pos, field->length - pos, "%d %d", tmpUint16(field), tmpUint16(field)); } strcat(tmp, ")"); From 6521354b5306afdf170f19db76ca9cffed77ac83 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 13 Jun 2024 19:33:43 +0800 Subject: [PATCH 46/57] fix: tmpGeometry format1 --- src/benchData.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/benchData.c b/src/benchData.c index 3cc906d2..dfb5acd8 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -518,8 +518,12 @@ int tmpGeometry(char *tmp, int iface, Field *field, int64_t k) { } int32_t pos = snprintf(tmp, field->length, "LINESTRING("); + char * format = ",%d %d"; for(int32_t i = 0; i < cnt; i++) { - pos += snprintf(tmp + pos, field->length - pos, "%d %d", tmpUint16(field), tmpUint16(field)); + if (i == 0) { + format = "%d %d"; + } + pos += snprintf(tmp + pos, field->length - pos, format, tmpUint16(field), tmpUint16(field)); } strcat(tmp, ")"); From 1310afc362ce41f6e1dbad4d921d7273c6434ba2 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 13 Jun 2024 19:37:07 +0800 Subject: [PATCH 47/57] fix: tmpGeometry format2 --- src/benchData.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/benchData.c b/src/benchData.c index dfb5acd8..ef8f0b75 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -518,11 +518,11 @@ int tmpGeometry(char *tmp, int iface, Field *field, int64_t k) { } int32_t pos = snprintf(tmp, field->length, "LINESTRING("); - char * format = ",%d %d"; + char * format = "%d %d,"; for(int32_t i = 0; i < cnt; i++) { - if (i == 0) { + if (i == cnt - 1) { format = "%d %d"; - } + } pos += snprintf(tmp + pos, field->length - pos, format, tmpUint16(field), tmpUint16(field)); } strcat(tmp, ")"); From 1278ac0035f8348f35f8fbf7eaf6956b999b0922 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 14 Jun 2024 15:09:30 +0800 Subject: [PATCH 48/57] fix: build error --- src/benchData.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/benchData.c b/src/benchData.c index ef8f0b75..b3fd211b 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -2302,12 +2302,13 @@ void generateSmlTaosJsonCols(tools_cJSON *array, tools_cJSON *tag, tmfree(buf); break; } - case TSDB_DATA_TYPE_GEOMETRY: + case TSDB_DATA_TYPE_GEOMETRY: { char *buf = (char *)benchCalloc(col->length + 1, 1, false); tmpGeometry(buf, stbInfo->iface, col, 0); tools_cJSON_AddStringToObject(value, "value", buf); tools_cJSON_AddStringToObject(value, "type", "geometry"); tmfree(buf); + } default: { double dblTmp = (double)col->min; if (col->max != col->min) { From d688eb2550a99a81d9118f91ada2d9925ad23ade Mon Sep 17 00:00:00 2001 From: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Date: Tue, 18 Jun 2024 19:10:09 +0800 Subject: [PATCH 49/57] Update taosdump.c --- src/taosdump.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/taosdump.c b/src/taosdump.c index e9475da9..6364da78 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -669,9 +669,9 @@ static void printVersion(FILE *file) { char taosdump_commit[] = TAOSDUMP_COMMIT_SHA1; - fprintf(file,"ntaosdump version: %s\ngit: %s\n", taostools_ver, taosdump_commit); + fprintf(file,"taosdump version: %s\ngit: %s\n", taostools_ver, taosdump_commit); #ifdef LINUX - printf("build: %s\n ", buildinfo); + printf("build: %s\n", buildinfo); #endif if (strlen(taosdump_status) > 0) { fprintf(file, "status:%s\n", taosdump_status); From 1f096e9a4cc3a8d704defb90699c656c19ccf26b Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 19 Jun 2024 10:57:31 +0800 Subject: [PATCH 50/57] fix: add get build info api call --- src/benchCommandOpt.c | 7 +------ src/taosdump.c | 8 +++----- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/src/benchCommandOpt.c b/src/benchCommandOpt.c index fa2fd435..a2e87f43 100644 --- a/src/benchCommandOpt.c +++ b/src/benchCommandOpt.c @@ -29,9 +29,6 @@ extern char g_configDir[MAX_PATH_LEN]; #endif -// libtaos.so -extern char buildinfo[]; - char *g_aggreFuncDemo[] = {"*", "count(*)", "avg(current)", @@ -50,9 +47,7 @@ void printVersion() { // version printf("taosBenchmark version: %s\ngit: %s\n", taosBenchmark_ver, taosBenchmark_commit); -#ifdef LINUX - printf("build: %s\n", buildinfo); -#endif + printf("build: %s\n", getBuildInfo()); if (strlen(taosBenchmark_status) > 0) { printf("status: %s\n", taosBenchmark_status); } diff --git a/src/taosdump.c b/src/taosdump.c index 6364da78..7674793a 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -656,8 +656,6 @@ static uint64_t getUniqueIDFromEpoch() { return id; } -// libtaos.so -extern char buildinfo[]; static void printVersion(FILE *file) { char taostools_longver[] = TAOSDUMP_TAG; @@ -670,9 +668,9 @@ static void printVersion(FILE *file) { char taosdump_commit[] = TAOSDUMP_COMMIT_SHA1; fprintf(file,"taosdump version: %s\ngit: %s\n", taostools_ver, taosdump_commit); -#ifdef LINUX - printf("build: %s\n", buildinfo); -#endif + + printf("build: %s\n", getBuildInfo()); + if (strlen(taosdump_status) > 0) { fprintf(file, "status:%s\n", taosdump_status); } From 75644c2145424e2af958b368d10906c3dd2deca4 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 20 Jun 2024 16:33:31 +0800 Subject: [PATCH 51/57] fix: remove windows buildInfo --- src/benchCommandOpt.c | 2 ++ src/taosdump.c | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/benchCommandOpt.c b/src/benchCommandOpt.c index a2e87f43..6eb30a0c 100644 --- a/src/benchCommandOpt.c +++ b/src/benchCommandOpt.c @@ -47,7 +47,9 @@ void printVersion() { // version printf("taosBenchmark version: %s\ngit: %s\n", taosBenchmark_ver, taosBenchmark_commit); +#ifndef WINDOWS printf("build: %s\n", getBuildInfo()); +#endif if (strlen(taosBenchmark_status) > 0) { printf("status: %s\n", taosBenchmark_status); } diff --git a/src/taosdump.c b/src/taosdump.c index 7674793a..6714e42d 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -668,8 +668,9 @@ static void printVersion(FILE *file) { char taosdump_commit[] = TAOSDUMP_COMMIT_SHA1; fprintf(file,"taosdump version: %s\ngit: %s\n", taostools_ver, taosdump_commit); - +#ifndef WINDOWS printf("build: %s\n", getBuildInfo()); +#endif if (strlen(taosdump_status) > 0) { fprintf(file, "status:%s\n", taosdump_status); From d7af721c24697346e60913819393960daefa1610 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 20 Jun 2024 19:28:39 +0800 Subject: [PATCH 52/57] fix: restore getBuildinfo function call --- src/benchCommandOpt.c | 2 -- src/taosdump.c | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/benchCommandOpt.c b/src/benchCommandOpt.c index 6eb30a0c..a2e87f43 100644 --- a/src/benchCommandOpt.c +++ b/src/benchCommandOpt.c @@ -47,9 +47,7 @@ void printVersion() { // version printf("taosBenchmark version: %s\ngit: %s\n", taosBenchmark_ver, taosBenchmark_commit); -#ifndef WINDOWS printf("build: %s\n", getBuildInfo()); -#endif if (strlen(taosBenchmark_status) > 0) { printf("status: %s\n", taosBenchmark_status); } diff --git a/src/taosdump.c b/src/taosdump.c index 6714e42d..d1f9f74d 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -668,9 +668,7 @@ static void printVersion(FILE *file) { char taosdump_commit[] = TAOSDUMP_COMMIT_SHA1; fprintf(file,"taosdump version: %s\ngit: %s\n", taostools_ver, taosdump_commit); -#ifndef WINDOWS printf("build: %s\n", getBuildInfo()); -#endif if (strlen(taosdump_status) > 0) { fprintf(file, "status:%s\n", taosdump_status); From 57c3b0c3b992cfc60364ea7134e2a7379651ca5b Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 26 Jun 2024 11:04:52 +0800 Subject: [PATCH 53/57] fix: window build error --- .github/workflows/3.0-windows-build.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/3.0-windows-build.yml b/.github/workflows/3.0-windows-build.yml index 8bc58fad..68a27bf8 100644 --- a/.github/workflows/3.0-windows-build.yml +++ b/.github/workflows/3.0-windows-build.yml @@ -5,12 +5,10 @@ on: - cron: "10 16 * * *" push: branches: - - develop - 3.0 - main pull_request: branches: - - develop - 3.0 - main @@ -122,7 +120,11 @@ jobs: with: toolchain: stable - - uses: ilammy/msvc-dev-cmd@v1 + - uses: actions/checkout@v2 + name: Setup MSVC build environment + env: + PATH: ${{ runner.tool_cache }}/msvc/2019/hostx64/x64/bin + run: echo "Setup MSVC environment" - name: Build & Install TDengine if: From d9178d22dc818025573204d4bdfda233469fe4b5 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 26 Jun 2024 11:15:45 +0800 Subject: [PATCH 54/57] fix: window build no found window --- .github/workflows/3.0-windows-build.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/3.0-windows-build.yml b/.github/workflows/3.0-windows-build.yml index 68a27bf8..a1538e39 100644 --- a/.github/workflows/3.0-windows-build.yml +++ b/.github/workflows/3.0-windows-build.yml @@ -120,12 +120,6 @@ jobs: with: toolchain: stable - - uses: actions/checkout@v2 - name: Setup MSVC build environment - env: - PATH: ${{ runner.tool_cache }}/msvc/2019/hostx64/x64/bin - run: echo "Setup MSVC environment" - - name: Build & Install TDengine if: (steps.changed-files-specific.outputs.any_changed == 'true' From e6a6e46ddb0f68e6448dba92c50af92435799324 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 26 Jun 2024 11:42:29 +0800 Subject: [PATCH 55/57] fix: window plat build error --- .github/workflows/3.0-windows-build.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/3.0-windows-build.yml b/.github/workflows/3.0-windows-build.yml index a1538e39..5deb3954 100644 --- a/.github/workflows/3.0-windows-build.yml +++ b/.github/workflows/3.0-windows-build.yml @@ -120,6 +120,12 @@ jobs: with: toolchain: stable + - name: Setup Visual Studio Build Tools + uses: microsoft/setup-msbuild@v1.0.2 + with: + vs-version: 2019 + components: 'Microsoft.VisualStudio.Workload.NativeDesktop,Microsoft.VisualStudio.Component.VC.Tools.x86.x64' + - name: Build & Install TDengine if: (steps.changed-files-specific.outputs.any_changed == 'true' From 315d4c7944f1a88490d7b677d413031bc7238773 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 26 Jun 2024 11:44:18 +0800 Subject: [PATCH 56/57] fix: window plat build error1 --- .github/workflows/3.0-windows-build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/3.0-windows-build.yml b/.github/workflows/3.0-windows-build.yml index 5deb3954..a41ba970 100644 --- a/.github/workflows/3.0-windows-build.yml +++ b/.github/workflows/3.0-windows-build.yml @@ -120,11 +120,11 @@ jobs: with: toolchain: stable - - name: Setup Visual Studio Build Tools - uses: microsoft/setup-msbuild@v1.0.2 - with: - vs-version: 2019 - components: 'Microsoft.VisualStudio.Workload.NativeDesktop,Microsoft.VisualStudio.Component.VC.Tools.x86.x64' + - name: Setup Visual Studio Build Tools + uses: microsoft/setup-msbuild@v1.0.2 + with: + vs-version: 2019 + components: 'Microsoft.VisualStudio.Workload.NativeDesktop,Microsoft.VisualStudio.Component.VC.Tools.x86.x64' - name: Build & Install TDengine if: From 690b6db8cc8eb5a9acc0a1321202553416febdcc Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 26 Jun 2024 13:11:29 +0800 Subject: [PATCH 57/57] fix: window plat build error1 --- .github/workflows/3.0-windows-build.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/3.0-windows-build.yml b/.github/workflows/3.0-windows-build.yml index a41ba970..68a0feeb 100644 --- a/.github/workflows/3.0-windows-build.yml +++ b/.github/workflows/3.0-windows-build.yml @@ -120,11 +120,7 @@ jobs: with: toolchain: stable - - name: Setup Visual Studio Build Tools - uses: microsoft/setup-msbuild@v1.0.2 - with: - vs-version: 2019 - components: 'Microsoft.VisualStudio.Workload.NativeDesktop,Microsoft.VisualStudio.Component.VC.Tools.x86.x64' + - uses: ilammy/msvc-dev-cmd@v1 - name: Build & Install TDengine if: