From b25d583aa04d8dfa1f437c013f8e49f0555103e8 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 5 Jul 2023 15:32:32 +0800 Subject: [PATCH 01/57] test: modify groupid --- src/benchTmq.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/benchTmq.c b/src/benchTmq.c index 861f2c8d..cd8c3c54 100644 --- a/src/benchTmq.c +++ b/src/benchTmq.c @@ -201,6 +201,7 @@ static void* tmqConsume(void* arg) { // "sequential" or "parallel" if (0 != strncasecmp(pConsumerInfo->createMode, "sequential", 10)) { + char* tPtr = pConsumerInfo->groupId; // "share" or "independent" char groupId[16] = {0}; if (0 != strncasecmp(pConsumerInfo->groupMode, "share", 5)) { @@ -210,11 +211,15 @@ static void* tmqConsume(void* arg) { memset(groupId, 0, sizeof(groupId)); rand_string(groupId, sizeof(groupId) - 1, 0); infoPrint("consumer id: %d generate rand group id: %s\n", pThreadInfo->id, groupId); - //pConsumerInfo->groupId = groupId; + tPtr = groupId; } } - buildConsumerAndSubscribe(pThreadInfo, groupId); + int ret = buildConsumerAndSubscribe(pThreadInfo, tPtr); + if (0 != ret) { + infoPrint("%s\n", "buildConsumerAndSubscribe() fail in tmqConsume()"); + return NULL; + } } int64_t totalMsgs = 0; @@ -342,7 +347,12 @@ int subscribeTestProcess() { // "sequential" or "parallel" if (0 == strncasecmp(pConsumerInfo->createMode, "sequential", 10)) { - buildConsumerAndSubscribe(pThreadInfo, pConsumerInfo->groupId); + int retVal = buildConsumerAndSubscribe(pThreadInfo, pConsumerInfo->groupId); + if (0 != retVal) { + infoPrint("%s\n", "buildConsumerAndSubscribe() fail!"); + ret = -1; + goto tmq_over; + } } pthread_create(pids + i, NULL, tmqConsume, pThreadInfo); } From 82fd3c071e510eb56e7b51d177ced3ccc96fa47f Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 5 Jul 2023 17:55:21 +0800 Subject: [PATCH 02/57] test: modify buffer size --- src/benchTmq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchTmq.c b/src/benchTmq.c index cd8c3c54..c92c6f9e 100644 --- a/src/benchTmq.c +++ b/src/benchTmq.c @@ -96,7 +96,7 @@ static tmq_list_t * buildTopicList() { } static int32_t data_msg_process(TAOS_RES* msg, tmqThreadInfo* pInfo, int32_t msgIndex) { - char* buf = (char*)calloc(1, 16*1024); + char* buf = (char*)calloc(1, 64*1024+8); if (NULL == buf) { errorPrint("consumer id %d calloc memory fail.\n", pInfo->id); return 0; From a2b5a6f162f2c7dd45d0e9a1a11b724911d2e209 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 18 Jul 2023 13:46:56 +0800 Subject: [PATCH 03/57] fix: replace taosws.dll.lib with taosws.lib --- src/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 07174fb7..e12895f4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -443,9 +443,9 @@ ELSE () IF (${WEBSOCKET}) INCLUDE_DIRECTORIES(/usr/local/include/) - SET(WEBSOCKET_LINK_FLAGS "taosws.dll.lib") + SET(WEBSOCKET_LINK_FLAGS "taosws.lib") IF (${CMAKE_PROJECT_NAME} STREQUAL "taos-tools") - MESSAGE("taosws.dll.lib need to be installed first") + MESSAGE("taosws.lib need to be installed first") ELSE () ADD_DEPENDENCIES(taosBenchmark taosws-rs) ADD_DEPENDENCIES(taosdump taosws-rs) From a2dea3803fb19a4a0425680f8d6904dd7be6439f Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 23 Jul 2023 08:23:17 +0800 Subject: [PATCH 04/57] fix : add funs express --- inc/bench.h | 15 ++++++++ src/benchData.c | 89 ++++++++++++++++++++++++++++++++++------------ src/benchJsonOpt.c | 67 ++++++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 22 deletions(-) diff --git a/inc/bench.h b/inc/bench.h index 275e07fc..7514c755 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -553,6 +553,15 @@ typedef struct SChildField { StmtData stmtData; } ChildField; +#define PI 3.141592654 +#define ATOR(x) (x*3.141592654/180) + +#define FUNTYPE_NONE 0 +#define FUNTYPE_SIN 1 +#define FUNTYPE_COS 2 + +#define FUNTYPE_CNT 2 + typedef struct SField { uint8_t type; char name[TSDB_COL_NAME_LEN + 1]; @@ -563,6 +572,12 @@ typedef struct SField { int64_t max; int64_t min; tools_cJSON * values; + + // fun + uint8_t funType; + float multiple; + int32_t addend; + bool sma; } Field; diff --git a/src/benchData.c b/src/benchData.c index d0907754..e0430de4 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -11,6 +11,7 @@ */ #include +#include #include const char charset[] = @@ -41,6 +42,14 @@ const char* locations_sml[] = { #include "benchLocations.h" #endif +// calc expression value like 10*sin(x) + 100 +float calc_expr_value(Field *field, int32_t angle) { + float radian = ATOR(angle); + float val = sin(radian); + return val; +} + + static int usc2utf8(char *p, int unic) { int ret = 0; if (unic <= 0x0000007F) { @@ -367,9 +376,12 @@ static int tmpStr(char *tmp, int iface, Field *field, int i) { return 0; } -FORCE_INLINE double tmpDouble(Field *field) { +FORCE_INLINE double tmpDoubleImpl(Field *field, int32_t angle) { double doubleTmp = (double)(field->min); - if (field->max != field->min) { + + if(field->funType != FUNTYPE_NONE) { + doubleTmp = calc_expr_value(field, angle); + } else if (field->max != field->min) { doubleTmp += ((taosRandom() % (field->max - field->min)) + taosRandom() % 1000000 / 1000000.0); @@ -377,6 +389,11 @@ FORCE_INLINE double tmpDouble(Field *field) { return doubleTmp; } +FORCE_INLINE double tmpDouble(Field *field) { + return tmpDoubleImpl(field, 0); +} + + FORCE_INLINE uint64_t tmpUint64(Field *field) { uint64_t ubigintTmp = field->min; if (field->max != field->min) { @@ -435,14 +452,20 @@ FORCE_INLINE uint16_t tmpUint16(Field *field) { return usmallintTmp; } -FORCE_INLINE int64_t tmpInt64(Field *field) { +FORCE_INLINE int64_t tmpInt64Impl(Field *field, int32_t angle) { int64_t bigintTmp = field->min; - if (field->min != field->max) { + if(field->funType != FUNTYPE_NONE) { + bigintTmp = calc_expr_value(field, angle); + } else if (field->min != field->max) { bigintTmp += (taosRandom() % (field->max - field->min)); } return bigintTmp; } +FORCE_INLINE int64_t tmpInt64(Field *field) { + return tmpInt64Impl(field, 0); +} + FORCE_INLINE float tmpFloat(Field *field) { float floatTmp = field->min; if (field->max != field->min) { @@ -452,30 +475,41 @@ FORCE_INLINE float tmpFloat(Field *field) { return floatTmp; } -static float tmpFloatI(Field *field, int i) { +static float tmpFloatImpl(Field *field, int i) { float floatTmp = (float)field->min; - if (field->max != field->min) { - floatTmp += ((taosRandom() % - (field->max - field->min)) - + (taosRandom() % 1000) / 1000.0); - } - if (g_arguments->demo_mode && i == 0) { - floatTmp = (float)(9.8 + 0.04 * (taosRandom() % 10) - + floatTmp / 1000000000); - } else if (g_arguments->demo_mode && i == 2) { - floatTmp = (float)((105 + taosRandom() % 10 - + floatTmp / 1000000000) / 360); + if(field->funType != FUNTYPE_NONE) { + floatTmp = calc_expr_value(field, angle); + } else { + if (field->max != field->min) { + floatTmp += ((taosRandom() % + (field->max - field->min)) + + (taosRandom() % 1000) / 1000.0); + } + if (g_arguments->demo_mode && i == 0) { + floatTmp = (float)(9.8 + 0.04 * (taosRandom() % 10) + + floatTmp / 1000000000); + } else if (g_arguments->demo_mode && i == 2) { + floatTmp = (float)((105 + taosRandom() % 10 + + floatTmp / 1000000000) / 360); + } } return floatTmp; } -static int tmpInt32(Field *field, int i) { +static float tmpFloatI(Field *field, int i) { + return tmpFloatImpl(field, i, 0); +} + +static int tmpInt32Impl(Field *field, int i, int angle) { int intTmp; if ((g_arguments->demo_mode) && (i == 0)) { unsigned int tmpRand = taosRandom(); intTmp = tmpRand % 10 + 1; } else if ((g_arguments->demo_mode) && (i == 1)) { intTmp = 105 + taosRandom() % 10; + } else if (field->funType != FUNTYPE_NONE) { + // calc from function + intTmp = calc_expr_value(field, angle); } else { if (field->min < (-1 * (RAND_MAX >> 1))) { field->min = -1 * (RAND_MAX >> 1); @@ -491,6 +525,10 @@ static int tmpInt32(Field *field, int i) { return intTmp; } +static int tmpInt32(Field *field, int i) { + return tmpInt32Impl(field, i, 0); +} + static int tmpJson(char *sampleDataBuf, int bufLen, int64_t pos, int fieldsSize, Field *field) { @@ -545,6 +583,8 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf, int64_t bufLen, int lenOfOneRow, BArray * fields, int64_t loop, bool tag) { + + int angle = 0; // 0 ~ 360 for (int64_t k = 0; k < loop; ++k) { int64_t pos = k * lenOfOneRow; int fieldsSize = fields->size; @@ -609,13 +649,13 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_INT: { - int32_t intTmp = tmpInt32(field, i); + int32_t intTmp = tmpInt32Impl(field, i, angle); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%d,", intTmp); break; } case TSDB_DATA_TYPE_BIGINT: { - int64_t bigintTmp = tmpInt64(field); + int64_t bigintTmp = tmpInt64Impl(field, angle); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%"PRId64",", bigintTmp); break; @@ -634,13 +674,13 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_FLOAT: { - float floatTmp = tmpFloatI(field, i); + float floatTmp = tmpFloatImpl(field, i, angle); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%f,", floatTmp); break; } case TSDB_DATA_TYPE_DOUBLE: { - double double_ = tmpDouble(field); + double double_ = tmpDoubleImpl(field, angle); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%f,", double_); break; @@ -675,7 +715,12 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf, } skip_sql: *(sampleDataBuf + pos - 1) = 0; - } + angle += 1; + if (angle > 360) { + // 360 is a circle + angle = 0; + } +} return 0; } diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index d89e3cb6..20e849ee 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -10,10 +10,63 @@ * FITNESS FOR A PARTICULAR PURPOSE. */ +#include #include extern char g_configDir[MAX_PATH_LEN]; +char funsName [FUNTYPE_CNT] [32] = { + "sin(", + "cos(" +} + +uint8_t parseFuns(char* funValue, float* multiple, int32_t* addend) { + // check valid + if (funValue == NULL || multiple == NULL || addend == NULL) { + return FUNTYPE_NONE; + } + + if(strlen(funValue) > 100) { + return FUNTYPE_NONE; + } + + //parse format 10*sin(x) + 100 + char value[128]; + strcpy(value, funValue); + + // multiple + char* key1 = strstr(value, "*"); + if(key1 == NULL) return FUNTYPE_NONE; + *key1 = 0; + * multiple = atof(value); + key1 += 1; + + // funType + uint8_t funType = FUNTYPE_NONE; + char* key2 = NULL; + for(int i=0; i < FUNTYPE_CNT; i++) { + key2 = strstr(key1, funsName[i]); + if(key2) { + funType = i; + key2 += strlen(funsName[i]); + break; + } + } + if (key2 == NULL) + return FUNTYPE_NONE; + + char* key3 = strstr(key2, "+"); + if(key3) { + *addend = atoi(key3 + 1); + } else { + key3 = strstr(key2, "-"); + if(key3) + *addend = atoi(key3 + 1) * -1; + } + + return funType; +} + static int getColumnAndTagTypeFromInsertJsonFile( tools_cJSON * superTblObj, SSuperTable *stbInfo) { int32_t code = -1; @@ -37,6 +90,10 @@ static int getColumnAndTagTypeFromInsertJsonFile( int64_t max = RAND_MAX >> 1; int64_t min = 0; int32_t length = 4; + // fun type + uint8_t funType = FUNTYPE_NONE; + float multiple = 0; + int32_t addend = 0; tools_cJSON *column = tools_cJSON_GetArrayItem(columnsObj, k); if (!tools_cJSON_IsObject(column)) { @@ -76,6 +133,12 @@ static int getColumnAndTagTypeFromInsertJsonFile( min = convertDatatypeToDefaultMin(type); } + // fun + tools_cJSON *fun = tools_cJSON_GetObjectItem(column, "fun"); + if (tools_cJSON_IsString(fun)) { + funType = parseFuns(fun->valuestring, &multiple, &addend); + } + tools_cJSON *dataValues = tools_cJSON_GetObjectItem(column, "values"); if (g_arguments->taosc_version == 3) { @@ -112,6 +175,10 @@ static int getColumnAndTagTypeFromInsertJsonFile( col->max = max; col->min = min; col->values = dataValues; + // fun + col->funType = funType; + col->multiple = multiple; + col->addend = addend; if (customName) { if (n >= 1) { snprintf(col->name, TSDB_COL_NAME_LEN, From f0c946edce5ede0553e6e89bdada0c55e4414375 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 23 Jul 2023 08:27:04 +0800 Subject: [PATCH 05/57] test: add insert_math.json case --- example/insert_math.json | 83 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 example/insert_math.json diff --git a/example/insert_math.json b/example/insert_math.json new file mode 100644 index 00000000..c519d463 --- /dev/null +++ b/example/insert_math.json @@ -0,0 +1,83 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "thread_count": 4, + "create_table_thread_count": 7, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 100, + "prepared_rand": 10000, + "chinese": "no", + "escape_character": "yes", + "databases": [ + { + "dbinfo": { + "name": "test", + "drop": "yes", + "precision": "ms" + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 10000, + "childtable_prefix": "d", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "rand", + "insert_mode": "taosc", + "non_stop_mode": "no", + "line_protocol": "line", + "insert_rows": 10000, + "childtable_limit": 10, + "childtable_offset": 100, + "interlace_rows": 0, + "insert_interval": 0, + "partial_col_num": 0, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 10, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "use_sample_ts": "no", + "tags_file": "", + "columns": [ + { + "type": "FLOAT", + "name": "current", + "count": 1, + "max": 12, + "min": 8 + }, + { "type": "INT", "name": "voltage", "max": 225, "min": 215, "fun": "10*sin(x)+100"}, + { "type": "FLOAT", "name": "phase", "max": 1, "min": 0 } + ], + "tags": [ + { + "type": "TINYINT", + "name": "groupid", + "max": 10, + "min": 1 + }, + { + "name": "location", + "type": "BINARY", + "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} From a72ebf5b53f16c072d765a751912cd37f7a1a7d9 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 23 Jul 2023 08:46:36 +0800 Subject: [PATCH 06/57] fix: add math.h library m --- src/CMakeLists.txt | 16 ++++++++-------- src/benchData.c | 2 +- src/benchJsonOpt.c | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 07174fb7..c4b3de61 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -308,8 +308,8 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin FIND_LIBRARY(LIBZ_LIBRARY z) MESSAGE(${ARGP_LIBRARY}) - TARGET_LINK_LIBRARIES(taosBenchmark taos pthread toolscJson $<$:${LIBZ_LIBRARY}> $<$:${ARGP_LIBRARY}> ${WEBSOCKET_LINK_FLAGS}) - TARGET_LINK_LIBRARIES(taosdump taos avro jansson atomic pthread argp $<$:${LIBZ_LIBRARY}> $<$:${ARGP_LIBRARY}> ${WEBSOCKET_LINK_FLAGS}) + TARGET_LINK_LIBRARIES(taosBenchmark taos pthread m toolscJson $<$:${LIBZ_LIBRARY}> $<$:${ARGP_LIBRARY}> ${WEBSOCKET_LINK_FLAGS}) + TARGET_LINK_LIBRARIES(taosdump taos avro jansson atomic pthread m argp $<$:${LIBZ_LIBRARY}> $<$:${ARGP_LIBRARY}> ${WEBSOCKET_LINK_FLAGS}) ELSEIF(${OS_ID} MATCHES "Darwin") ADD_LIBRARY(argp STATIC IMPORTED) IF (CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") @@ -319,11 +319,11 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin SET_PROPERTY(TARGET argp PROPERTY IMPORTED_LOCATION "/usr/local/lib/libargp.a") INCLUDE_DIRECTORIES(/usr/local/include/) ENDIF () - TARGET_LINK_LIBRARIES(taosBenchmark taos pthread toolscJson argp ${WEBSOCKET_LINK_FLAGS}) + TARGET_LINK_LIBRARIES(taosBenchmark taos pthread m toolscJson argp ${WEBSOCKET_LINK_FLAGS}) ElSE () MESSAGE("${Yellow} DEBUG mode use shared avro library to link for debug ${ColourReset}") - TARGET_LINK_LIBRARIES(taosdump taos avro jansson atomic pthread ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}) - TARGET_LINK_LIBRARIES(taosBenchmark taos pthread toolscJson ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}) + TARGET_LINK_LIBRARIES(taosdump taos avro jansson atomic pthread m ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}) + TARGET_LINK_LIBRARIES(taosBenchmark taos pthread m toolscJson ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}) ENDIF() ELSE () @@ -362,7 +362,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin INCLUDE_DIRECTORIES(/usr/local/include/) ENDIF () - TARGET_LINK_LIBRARIES(taosBenchmark taos pthread toolscJson argp ${WEBSOCKET_LINK_FLAGS}) + TARGET_LINK_LIBRARIES(taosBenchmark taos pthread m toolscJson argp ${WEBSOCKET_LINK_FLAGS}) ELSE () EXECUTE_PROCESS ( COMMAND sh -c "awk -F= '/^ID=/{print $2}' /etc/os-release |tr -d '\n' | tr -d '\"'" @@ -414,10 +414,10 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin MESSAGE(${LIBZ_LIBRARY}) TARGET_LINK_LIBRARIES(taosdump taos avro jansson snappy stdc++ lzma atomic pthread $<$:${LIBZ_LIBRARY}> $<$:${ARGP_LIBRARY}> ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}) - TARGET_LINK_LIBRARIES(taosBenchmark taos pthread toolscJson $<$:${LIBZ_LIBRARY}> $<$:${ARGP_LIBRARY}> ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}) + TARGET_LINK_LIBRARIES(taosBenchmark taos pthread m toolscJson $<$:${LIBZ_LIBRARY}> $<$:${ARGP_LIBRARY}> ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}) ELSE() TARGET_LINK_LIBRARIES(taosdump taos avro jansson snappy stdc++ lzma libz-static atomic pthread ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}) - TARGET_LINK_LIBRARIES(taosBenchmark taos pthread toolscJson ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}) + TARGET_LINK_LIBRARIES(taosBenchmark taos pthread m toolscJson ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}) ENDIF() ENDIF () diff --git a/src/benchData.c b/src/benchData.c index e0430de4..c82f6769 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -475,7 +475,7 @@ FORCE_INLINE float tmpFloat(Field *field) { return floatTmp; } -static float tmpFloatImpl(Field *field, int i) { +static float tmpFloatImpl(Field *field, int i, int32_t angle) { float floatTmp = (float)field->min; if(field->funType != FUNTYPE_NONE) { floatTmp = calc_expr_value(field, angle); diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 20e849ee..d7bbafaf 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -18,7 +18,7 @@ extern char g_configDir[MAX_PATH_LEN]; char funsName [FUNTYPE_CNT] [32] = { "sin(", "cos(" -} +}; uint8_t parseFuns(char* funValue, float* multiple, int32_t* addend) { // check valid From 90f86c0cbd3ad848e604824856c4c6fcf1edd2e7 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 23 Jul 2023 09:06:06 +0800 Subject: [PATCH 07/57] fix: funType = i+1 --- src/benchJsonOpt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index d7bbafaf..5cc038f2 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -47,7 +47,7 @@ uint8_t parseFuns(char* funValue, float* multiple, int32_t* addend) { for(int i=0; i < FUNTYPE_CNT; i++) { key2 = strstr(key1, funsName[i]); if(key2) { - funType = i; + funType = i + 1; key2 += strlen(funsName[i]); break; } From 5096d40ddff704e6daedaf86c5d8d2f55f3457c1 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 23 Jul 2023 09:47:41 +0800 Subject: [PATCH 08/57] fix: modify json case --- example/insert_math.json | 8 ++++---- src/benchData.c | 8 +++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/example/insert_math.json b/example/insert_math.json index c519d463..51d4bf4a 100644 --- a/example/insert_math.json +++ b/example/insert_math.json @@ -13,7 +13,7 @@ "insert_interval": 0, "interlace_rows": 100, "num_of_records_per_req": 100, - "prepared_rand": 10000, + "prepared_rand": 3600, "chinese": "no", "escape_character": "yes", "databases": [ @@ -27,7 +27,7 @@ { "name": "meters", "child_table_exists": "no", - "childtable_count": 10000, + "childtable_count": 100, "childtable_prefix": "d", "auto_create_table": "no", "batch_create_tbl_num": 5, @@ -57,8 +57,8 @@ "max": 12, "min": 8 }, - { "type": "INT", "name": "voltage", "max": 225, "min": 215, "fun": "10*sin(x)+100"}, - { "type": "FLOAT", "name": "phase", "max": 1, "min": 0 } + { "type": "INT", "name": "voltage", "max": 225, "min": 215, "fun": "1000*sin(x)+100"}, + { "type": "FLOAT", "name": "phase", "max": 1, "min": 0, "fun": "100*sin(x)+20"} ], "tags": [ { diff --git a/src/benchData.c b/src/benchData.c index c82f6769..c1be8eaf 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -45,7 +45,13 @@ const char* locations_sml[] = { // calc expression value like 10*sin(x) + 100 float calc_expr_value(Field *field, int32_t angle) { float radian = ATOR(angle); - float val = sin(radian); + float funVal = 0; + if (field->funType == FUNTYPE_SIN) + funVal = sin(radian); + else if (field->funType == FUNTYPE_COS) + funVal = cos(radian); + + float val = field->multiple * funVal + field->addend; return val; } From 9720b264ffb8583af6db65d5bace4dc8bd2b3de8 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 23 Jul 2023 11:11:52 +0800 Subject: [PATCH 09/57] fix: add random number --- example/insert_math.json | 6 +++--- inc/bench.h | 1 + src/benchData.c | 7 ++++++- src/benchJsonOpt.c | 16 +++++++++++++--- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/example/insert_math.json b/example/insert_math.json index 51d4bf4a..37d492fc 100644 --- a/example/insert_math.json +++ b/example/insert_math.json @@ -27,7 +27,7 @@ { "name": "meters", "child_table_exists": "no", - "childtable_count": 100, + "childtable_count": 10, "childtable_prefix": "d", "auto_create_table": "no", "batch_create_tbl_num": 5, @@ -57,8 +57,8 @@ "max": 12, "min": 8 }, - { "type": "INT", "name": "voltage", "max": 225, "min": 215, "fun": "1000*sin(x)+100"}, - { "type": "FLOAT", "name": "phase", "max": 1, "min": 0, "fun": "100*sin(x)+20"} + { "type": "INT", "name": "voltage", "max": 225, "min": 215, "fun": "1000*sin(x)+100*random(10)"}, + { "type": "FLOAT", "name": "phase", "max": 1, "min": 0, "fun": "10*sin(x)+1*random(5)"} ], "tags": [ { diff --git a/inc/bench.h b/inc/bench.h index 7514c755..48a23d19 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -577,6 +577,7 @@ typedef struct SField { uint8_t funType; float multiple; int32_t addend; + int32_t random; bool sma; } Field; diff --git a/src/benchData.c b/src/benchData.c index c1be8eaf..b6ec2e52 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -51,7 +51,12 @@ float calc_expr_value(Field *field, int32_t angle) { else if (field->funType == FUNTYPE_COS) funVal = cos(radian); - float val = field->multiple * funVal + field->addend; + float val = field->multiple * funVal + field->addend; + if (field->random >0) { + float rate = taosRandom() % field->random; + val += field->addend * (rate/100); + } + return val; } diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 5cc038f2..7610636e 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -20,7 +20,7 @@ char funsName [FUNTYPE_CNT] [32] = { "cos(" }; -uint8_t parseFuns(char* funValue, float* multiple, int32_t* addend) { +uint8_t parseFuns(char* funValue, float* multiple, int32_t* addend, int32_t* random) { // check valid if (funValue == NULL || multiple == NULL || addend == NULL) { return FUNTYPE_NONE; @@ -30,7 +30,7 @@ uint8_t parseFuns(char* funValue, float* multiple, int32_t* addend) { return FUNTYPE_NONE; } - //parse format 10*sin(x) + 100 + //parse format 10*sin(x) + 100 * random(5) char value[128]; strcpy(value, funValue); @@ -63,6 +63,13 @@ uint8_t parseFuns(char* funValue, float* multiple, int32_t* addend) { if(key3) *addend = atoi(key3 + 1) * -1; } + key3 += 1; + + // random + char* key4 = strstr(key3, "*"); + if(key4) { + *random = aoti(key4 + 1); + } return funType; } @@ -94,6 +101,7 @@ static int getColumnAndTagTypeFromInsertJsonFile( uint8_t funType = FUNTYPE_NONE; float multiple = 0; int32_t addend = 0; + int32_t random = 0; tools_cJSON *column = tools_cJSON_GetArrayItem(columnsObj, k); if (!tools_cJSON_IsObject(column)) { @@ -136,7 +144,7 @@ static int getColumnAndTagTypeFromInsertJsonFile( // fun tools_cJSON *fun = tools_cJSON_GetObjectItem(column, "fun"); if (tools_cJSON_IsString(fun)) { - funType = parseFuns(fun->valuestring, &multiple, &addend); + funType = parseFuns(fun->valuestring, &multiple, &addend, &random); } tools_cJSON *dataValues = tools_cJSON_GetObjectItem(column, "values"); @@ -179,6 +187,8 @@ static int getColumnAndTagTypeFromInsertJsonFile( col->funType = funType; col->multiple = multiple; col->addend = addend; + col->radom = random; + if (customName) { if (n >= 1) { snprintf(col->name, TSDB_COL_NAME_LEN, From 86275b669c95ca402f4aa61f39f766f3ffd70698 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 23 Jul 2023 11:17:52 +0800 Subject: [PATCH 10/57] fix:build error --- src/benchJsonOpt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 7610636e..b452d2df 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -68,7 +68,7 @@ uint8_t parseFuns(char* funValue, float* multiple, int32_t* addend, int32_t* ran // random char* key4 = strstr(key3, "*"); if(key4) { - *random = aoti(key4 + 1); + *random = atoi(key4 + 1); } return funType; @@ -187,7 +187,7 @@ static int getColumnAndTagTypeFromInsertJsonFile( col->funType = funType; col->multiple = multiple; col->addend = addend; - col->radom = random; + col->random = random; if (customName) { if (n >= 1) { From f7b99d0b8ae753729bd00de6b304add2a10726e9 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 23 Jul 2023 11:21:22 +0800 Subject: [PATCH 11/57] fix: find key4 --- src/benchJsonOpt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index b452d2df..1205f6a8 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -66,9 +66,9 @@ uint8_t parseFuns(char* funValue, float* multiple, int32_t* addend, int32_t* ran key3 += 1; // random - char* key4 = strstr(key3, "*"); + char* key4 = strstr(key3, "*random("); if(key4) { - *random = atoi(key4 + 1); + *random = atoi(key4 + 8); } return funType; From 044523a55376870aacff0567212ce6dc7b242eec Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 23 Jul 2023 12:42:20 +0800 Subject: [PATCH 12/57] fix: case case insert_math.json --- example/insert_math.json | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/example/insert_math.json b/example/insert_math.json index 37d492fc..0fdbd8fb 100644 --- a/example/insert_math.json +++ b/example/insert_math.json @@ -27,7 +27,7 @@ { "name": "meters", "child_table_exists": "no", - "childtable_count": 10, + "childtable_count": 1000, "childtable_prefix": "d", "auto_create_table": "no", "batch_create_tbl_num": 5, @@ -35,7 +35,7 @@ "insert_mode": "taosc", "non_stop_mode": "no", "line_protocol": "line", - "insert_rows": 10000, + "insert_rows": 1000, "childtable_limit": 10, "childtable_offset": 100, "interlace_rows": 0, @@ -50,15 +50,9 @@ "use_sample_ts": "no", "tags_file": "", "columns": [ - { - "type": "FLOAT", - "name": "current", - "count": 1, - "max": 12, - "min": 8 - }, - { "type": "INT", "name": "voltage", "max": 225, "min": 215, "fun": "1000*sin(x)+100*random(10)"}, - { "type": "FLOAT", "name": "phase", "max": 1, "min": 0, "fun": "10*sin(x)+1*random(5)"} + { "type": "FLOAT", "name": "current", "fun": "1*sin(x)+10*random(15)"}, + { "type": "INT", "name": "voltage", "fun": "40*sin(x)+200*random(10)"}, + { "type": "FLOAT", "name": "phase", "fun": "1*sin(x)+1*random(3)"} ], "tags": [ { From cfcb8531721bac7122162bdab2f9cebc4aeff40a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 23 Jul 2023 23:06:37 +0800 Subject: [PATCH 13/57] test: move fun to default config --- src/benchCommandOpt.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/benchCommandOpt.c b/src/benchCommandOpt.c index b776117b..c295d5c0 100644 --- a/src/benchCommandOpt.c +++ b/src/benchCommandOpt.c @@ -137,11 +137,28 @@ static void initStable() { tstrncpy(c3->name, "phase", TSDB_COL_NAME_LEN + 1); c1->min = 9; - c1->max = 10; + c1->max = 10; + // fun = "1*sin(x)+10*random(15)" + c1->funType = FUNTYPE_SIN; + c1->multiple = 1; + c1->random = 15; + c1->addend = 10; + c2->min = 110; c2->max = 119; + // fun = "40*sin(x)+200*random(10)" + c2->funType = FUNTYPE_SIN; + c2->multiple = 40; + c2->random = 10; + c2->addend = 200; + c3->min = 115; c3->max = 125; + // fun = "1*sin(x)+1*random(3)" + c3->funType = FUNTYPE_SIN; + c3->multiple = 1; + c3->random = 3; + c3->addend = 1; stbInfo->tags = benchArrayInit(2, sizeof(Field)); for (int i = 0; i < 2; ++i) { From 35065733b9bc3b9726899ac47f5254e9ef93c975 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 23 Jul 2023 23:54:04 +0800 Subject: [PATCH 14/57] test: prepare_rand set 360 --- src/benchCommandOpt.c | 2 +- src/benchData.c | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/benchCommandOpt.c b/src/benchCommandOpt.c index c295d5c0..694c3258 100644 --- a/src/benchCommandOpt.c +++ b/src/benchCommandOpt.c @@ -366,7 +366,7 @@ void modifyArgument() { if (g_arguments->demo_mode && TAOSC_IFACE == g_arguments->iface) { g_arguments->mistMode = true; - g_arguments->prepared_rand = 57; + g_arguments->prepared_rand = 360; } } diff --git a/src/benchData.c b/src/benchData.c index b6ec2e52..6841b968 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -513,14 +513,14 @@ static float tmpFloatI(Field *field, int i) { static int tmpInt32Impl(Field *field, int i, int angle) { int intTmp; - if ((g_arguments->demo_mode) && (i == 0)) { + if (field->funType != FUNTYPE_NONE) { + // calc from function + intTmp = calc_expr_value(field, angle); + } else if ((g_arguments->demo_mode) && (i == 0)) { unsigned int tmpRand = taosRandom(); intTmp = tmpRand % 10 + 1; } else if ((g_arguments->demo_mode) && (i == 1)) { intTmp = 105 + taosRandom() % 10; - } else if (field->funType != FUNTYPE_NONE) { - // calc from function - intTmp = calc_expr_value(field, angle); } else { if (field->min < (-1 * (RAND_MAX >> 1))) { field->min = -1 * (RAND_MAX >> 1); From 3e0446bb9ff069c15f53892718ff4e23a0941591 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 24 Jul 2023 10:55:32 +0800 Subject: [PATCH 15/57] test: calculate angle according with timestamp and step --- src/benchData.c | 7 +++---- src/benchJsonOpt.c | 12 ++++++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/benchData.c b/src/benchData.c index 6841b968..54b54dd8 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -595,7 +595,7 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf, int lenOfOneRow, BArray * fields, int64_t loop, bool tag) { - int angle = 0; // 0 ~ 360 + int angle = stbInfo->startTimestamp % 360; // 0 ~ 360 for (int64_t k = 0; k < loop; ++k) { int64_t pos = k * lenOfOneRow; int fieldsSize = fields->size; @@ -726,10 +726,9 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf, } skip_sql: *(sampleDataBuf + pos - 1) = 0; - angle += 1; + angle += stbInfo->timestamp_step; if (angle > 360) { - // 360 is a circle - angle = 0; + angle -= 360; } } diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 1205f6a8..d3303645 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -26,13 +26,21 @@ uint8_t parseFuns(char* funValue, float* multiple, int32_t* addend, int32_t* ran return FUNTYPE_NONE; } - if(strlen(funValue) > 100) { + size_t len = strlen(funValue); + if(len > 100) { return FUNTYPE_NONE; } //parse format 10*sin(x) + 100 * random(5) char value[128]; - strcpy(value, funValue); + size_t n = 0; + for (size_t i = 0; i < len; i++) { + if (funValue[i] != ' ') { + value[n++] = funValue[i]; + } + } + // set end + value[n] = 0; // multiple char* key1 = strstr(value, "*"); From d53f4eb426f79acf9f2b00678d907e331636c5ba Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 24 Jul 2023 21:24:33 +0800 Subject: [PATCH 16/57] add angle step config with -A and angle_step --- inc/bench.h | 1 + src/benchCommandOpt.c | 1 + src/benchData.c | 2 +- src/benchJsonOpt.c | 7 +++++++ src/benchSys.c | 15 +++++++++++++++ 5 files changed, 25 insertions(+), 1 deletion(-) diff --git a/inc/bench.h b/inc/bench.h index 48a23d19..c2a5e529 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -682,6 +682,7 @@ typedef struct SSuperTable_S { uint64_t insert_interval; uint64_t insertRows; uint64_t timestamp_step; + uint64_t angle_step; int64_t startTimestamp; int64_t specifiedColumns; char sampleFile[MAX_FILE_NAME_LEN]; diff --git a/src/benchCommandOpt.c b/src/benchCommandOpt.c index 694c3258..2ef67d77 100644 --- a/src/benchCommandOpt.c +++ b/src/benchCommandOpt.c @@ -183,6 +183,7 @@ static void initStable() { stbInfo->insert_interval = 0; stbInfo->timestamp_step = 1; + stbInfo->angle_step = 1; stbInfo->interlaceRows = 0; stbInfo->childTblCount = DEFAULT_CHILDTABLES; stbInfo->childTblLimit = 0; diff --git a/src/benchData.c b/src/benchData.c index 54b54dd8..a196ada3 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -726,7 +726,7 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf, } skip_sql: *(sampleDataBuf + pos - 1) = 0; - angle += stbInfo->timestamp_step; + angle += stbInfo->timestamp_step/stbInfo->angle_step; if (angle > 360) { angle -= 360; } diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index d3303645..eb04ce92 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -564,6 +564,7 @@ static int getStableInfo(tools_cJSON *dbinfos, int index) { superTable->tcpTransfer = false; superTable->childTblOffset = 0; superTable->timestamp_step = 1; + superTable->angle_step = 1; superTable->useSampleTs = false; superTable->non_stop = false; superTable->insertRows = 0; @@ -819,6 +820,12 @@ static int getStableInfo(tools_cJSON *dbinfos, int index) { superTable->timestamp_step = timestampStep->valueint; } + tools_cJSON *angleStep = + tools_cJSON_GetObjectItem(stbInfo, "angle_step"); + if (tools_cJSON_IsNumber(angleStep)) { + superTable->angle_step = timestampStep->valueint; + } + tools_cJSON *keepTrying = tools_cJSON_GetObjectItem(stbInfo, "keep_trying"); if (tools_cJSON_IsNumber(keepTrying)) { diff --git a/src/benchSys.c b/src/benchSys.c index 430fd92a..854d8729 100644 --- a/src/benchSys.c +++ b/src/benchSys.c @@ -392,6 +392,21 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { } break; + // angle step + case 'A': + if (!toolsIsStringNumber(arg)) { + errorPrintReqArg2(CUS_PROMPT"Benchmark", "A"); + } + + stbInfo->angle_step = atol(arg); + if (stbInfo->angle_step <= 0) { + errorPrint( + "Invalid -A: %s, will auto set to default(1)\n", + arg); + stbInfo->angle_step = 1; + } + break; + case 'B': if (!toolsIsStringNumber(arg)) { errorPrintReqArg2(CUS_PROMPT"Benchmark", "B"); From 40ea9cf900bc604d0e309faa55e028ba04567ad6 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 24 Jul 2023 21:28:17 +0800 Subject: [PATCH 17/57] test: A change to H --- src/benchSys.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/benchSys.c b/src/benchSys.c index 854d8729..e5b952c9 100644 --- a/src/benchSys.c +++ b/src/benchSys.c @@ -393,15 +393,15 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { break; // angle step - case 'A': + case 'H': if (!toolsIsStringNumber(arg)) { - errorPrintReqArg2(CUS_PROMPT"Benchmark", "A"); + errorPrintReqArg2(CUS_PROMPT"Benchmark", "H"); } stbInfo->angle_step = atol(arg); if (stbInfo->angle_step <= 0) { errorPrint( - "Invalid -A: %s, will auto set to default(1)\n", + "Invalid -H: %s, will auto set to default(1)\n", arg); stbInfo->angle_step = 1; } From 13f7d2a9590ece9cc6994e12ba74940d59c368bc Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 24 Jul 2023 21:35:17 +0800 Subject: [PATCH 18/57] test: add option angle-step --- inc/bench.h | 1 + src/benchSys.c | 1 + 2 files changed, 2 insertions(+) diff --git a/inc/bench.h b/inc/bench.h index c2a5e529..764d3409 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -201,6 +201,7 @@ typedef unsigned __int32 uint32_t; #define BENCH_INTERVAL \ "Insert interval for interlace mode in milliseconds, default is 0." #define BENCH_STEP "Timestamp step in milliseconds, default is 1." +#define ANGLE_STEP "Angle step in milliseconds, default is 1." #define BENCH_SUPPLEMENT \ "Supplementally insert data without create " \ "database and table, optional, default is off." diff --git a/src/benchSys.c b/src/benchSys.c index e5b952c9..3b8a647f 100644 --- a/src/benchSys.c +++ b/src/benchSys.c @@ -182,6 +182,7 @@ static struct argp_option bench_options[] = { {"threads", 'T', "NUMBER", 0, BENCH_THREAD}, {"insert-interval", 'i', "NUMBER", 0, BENCH_INTERVAL}, {"time-step", 'S', "NUMBER", 0, BENCH_STEP}, + {"angle-step", 'H', "NUMBER", 0, ANGLE_STEP}, {"start-timestamp", 's', "NUMBER", 0, BENCH_START_TIMESTAMP}, {"supplement-insert", 'U', 0, 0, BENCH_SUPPLEMENT}, {"interlace-rows", 'B', "NUMBER", 0, BENCH_INTERLACE}, From c2d977e6c92d77174c945573632c8adfe2f428e9 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 25 Jul 2023 11:04:39 +0800 Subject: [PATCH 19/57] adjust insert_math.json to secnond --- example/insert_math.json | 5 +++-- src/benchCommandOpt.c | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/example/insert_math.json b/example/insert_math.json index 0fdbd8fb..f272c492 100644 --- a/example/insert_math.json +++ b/example/insert_math.json @@ -43,14 +43,15 @@ "partial_col_num": 0, "disorder_ratio": 0, "disorder_range": 1000, - "timestamp_step": 10, + "timestamp_step": 1000, + "angle_step": 500, "start_timestamp": "2020-10-01 00:00:00.000", "sample_format": "csv", "sample_file": "./sample.csv", "use_sample_ts": "no", "tags_file": "", "columns": [ - { "type": "FLOAT", "name": "current", "fun": "1*sin(x)+10*random(15)"}, + { "type": "FLOAT", "name": "current", "fun": "3*sin(x)+10*random(2)"}, { "type": "INT", "name": "voltage", "fun": "40*sin(x)+200*random(10)"}, { "type": "FLOAT", "name": "phase", "fun": "1*sin(x)+1*random(3)"} ], diff --git a/src/benchCommandOpt.c b/src/benchCommandOpt.c index 2ef67d77..8f1fb171 100644 --- a/src/benchCommandOpt.c +++ b/src/benchCommandOpt.c @@ -138,10 +138,10 @@ static void initStable() { c1->min = 9; c1->max = 10; - // fun = "1*sin(x)+10*random(15)" + // fun = "3*sin(x)+10*random(2)" c1->funType = FUNTYPE_SIN; - c1->multiple = 1; - c1->random = 15; + c1->multiple = 3; + c1->random = 2; c1->addend = 10; c2->min = 110; From 44503413fb110595866e29c2acc17ef03adfbaef Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 25 Jul 2023 16:36:28 +0800 Subject: [PATCH 20/57] feat: add progress on export data --- src/taosdump.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/taosdump.c b/src/taosdump.c index 569bd660..53643cc0 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -118,6 +118,10 @@ static int g_majorVersionOfClient = 0; static int g_maxFilesPerDir = 100000; volatile int64_t g_countOfDataFile = 0; +// progress +static int64_t g_tableCount = 0; +static int64_t g_tableDone = 0; + static void print_json_aux(json_t *element, int indent); // for tstrncpy buffer overflow @@ -5030,7 +5034,7 @@ static int64_t writeResultToAvroNative( currentPercent = ((offset) * 100 / queryCount); if (currentPercent > percentComplete) { - infoPrint("%d%% of %s\n", currentPercent, tbName); + infoPrint("[%" PRId64 "/%" PRId64 "] write avro %d%% of %s\n", g_tableDone, g_tableCount, currentPercent, tbName); percentComplete = currentPercent; } } while (offset < queryCount); @@ -10470,6 +10474,9 @@ static void dumpNormalTablesOfStbNative( fp); } + // update progress + atomic_add_fetch_64(&g_tableDone, 1); + if (count < 0) { break; } else { @@ -10568,6 +10575,10 @@ static int64_t dumpNtbOfStbByThreads( } #endif + // set progress to global + g_tableCount = ntbCount; + g_tableDone = 0; + infoPrint("%s() LN%d, %s's %s's total normal table count: %"PRId64"\n", __func__, __LINE__, dbInfo->name, stbName, ntbCount); if (ntbCount <= 0) { From 6f4521fef272e10417240852b62583669a34a504 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 25 Jul 2023 16:46:48 +0800 Subject: [PATCH 21/57] feat: show start from 1 index --- src/taosdump.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/taosdump.c b/src/taosdump.c index 53643cc0..f44df128 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -5034,7 +5034,7 @@ static int64_t writeResultToAvroNative( currentPercent = ((offset) * 100 / queryCount); if (currentPercent > percentComplete) { - infoPrint("[%" PRId64 "/%" PRId64 "] write avro %d%% of %s\n", g_tableDone, g_tableCount, currentPercent, tbName); + infoPrint("[%" PRId64 "/%" PRId64 "] write avro %d%% of %s\n", g_tableDone + 1, g_tableCount, currentPercent, tbName); percentComplete = currentPercent; } } while (offset < queryCount); From 70e4a46550ea2ebf6e521c9bfe6c0c4207ca1add Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 25 Jul 2023 16:54:24 +0800 Subject: [PATCH 22/57] feat: add connect and close tips --- src/taosdump.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/taosdump.c b/src/taosdump.c index f44df128..f3f5cc99 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -10654,6 +10654,7 @@ static int64_t dumpNtbOfStbByThreads( ASSERT(pids); ASSERT(infos); + infoPrint("create %d thread(s) and connect to server...\n", threads); threadInfo *pThreadInfo; for (int32_t i = 0; i < threads; i++) { pThreadInfo = infos + i; @@ -10723,6 +10724,7 @@ static int64_t dumpNtbOfStbByThreads( } } + infoPrint("close taos connections...\n"); for (int32_t i = 0; i < threads; i++) { pThreadInfo = infos + i; taos_close(pThreadInfo->taos); From c30c791fb4610fffee07784bdc2c9c0ec921c3c1 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 25 Jul 2023 16:56:11 +0800 Subject: [PATCH 23/57] feat: add connect and close tips --- src/taosdump.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/taosdump.c b/src/taosdump.c index f3f5cc99..6a89429f 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -10724,7 +10724,7 @@ static int64_t dumpNtbOfStbByThreads( } } - infoPrint("close taos connections...\n"); + infoPrint("%s\n","close taos connections..."); for (int32_t i = 0; i < threads; i++) { pThreadInfo = infos + i; taos_close(pThreadInfo->taos); From 3b827faafd82143bb35fedaf13b4b6c0c5b31fdb Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 25 Jul 2023 16:58:33 +0800 Subject: [PATCH 24/57] feat: add connect and close tips --- src/taosdump.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/taosdump.c b/src/taosdump.c index 6a89429f..adb8f95e 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -10654,7 +10654,7 @@ static int64_t dumpNtbOfStbByThreads( ASSERT(pids); ASSERT(infos); - infoPrint("create %d thread(s) and connect to server...\n", threads); + infoPrint("create %d thread(s) to export data ...\n", threads); threadInfo *pThreadInfo; for (int32_t i = 0; i < threads; i++) { pThreadInfo = infos + i; From e7d2a72a6247c53b5286fbb12bd2cb7b67d22438 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 28 Jul 2023 17:20:26 +0800 Subject: [PATCH 25/57] feat(taosdump): add dbname.stbname to tip --- src/benchJsonOpt.c | 2 +- src/taosdump.c | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index eb04ce92..535d960b 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -823,7 +823,7 @@ static int getStableInfo(tools_cJSON *dbinfos, int index) { tools_cJSON *angleStep = tools_cJSON_GetObjectItem(stbInfo, "angle_step"); if (tools_cJSON_IsNumber(angleStep)) { - superTable->angle_step = timestampStep->valueint; + superTable->angle_step = angleStep->valueint; } tools_cJSON *keepTrying = diff --git a/src/taosdump.c b/src/taosdump.c index adb8f95e..d9806c43 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -121,6 +121,8 @@ volatile int64_t g_countOfDataFile = 0; // progress static int64_t g_tableCount = 0; static int64_t g_tableDone = 0; +static char g_dbName[TSDB_DB_NAME_LEN]= ""; +static char g_stbName[TSDB_TABLE_NAME_LEN] = ""; static void print_json_aux(json_t *element, int indent); @@ -5034,7 +5036,7 @@ static int64_t writeResultToAvroNative( currentPercent = ((offset) * 100 / queryCount); if (currentPercent > percentComplete) { - infoPrint("[%" PRId64 "/%" PRId64 "] write avro %d%% of %s\n", g_tableDone + 1, g_tableCount, currentPercent, tbName); + infoPrint("%s.%s [%" PRId64 "/%" PRId64 "] write avro %d%% of %s\n", g_dbName, g_stbName ,g_tableDone + 1, g_tableCount, currentPercent, tbName); percentComplete = currentPercent; } } while (offset < queryCount); @@ -10578,6 +10580,8 @@ static int64_t dumpNtbOfStbByThreads( // set progress to global g_tableCount = ntbCount; g_tableDone = 0; + strcpy(g_dbName, dbInfo->name); + strcpy(g_stbName, stbName); infoPrint("%s() LN%d, %s's %s's total normal table count: %"PRId64"\n", __func__, __LINE__, dbInfo->name, stbName, ntbCount); From 34ad3ea2b45ac0e43a8f9fa183de520ecbec0d6c Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 15 Aug 2023 10:09:24 +0800 Subject: [PATCH 26/57] fix: deal dot in table name --- src/taosdump.c | 59 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/src/taosdump.c b/src/taosdump.c index d9806c43..f2fff1be 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -1797,6 +1797,21 @@ static int getDumpDbCount() { return count; } +bool replaceCopy(char *des, char *src) { + size_t len = strlen(src); + bool replace = false; + for (size_t i = 0; i <= len; i++) { + if (src[i] == '.') { + des[i] = '_'; + replace = true; + } else { + des[i] = src[i]; + } + } + + return replace; +} + static int dumpCreateMTableClause( const char* dbName, const char *stable, @@ -1817,11 +1832,18 @@ static int dumpCreateMTableClause( char *pstr = NULL; pstr = tmpBuf; + // outName is output to file table name + char * outName = tableDes->name; + char tableName[TSDB_TABLE_NAME_LEN+1]; + if(replaceCopy(tableName, tableDes->name)) { + outName = tableName; + } + pstr += snprintf(tmpBuf, TSDB_DEFAULT_PKT_SIZE, g_args.db_escape_char ? "CREATE TABLE IF NOT EXISTS `%s`.%s%s%s USING `%s`.%s%s%s TAGS(" : "CREATE TABLE IF NOT EXISTS %s.%s%s%s USING %s.%s%s%s TAGS(", - dbName, g_escapeChar, tableDes->name, g_escapeChar, + dbName, g_escapeChar, outName, g_escapeChar, dbName, g_escapeChar, stable, g_escapeChar); for (; counter < numColsAndTags; counter++) { @@ -2921,11 +2943,18 @@ static int convertTableDesToSql( char* pstr = *buffer; + // outName is output to file table name + char * outName = tableDes->name; + char tableName[TSDB_TABLE_NAME_LEN+1]; + if(replaceCopy(tableName, tableDes->name)) { + outName = tableName; + } + pstr += sprintf(pstr, g_args.db_escape_char ? "CREATE TABLE IF NOT EXISTS `%s`.%s%s%s" : "CREATE TABLE IF NOT EXISTS %s.%s%s%s", - dbName, g_escapeChar, tableDes->name, g_escapeChar); + dbName, g_escapeChar, outName, g_escapeChar); for (; counter < tableDes->columns; counter++) { if (tableDes->cols[counter].note[0] != '\0') break; @@ -3311,7 +3340,9 @@ static int dumpCreateTableClauseAvro( } avro_value_set_branch(&value, 1, &branch); - avro_value_set_string(&branch, tableDes->name); + char tableName[TSDB_TABLE_NAME_LEN+1]; + replaceCopy(tableName, tableDes->name); + avro_value_set_string(&branch, tableName); if (0 != avro_value_get_by_name( &record, "sql", &value, NULL)) { @@ -3829,12 +3860,16 @@ static int convertTbDesToJsonImpl( const char *tbName, TableDes *tableDes, char **jsonSchema, bool isColumn) { + + char tableName[TSDB_TABLE_NAME_LEN + 1]; + replaceCopy(tableName, (char*) tbName); + char *pstr = *jsonSchema; pstr += sprintf(pstr, "{\"type\":\"record\",\"name\":\"%s.%s\",\"fields\":[", namespace, - (isColumn)?(g_args.loose_mode?tbName:"_record") - :(g_args.loose_mode?tbName:"_stb")); + (isColumn)?(g_args.loose_mode?tableName:"_record") + :(g_args.loose_mode?tableName:"_stb")); int iterate = 0; if (g_args.loose_mode) { @@ -4934,6 +4969,9 @@ static int64_t writeResultToAvroNative( return 0; } + char tableName[TSDB_TABLE_NAME_LEN + 1]; + replaceCopy(tableName, (char*) tbName); + avro_schema_t schema; RecordSchema *recordSchema; avro_file_writer_t db; @@ -4997,7 +5035,7 @@ static int64_t writeResultToAvroNative( break; } avro_value_set_branch(&avro_value, 1, &branch); - avro_value_set_string(&branch, tbName); + avro_value_set_string(&branch, tableName); } for (int32_t col = 0; col < numFields; col++) { @@ -8560,7 +8598,9 @@ static int createMTableAvroHeadImp( } avro_value_set_branch(&value, 1, &branch); - avro_value_set_string(&branch, stable); + char stableName[TSDB_TABLE_NAME_LEN + 1]; + replaceCopy(stableName, (char *)stable); + avro_value_set_string(&branch, stableName); } if (0 != avro_value_get_by_name( @@ -8571,7 +8611,9 @@ static int createMTableAvroHeadImp( } avro_value_set_branch(&value, 1, &branch); - avro_value_set_string(&branch, tbName); + char tableName[TSDB_TABLE_NAME_LEN + 1]; + replaceCopy(tableName, (char*)tbName); + avro_value_set_string(&branch, tableName); TableDes *subTableDes = (TableDes *) calloc(1, sizeof(TableDes) + sizeof(ColDes) * (stbTableDes->columns + stbTableDes->tags)); @@ -13062,3 +13104,4 @@ int main(int argc, char *argv[]) { } return ret; } + From e95531dcf2b47b149dcc3bdeb850d61873e3efdc Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 15 Aug 2023 11:02:38 +0800 Subject: [PATCH 27/57] add Z --nodrop database for command --- inc/bench.h | 1 + src/benchSys.c | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/inc/bench.h b/inc/bench.h index 764d3409..74f4e226 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -249,6 +249,7 @@ typedef unsigned __int32 uint32_t; "Specify interval between keep trying insert. " \ "Valid value is a positive number. Only valid " \ "when keep trying be enabled." +#define BENCH_NODROP "Do not drop database." #ifdef WEBSOCKET #define BENCH_DSN "The dsn to connect the cloud service." diff --git a/src/benchSys.c b/src/benchSys.c index 3b8a647f..ce807696 100644 --- a/src/benchSys.c +++ b/src/benchSys.c @@ -218,6 +218,7 @@ static struct argp_option bench_options[] = { {"vgroups", 'v', "NUMBER", 0, BENCH_VGROUPS}, #endif {"version", 'V', 0, 0, BENCH_VERSION}, + {"nodrop", 'Z', 0, 0, BENCH_NODROP}, {0} }; @@ -673,7 +674,9 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { g_arguments->inputted_vgroups = atoi(arg); break; #endif - + case 'Q': + database->drop = false; + break; case 'V': printVersion(); exit(0); From 84a30986ce81645c68c7eca4f5e24f89b915b1aa Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 15 Aug 2023 11:27:03 +0800 Subject: [PATCH 28/57] test: option modify to upper letter Q --- src/benchSys.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchSys.c b/src/benchSys.c index ce807696..e899af68 100644 --- a/src/benchSys.c +++ b/src/benchSys.c @@ -218,7 +218,7 @@ static struct argp_option bench_options[] = { {"vgroups", 'v', "NUMBER", 0, BENCH_VGROUPS}, #endif {"version", 'V', 0, 0, BENCH_VERSION}, - {"nodrop", 'Z', 0, 0, BENCH_NODROP}, + {"nodrop", 'Q', 0, 0, BENCH_NODROP}, {0} }; From 39470973b28034347c0e70a22ca0f046c349e3fe Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 15 Aug 2023 14:32:51 +0800 Subject: [PATCH 29/57] feat: add option -R --dot-replace --- src/taosdump.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/taosdump.c b/src/taosdump.c index f2fff1be..ec1e2ecb 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -493,6 +493,7 @@ static struct argp_option options[] = { "websocket to interact."}, #endif {"debug", 'g', 0, 0, "Print debug info.", 15}, + {"dot-replace", 'R', 0, 0, "Repalce dot character with underline character in the table name.", 10}, {0} }; @@ -543,6 +544,7 @@ typedef struct arguments { bool debug_print; bool verbose_print; bool performance_print; + bool dotReplace; int dumpDbCount; @@ -1061,6 +1063,11 @@ static void parse_args( || (0 == strcmp(argv[i], "--lose-mode"))) { g_args.loose_mode = true; strcpy(argv[i], ""); + // dot replace + } else if ((strcmp(argv[i], "-R") == 0) + || (0 == strcmp(argv[i], "--dot-replace"))) { + g_args.dotReplace = true; + strcpy(argv[i], ""); } else if (strcmp(argv[i], "-gg") == 0) { arguments->verbose_print = true; strcpy(argv[i], ""); From 21c7077320e5ccc36ccfebae95e71291537cea29 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 15 Aug 2023 15:46:50 +0800 Subject: [PATCH 30/57] feat: apply -R --dot-replace option to code --- src/taosdump.c | 45 +++++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/src/taosdump.c b/src/taosdump.c index ec1e2ecb..a8a87f38 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -1842,7 +1842,7 @@ static int dumpCreateMTableClause( // outName is output to file table name char * outName = tableDes->name; char tableName[TSDB_TABLE_NAME_LEN+1]; - if(replaceCopy(tableName, tableDes->name)) { + if(g_args.dotReplace && replaceCopy(tableName, tableDes->name)) { outName = tableName; } @@ -2953,7 +2953,7 @@ static int convertTableDesToSql( // outName is output to file table name char * outName = tableDes->name; char tableName[TSDB_TABLE_NAME_LEN+1]; - if(replaceCopy(tableName, tableDes->name)) { + if(g_args.dotReplace && replaceCopy(tableName, tableDes->name)) { outName = tableName; } @@ -3347,9 +3347,13 @@ static int dumpCreateTableClauseAvro( } avro_value_set_branch(&value, 1, &branch); - char tableName[TSDB_TABLE_NAME_LEN+1]; - replaceCopy(tableName, tableDes->name); - avro_value_set_string(&branch, tableName); + if(g_args.dotReplace) { + char tableName[TSDB_TABLE_NAME_LEN+1]; + replaceCopy(tableName, tableDes->name); + avro_value_set_string(&branch, tableName); + } else { + avro_value_set_string(&branch, tableDes->name); + } if (0 != avro_value_get_by_name( &record, "sql", &value, NULL)) { @@ -3868,15 +3872,18 @@ static int convertTbDesToJsonImpl( TableDes *tableDes, char **jsonSchema, bool isColumn) { + char* outName = tbName char tableName[TSDB_TABLE_NAME_LEN + 1]; - replaceCopy(tableName, (char*) tbName); + if(g_args.dotReplace && replaceCopy(tableName, (char*)tbName)) { + outName = tableName; + } char *pstr = *jsonSchema; pstr += sprintf(pstr, "{\"type\":\"record\",\"name\":\"%s.%s\",\"fields\":[", namespace, - (isColumn)?(g_args.loose_mode?tableName:"_record") - :(g_args.loose_mode?tableName:"_stb")); + (isColumn)?(g_args.loose_mode?outName:"_record") + :(g_args.loose_mode?outName:"_stb")); int iterate = 0; if (g_args.loose_mode) { @@ -4976,8 +4983,11 @@ static int64_t writeResultToAvroNative( return 0; } + char* outName = tbName char tableName[TSDB_TABLE_NAME_LEN + 1]; - replaceCopy(tableName, (char*) tbName); + if(g_args.dotReplace && replaceCopy(tableName, (char*)tbName)) { + outName = tableName; + } avro_schema_t schema; RecordSchema *recordSchema; @@ -5042,7 +5052,7 @@ static int64_t writeResultToAvroNative( break; } avro_value_set_branch(&avro_value, 1, &branch); - avro_value_set_string(&branch, tableName); + avro_value_set_string(&branch, outName); } for (int32_t col = 0; col < numFields; col++) { @@ -8605,9 +8615,12 @@ static int createMTableAvroHeadImp( } avro_value_set_branch(&value, 1, &branch); + char* outSName = stable char stableName[TSDB_TABLE_NAME_LEN + 1]; - replaceCopy(stableName, (char *)stable); - avro_value_set_string(&branch, stableName); + if(g_args.dotReplace && replaceCopy(stableName, (char*)stable)) { + outSName = stableName; + } + avro_value_set_string(&branch, outSName); } if (0 != avro_value_get_by_name( @@ -8618,9 +8631,13 @@ static int createMTableAvroHeadImp( } avro_value_set_branch(&value, 1, &branch); + + char* outName = tbName char tableName[TSDB_TABLE_NAME_LEN + 1]; - replaceCopy(tableName, (char*)tbName); - avro_value_set_string(&branch, tableName); + if(g_args.dotReplace && replaceCopy(tableName, (char*)tbName)) { + outName = tableName; + } + avro_value_set_string(&branch, outName); TableDes *subTableDes = (TableDes *) calloc(1, sizeof(TableDes) + sizeof(ColDes) * (stbTableDes->columns + stbTableDes->tags)); From 1f810f2537997fc6b13957ea329b8b18074909d3 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 15 Aug 2023 15:49:14 +0800 Subject: [PATCH 31/57] fix: build error --- src/taosdump.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/taosdump.c b/src/taosdump.c index a8a87f38..0d6560d9 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -3872,7 +3872,7 @@ static int convertTbDesToJsonImpl( TableDes *tableDes, char **jsonSchema, bool isColumn) { - char* outName = tbName + char* outName = tbName; char tableName[TSDB_TABLE_NAME_LEN + 1]; if(g_args.dotReplace && replaceCopy(tableName, (char*)tbName)) { outName = tableName; @@ -4983,7 +4983,7 @@ static int64_t writeResultToAvroNative( return 0; } - char* outName = tbName + char* outName = tbName; char tableName[TSDB_TABLE_NAME_LEN + 1]; if(g_args.dotReplace && replaceCopy(tableName, (char*)tbName)) { outName = tableName; @@ -8615,7 +8615,7 @@ static int createMTableAvroHeadImp( } avro_value_set_branch(&value, 1, &branch); - char* outSName = stable + char* outSName = stable; char stableName[TSDB_TABLE_NAME_LEN + 1]; if(g_args.dotReplace && replaceCopy(stableName, (char*)stable)) { outSName = stableName; @@ -8632,7 +8632,7 @@ static int createMTableAvroHeadImp( avro_value_set_branch(&value, 1, &branch); - char* outName = tbName + char* outName = tbName; char tableName[TSDB_TABLE_NAME_LEN + 1]; if(g_args.dotReplace && replaceCopy(tableName, (char*)tbName)) { outName = tableName; From 38fcf76e204f7f9dea2c4d2ebdf7dad2c59d261d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 15 Aug 2023 15:52:26 +0800 Subject: [PATCH 32/57] fix: build error1 --- src/taosdump.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/taosdump.c b/src/taosdump.c index 0d6560d9..19f70541 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -3872,7 +3872,7 @@ static int convertTbDesToJsonImpl( TableDes *tableDes, char **jsonSchema, bool isColumn) { - char* outName = tbName; + char* outName = (char*)tbName; char tableName[TSDB_TABLE_NAME_LEN + 1]; if(g_args.dotReplace && replaceCopy(tableName, (char*)tbName)) { outName = tableName; @@ -4983,7 +4983,7 @@ static int64_t writeResultToAvroNative( return 0; } - char* outName = tbName; + char* outName = (char*)tbName; char tableName[TSDB_TABLE_NAME_LEN + 1]; if(g_args.dotReplace && replaceCopy(tableName, (char*)tbName)) { outName = tableName; @@ -8615,7 +8615,7 @@ static int createMTableAvroHeadImp( } avro_value_set_branch(&value, 1, &branch); - char* outSName = stable; + char* outSName = (char*)stable; char stableName[TSDB_TABLE_NAME_LEN + 1]; if(g_args.dotReplace && replaceCopy(stableName, (char*)stable)) { outSName = stableName; @@ -8632,7 +8632,7 @@ static int createMTableAvroHeadImp( avro_value_set_branch(&value, 1, &branch); - char* outName = tbName; + char* outName = (char*)tbName; char tableName[TSDB_TABLE_NAME_LEN + 1]; if(g_args.dotReplace && replaceCopy(tableName, (char*)tbName)) { outName = tableName; From 48a430b776ac85edd9386799355c07b215ef231c Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 15 Aug 2023 17:58:26 +0800 Subject: [PATCH 33/57] fix: struct init warning --- src/taosdump.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/taosdump.c b/src/taosdump.c index 19f70541..2ac2d41b 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -609,6 +609,7 @@ struct arguments g_args = { false, // debug_print false, // verbose_print false, // performance_print + false, // dotRepalce 0, // dumpDbCount #ifdef WEBSOCKET false, // restful From 06af9192e21eb9df55293c18fb37ea4812b51338 Mon Sep 17 00:00:00 2001 From: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Date: Tue, 15 Aug 2023 18:03:33 +0800 Subject: [PATCH 34/57] fix: struct init warning (#697) --- src/taosdump.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/taosdump.c b/src/taosdump.c index 19f70541..2ac2d41b 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -609,6 +609,7 @@ struct arguments g_args = { false, // debug_print false, // verbose_print false, // performance_print + false, // dotRepalce 0, // dumpDbCount #ifdef WEBSOCKET false, // restful From 0c581dd67ece00aa56361c58b721a99988d6c317 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 16 Aug 2023 17:32:03 +0800 Subject: [PATCH 35/57] fix: memory leak and add cloud generate json file --- example/insertDemo.json | 92 +++++++++++++++++++++++++++++++++++++++++ src/benchUtil.c | 29 ++++++++----- 2 files changed, 110 insertions(+), 11 deletions(-) create mode 100644 example/insertDemo.json diff --git a/example/insertDemo.json b/example/insertDemo.json new file mode 100644 index 00000000..d76ff014 --- /dev/null +++ b/example/insertDemo.json @@ -0,0 +1,92 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "thread_count": 4, + "create_table_thread_count": 7, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 20000, + "prepared_rand": 10000, + "chinese": "no", + "escape_character": "yes", + "databases": [ + { + "dbinfo": { + "name": "smartmeters", + "drop": "yes", + "vgroups": 4, + "duration": 3650, + "pages": 512, + "minrows":10000, + "maxrows":20000, + "stt_trigger":1, + "wal_retention_period": 10, + "wal_retention_size": 100, + "cachemodel": "'both'", + "precision": "ms" + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 100000, + "childtable_prefix": "d", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "rand", + "insert_mode": "taosc", + "non_stop_mode": "no", + "line_protocol": "line", + "insert_rows": 100000, + "childtable_limit": 10, + "childtable_offset": 100, + "interlace_rows": 0, + "insert_interval": 0, + "partial_col_num": 0, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 900000, + "angle_step": 180000, + "start_timestamp": "2020-06-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "use_sample_ts": "no", + "tags_file": "", + "columns": [ + { "type": "FLOAT", "name": "current", "fun": "3*sin(x)+10*random(2)"}, + { "type": "INT", "name": "voltage", "fun": "40*sin(x)+200*random(10)"}, + { "type": "FLOAT", "name": "phase", "fun": "1*sin(x)+1*random(3)"} + ], + "tags": [ + { + "type": "TINYINT", + "name": "groupid", + "max": 10, + "min": 1 + }, + { + "name": "location", + "type": "BINARY", + "len": 16, + "values": ["BJ.HaiDian", "BJ.ChaoYang","BJ.FengTai", + "BJ.DongCheng", "BJ.XiCheng", "Bj.YanQing", "BJ.MenTouGou", + "BJ.HuaiRou", "BJ.PingGu", "BJ.MiYun", + "SH.HuangPu", "SH.XuHui", "SH.ChangNing", "SH.JingAn", "SH.PuTuo", + "SH.HongKou", "SH.YangPu", "SH.MinHang", "SH.BaoShan", "SH.JiaDing", + "SH.JinShan", "SH.SongJiang", "SH.QingPu", "SH.FengXian", "SH.ChongMing", + "SH.PuDong" + ] + } + ] + } + ] + } + ] +} diff --git a/src/benchUtil.c b/src/benchUtil.c index 07b1358f..6eb97972 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -583,17 +583,20 @@ static int getServerVersionRestImpl(int16_t rest_port, int sockfd) { } tools_cJSON* dataObj = tools_cJSON_GetObjectItem(resObj, "data"); if (!tools_cJSON_IsArray(dataObj)) { - errorPrint("Invalid or miss 'data' key in json: %s\n", - tools_cJSON_Print(resObj)); + char* pstr = tools_cJSON_Print(resObj); + errorPrint("Invalid or miss 'data' key in json: %s\n", pstr ? pstr : "null"); + tmfree(pstr); tools_cJSON_Delete(resObj); goto free_of_getversion; } tools_cJSON *versionObj = tools_cJSON_GetArrayItem(dataObj, 0); tools_cJSON *versionStrObj = tools_cJSON_GetArrayItem(versionObj, 0); server_ver = atoi(versionStrObj->valuestring); + char* pstr = tools_cJSON_Print(versionStrObj); debugPrint("versionStrObj: %s, version: %s, server_ver: %d\n", - tools_cJSON_Print(versionStrObj), + pstr ? pstr : "null", versionStrObj->valuestring, server_ver); + tmfree(pstr); tools_cJSON_Delete(resObj); } free_of_getversion: @@ -627,8 +630,9 @@ static int getCodeFromResp(char *responseBuf) { } tools_cJSON* codeObj = tools_cJSON_GetObjectItem(resObj, "code"); if (!tools_cJSON_IsNumber(codeObj)) { - errorPrint("Invalid or miss 'code' key in json: %s\n", - tools_cJSON_Print(resObj)); + char* pstr = tools_cJSON_Print(resObj); + errorPrint("Invalid or miss 'code' key in json: %s\n", pstr ? pstr : "null"); + tmfree(pstr); tools_cJSON_Delete(resObj); return -1; } @@ -638,8 +642,9 @@ static int getCodeFromResp(char *responseBuf) { if (codeObj->valueint != 0) { tools_cJSON* desc = tools_cJSON_GetObjectItem(resObj, "desc"); if (!tools_cJSON_IsString(desc)) { - errorPrint("Invalid or miss 'desc' key in json: %s\n", - tools_cJSON_Print(resObj)); + char* pstr = tools_cJSON_Print(resObj); + errorPrint("Invalid or miss 'desc' key in json: %s\n", pstr ? pstr : "null"); + tmfree(pstr); return -1; } errorPrint("response, code: %d, reason: %s\n", @@ -732,8 +737,9 @@ int postProceSql(char *sqlstr, char* dbName, int precision, int iface, && (TSDB_SML_JSON_PROTOCOL != protocol) && (SML_JSON_TAOS_FORMAT != protocol) ) { - errorPrint("Invalid or miss 'code' key in json: %s\n", - tools_cJSON_Print(resObj)); + char* pstr = tools_cJSON_Print(resObj); + errorPrint("Invalid or miss 'code' key in json: %s\n", pstr ? pstr : "null"); + tmfree(pstr); tools_cJSON_Delete(resObj); goto free_of_post; } @@ -751,8 +757,9 @@ int postProceSql(char *sqlstr, char* dbName, int precision, int iface, && (codeObj->valueint != 0) && (codeObj->valueint != 200)) { tools_cJSON* desc = tools_cJSON_GetObjectItem(resObj, "desc"); if (!tools_cJSON_IsString(desc)) { - errorPrint("Invalid or miss 'desc' key in json: %s\n", - tools_cJSON_Print(resObj)); + char* pstr = tools_cJSON_Print(resObj); + errorPrint("Invalid or miss 'desc' key in json: %s\n", pstr ? pstr : "null"); + tmfree(pstr); } else { errorPrint("insert mode response, code: %d, reason: %s\n", (int)codeObj->valueint, desc->valuestring); From 9cdef4e565664a8ebaac4e1bfb03d3ba72644c36 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 17 Aug 2023 17:09:19 +0800 Subject: [PATCH 36/57] chore: update taos-tools version (#699) --- VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index c4f35d94..5754c92f 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ -taosbenchmark-3.2.2 -taosdump-2.5.2 +taosbenchmark-3.2.3 +taosdump-2.5.3 2.5.3 From 0cdfcf3059b0782d08705475ec2a22f98d358ba4 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 23 Aug 2023 17:37:49 +0800 Subject: [PATCH 37/57] fix: dot-replace change option from R to Q --- src/taosdump.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/taosdump.c b/src/taosdump.c index 2ac2d41b..e1de127a 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -493,7 +493,7 @@ static struct argp_option options[] = { "websocket to interact."}, #endif {"debug", 'g', 0, 0, "Print debug info.", 15}, - {"dot-replace", 'R', 0, 0, "Repalce dot character with underline character in the table name.", 10}, + {"dot-replace", 'Q', 0, 0, "Repalce dot character with underline character in the table name.", 10}, {0} }; @@ -1065,7 +1065,7 @@ static void parse_args( g_args.loose_mode = true; strcpy(argv[i], ""); // dot replace - } else if ((strcmp(argv[i], "-R") == 0) + } else if ((strcmp(argv[i], "-Q") == 0) || (0 == strcmp(argv[i], "--dot-replace"))) { g_args.dotReplace = true; strcpy(argv[i], ""); From ba0b1c273dcf911f414f2e118c39cda056c6403d Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 30 Aug 2023 10:07:44 +0800 Subject: [PATCH 38/57] fix: --help on linux (#701) --- src/benchSys.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchSys.c b/src/benchSys.c index e899af68..456bf3af 100644 --- a/src/benchSys.c +++ b/src/benchSys.c @@ -217,7 +217,7 @@ static struct argp_option bench_options[] = { #ifdef TD_VER_COMPATIBLE_3_0_0_0 {"vgroups", 'v', "NUMBER", 0, BENCH_VGROUPS}, #endif - {"version", 'V', 0, 0, BENCH_VERSION}, +// {"version", 'V', 0, 0, BENCH_VERSION}, {"nodrop", 'Q', 0, 0, BENCH_NODROP}, {0} }; From 15016ce86c1832de5fc9c40174b019f158467555 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 12 Sep 2023 11:09:19 +0800 Subject: [PATCH 39/57] fix: if child table count error tips --- src/benchInsert.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index adf66b69..5c3f0f24 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -3528,7 +3528,11 @@ int insertTestProcess() { } } } - fillChildTblName(database, stbInfo); + // check fill child table count valid + if(fillChildTblName(database, stbInfo) <= 0) { + errorPrint(" fill child table is zero, please check parameters in json is correct. database:%s stb: %s \n", database->dbName, stbInfo->stbName); + return -1; + } if (0 != prepareSampleData(database, stbInfo)) { return -1; } From 9c910faa3c9bb7a02cea5f1b6253fb00c40481f3 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 13 Sep 2023 14:31:48 +0800 Subject: [PATCH 40/57] feat: support dbname rename during importing --- src/taosdump.c | 188 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 186 insertions(+), 2 deletions(-) diff --git a/src/taosdump.c b/src/taosdump.c index e1de127a..f3ebdc26 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -494,12 +494,20 @@ static struct argp_option options[] = { #endif {"debug", 'g', 0, 0, "Print debug info.", 15}, {"dot-replace", 'Q', 0, 0, "Repalce dot character with underline character in the table name.", 10}, + {"rename-dbname", 'W', 0, 0, "rename database name with new name. format demo: db1=newDB1|db2=newDB2", 10}, {0} }; #define HUMAN_TIME_LEN 28 #define DUMP_DIR_LEN (MAX_DIR_LEN - (TSDB_DB_NAME_LEN + 10)) +// rename db +typedef struct SRenameDB { + char* old; + char* new; + SRenameDB* next; +} + /* Used by main to communicate with parse_opt. */ typedef struct arguments { // connection option @@ -545,9 +553,7 @@ typedef struct arguments { bool verbose_print; bool performance_print; bool dotReplace; - int dumpDbCount; - #ifdef WEBSOCKET bool restful; bool cloud; @@ -557,6 +563,10 @@ typedef struct arguments { int cloudPort; char cloudHost[MAX_HOSTNAME_LEN]; #endif + + // put rename db string + char * renameBuf; + SRenameDB * renameHead; } SArguments; static resultStatistics g_resultStatistics = {0}; @@ -611,6 +621,7 @@ struct arguments g_args = { false, // performance_print false, // dotRepalce 0, // dumpDbCount + #ifdef WEBSOCKET false, // restful false, // cloud @@ -620,6 +631,9 @@ struct arguments g_args = { 0, // cloudPort {0}, // cloudHost #endif // WEBSOCKET + + NULL, // renameBuf + NULL // renameHead }; @@ -781,6 +795,71 @@ int64_t getEndTime(int precision) { return end_time; } +SRenameDB* newNode(char* first, SRenameDB* prev) { + SRenameDB* node = (SRenameDB*) malloc(sizeof(SRenameDB)); + memset(node, 0, sizeof(SRenameDB)); + node->old = first; + // link to list + if(prev) { + prev->next = node; + } + + return node; +} + +void setRenameDbs(char* arg) { + // malloc new + int len = strlen(arg); + if(len <= 2) { + return ; + } + len += 1; // include \0 + + // malloc + char* p = malloc(len); + int j = 0; // j is p pos + for (int i = 0; i < len; i++) { + if (arg[i] == ' ') { + // do nothing + } else if (arg[i] == '=' || arg[i] == '|') { + // set zero + p[j++] = 0; + } else { + // copy + p[j++] = arg[i]; + } + } + + // splite + SRenameDB* node = newNode(p); + g_args.renameHead = node; + for (int k = 0; k < j; k++) { + if(p[k] == 0 && k + 1 != j && k > 0) { + // string end and not last end + char* name = p + (k - 1); + if (node->new == NULL) { + node->new = name; + } else { + node = newNode(name, node); + } + } + } + + // end + g_args.renameBuf = p; +} + +// find newName +char* findNewName(char* oldName) { + SRenameDB* node = g_args.renameHead; + while(node) { + if (strcmp(node->old, oldName) == 0) { + return node->new; + } + node = node->next; + } +} + /* Parse a single option. */ static error_t parse_opt(int key, char *arg, struct argp_state *state) { /* Get the input argument from argp_parse, which we @@ -967,6 +1046,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { } state->next = state->argc; break; + case 'W': + setRenameDbs(arg); + break; default: return ARGP_ERR_UNKNOWN; @@ -7320,6 +7402,13 @@ static int64_t dumpInOneAvroFile( } const char *namespace = avro_schema_namespace((const avro_schema_t)schema); + if(g_args.renameHead) { + char* newDbName = findNewName(namespace); + if(newDbName) { + infoPrint(" ------- rename DB Name %s to %s ------\n", namespace, newDbName); + namespace = newDbName; + } + } debugPrint("%s() LN%d, Namespace: %s\n", __func__, __LINE__, namespace); @@ -10012,6 +10101,71 @@ bool convertDbClauseForV3(char **cmd) { return true; } +// repalce old name with new +char * replaceNewName(char* cmd, int len) { + // database name left char and right char + int nLeftSql = len; + char left = cmd[len]; + char right = "." ; + if(left == '`') { + right = left; + nLeftSql += 1; + } else { + right = "."; + } + + // get old database name + char oldName[TSDB_DB_NAME_LEN]; + char* s = &cmd[len + 1]; + char* e = strstr(s, right); + if(e == NULL) { + return NULL; + } + + int oldLen = e - s; + if(oldLen + 1 > TSDB_DB_NAME_LEN) { + return NULL; + } + memcpy(oldName, s, oldLen); + oldName[oldLen] = 0; + + // macth new database + char* newName = findNewName(oldName); + if(newName == NULL){ + return NULL; + } + + // malloc new buff put new sql with new name + int newLen = strlen(cmd) + (strlen(newName) - oldLen) + 1; + char* newCmd = (char *)malloc(newLen); + memset(newCmd, 0, newLen); + + // copy left + newName + right from cmd + memcpy(newCmd, cmd, nLeftSql); // left sql + strcat(newCmd, newName); // newName + strcat(newCmd, e); // right sql + + return newCmd; +} + +// if have database name rename, return new sql with new database name +// retrn value need call free() to free memory +char * afterRenameSql(char *cmd) { + // match pattern + const char* CREATE_DB = "CREATE DATABASE IF NOT EXISTS "; + const char* CREATE_TB = "CREATE TABLE IF NOT EXISTS "; + + char* pres[] = {CREATE_DB, CREATE_TB}; + for (int i = 0; i < sizeof(pres); i++ ) { + int len = strlen(pres[i]) + if (strncmp(cmd, pres[i], len) == 0) { + // found + return replaceNewName(cmd, len); + } + } + return NULL; +} + // dumpIn support multi threads functions static int64_t dumpInOneDebugFile( void* taos, FILE* fp, @@ -10090,7 +10244,19 @@ static int64_t dumpInOneDebugFile( ret = queryDbImplWS(taos, cmd); } else { #endif + if(g_args.renameHead) { + // have rename database options + char *newSql = afterRenameSql(cmd); + if(newSql) { + ret = queryDbImplNative(taos, newSql); + free(newSql); + } else { + ret = queryDbImplNative(taos, cmd); + } + } else { ret = queryDbImplNative(taos, cmd); + } + #ifdef WEBSOCKET } #endif @@ -13127,6 +13293,24 @@ int main(int argc, char *argv[]) { } else { ret = dumpEntry(); } + + // free buf + if (arguments.renameBuf) { + free(arguments.renameBuf); + arguments.renameBuf = NULL; + } + + // free node + SRenameDB* node = arguments.renameHead; + arguments.renameHead = NULL; + while(node) { + SRenameDB* next = node->next; + free(node); + node = next; + } + + + return ret; } From 67d3eb7b6e3fb7d979e12722d17fe30678ffad81 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 13 Sep 2023 15:13:46 +0800 Subject: [PATCH 41/57] fix: build error --- src/taosdump.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/taosdump.c b/src/taosdump.c index f3ebdc26..e03d778b 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -502,11 +502,12 @@ static struct argp_option options[] = { #define DUMP_DIR_LEN (MAX_DIR_LEN - (TSDB_DB_NAME_LEN + 10)) // rename db +struct SRenameDB; typedef struct SRenameDB { char* old; char* new; - SRenameDB* next; -} + void* next; +}SRenameDB; /* Used by main to communicate with parse_opt. */ typedef struct arguments { @@ -831,7 +832,7 @@ void setRenameDbs(char* arg) { } // splite - SRenameDB* node = newNode(p); + SRenameDB* node = newNode(p, NULL); g_args.renameHead = node; for (int k = 0; k < j; k++) { if(p[k] == 0 && k + 1 != j && k > 0) { @@ -857,7 +858,7 @@ char* findNewName(char* oldName) { return node->new; } node = node->next; - } + }; } /* Parse a single option. */ @@ -7403,7 +7404,7 @@ static int64_t dumpInOneAvroFile( const char *namespace = avro_schema_namespace((const avro_schema_t)schema); if(g_args.renameHead) { - char* newDbName = findNewName(namespace); + char* newDbName = findNewName((char *)namespace); if(newDbName) { infoPrint(" ------- rename DB Name %s to %s ------\n", namespace, newDbName); namespace = newDbName; @@ -10106,18 +10107,16 @@ char * replaceNewName(char* cmd, int len) { // database name left char and right char int nLeftSql = len; char left = cmd[len]; - char right = "." ; + char right = '.'; if(left == '`') { right = left; nLeftSql += 1; - } else { - right = "."; } // get old database name char oldName[TSDB_DB_NAME_LEN]; char* s = &cmd[len + 1]; - char* e = strstr(s, right); + char* e = strchr(s, right); if(e == NULL) { return NULL; } @@ -10155,9 +10154,9 @@ char * afterRenameSql(char *cmd) { const char* CREATE_DB = "CREATE DATABASE IF NOT EXISTS "; const char* CREATE_TB = "CREATE TABLE IF NOT EXISTS "; - char* pres[] = {CREATE_DB, CREATE_TB}; + const char* pres[] = {CREATE_DB, CREATE_TB}; for (int i = 0; i < sizeof(pres); i++ ) { - int len = strlen(pres[i]) + int len = strlen(pres[i]); if (strncmp(cmd, pres[i], len) == 0) { // found return replaceNewName(cmd, len); From ba01746100201537b8f0b331f8b57a6765692748 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 13 Sep 2023 15:16:03 +0800 Subject: [PATCH 42/57] fix: build error1 --- src/taosdump.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/taosdump.c b/src/taosdump.c index e03d778b..a7d6f5f1 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -857,7 +857,7 @@ char* findNewName(char* oldName) { if (strcmp(node->old, oldName) == 0) { return node->new; } - node = node->next; + node = (SRenameDB* )node->next; }; } @@ -13294,16 +13294,16 @@ int main(int argc, char *argv[]) { } // free buf - if (arguments.renameBuf) { - free(arguments.renameBuf); - arguments.renameBuf = NULL; + if (g_args.renameBuf) { + free(g_args.renameBuf); + g_args.renameBuf = NULL; } // free node - SRenameDB* node = arguments.renameHead; - arguments.renameHead = NULL; + SRenameDB* node = g_args.renameHead; + g_args.renameHead = NULL; while(node) { - SRenameDB* next = node->next; + SRenameDB* next = (SRenameDB*)node->next; free(node); node = next; } From 089086b422e108bf00eddfe0cf247a0cbbaade79 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 13 Sep 2023 15:18:55 +0800 Subject: [PATCH 43/57] fix: build error2 --- src/taosdump.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/taosdump.c b/src/taosdump.c index a7d6f5f1..bdfa7e2c 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -858,7 +858,8 @@ char* findNewName(char* oldName) { return node->new; } node = (SRenameDB* )node->next; - }; + } + return NULL; } /* Parse a single option. */ From 09718ff7999f0a52e65780a1d67c2397598e50d6 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 13 Sep 2023 15:46:20 +0800 Subject: [PATCH 44/57] fix: add option name --- src/taosdump.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/taosdump.c b/src/taosdump.c index bdfa7e2c..38506d04 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -494,7 +494,7 @@ static struct argp_option options[] = { #endif {"debug", 'g', 0, 0, "Print debug info.", 15}, {"dot-replace", 'Q', 0, 0, "Repalce dot character with underline character in the table name.", 10}, - {"rename-dbname", 'W', 0, 0, "rename database name with new name. format demo: db1=newDB1|db2=newDB2", 10}, + {"rename-dbname", 'W', "rename", 0, "rename database name with new name. format demo: db1=newDB1|db2=newDB2", 10}, {0} }; @@ -809,6 +809,7 @@ SRenameDB* newNode(char* first, SRenameDB* prev) { } void setRenameDbs(char* arg) { + if (arg == NULL) return ; // malloc new int len = strlen(arg); if(len <= 2) { From 44185a4c0f8319d4b2bfda56a4bc02646d791c9f Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 13 Sep 2023 15:52:27 +0800 Subject: [PATCH 45/57] fix: add option name1 --- src/taosdump.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/taosdump.c b/src/taosdump.c index 38506d04..e727cc5f 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -838,7 +838,7 @@ void setRenameDbs(char* arg) { for (int k = 0; k < j; k++) { if(p[k] == 0 && k + 1 != j && k > 0) { // string end and not last end - char* name = p + (k - 1); + char* name = &p[k] + 1; if (node->new == NULL) { node->new = name; } else { From eb3b2c4c5a2190f87a0521f5e861ea8cbaaa3852 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 13 Sep 2023 16:16:02 +0800 Subject: [PATCH 46/57] fix: left sql length --- src/taosdump.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/taosdump.c b/src/taosdump.c index e727cc5f..04a5436b 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -10117,7 +10117,7 @@ char * replaceNewName(char* cmd, int len) { // get old database name char oldName[TSDB_DB_NAME_LEN]; - char* s = &cmd[len + 1]; + char* s = &cmd[nLeftSql]; char* e = strchr(s, right); if(e == NULL) { return NULL; From 765e7b0a89b3d870002b2f7c8cd78ee96231d212 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 13 Sep 2023 17:21:04 +0800 Subject: [PATCH 47/57] fix: rename normal table sql --- src/taosdump.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/taosdump.c b/src/taosdump.c index 04a5436b..7a140682 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -6092,6 +6092,13 @@ static int64_t dumpInAvroNtbImpl( __func__, __LINE__); continue; } + + char* newBuf = afterRenameSql(buf); + if(newBuf) { + infoPrint(" rename database name for create normal table sql: \n old=%s\n new=%s\n", buf, newBuf); + buf = newBuf; + } + #ifdef WEBSOCKET if (g_args.cloud || g_args.restful) { WS_RES *ws_res = ws_query_timeout(taos, buf, g_args.ws_timeout); @@ -6111,6 +6118,9 @@ static int64_t dumpInAvroNtbImpl( } else { #endif TAOS_RES *res = taos_query(taos, buf); + if(newBuf) { + free(newBuf); + } int code = taos_errno(res); if (0 != code) { errorPrint("%s() LN%d," @@ -10119,8 +10129,15 @@ char * replaceNewName(char* cmd, int len) { char oldName[TSDB_DB_NAME_LEN]; char* s = &cmd[nLeftSql]; char* e = strchr(s, right); - if(e == NULL) { + char* e1 = strchr(s, ' '); + if(e == NULL && e1 == NULL) { return NULL; + } else if(e == NULL && e1) { + e = e1; + } else if(e && e1 ) { + if (e > e1) { + e = e1; + } } int oldLen = e - s; From bfb822d421b291ac9562133350b7d28546d078c9 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 13 Sep 2023 17:24:47 +0800 Subject: [PATCH 48/57] fix: rename normal table sql --- src/taosdump.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/taosdump.c b/src/taosdump.c index 7a140682..0227e1c0 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -1905,6 +1905,24 @@ bool replaceCopy(char *des, char *src) { return replace; } +// if have database name rename, return new sql with new database name +// retrn value need call free() to free memory +char * afterRenameSql(char *cmd) { + // match pattern + const char* CREATE_DB = "CREATE DATABASE IF NOT EXISTS "; + const char* CREATE_TB = "CREATE TABLE IF NOT EXISTS "; + + const char* pres[] = {CREATE_DB, CREATE_TB}; + for (int i = 0; i < sizeof(pres); i++ ) { + int len = strlen(pres[i]); + if (strncmp(cmd, pres[i], len) == 0) { + // found + return replaceNewName(cmd, len); + } + } + return NULL; +} + static int dumpCreateMTableClause( const char* dbName, const char *stable, @@ -10166,24 +10184,6 @@ char * replaceNewName(char* cmd, int len) { return newCmd; } -// if have database name rename, return new sql with new database name -// retrn value need call free() to free memory -char * afterRenameSql(char *cmd) { - // match pattern - const char* CREATE_DB = "CREATE DATABASE IF NOT EXISTS "; - const char* CREATE_TB = "CREATE TABLE IF NOT EXISTS "; - - const char* pres[] = {CREATE_DB, CREATE_TB}; - for (int i = 0; i < sizeof(pres); i++ ) { - int len = strlen(pres[i]); - if (strncmp(cmd, pres[i], len) == 0) { - // found - return replaceNewName(cmd, len); - } - } - return NULL; -} - // dumpIn support multi threads functions static int64_t dumpInOneDebugFile( void* taos, FILE* fp, From 81563d909a76d042b63f5a1c09a63e287eedbf30 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 13 Sep 2023 17:28:38 +0800 Subject: [PATCH 49/57] fix: adjust new function position --- src/taosdump.c | 170 ++++++++++++++++++++++++------------------------- 1 file changed, 85 insertions(+), 85 deletions(-) diff --git a/src/taosdump.c b/src/taosdump.c index 0227e1c0..bb542b3c 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -863,6 +863,91 @@ char* findNewName(char* oldName) { return NULL; } +bool replaceCopy(char *des, char *src) { + size_t len = strlen(src); + bool replace = false; + for (size_t i = 0; i <= len; i++) { + if (src[i] == '.') { + des[i] = '_'; + replace = true; + } else { + des[i] = src[i]; + } + } + + return replace; +} + +// repalce old name with new +char * replaceNewName(char* cmd, int len) { + // database name left char and right char + int nLeftSql = len; + char left = cmd[len]; + char right = '.'; + if(left == '`') { + right = left; + nLeftSql += 1; + } + + // get old database name + char oldName[TSDB_DB_NAME_LEN]; + char* s = &cmd[nLeftSql]; + char* e = strchr(s, right); + char* e1 = strchr(s, ' '); + if(e == NULL && e1 == NULL) { + return NULL; + } else if(e == NULL && e1) { + e = e1; + } else if(e && e1 ) { + if (e > e1) { + e = e1; + } + } + + int oldLen = e - s; + if(oldLen + 1 > TSDB_DB_NAME_LEN) { + return NULL; + } + memcpy(oldName, s, oldLen); + oldName[oldLen] = 0; + + // macth new database + char* newName = findNewName(oldName); + if(newName == NULL){ + return NULL; + } + + // malloc new buff put new sql with new name + int newLen = strlen(cmd) + (strlen(newName) - oldLen) + 1; + char* newCmd = (char *)malloc(newLen); + memset(newCmd, 0, newLen); + + // copy left + newName + right from cmd + memcpy(newCmd, cmd, nLeftSql); // left sql + strcat(newCmd, newName); // newName + strcat(newCmd, e); // right sql + + return newCmd; +} + +// if have database name rename, return new sql with new database name +// retrn value need call free() to free memory +char * afterRenameSql(char *cmd) { + // match pattern + const char* CREATE_DB = "CREATE DATABASE IF NOT EXISTS "; + const char* CREATE_TB = "CREATE TABLE IF NOT EXISTS "; + + const char* pres[] = {CREATE_DB, CREATE_TB}; + for (int i = 0; i < sizeof(pres); i++ ) { + int len = strlen(pres[i]); + if (strncmp(cmd, pres[i], len) == 0) { + // found + return replaceNewName(cmd, len); + } + } + return NULL; +} + /* Parse a single option. */ static error_t parse_opt(int key, char *arg, struct argp_state *state) { /* Get the input argument from argp_parse, which we @@ -1890,39 +1975,6 @@ static int getDumpDbCount() { return count; } -bool replaceCopy(char *des, char *src) { - size_t len = strlen(src); - bool replace = false; - for (size_t i = 0; i <= len; i++) { - if (src[i] == '.') { - des[i] = '_'; - replace = true; - } else { - des[i] = src[i]; - } - } - - return replace; -} - -// if have database name rename, return new sql with new database name -// retrn value need call free() to free memory -char * afterRenameSql(char *cmd) { - // match pattern - const char* CREATE_DB = "CREATE DATABASE IF NOT EXISTS "; - const char* CREATE_TB = "CREATE TABLE IF NOT EXISTS "; - - const char* pres[] = {CREATE_DB, CREATE_TB}; - for (int i = 0; i < sizeof(pres); i++ ) { - int len = strlen(pres[i]); - if (strncmp(cmd, pres[i], len) == 0) { - // found - return replaceNewName(cmd, len); - } - } - return NULL; -} - static int dumpCreateMTableClause( const char* dbName, const char *stable, @@ -10132,58 +10184,6 @@ bool convertDbClauseForV3(char **cmd) { return true; } -// repalce old name with new -char * replaceNewName(char* cmd, int len) { - // database name left char and right char - int nLeftSql = len; - char left = cmd[len]; - char right = '.'; - if(left == '`') { - right = left; - nLeftSql += 1; - } - - // get old database name - char oldName[TSDB_DB_NAME_LEN]; - char* s = &cmd[nLeftSql]; - char* e = strchr(s, right); - char* e1 = strchr(s, ' '); - if(e == NULL && e1 == NULL) { - return NULL; - } else if(e == NULL && e1) { - e = e1; - } else if(e && e1 ) { - if (e > e1) { - e = e1; - } - } - - int oldLen = e - s; - if(oldLen + 1 > TSDB_DB_NAME_LEN) { - return NULL; - } - memcpy(oldName, s, oldLen); - oldName[oldLen] = 0; - - // macth new database - char* newName = findNewName(oldName); - if(newName == NULL){ - return NULL; - } - - // malloc new buff put new sql with new name - int newLen = strlen(cmd) + (strlen(newName) - oldLen) + 1; - char* newCmd = (char *)malloc(newLen); - memset(newCmd, 0, newLen); - - // copy left + newName + right from cmd - memcpy(newCmd, cmd, nLeftSql); // left sql - strcat(newCmd, newName); // newName - strcat(newCmd, e); // right sql - - return newCmd; -} - // dumpIn support multi threads functions static int64_t dumpInOneDebugFile( void* taos, FILE* fp, From 97542a16a93e790beed2d8d1aea7588841076504 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 13 Sep 2023 17:41:25 +0800 Subject: [PATCH 50/57] fix: support websocket --- src/taosdump.c | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/taosdump.c b/src/taosdump.c index bb542b3c..e9626260 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -10255,24 +10255,22 @@ static int64_t dumpInOneDebugFile( } int ret; + char *newSql = NULL; + + if(g_args.renameHead) { + // have rename database options + newSql = afterRenameSql(cmd); + } debugPrint("%s() LN%d, cmd: %s\n", __func__, __LINE__, cmd); #ifdef WEBSOCKET if (g_args.cloud || g_args.restful) { - ret = queryDbImplWS(taos, cmd); + ret = queryDbImplWS(taos, newSql?newSql:cmd); } else { #endif - if(g_args.renameHead) { - // have rename database options - char *newSql = afterRenameSql(cmd); - if(newSql) { - ret = queryDbImplNative(taos, newSql); - free(newSql); - } else { - ret = queryDbImplNative(taos, cmd); - } - } else { - ret = queryDbImplNative(taos, cmd); + ret = queryDbImplNative(taos, newSql?newSql:cmd); + if(newSql) { + free(newSql); } #ifdef WEBSOCKET From aa652430f5b76c3853dba4e60e3681909eaa5cc3 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 13 Sep 2023 19:22:10 +0800 Subject: [PATCH 51/57] feat: taosdump add rename database option -W case --- tests/taosdump/native/taosdumpDbNtb.py | 6 +++--- tests/taosdump/native/taosdumpDbStb.py | 6 +++--- tests/taosdump/native/taosdumpDbWithNonRoot.py | 6 +++--- tests/taosdump/native/taosdumpEscapedDb.py | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/taosdump/native/taosdumpDbNtb.py b/tests/taosdump/native/taosdumpDbNtb.py index 99b2922a..d9f149f3 100644 --- a/tests/taosdump/native/taosdumpDbNtb.py +++ b/tests/taosdump/native/taosdumpDbNtb.py @@ -107,7 +107,7 @@ def run(self): tdSql.execute("drop database db") # sys.exit(1) - os.system("%s -i %s -T 1" % (binPath, self.tmpdir)) + os.system("%s -i %s -T 1 -W db=newdb" % (binPath, self.tmpdir)) tdSql.query("show databases") dbresult = tdSql.queryResult @@ -115,13 +115,13 @@ def run(self): found = False for i in range(len(dbresult)): print("Found db: %s" % dbresult[i][0]) - if dbresult[i][0] == "db": + if dbresult[i][0] == "newdb": found = True break assert found == True - tdSql.execute("use db") + tdSql.execute("use newdb") tdSql.query("show stables") tdSql.checkRows(1) tdSql.checkData(0, 0, "st") diff --git a/tests/taosdump/native/taosdumpDbStb.py b/tests/taosdump/native/taosdumpDbStb.py index 025a1a2c..bdc3f89d 100644 --- a/tests/taosdump/native/taosdumpDbStb.py +++ b/tests/taosdump/native/taosdumpDbStb.py @@ -107,7 +107,7 @@ def run(self): tdSql.execute("drop database db") # sys.exit(1) - os.system("%s -i %s -T 1" % (binPath, self.tmpdir)) + os.system("%s -i %s -T 1 -W db=newdb" % (binPath, self.tmpdir)) tdSql.query("show databases") dbresult = tdSql.queryResult @@ -115,13 +115,13 @@ def run(self): found = False for i in range(len(dbresult)): print("Found db: %s" % dbresult[i][0]) - if dbresult[i][0] == "db": + if dbresult[i][0] == "newdb": found = True break assert found == True - tdSql.execute("use db") + tdSql.execute("use newdb") tdSql.query("show stables") tdSql.checkRows(1) tdSql.checkData(0, 0, "st") diff --git a/tests/taosdump/native/taosdumpDbWithNonRoot.py b/tests/taosdump/native/taosdumpDbWithNonRoot.py index bdc74611..6bb7eb1d 100644 --- a/tests/taosdump/native/taosdumpDbWithNonRoot.py +++ b/tests/taosdump/native/taosdumpDbWithNonRoot.py @@ -109,7 +109,7 @@ def run(self): tdSql.execute("drop database db") # sys.exit(1) - os.system("%s -i %s -T 1" % (binPath, self.tmpdir)) + os.system("%s -i %s -T 1 -W db=newdb" % (binPath, self.tmpdir)) tdSql.query("show databases") dbresult = tdSql.queryResult @@ -117,13 +117,13 @@ def run(self): found = False for i in range(len(dbresult)): print("Found db: %s" % dbresult[i][0]) - if dbresult[i][0] == "db": + if dbresult[i][0] == "newdb": found = True break assert found == True - tdSql.execute("use db") + tdSql.execute("use newdb") tdSql.query("show stables") tdSql.checkRows(1) tdSql.checkData(0, 0, "st") diff --git a/tests/taosdump/native/taosdumpEscapedDb.py b/tests/taosdump/native/taosdumpEscapedDb.py index 19fb8df7..f9bb4607 100644 --- a/tests/taosdump/native/taosdumpEscapedDb.py +++ b/tests/taosdump/native/taosdumpEscapedDb.py @@ -93,7 +93,7 @@ def run(self): tdSql.execute("drop database `Db`") # sys.exit(1) - os.system("%s -e -i %s -T 1" % (binPath, self.tmpdir)) + os.system("%s -e -i %s -T 1 -W Db=NewDb" % (binPath, self.tmpdir)) tdSql.query("show databases") dbresult = tdSql.queryResult @@ -101,13 +101,13 @@ def run(self): found = False for i in range(len(dbresult)): print("Found db: %s" % dbresult[i][0]) - if dbresult[i][0] == "Db": + if dbresult[i][0] == "NewDb": found = True break assert found == True - tdSql.execute("use `Db`") + tdSql.execute("use `NewDb`") tdSql.query("show stables") tdSql.checkRows(1) tdSql.checkData(0, 0, "st") From d302253c8603c92d9643047804e4e06b33822d71 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 13 Sep 2023 19:36:55 +0800 Subject: [PATCH 52/57] fix: add old case --- tests/taosdump/old/taosdumpTest.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/taosdump/old/taosdumpTest.py b/tests/taosdump/old/taosdumpTest.py index 3f4dad35..0d2f8f22 100644 --- a/tests/taosdump/old/taosdumpTest.py +++ b/tests/taosdump/old/taosdumpTest.py @@ -107,10 +107,10 @@ def run(self): tdSql.query("select * from information_schema.ins_databases") tdSql.checkRows(2) - os.system("%s -i ./taosdumptest/tmp1" % binPath) - os.system("%s -i ./taosdumptest/tmp2" % binPath) + os.system("%s -W db=newdb -i ./taosdumptest/tmp1" % binPath) + os.system("%s -W \"db=newdb|db1=newdb1\" -i ./taosdumptest/tmp2" % binPath) - tdSql.execute("use db") + tdSql.execute("use newdb") tdSql.query("select * from information_schema.ins_databases") tdSql.checkRows(4) dbresult = tdSql.queryResult @@ -119,7 +119,7 @@ def run(self): isCommunity = self.checkCommunity() print("iscommunity: %d" % isCommunity) for i in range(len(dbresult)): - if dbresult[i][0] == "db": + if dbresult[i][0] == "newdb": print(dbresult[i]) print(type(dbresult[i][6])) print(type(dbresult[i][7])) @@ -127,7 +127,7 @@ def run(self): assert dbresult[i][6] == "15840m" print((dbresult[i][7])) assert dbresult[i][7] == "5254560m,5254560m,5254560m" - if dbresult[i][0] == "db1": + if dbresult[i][0] == "newdb1": print((dbresult[i][6])) assert dbresult[i][6] == "17280m" print((dbresult[i][7])) @@ -158,8 +158,8 @@ def run(self): # drop all databases，boundary value testing. # length(databasename)<=32;length(tablesname)<=192 - tdSql.execute("drop database db") - tdSql.execute("drop database db1") + tdSql.execute("drop database newdb") + tdSql.execute("drop database newdb1") os.system("rm -rf ./taosdumptest/tmp1") os.system("rm -rf ./taosdumptest/tmp2") os.makedirs("./taosdumptest/tmp1") @@ -194,8 +194,8 @@ def run(self): % binPath ) tdSql.execute("drop database db12312313231231321312312312_323") - os.system("%s -i ./taosdumptest/tmp1" % binPath) - tdSql.execute("use db12312313231231321312312312_323") + os.system("%s -W db12312313231231321312312312_323=db12312313231231321312312312_323abc -i ./taosdumptest/tmp1" % binPath) + tdSql.execute("use db12312313231231321312312312_323abc") tdSql.query("show stables") tdSql.checkRows(2) os.system("rm -rf ./taosdumptest/tmp1") From 330f11a42e05fe1c60818ac4a50b3a4ad82d0aea Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 13 Sep 2023 20:09:31 +0800 Subject: [PATCH 53/57] fix: case failed for escapedb.py --- tests/taosdump/native/taosdumpEscapedDb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/taosdump/native/taosdumpEscapedDb.py b/tests/taosdump/native/taosdumpEscapedDb.py index f9bb4607..77a17f76 100644 --- a/tests/taosdump/native/taosdumpEscapedDb.py +++ b/tests/taosdump/native/taosdumpEscapedDb.py @@ -112,7 +112,7 @@ def run(self): tdSql.checkRows(1) tdSql.checkData(0, 0, "st") - tdSql.query("select count(*) from `Db`.st") + tdSql.query("select count(*) from `NewDb`.st") tdSql.checkData(0, 0, 1) def stop(self): From 7360894abf6c8a3be8e87fb73e8e73a95dd90333 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 13 Sep 2023 21:02:53 +0800 Subject: [PATCH 54/57] fix: help information and -p can crash --- src/taosdump.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/taosdump.c b/src/taosdump.c index e9626260..0882cd1b 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -437,7 +437,7 @@ static struct argp_option options[] = { "Server host from which to dump data. Default is localhost.", 0}, {"user", 'u', "USER", 0, "User name used to connect to server. Default is root.", 0}, - {"password", 'p', 0, 0, + {"password", 'p', "PASS", 0, "User password to connect to server. Default is taosdata.", 0}, {"port", 'P', "PORT", 0, "Port to connect", 0}, // input/output file @@ -494,7 +494,7 @@ static struct argp_option options[] = { #endif {"debug", 'g', 0, 0, "Print debug info.", 15}, {"dot-replace", 'Q', 0, 0, "Repalce dot character with underline character in the table name.", 10}, - {"rename-dbname", 'W', "rename", 0, "rename database name with new name. format demo: db1=newDB1|db2=newDB2", 10}, + {"rename-dbname", 'W', "rename", 0, "rename database name with new name. format demo: -W \"db1=newDB1|db2=newDB2\" means rename db1 to newDB1 and rename db2 to newDB2", 10}, {0} }; From 97cca0439257d361c0a947d48fdc98e57e40af7a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 13 Sep 2023 21:06:25 +0800 Subject: [PATCH 55/57] fix: help information and reset -p --- src/taosdump.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/taosdump.c b/src/taosdump.c index 0882cd1b..a12bb6d4 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -437,7 +437,7 @@ static struct argp_option options[] = { "Server host from which to dump data. Default is localhost.", 0}, {"user", 'u', "USER", 0, "User name used to connect to server. Default is root.", 0}, - {"password", 'p', "PASS", 0, + {"password", 'p', 0, 0, "User password to connect to server. Default is taosdata.", 0}, {"port", 'P', "PORT", 0, "Port to connect", 0}, // input/output file From 4106fb74d9774ef27ee7c0a60d0bf2a853481b93 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 13 Sep 2023 21:23:21 +0800 Subject: [PATCH 56/57] fix: help information --- src/taosdump.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/taosdump.c b/src/taosdump.c index a12bb6d4..97f5ed99 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -494,7 +494,7 @@ static struct argp_option options[] = { #endif {"debug", 'g', 0, 0, "Print debug info.", 15}, {"dot-replace", 'Q', 0, 0, "Repalce dot character with underline character in the table name.", 10}, - {"rename-dbname", 'W', "rename", 0, "rename database name with new name. format demo: -W \"db1=newDB1|db2=newDB2\" means rename db1 to newDB1 and rename db2 to newDB2", 10}, + {"rename", 'W', "RENAME-LIST", 0, "Rename database name with new name during importing data. RENAME-LIST: \"db1=newDB1|db2=newDB2\" means rename db1 to newDB1 and rename db2 to newDB2", 10}, {0} }; From 4148259f491c3df3531750916a9fc41de63cbe7f Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 14 Sep 2023 11:09:56 +0800 Subject: [PATCH 57/57] fix: modify version to 2.5.4 --- VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 5754c92f..2d9d10b2 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ taosbenchmark-3.2.3 -taosdump-2.5.3 -2.5.3 +taosdump-2.5.4 +2.5.4