diff --git a/VERSION b/VERSION index c4f35d94..2d9d10b2 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ -taosbenchmark-3.2.2 -taosdump-2.5.2 -2.5.3 +taosbenchmark-3.2.3 +taosdump-2.5.4 +2.5.4 diff --git a/example/insertDemo.json b/example/insertDemo.json new file mode 100644 index 00000000..d76ff014 --- /dev/null +++ b/example/insertDemo.json @@ -0,0 +1,92 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "thread_count": 4, + "create_table_thread_count": 7, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 20000, + "prepared_rand": 10000, + "chinese": "no", + "escape_character": "yes", + "databases": [ + { + "dbinfo": { + "name": "smartmeters", + "drop": "yes", + "vgroups": 4, + "duration": 3650, + "pages": 512, + "minrows":10000, + "maxrows":20000, + "stt_trigger":1, + "wal_retention_period": 10, + "wal_retention_size": 100, + "cachemodel": "'both'", + "precision": "ms" + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 100000, + "childtable_prefix": "d", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "rand", + "insert_mode": "taosc", + "non_stop_mode": "no", + "line_protocol": "line", + "insert_rows": 100000, + "childtable_limit": 10, + "childtable_offset": 100, + "interlace_rows": 0, + "insert_interval": 0, + "partial_col_num": 0, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 900000, + "angle_step": 180000, + "start_timestamp": "2020-06-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "use_sample_ts": "no", + "tags_file": "", + "columns": [ + { "type": "FLOAT", "name": "current", "fun": "3*sin(x)+10*random(2)"}, + { "type": "INT", "name": "voltage", "fun": "40*sin(x)+200*random(10)"}, + { "type": "FLOAT", "name": "phase", "fun": "1*sin(x)+1*random(3)"} + ], + "tags": [ + { + "type": "TINYINT", + "name": "groupid", + "max": 10, + "min": 1 + }, + { + "name": "location", + "type": "BINARY", + "len": 16, + "values": ["BJ.HaiDian", "BJ.ChaoYang","BJ.FengTai", + "BJ.DongCheng", "BJ.XiCheng", "Bj.YanQing", "BJ.MenTouGou", + "BJ.HuaiRou", "BJ.PingGu", "BJ.MiYun", + "SH.HuangPu", "SH.XuHui", "SH.ChangNing", "SH.JingAn", "SH.PuTuo", + "SH.HongKou", "SH.YangPu", "SH.MinHang", "SH.BaoShan", "SH.JiaDing", + "SH.JinShan", "SH.SongJiang", "SH.QingPu", "SH.FengXian", "SH.ChongMing", + "SH.PuDong" + ] + } + ] + } + ] + } + ] +} diff --git a/example/insert_math.json b/example/insert_math.json new file mode 100644 index 00000000..f272c492 --- /dev/null +++ b/example/insert_math.json @@ -0,0 +1,78 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "thread_count": 4, + "create_table_thread_count": 7, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 100, + "prepared_rand": 3600, + "chinese": "no", + "escape_character": "yes", + "databases": [ + { + "dbinfo": { + "name": "test", + "drop": "yes", + "precision": "ms" + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 1000, + "childtable_prefix": "d", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "rand", + "insert_mode": "taosc", + "non_stop_mode": "no", + "line_protocol": "line", + "insert_rows": 1000, + "childtable_limit": 10, + "childtable_offset": 100, + "interlace_rows": 0, + "insert_interval": 0, + "partial_col_num": 0, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1000, + "angle_step": 500, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "use_sample_ts": "no", + "tags_file": "", + "columns": [ + { "type": "FLOAT", "name": "current", "fun": "3*sin(x)+10*random(2)"}, + { "type": "INT", "name": "voltage", "fun": "40*sin(x)+200*random(10)"}, + { "type": "FLOAT", "name": "phase", "fun": "1*sin(x)+1*random(3)"} + ], + "tags": [ + { + "type": "TINYINT", + "name": "groupid", + "max": 10, + "min": 1 + }, + { + "name": "location", + "type": "BINARY", + "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/inc/bench.h b/inc/bench.h index 275e07fc..74f4e226 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -201,6 +201,7 @@ typedef unsigned __int32 uint32_t; #define BENCH_INTERVAL \ "Insert interval for interlace mode in milliseconds, default is 0." #define BENCH_STEP "Timestamp step in milliseconds, default is 1." +#define ANGLE_STEP "Angle step in milliseconds, default is 1." #define BENCH_SUPPLEMENT \ "Supplementally insert data without create " \ "database and table, optional, default is off." @@ -248,6 +249,7 @@ typedef unsigned __int32 uint32_t; "Specify interval between keep trying insert. " \ "Valid value is a positive number. Only valid " \ "when keep trying be enabled." +#define BENCH_NODROP "Do not drop database." #ifdef WEBSOCKET #define BENCH_DSN "The dsn to connect the cloud service." @@ -553,6 +555,15 @@ typedef struct SChildField { StmtData stmtData; } ChildField; +#define PI 3.141592654 +#define ATOR(x) (x*3.141592654/180) + +#define FUNTYPE_NONE 0 +#define FUNTYPE_SIN 1 +#define FUNTYPE_COS 2 + +#define FUNTYPE_CNT 2 + typedef struct SField { uint8_t type; char name[TSDB_COL_NAME_LEN + 1]; @@ -563,6 +574,13 @@ typedef struct SField { int64_t max; int64_t min; tools_cJSON * values; + + // fun + uint8_t funType; + float multiple; + int32_t addend; + int32_t random; + bool sma; } Field; @@ -666,6 +684,7 @@ typedef struct SSuperTable_S { uint64_t insert_interval; uint64_t insertRows; uint64_t timestamp_step; + uint64_t angle_step; int64_t startTimestamp; int64_t specifiedColumns; char sampleFile[MAX_FILE_NAME_LEN]; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 07174fb7..06fda650 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -308,8 +308,8 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin FIND_LIBRARY(LIBZ_LIBRARY z) MESSAGE(${ARGP_LIBRARY}) - TARGET_LINK_LIBRARIES(taosBenchmark taos pthread toolscJson $<$:${LIBZ_LIBRARY}> $<$:${ARGP_LIBRARY}> ${WEBSOCKET_LINK_FLAGS}) - TARGET_LINK_LIBRARIES(taosdump taos avro jansson atomic pthread argp $<$:${LIBZ_LIBRARY}> $<$:${ARGP_LIBRARY}> ${WEBSOCKET_LINK_FLAGS}) + TARGET_LINK_LIBRARIES(taosBenchmark taos pthread m toolscJson $<$:${LIBZ_LIBRARY}> $<$:${ARGP_LIBRARY}> ${WEBSOCKET_LINK_FLAGS}) + TARGET_LINK_LIBRARIES(taosdump taos avro jansson atomic pthread m argp $<$:${LIBZ_LIBRARY}> $<$:${ARGP_LIBRARY}> ${WEBSOCKET_LINK_FLAGS}) ELSEIF(${OS_ID} MATCHES "Darwin") ADD_LIBRARY(argp STATIC IMPORTED) IF (CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") @@ -319,11 +319,11 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin SET_PROPERTY(TARGET argp PROPERTY IMPORTED_LOCATION "/usr/local/lib/libargp.a") INCLUDE_DIRECTORIES(/usr/local/include/) ENDIF () - TARGET_LINK_LIBRARIES(taosBenchmark taos pthread toolscJson argp ${WEBSOCKET_LINK_FLAGS}) + TARGET_LINK_LIBRARIES(taosBenchmark taos pthread m toolscJson argp ${WEBSOCKET_LINK_FLAGS}) ElSE () MESSAGE("${Yellow} DEBUG mode use shared avro library to link for debug ${ColourReset}") - TARGET_LINK_LIBRARIES(taosdump taos avro jansson atomic pthread ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}) - TARGET_LINK_LIBRARIES(taosBenchmark taos pthread toolscJson ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}) + TARGET_LINK_LIBRARIES(taosdump taos avro jansson atomic pthread m ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}) + TARGET_LINK_LIBRARIES(taosBenchmark taos pthread m toolscJson ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}) ENDIF() ELSE () @@ -362,7 +362,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin INCLUDE_DIRECTORIES(/usr/local/include/) ENDIF () - TARGET_LINK_LIBRARIES(taosBenchmark taos pthread toolscJson argp ${WEBSOCKET_LINK_FLAGS}) + TARGET_LINK_LIBRARIES(taosBenchmark taos pthread m toolscJson argp ${WEBSOCKET_LINK_FLAGS}) ELSE () EXECUTE_PROCESS ( COMMAND sh -c "awk -F= '/^ID=/{print $2}' /etc/os-release |tr -d '\n' | tr -d '\"'" @@ -414,10 +414,10 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin MESSAGE(${LIBZ_LIBRARY}) TARGET_LINK_LIBRARIES(taosdump taos avro jansson snappy stdc++ lzma atomic pthread $<$:${LIBZ_LIBRARY}> $<$:${ARGP_LIBRARY}> ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}) - TARGET_LINK_LIBRARIES(taosBenchmark taos pthread toolscJson $<$:${LIBZ_LIBRARY}> $<$:${ARGP_LIBRARY}> ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}) + TARGET_LINK_LIBRARIES(taosBenchmark taos pthread m toolscJson $<$:${LIBZ_LIBRARY}> $<$:${ARGP_LIBRARY}> ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}) ELSE() TARGET_LINK_LIBRARIES(taosdump taos avro jansson snappy stdc++ lzma libz-static atomic pthread ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}) - TARGET_LINK_LIBRARIES(taosBenchmark taos pthread toolscJson ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}) + TARGET_LINK_LIBRARIES(taosBenchmark taos pthread m toolscJson ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}) ENDIF() ENDIF () @@ -443,9 +443,9 @@ ELSE () IF (${WEBSOCKET}) INCLUDE_DIRECTORIES(/usr/local/include/) - SET(WEBSOCKET_LINK_FLAGS "taosws.dll.lib") + SET(WEBSOCKET_LINK_FLAGS "taosws.lib") IF (${CMAKE_PROJECT_NAME} STREQUAL "taos-tools") - MESSAGE("taosws.dll.lib need to be installed first") + MESSAGE("taosws.lib need to be installed first") ELSE () ADD_DEPENDENCIES(taosBenchmark taosws-rs) ADD_DEPENDENCIES(taosdump taosws-rs) diff --git a/src/benchCommandOpt.c b/src/benchCommandOpt.c index b776117b..8f1fb171 100644 --- a/src/benchCommandOpt.c +++ b/src/benchCommandOpt.c @@ -137,11 +137,28 @@ static void initStable() { tstrncpy(c3->name, "phase", TSDB_COL_NAME_LEN + 1); c1->min = 9; - c1->max = 10; + c1->max = 10; + // fun = "3*sin(x)+10*random(2)" + c1->funType = FUNTYPE_SIN; + c1->multiple = 3; + c1->random = 2; + c1->addend = 10; + c2->min = 110; c2->max = 119; + // fun = "40*sin(x)+200*random(10)" + c2->funType = FUNTYPE_SIN; + c2->multiple = 40; + c2->random = 10; + c2->addend = 200; + c3->min = 115; c3->max = 125; + // fun = "1*sin(x)+1*random(3)" + c3->funType = FUNTYPE_SIN; + c3->multiple = 1; + c3->random = 3; + c3->addend = 1; stbInfo->tags = benchArrayInit(2, sizeof(Field)); for (int i = 0; i < 2; ++i) { @@ -166,6 +183,7 @@ static void initStable() { stbInfo->insert_interval = 0; stbInfo->timestamp_step = 1; + stbInfo->angle_step = 1; stbInfo->interlaceRows = 0; stbInfo->childTblCount = DEFAULT_CHILDTABLES; stbInfo->childTblLimit = 0; @@ -349,7 +367,7 @@ void modifyArgument() { if (g_arguments->demo_mode && TAOSC_IFACE == g_arguments->iface) { g_arguments->mistMode = true; - g_arguments->prepared_rand = 57; + g_arguments->prepared_rand = 360; } } diff --git a/src/benchData.c b/src/benchData.c index d0907754..a196ada3 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -11,6 +11,7 @@ */ #include +#include #include const char charset[] = @@ -41,6 +42,25 @@ const char* locations_sml[] = { #include "benchLocations.h" #endif +// calc expression value like 10*sin(x) + 100 +float calc_expr_value(Field *field, int32_t angle) { + float radian = ATOR(angle); + float funVal = 0; + if (field->funType == FUNTYPE_SIN) + funVal = sin(radian); + else if (field->funType == FUNTYPE_COS) + funVal = cos(radian); + + float val = field->multiple * funVal + field->addend; + if (field->random >0) { + float rate = taosRandom() % field->random; + val += field->addend * (rate/100); + } + + return val; +} + + static int usc2utf8(char *p, int unic) { int ret = 0; if (unic <= 0x0000007F) { @@ -367,9 +387,12 @@ static int tmpStr(char *tmp, int iface, Field *field, int i) { return 0; } -FORCE_INLINE double tmpDouble(Field *field) { +FORCE_INLINE double tmpDoubleImpl(Field *field, int32_t angle) { double doubleTmp = (double)(field->min); - if (field->max != field->min) { + + if(field->funType != FUNTYPE_NONE) { + doubleTmp = calc_expr_value(field, angle); + } else if (field->max != field->min) { doubleTmp += ((taosRandom() % (field->max - field->min)) + taosRandom() % 1000000 / 1000000.0); @@ -377,6 +400,11 @@ FORCE_INLINE double tmpDouble(Field *field) { return doubleTmp; } +FORCE_INLINE double tmpDouble(Field *field) { + return tmpDoubleImpl(field, 0); +} + + FORCE_INLINE uint64_t tmpUint64(Field *field) { uint64_t ubigintTmp = field->min; if (field->max != field->min) { @@ -435,14 +463,20 @@ FORCE_INLINE uint16_t tmpUint16(Field *field) { return usmallintTmp; } -FORCE_INLINE int64_t tmpInt64(Field *field) { +FORCE_INLINE int64_t tmpInt64Impl(Field *field, int32_t angle) { int64_t bigintTmp = field->min; - if (field->min != field->max) { + if(field->funType != FUNTYPE_NONE) { + bigintTmp = calc_expr_value(field, angle); + } else if (field->min != field->max) { bigintTmp += (taosRandom() % (field->max - field->min)); } return bigintTmp; } +FORCE_INLINE int64_t tmpInt64(Field *field) { + return tmpInt64Impl(field, 0); +} + FORCE_INLINE float tmpFloat(Field *field) { float floatTmp = field->min; if (field->max != field->min) { @@ -452,26 +486,37 @@ FORCE_INLINE float tmpFloat(Field *field) { return floatTmp; } -static float tmpFloatI(Field *field, int i) { +static float tmpFloatImpl(Field *field, int i, int32_t angle) { float floatTmp = (float)field->min; - if (field->max != field->min) { - floatTmp += ((taosRandom() % - (field->max - field->min)) - + (taosRandom() % 1000) / 1000.0); - } - if (g_arguments->demo_mode && i == 0) { - floatTmp = (float)(9.8 + 0.04 * (taosRandom() % 10) - + floatTmp / 1000000000); - } else if (g_arguments->demo_mode && i == 2) { - floatTmp = (float)((105 + taosRandom() % 10 - + floatTmp / 1000000000) / 360); + if(field->funType != FUNTYPE_NONE) { + floatTmp = calc_expr_value(field, angle); + } else { + if (field->max != field->min) { + floatTmp += ((taosRandom() % + (field->max - field->min)) + + (taosRandom() % 1000) / 1000.0); + } + if (g_arguments->demo_mode && i == 0) { + floatTmp = (float)(9.8 + 0.04 * (taosRandom() % 10) + + floatTmp / 1000000000); + } else if (g_arguments->demo_mode && i == 2) { + floatTmp = (float)((105 + taosRandom() % 10 + + floatTmp / 1000000000) / 360); + } } return floatTmp; } -static int tmpInt32(Field *field, int i) { +static float tmpFloatI(Field *field, int i) { + return tmpFloatImpl(field, i, 0); +} + +static int tmpInt32Impl(Field *field, int i, int angle) { int intTmp; - if ((g_arguments->demo_mode) && (i == 0)) { + if (field->funType != FUNTYPE_NONE) { + // calc from function + intTmp = calc_expr_value(field, angle); + } else if ((g_arguments->demo_mode) && (i == 0)) { unsigned int tmpRand = taosRandom(); intTmp = tmpRand % 10 + 1; } else if ((g_arguments->demo_mode) && (i == 1)) { @@ -491,6 +536,10 @@ static int tmpInt32(Field *field, int i) { return intTmp; } +static int tmpInt32(Field *field, int i) { + return tmpInt32Impl(field, i, 0); +} + static int tmpJson(char *sampleDataBuf, int bufLen, int64_t pos, int fieldsSize, Field *field) { @@ -545,6 +594,8 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf, int64_t bufLen, int lenOfOneRow, BArray * fields, int64_t loop, bool tag) { + + int angle = stbInfo->startTimestamp % 360; // 0 ~ 360 for (int64_t k = 0; k < loop; ++k) { int64_t pos = k * lenOfOneRow; int fieldsSize = fields->size; @@ -609,13 +660,13 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_INT: { - int32_t intTmp = tmpInt32(field, i); + int32_t intTmp = tmpInt32Impl(field, i, angle); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%d,", intTmp); break; } case TSDB_DATA_TYPE_BIGINT: { - int64_t bigintTmp = tmpInt64(field); + int64_t bigintTmp = tmpInt64Impl(field, angle); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%"PRId64",", bigintTmp); break; @@ -634,13 +685,13 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_FLOAT: { - float floatTmp = tmpFloatI(field, i); + float floatTmp = tmpFloatImpl(field, i, angle); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%f,", floatTmp); break; } case TSDB_DATA_TYPE_DOUBLE: { - double double_ = tmpDouble(field); + double double_ = tmpDoubleImpl(field, angle); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%f,", double_); break; @@ -675,7 +726,11 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf, } skip_sql: *(sampleDataBuf + pos - 1) = 0; - } + angle += stbInfo->timestamp_step/stbInfo->angle_step; + if (angle > 360) { + angle -= 360; + } +} return 0; } diff --git a/src/benchInsert.c b/src/benchInsert.c index adf66b69..5c3f0f24 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -3528,7 +3528,11 @@ int insertTestProcess() { } } } - fillChildTblName(database, stbInfo); + // check fill child table count valid + if(fillChildTblName(database, stbInfo) <= 0) { + errorPrint(" fill child table is zero, please check parameters in json is correct. database:%s stb: %s \n", database->dbName, stbInfo->stbName); + return -1; + } if (0 != prepareSampleData(database, stbInfo)) { return -1; } diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index d89e3cb6..535d960b 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -10,10 +10,78 @@ * FITNESS FOR A PARTICULAR PURPOSE. */ +#include #include extern char g_configDir[MAX_PATH_LEN]; +char funsName [FUNTYPE_CNT] [32] = { + "sin(", + "cos(" +}; + +uint8_t parseFuns(char* funValue, float* multiple, int32_t* addend, int32_t* random) { + // check valid + if (funValue == NULL || multiple == NULL || addend == NULL) { + return FUNTYPE_NONE; + } + + size_t len = strlen(funValue); + if(len > 100) { + return FUNTYPE_NONE; + } + + //parse format 10*sin(x) + 100 * random(5) + char value[128]; + size_t n = 0; + for (size_t i = 0; i < len; i++) { + if (funValue[i] != ' ') { + value[n++] = funValue[i]; + } + } + // set end + value[n] = 0; + + // multiple + char* key1 = strstr(value, "*"); + if(key1 == NULL) return FUNTYPE_NONE; + *key1 = 0; + * multiple = atof(value); + key1 += 1; + + // funType + uint8_t funType = FUNTYPE_NONE; + char* key2 = NULL; + for(int i=0; i < FUNTYPE_CNT; i++) { + key2 = strstr(key1, funsName[i]); + if(key2) { + funType = i + 1; + key2 += strlen(funsName[i]); + break; + } + } + if (key2 == NULL) + return FUNTYPE_NONE; + + char* key3 = strstr(key2, "+"); + if(key3) { + *addend = atoi(key3 + 1); + } else { + key3 = strstr(key2, "-"); + if(key3) + *addend = atoi(key3 + 1) * -1; + } + key3 += 1; + + // random + char* key4 = strstr(key3, "*random("); + if(key4) { + *random = atoi(key4 + 8); + } + + return funType; +} + static int getColumnAndTagTypeFromInsertJsonFile( tools_cJSON * superTblObj, SSuperTable *stbInfo) { int32_t code = -1; @@ -37,6 +105,11 @@ static int getColumnAndTagTypeFromInsertJsonFile( int64_t max = RAND_MAX >> 1; int64_t min = 0; int32_t length = 4; + // fun type + uint8_t funType = FUNTYPE_NONE; + float multiple = 0; + int32_t addend = 0; + int32_t random = 0; tools_cJSON *column = tools_cJSON_GetArrayItem(columnsObj, k); if (!tools_cJSON_IsObject(column)) { @@ -76,6 +149,12 @@ static int getColumnAndTagTypeFromInsertJsonFile( min = convertDatatypeToDefaultMin(type); } + // fun + tools_cJSON *fun = tools_cJSON_GetObjectItem(column, "fun"); + if (tools_cJSON_IsString(fun)) { + funType = parseFuns(fun->valuestring, &multiple, &addend, &random); + } + tools_cJSON *dataValues = tools_cJSON_GetObjectItem(column, "values"); if (g_arguments->taosc_version == 3) { @@ -112,6 +191,12 @@ static int getColumnAndTagTypeFromInsertJsonFile( col->max = max; col->min = min; col->values = dataValues; + // fun + col->funType = funType; + col->multiple = multiple; + col->addend = addend; + col->random = random; + if (customName) { if (n >= 1) { snprintf(col->name, TSDB_COL_NAME_LEN, @@ -479,6 +564,7 @@ static int getStableInfo(tools_cJSON *dbinfos, int index) { superTable->tcpTransfer = false; superTable->childTblOffset = 0; superTable->timestamp_step = 1; + superTable->angle_step = 1; superTable->useSampleTs = false; superTable->non_stop = false; superTable->insertRows = 0; @@ -734,6 +820,12 @@ static int getStableInfo(tools_cJSON *dbinfos, int index) { superTable->timestamp_step = timestampStep->valueint; } + tools_cJSON *angleStep = + tools_cJSON_GetObjectItem(stbInfo, "angle_step"); + if (tools_cJSON_IsNumber(angleStep)) { + superTable->angle_step = angleStep->valueint; + } + tools_cJSON *keepTrying = tools_cJSON_GetObjectItem(stbInfo, "keep_trying"); if (tools_cJSON_IsNumber(keepTrying)) { diff --git a/src/benchSys.c b/src/benchSys.c index 430fd92a..456bf3af 100644 --- a/src/benchSys.c +++ b/src/benchSys.c @@ -182,6 +182,7 @@ static struct argp_option bench_options[] = { {"threads", 'T', "NUMBER", 0, BENCH_THREAD}, {"insert-interval", 'i', "NUMBER", 0, BENCH_INTERVAL}, {"time-step", 'S', "NUMBER", 0, BENCH_STEP}, + {"angle-step", 'H', "NUMBER", 0, ANGLE_STEP}, {"start-timestamp", 's', "NUMBER", 0, BENCH_START_TIMESTAMP}, {"supplement-insert", 'U', 0, 0, BENCH_SUPPLEMENT}, {"interlace-rows", 'B', "NUMBER", 0, BENCH_INTERLACE}, @@ -216,7 +217,8 @@ static struct argp_option bench_options[] = { #ifdef TD_VER_COMPATIBLE_3_0_0_0 {"vgroups", 'v', "NUMBER", 0, BENCH_VGROUPS}, #endif - {"version", 'V', 0, 0, BENCH_VERSION}, +// {"version", 'V', 0, 0, BENCH_VERSION}, + {"nodrop", 'Q', 0, 0, BENCH_NODROP}, {0} }; @@ -392,6 +394,21 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { } break; + // angle step + case 'H': + if (!toolsIsStringNumber(arg)) { + errorPrintReqArg2(CUS_PROMPT"Benchmark", "H"); + } + + stbInfo->angle_step = atol(arg); + if (stbInfo->angle_step <= 0) { + errorPrint( + "Invalid -H: %s, will auto set to default(1)\n", + arg); + stbInfo->angle_step = 1; + } + break; + case 'B': if (!toolsIsStringNumber(arg)) { errorPrintReqArg2(CUS_PROMPT"Benchmark", "B"); @@ -657,7 +674,9 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { g_arguments->inputted_vgroups = atoi(arg); break; #endif - + case 'Q': + database->drop = false; + break; case 'V': printVersion(); exit(0); diff --git a/src/benchTmq.c b/src/benchTmq.c index 861f2c8d..c92c6f9e 100644 --- a/src/benchTmq.c +++ b/src/benchTmq.c @@ -96,7 +96,7 @@ static tmq_list_t * buildTopicList() { } static int32_t data_msg_process(TAOS_RES* msg, tmqThreadInfo* pInfo, int32_t msgIndex) { - char* buf = (char*)calloc(1, 16*1024); + char* buf = (char*)calloc(1, 64*1024+8); if (NULL == buf) { errorPrint("consumer id %d calloc memory fail.\n", pInfo->id); return 0; @@ -201,6 +201,7 @@ static void* tmqConsume(void* arg) { // "sequential" or "parallel" if (0 != strncasecmp(pConsumerInfo->createMode, "sequential", 10)) { + char* tPtr = pConsumerInfo->groupId; // "share" or "independent" char groupId[16] = {0}; if (0 != strncasecmp(pConsumerInfo->groupMode, "share", 5)) { @@ -210,11 +211,15 @@ static void* tmqConsume(void* arg) { memset(groupId, 0, sizeof(groupId)); rand_string(groupId, sizeof(groupId) - 1, 0); infoPrint("consumer id: %d generate rand group id: %s\n", pThreadInfo->id, groupId); - //pConsumerInfo->groupId = groupId; + tPtr = groupId; } } - buildConsumerAndSubscribe(pThreadInfo, groupId); + int ret = buildConsumerAndSubscribe(pThreadInfo, tPtr); + if (0 != ret) { + infoPrint("%s\n", "buildConsumerAndSubscribe() fail in tmqConsume()"); + return NULL; + } } int64_t totalMsgs = 0; @@ -342,7 +347,12 @@ int subscribeTestProcess() { // "sequential" or "parallel" if (0 == strncasecmp(pConsumerInfo->createMode, "sequential", 10)) { - buildConsumerAndSubscribe(pThreadInfo, pConsumerInfo->groupId); + int retVal = buildConsumerAndSubscribe(pThreadInfo, pConsumerInfo->groupId); + if (0 != retVal) { + infoPrint("%s\n", "buildConsumerAndSubscribe() fail!"); + ret = -1; + goto tmq_over; + } } pthread_create(pids + i, NULL, tmqConsume, pThreadInfo); } diff --git a/src/benchUtil.c b/src/benchUtil.c index 07b1358f..6eb97972 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -583,17 +583,20 @@ static int getServerVersionRestImpl(int16_t rest_port, int sockfd) { } tools_cJSON* dataObj = tools_cJSON_GetObjectItem(resObj, "data"); if (!tools_cJSON_IsArray(dataObj)) { - errorPrint("Invalid or miss 'data' key in json: %s\n", - tools_cJSON_Print(resObj)); + char* pstr = tools_cJSON_Print(resObj); + errorPrint("Invalid or miss 'data' key in json: %s\n", pstr ? pstr : "null"); + tmfree(pstr); tools_cJSON_Delete(resObj); goto free_of_getversion; } tools_cJSON *versionObj = tools_cJSON_GetArrayItem(dataObj, 0); tools_cJSON *versionStrObj = tools_cJSON_GetArrayItem(versionObj, 0); server_ver = atoi(versionStrObj->valuestring); + char* pstr = tools_cJSON_Print(versionStrObj); debugPrint("versionStrObj: %s, version: %s, server_ver: %d\n", - tools_cJSON_Print(versionStrObj), + pstr ? pstr : "null", versionStrObj->valuestring, server_ver); + tmfree(pstr); tools_cJSON_Delete(resObj); } free_of_getversion: @@ -627,8 +630,9 @@ static int getCodeFromResp(char *responseBuf) { } tools_cJSON* codeObj = tools_cJSON_GetObjectItem(resObj, "code"); if (!tools_cJSON_IsNumber(codeObj)) { - errorPrint("Invalid or miss 'code' key in json: %s\n", - tools_cJSON_Print(resObj)); + char* pstr = tools_cJSON_Print(resObj); + errorPrint("Invalid or miss 'code' key in json: %s\n", pstr ? pstr : "null"); + tmfree(pstr); tools_cJSON_Delete(resObj); return -1; } @@ -638,8 +642,9 @@ static int getCodeFromResp(char *responseBuf) { if (codeObj->valueint != 0) { tools_cJSON* desc = tools_cJSON_GetObjectItem(resObj, "desc"); if (!tools_cJSON_IsString(desc)) { - errorPrint("Invalid or miss 'desc' key in json: %s\n", - tools_cJSON_Print(resObj)); + char* pstr = tools_cJSON_Print(resObj); + errorPrint("Invalid or miss 'desc' key in json: %s\n", pstr ? pstr : "null"); + tmfree(pstr); return -1; } errorPrint("response, code: %d, reason: %s\n", @@ -732,8 +737,9 @@ int postProceSql(char *sqlstr, char* dbName, int precision, int iface, && (TSDB_SML_JSON_PROTOCOL != protocol) && (SML_JSON_TAOS_FORMAT != protocol) ) { - errorPrint("Invalid or miss 'code' key in json: %s\n", - tools_cJSON_Print(resObj)); + char* pstr = tools_cJSON_Print(resObj); + errorPrint("Invalid or miss 'code' key in json: %s\n", pstr ? pstr : "null"); + tmfree(pstr); tools_cJSON_Delete(resObj); goto free_of_post; } @@ -751,8 +757,9 @@ int postProceSql(char *sqlstr, char* dbName, int precision, int iface, && (codeObj->valueint != 0) && (codeObj->valueint != 200)) { tools_cJSON* desc = tools_cJSON_GetObjectItem(resObj, "desc"); if (!tools_cJSON_IsString(desc)) { - errorPrint("Invalid or miss 'desc' key in json: %s\n", - tools_cJSON_Print(resObj)); + char* pstr = tools_cJSON_Print(resObj); + errorPrint("Invalid or miss 'desc' key in json: %s\n", pstr ? pstr : "null"); + tmfree(pstr); } else { errorPrint("insert mode response, code: %d, reason: %s\n", (int)codeObj->valueint, desc->valuestring); diff --git a/src/taosdump.c b/src/taosdump.c index 569bd660..97f5ed99 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -118,6 +118,12 @@ static int g_majorVersionOfClient = 0; static int g_maxFilesPerDir = 100000; volatile int64_t g_countOfDataFile = 0; +// progress +static int64_t g_tableCount = 0; +static int64_t g_tableDone = 0; +static char g_dbName[TSDB_DB_NAME_LEN]= ""; +static char g_stbName[TSDB_TABLE_NAME_LEN] = ""; + static void print_json_aux(json_t *element, int indent); // for tstrncpy buffer overflow @@ -431,7 +437,7 @@ static struct argp_option options[] = { "Server host from which to dump data. Default is localhost.", 0}, {"user", 'u', "USER", 0, "User name used to connect to server. Default is root.", 0}, - {"password", 'p', 0, 0, + {"password", 'p', 0, 0, "User password to connect to server. Default is taosdata.", 0}, {"port", 'P', "PORT", 0, "Port to connect", 0}, // input/output file @@ -487,12 +493,22 @@ static struct argp_option options[] = { "websocket to interact."}, #endif {"debug", 'g', 0, 0, "Print debug info.", 15}, + {"dot-replace", 'Q', 0, 0, "Repalce dot character with underline character in the table name.", 10}, + {"rename", 'W', "RENAME-LIST", 0, "Rename database name with new name during importing data. RENAME-LIST: \"db1=newDB1|db2=newDB2\" means rename db1 to newDB1 and rename db2 to newDB2", 10}, {0} }; #define HUMAN_TIME_LEN 28 #define DUMP_DIR_LEN (MAX_DIR_LEN - (TSDB_DB_NAME_LEN + 10)) +// rename db +struct SRenameDB; +typedef struct SRenameDB { + char* old; + char* new; + void* next; +}SRenameDB; + /* Used by main to communicate with parse_opt. */ typedef struct arguments { // connection option @@ -537,9 +553,8 @@ typedef struct arguments { bool debug_print; bool verbose_print; bool performance_print; - + bool dotReplace; int dumpDbCount; - #ifdef WEBSOCKET bool restful; bool cloud; @@ -549,6 +564,10 @@ typedef struct arguments { int cloudPort; char cloudHost[MAX_HOSTNAME_LEN]; #endif + + // put rename db string + char * renameBuf; + SRenameDB * renameHead; } SArguments; static resultStatistics g_resultStatistics = {0}; @@ -601,7 +620,9 @@ struct arguments g_args = { false, // debug_print false, // verbose_print false, // performance_print + false, // dotRepalce 0, // dumpDbCount + #ifdef WEBSOCKET false, // restful false, // cloud @@ -611,6 +632,9 @@ struct arguments g_args = { 0, // cloudPort {0}, // cloudHost #endif // WEBSOCKET + + NULL, // renameBuf + NULL // renameHead }; @@ -772,6 +796,158 @@ int64_t getEndTime(int precision) { return end_time; } +SRenameDB* newNode(char* first, SRenameDB* prev) { + SRenameDB* node = (SRenameDB*) malloc(sizeof(SRenameDB)); + memset(node, 0, sizeof(SRenameDB)); + node->old = first; + // link to list + if(prev) { + prev->next = node; + } + + return node; +} + +void setRenameDbs(char* arg) { + if (arg == NULL) return ; + // malloc new + int len = strlen(arg); + if(len <= 2) { + return ; + } + len += 1; // include \0 + + // malloc + char* p = malloc(len); + int j = 0; // j is p pos + for (int i = 0; i < len; i++) { + if (arg[i] == ' ') { + // do nothing + } else if (arg[i] == '=' || arg[i] == '|') { + // set zero + p[j++] = 0; + } else { + // copy + p[j++] = arg[i]; + } + } + + // splite + SRenameDB* node = newNode(p, NULL); + g_args.renameHead = node; + for (int k = 0; k < j; k++) { + if(p[k] == 0 && k + 1 != j && k > 0) { + // string end and not last end + char* name = &p[k] + 1; + if (node->new == NULL) { + node->new = name; + } else { + node = newNode(name, node); + } + } + } + + // end + g_args.renameBuf = p; +} + +// find newName +char* findNewName(char* oldName) { + SRenameDB* node = g_args.renameHead; + while(node) { + if (strcmp(node->old, oldName) == 0) { + return node->new; + } + node = (SRenameDB* )node->next; + } + return NULL; +} + +bool replaceCopy(char *des, char *src) { + size_t len = strlen(src); + bool replace = false; + for (size_t i = 0; i <= len; i++) { + if (src[i] == '.') { + des[i] = '_'; + replace = true; + } else { + des[i] = src[i]; + } + } + + return replace; +} + +// repalce old name with new +char * replaceNewName(char* cmd, int len) { + // database name left char and right char + int nLeftSql = len; + char left = cmd[len]; + char right = '.'; + if(left == '`') { + right = left; + nLeftSql += 1; + } + + // get old database name + char oldName[TSDB_DB_NAME_LEN]; + char* s = &cmd[nLeftSql]; + char* e = strchr(s, right); + char* e1 = strchr(s, ' '); + if(e == NULL && e1 == NULL) { + return NULL; + } else if(e == NULL && e1) { + e = e1; + } else if(e && e1 ) { + if (e > e1) { + e = e1; + } + } + + int oldLen = e - s; + if(oldLen + 1 > TSDB_DB_NAME_LEN) { + return NULL; + } + memcpy(oldName, s, oldLen); + oldName[oldLen] = 0; + + // macth new database + char* newName = findNewName(oldName); + if(newName == NULL){ + return NULL; + } + + // malloc new buff put new sql with new name + int newLen = strlen(cmd) + (strlen(newName) - oldLen) + 1; + char* newCmd = (char *)malloc(newLen); + memset(newCmd, 0, newLen); + + // copy left + newName + right from cmd + memcpy(newCmd, cmd, nLeftSql); // left sql + strcat(newCmd, newName); // newName + strcat(newCmd, e); // right sql + + return newCmd; +} + +// if have database name rename, return new sql with new database name +// retrn value need call free() to free memory +char * afterRenameSql(char *cmd) { + // match pattern + const char* CREATE_DB = "CREATE DATABASE IF NOT EXISTS "; + const char* CREATE_TB = "CREATE TABLE IF NOT EXISTS "; + + const char* pres[] = {CREATE_DB, CREATE_TB}; + for (int i = 0; i < sizeof(pres); i++ ) { + int len = strlen(pres[i]); + if (strncmp(cmd, pres[i], len) == 0) { + // found + return replaceNewName(cmd, len); + } + } + return NULL; +} + /* Parse a single option. */ static error_t parse_opt(int key, char *arg, struct argp_state *state) { /* Get the input argument from argp_parse, which we @@ -958,6 +1134,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { } state->next = state->argc; break; + case 'W': + setRenameDbs(arg); + break; default: return ARGP_ERR_UNKNOWN; @@ -1055,6 +1234,11 @@ static void parse_args( || (0 == strcmp(argv[i], "--lose-mode"))) { g_args.loose_mode = true; strcpy(argv[i], ""); + // dot replace + } else if ((strcmp(argv[i], "-Q") == 0) + || (0 == strcmp(argv[i], "--dot-replace"))) { + g_args.dotReplace = true; + strcpy(argv[i], ""); } else if (strcmp(argv[i], "-gg") == 0) { arguments->verbose_print = true; strcpy(argv[i], ""); @@ -1811,11 +1995,18 @@ static int dumpCreateMTableClause( char *pstr = NULL; pstr = tmpBuf; + // outName is output to file table name + char * outName = tableDes->name; + char tableName[TSDB_TABLE_NAME_LEN+1]; + if(g_args.dotReplace && replaceCopy(tableName, tableDes->name)) { + outName = tableName; + } + pstr += snprintf(tmpBuf, TSDB_DEFAULT_PKT_SIZE, g_args.db_escape_char ? "CREATE TABLE IF NOT EXISTS `%s`.%s%s%s USING `%s`.%s%s%s TAGS(" : "CREATE TABLE IF NOT EXISTS %s.%s%s%s USING %s.%s%s%s TAGS(", - dbName, g_escapeChar, tableDes->name, g_escapeChar, + dbName, g_escapeChar, outName, g_escapeChar, dbName, g_escapeChar, stable, g_escapeChar); for (; counter < numColsAndTags; counter++) { @@ -2915,11 +3106,18 @@ static int convertTableDesToSql( char* pstr = *buffer; + // outName is output to file table name + char * outName = tableDes->name; + char tableName[TSDB_TABLE_NAME_LEN+1]; + if(g_args.dotReplace && replaceCopy(tableName, tableDes->name)) { + outName = tableName; + } + pstr += sprintf(pstr, g_args.db_escape_char ? "CREATE TABLE IF NOT EXISTS `%s`.%s%s%s" : "CREATE TABLE IF NOT EXISTS %s.%s%s%s", - dbName, g_escapeChar, tableDes->name, g_escapeChar); + dbName, g_escapeChar, outName, g_escapeChar); for (; counter < tableDes->columns; counter++) { if (tableDes->cols[counter].note[0] != '\0') break; @@ -3305,7 +3503,13 @@ static int dumpCreateTableClauseAvro( } avro_value_set_branch(&value, 1, &branch); - avro_value_set_string(&branch, tableDes->name); + if(g_args.dotReplace) { + char tableName[TSDB_TABLE_NAME_LEN+1]; + replaceCopy(tableName, tableDes->name); + avro_value_set_string(&branch, tableName); + } else { + avro_value_set_string(&branch, tableDes->name); + } if (0 != avro_value_get_by_name( &record, "sql", &value, NULL)) { @@ -3823,12 +4027,19 @@ static int convertTbDesToJsonImpl( const char *tbName, TableDes *tableDes, char **jsonSchema, bool isColumn) { + + char* outName = (char*)tbName; + char tableName[TSDB_TABLE_NAME_LEN + 1]; + if(g_args.dotReplace && replaceCopy(tableName, (char*)tbName)) { + outName = tableName; + } + char *pstr = *jsonSchema; pstr += sprintf(pstr, "{\"type\":\"record\",\"name\":\"%s.%s\",\"fields\":[", namespace, - (isColumn)?(g_args.loose_mode?tbName:"_record") - :(g_args.loose_mode?tbName:"_stb")); + (isColumn)?(g_args.loose_mode?outName:"_record") + :(g_args.loose_mode?outName:"_stb")); int iterate = 0; if (g_args.loose_mode) { @@ -4928,6 +5139,12 @@ static int64_t writeResultToAvroNative( return 0; } + char* outName = (char*)tbName; + char tableName[TSDB_TABLE_NAME_LEN + 1]; + if(g_args.dotReplace && replaceCopy(tableName, (char*)tbName)) { + outName = tableName; + } + avro_schema_t schema; RecordSchema *recordSchema; avro_file_writer_t db; @@ -4991,7 +5208,7 @@ static int64_t writeResultToAvroNative( break; } avro_value_set_branch(&avro_value, 1, &branch); - avro_value_set_string(&branch, tbName); + avro_value_set_string(&branch, outName); } for (int32_t col = 0; col < numFields; col++) { @@ -5030,7 +5247,7 @@ static int64_t writeResultToAvroNative( currentPercent = ((offset) * 100 / queryCount); if (currentPercent > percentComplete) { - infoPrint("%d%% of %s\n", currentPercent, tbName); + infoPrint("%s.%s [%" PRId64 "/%" PRId64 "] write avro %d%% of %s\n", g_dbName, g_stbName ,g_tableDone + 1, g_tableCount, currentPercent, tbName); percentComplete = currentPercent; } } while (offset < queryCount); @@ -5945,6 +6162,13 @@ static int64_t dumpInAvroNtbImpl( __func__, __LINE__); continue; } + + char* newBuf = afterRenameSql(buf); + if(newBuf) { + infoPrint(" rename database name for create normal table sql: \n old=%s\n new=%s\n", buf, newBuf); + buf = newBuf; + } + #ifdef WEBSOCKET if (g_args.cloud || g_args.restful) { WS_RES *ws_res = ws_query_timeout(taos, buf, g_args.ws_timeout); @@ -5964,6 +6188,9 @@ static int64_t dumpInAvroNtbImpl( } else { #endif TAOS_RES *res = taos_query(taos, buf); + if(newBuf) { + free(newBuf); + } int code = taos_errno(res); if (0 != code) { errorPrint("%s() LN%d," @@ -7258,6 +7485,13 @@ static int64_t dumpInOneAvroFile( } const char *namespace = avro_schema_namespace((const avro_schema_t)schema); + if(g_args.renameHead) { + char* newDbName = findNewName((char *)namespace); + if(newDbName) { + infoPrint(" ------- rename DB Name %s to %s ------\n", namespace, newDbName); + namespace = newDbName; + } + } debugPrint("%s() LN%d, Namespace: %s\n", __func__, __LINE__, namespace); @@ -8554,7 +8788,12 @@ static int createMTableAvroHeadImp( } avro_value_set_branch(&value, 1, &branch); - avro_value_set_string(&branch, stable); + char* outSName = (char*)stable; + char stableName[TSDB_TABLE_NAME_LEN + 1]; + if(g_args.dotReplace && replaceCopy(stableName, (char*)stable)) { + outSName = stableName; + } + avro_value_set_string(&branch, outSName); } if (0 != avro_value_get_by_name( @@ -8565,7 +8804,13 @@ static int createMTableAvroHeadImp( } avro_value_set_branch(&value, 1, &branch); - avro_value_set_string(&branch, tbName); + + char* outName = (char*)tbName; + char tableName[TSDB_TABLE_NAME_LEN + 1]; + if(g_args.dotReplace && replaceCopy(tableName, (char*)tbName)) { + outName = tableName; + } + avro_value_set_string(&branch, outName); TableDes *subTableDes = (TableDes *) calloc(1, sizeof(TableDes) + sizeof(ColDes) * (stbTableDes->columns + stbTableDes->tags)); @@ -10010,14 +10255,24 @@ static int64_t dumpInOneDebugFile( } int ret; + char *newSql = NULL; + + if(g_args.renameHead) { + // have rename database options + newSql = afterRenameSql(cmd); + } debugPrint("%s() LN%d, cmd: %s\n", __func__, __LINE__, cmd); #ifdef WEBSOCKET if (g_args.cloud || g_args.restful) { - ret = queryDbImplWS(taos, cmd); + ret = queryDbImplWS(taos, newSql?newSql:cmd); } else { #endif - ret = queryDbImplNative(taos, cmd); + ret = queryDbImplNative(taos, newSql?newSql:cmd); + if(newSql) { + free(newSql); + } + #ifdef WEBSOCKET } #endif @@ -10470,6 +10725,9 @@ static void dumpNormalTablesOfStbNative( fp); } + // update progress + atomic_add_fetch_64(&g_tableDone, 1); + if (count < 0) { break; } else { @@ -10568,6 +10826,12 @@ static int64_t dumpNtbOfStbByThreads( } #endif + // set progress to global + g_tableCount = ntbCount; + g_tableDone = 0; + strcpy(g_dbName, dbInfo->name); + strcpy(g_stbName, stbName); + infoPrint("%s() LN%d, %s's %s's total normal table count: %"PRId64"\n", __func__, __LINE__, dbInfo->name, stbName, ntbCount); if (ntbCount <= 0) { @@ -10643,6 +10907,7 @@ static int64_t dumpNtbOfStbByThreads( ASSERT(pids); ASSERT(infos); + infoPrint("create %d thread(s) to export data ...\n", threads); threadInfo *pThreadInfo; for (int32_t i = 0; i < threads; i++) { pThreadInfo = infos + i; @@ -10712,6 +10977,7 @@ static int64_t dumpNtbOfStbByThreads( } } + infoPrint("%s\n","close taos connections..."); for (int32_t i = 0; i < threads; i++) { pThreadInfo = infos + i; taos_close(pThreadInfo->taos); @@ -13043,5 +13309,24 @@ int main(int argc, char *argv[]) { } else { ret = dumpEntry(); } + + // free buf + if (g_args.renameBuf) { + free(g_args.renameBuf); + g_args.renameBuf = NULL; + } + + // free node + SRenameDB* node = g_args.renameHead; + g_args.renameHead = NULL; + while(node) { + SRenameDB* next = (SRenameDB*)node->next; + free(node); + node = next; + } + + + return ret; } + diff --git a/tests/taosdump/native/taosdumpDbNtb.py b/tests/taosdump/native/taosdumpDbNtb.py index 99b2922a..d9f149f3 100644 --- a/tests/taosdump/native/taosdumpDbNtb.py +++ b/tests/taosdump/native/taosdumpDbNtb.py @@ -107,7 +107,7 @@ def run(self): tdSql.execute("drop database db") # sys.exit(1) - os.system("%s -i %s -T 1" % (binPath, self.tmpdir)) + os.system("%s -i %s -T 1 -W db=newdb" % (binPath, self.tmpdir)) tdSql.query("show databases") dbresult = tdSql.queryResult @@ -115,13 +115,13 @@ def run(self): found = False for i in range(len(dbresult)): print("Found db: %s" % dbresult[i][0]) - if dbresult[i][0] == "db": + if dbresult[i][0] == "newdb": found = True break assert found == True - tdSql.execute("use db") + tdSql.execute("use newdb") tdSql.query("show stables") tdSql.checkRows(1) tdSql.checkData(0, 0, "st") diff --git a/tests/taosdump/native/taosdumpDbStb.py b/tests/taosdump/native/taosdumpDbStb.py index 025a1a2c..bdc3f89d 100644 --- a/tests/taosdump/native/taosdumpDbStb.py +++ b/tests/taosdump/native/taosdumpDbStb.py @@ -107,7 +107,7 @@ def run(self): tdSql.execute("drop database db") # sys.exit(1) - os.system("%s -i %s -T 1" % (binPath, self.tmpdir)) + os.system("%s -i %s -T 1 -W db=newdb" % (binPath, self.tmpdir)) tdSql.query("show databases") dbresult = tdSql.queryResult @@ -115,13 +115,13 @@ def run(self): found = False for i in range(len(dbresult)): print("Found db: %s" % dbresult[i][0]) - if dbresult[i][0] == "db": + if dbresult[i][0] == "newdb": found = True break assert found == True - tdSql.execute("use db") + tdSql.execute("use newdb") tdSql.query("show stables") tdSql.checkRows(1) tdSql.checkData(0, 0, "st") diff --git a/tests/taosdump/native/taosdumpDbWithNonRoot.py b/tests/taosdump/native/taosdumpDbWithNonRoot.py index bdc74611..6bb7eb1d 100644 --- a/tests/taosdump/native/taosdumpDbWithNonRoot.py +++ b/tests/taosdump/native/taosdumpDbWithNonRoot.py @@ -109,7 +109,7 @@ def run(self): tdSql.execute("drop database db") # sys.exit(1) - os.system("%s -i %s -T 1" % (binPath, self.tmpdir)) + os.system("%s -i %s -T 1 -W db=newdb" % (binPath, self.tmpdir)) tdSql.query("show databases") dbresult = tdSql.queryResult @@ -117,13 +117,13 @@ def run(self): found = False for i in range(len(dbresult)): print("Found db: %s" % dbresult[i][0]) - if dbresult[i][0] == "db": + if dbresult[i][0] == "newdb": found = True break assert found == True - tdSql.execute("use db") + tdSql.execute("use newdb") tdSql.query("show stables") tdSql.checkRows(1) tdSql.checkData(0, 0, "st") diff --git a/tests/taosdump/native/taosdumpEscapedDb.py b/tests/taosdump/native/taosdumpEscapedDb.py index 19fb8df7..77a17f76 100644 --- a/tests/taosdump/native/taosdumpEscapedDb.py +++ b/tests/taosdump/native/taosdumpEscapedDb.py @@ -93,7 +93,7 @@ def run(self): tdSql.execute("drop database `Db`") # sys.exit(1) - os.system("%s -e -i %s -T 1" % (binPath, self.tmpdir)) + os.system("%s -e -i %s -T 1 -W Db=NewDb" % (binPath, self.tmpdir)) tdSql.query("show databases") dbresult = tdSql.queryResult @@ -101,18 +101,18 @@ def run(self): found = False for i in range(len(dbresult)): print("Found db: %s" % dbresult[i][0]) - if dbresult[i][0] == "Db": + if dbresult[i][0] == "NewDb": found = True break assert found == True - tdSql.execute("use `Db`") + tdSql.execute("use `NewDb`") tdSql.query("show stables") tdSql.checkRows(1) tdSql.checkData(0, 0, "st") - tdSql.query("select count(*) from `Db`.st") + tdSql.query("select count(*) from `NewDb`.st") tdSql.checkData(0, 0, 1) def stop(self): diff --git a/tests/taosdump/old/taosdumpTest.py b/tests/taosdump/old/taosdumpTest.py index 3f4dad35..0d2f8f22 100644 --- a/tests/taosdump/old/taosdumpTest.py +++ b/tests/taosdump/old/taosdumpTest.py @@ -107,10 +107,10 @@ def run(self): tdSql.query("select * from information_schema.ins_databases") tdSql.checkRows(2) - os.system("%s -i ./taosdumptest/tmp1" % binPath) - os.system("%s -i ./taosdumptest/tmp2" % binPath) + os.system("%s -W db=newdb -i ./taosdumptest/tmp1" % binPath) + os.system("%s -W \"db=newdb|db1=newdb1\" -i ./taosdumptest/tmp2" % binPath) - tdSql.execute("use db") + tdSql.execute("use newdb") tdSql.query("select * from information_schema.ins_databases") tdSql.checkRows(4) dbresult = tdSql.queryResult @@ -119,7 +119,7 @@ def run(self): isCommunity = self.checkCommunity() print("iscommunity: %d" % isCommunity) for i in range(len(dbresult)): - if dbresult[i][0] == "db": + if dbresult[i][0] == "newdb": print(dbresult[i]) print(type(dbresult[i][6])) print(type(dbresult[i][7])) @@ -127,7 +127,7 @@ def run(self): assert dbresult[i][6] == "15840m" print((dbresult[i][7])) assert dbresult[i][7] == "5254560m,5254560m,5254560m" - if dbresult[i][0] == "db1": + if dbresult[i][0] == "newdb1": print((dbresult[i][6])) assert dbresult[i][6] == "17280m" print((dbresult[i][7])) @@ -158,8 +158,8 @@ def run(self): # drop all databases，boundary value testing. # length(databasename)<=32;length(tablesname)<=192 - tdSql.execute("drop database db") - tdSql.execute("drop database db1") + tdSql.execute("drop database newdb") + tdSql.execute("drop database newdb1") os.system("rm -rf ./taosdumptest/tmp1") os.system("rm -rf ./taosdumptest/tmp2") os.makedirs("./taosdumptest/tmp1") @@ -194,8 +194,8 @@ def run(self): % binPath ) tdSql.execute("drop database db12312313231231321312312312_323") - os.system("%s -i ./taosdumptest/tmp1" % binPath) - tdSql.execute("use db12312313231231321312312312_323") + os.system("%s -W db12312313231231321312312312_323=db12312313231231321312312312_323abc -i ./taosdumptest/tmp1" % binPath) + tdSql.execute("use db12312313231231321312312312_323abc") tdSql.query("show stables") tdSql.checkRows(2) os.system("rm -rf ./taosdumptest/tmp1")