From 162164340825daff3c3a755e1880387b33143585 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 21 Sep 2023 13:04:05 +0800 Subject: [PATCH 01/12] fix: fillchildtable report waring when error --- src/benchInsert.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index e7e9bab6..d1dea989 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -3542,8 +3542,8 @@ int insertTestProcess() { } // check fill child table count valid if(fillChildTblName(database, stbInfo) <= 0) { - errorPrint(" fill child table is zero, please check parameters in json is correct. database:%s stb: %s \n", database->dbName, stbInfo->stbName); - return -1; + infoPrint(" warning fill child table is zero, please check parameters in json is correct. database:%s stb: %s \n", database->dbName, stbInfo->stbName); + return 0; } if (0 != prepareSampleData(database, stbInfo)) { return -1; From ff5ea1a0f67bbbf3e425cdf59216f7a2df27a7af Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 21 Sep 2023 14:18:10 +0800 Subject: [PATCH 02/12] fix: do not return 1 --- src/benchInsert.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index d1dea989..6d0cc253 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -3542,8 +3542,7 @@ int insertTestProcess() { } // check fill child table count valid if(fillChildTblName(database, stbInfo) <= 0) { - infoPrint(" warning fill child table is zero, please check parameters in json is correct. database:%s stb: %s \n", database->dbName, stbInfo->stbName); - return 0; + infoPrint(" warning fill childs table count is zero, please check parameters in json is correct. database:%s stb: %s \n", database->dbName, stbInfo->stbName); } if (0 != prepareSampleData(database, stbInfo)) { return -1; From 09f05158e0bbc3c07cadb30de4aa6e592bedde46 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 21 Sep 2023 18:46:04 +0800 Subject: [PATCH 03/12] fix: set tbname for every one --- src/taosdump.c | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/taosdump.c b/src/taosdump.c index 97f5ed99..cc2387e3 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -6969,8 +6969,10 @@ static int64_t dumpInAvroDataImpl( avro_value_get_current_branch(&tbname_value, &tbname_branch); size_t tbname_size; + char * avroName = NULL; avro_value_get_string(&tbname_branch, - (const char **)&tbName, &tbname_size); + (const char **)&avroName, &tbname_size); + tbName = strdup(avroName); } else { tbName = malloc(TSDB_TABLE_NAME_LEN+1); ASSERT(tbName); @@ -7040,7 +7042,7 @@ static int64_t dumpInAvroDataImpl( "reason: %s\n", escapedTbName, taos_stmt_errstr(stmt)); free(escapedTbName); - freeTbNameIfLooseMode(tbName); + free(tbName); tbName = NULL; continue; } @@ -7061,8 +7063,28 @@ static int64_t dumpInAvroDataImpl( #ifdef WEBSOCKET } #endif - } + } } // tbName +#ifndef TD_VER_COMPATIBLE_3_0_0_0 + else { + // 2.6 need call taos_stmt_set_tbname every loop + const int escapedTbNameLen = TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN + 3; + char *escapedTbName = calloc(1, escapedTbNameLen); + snprintf(escapedTbName, escapedTbNameLen, "%s%s%s", + g_escapeChar, tbName, g_escapeChar); + + if (0 != taos_stmt_set_tbname(stmt, escapedTbName)) { + errorPrint("Failed to execute taos_stmt_set_tbname(%s)." + "reason: %s\n", + escapedTbName, taos_stmt_errstr(stmt)); + free(escapedTbName); + freeTbNameIfLooseMode(tbName); + tbName = NULL; + continue; + } + free(escapedTbName); + } +#endif debugPrint("%s() LN%d, count: %"PRId64"\n", __func__, __LINE__, count); @@ -7343,7 +7365,7 @@ static int64_t dumpInAvroDataImpl( } } - freeTbNameIfLooseMode(tbName); + free(tbName); avro_value_decref(&value); avro_value_iface_decref(value_class); tfree(bindArray); From b8e479d5f81f64bcefa479e221274b812a8be74f Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 22 Sep 2023 15:38:56 +0800 Subject: [PATCH 04/12] fix: add dbname tbname output in error information --- src/taosdump.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/taosdump.c b/src/taosdump.c index cc2387e3..e0fc0ecb 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -5203,8 +5203,8 @@ static int64_t writeResultToAvroNative( if (0 != avro_value_get_by_name( &record, "tbname", &avro_value, NULL)) { errorPrint("%s() LN%d, avro_value_get_by_name(tbname) " - "failed\n", - __func__, __LINE__); + "failed dbName=%s tbName=%s\n", + __func__, __LINE__, dbName, tbName); break; } avro_value_set_branch(&avro_value, 1, &branch); @@ -5224,9 +5224,9 @@ static int64_t writeResultToAvroNative( if (0 != avro_file_writer_append_value(db, &record)) { errorPrint("%s() LN%d, " - "Unable to write record to file. Message: %s\n", + "Unable to write record to file. Message: %s dbName=%s tbName=%s\n", __func__, __LINE__, - avro_strerror()); + avro_strerror(), dbName, tbName); failed--; } else { success++; @@ -5237,9 +5237,9 @@ static int64_t writeResultToAvroNative( } if (countInBatch != limit) { - errorPrint("%s() LN%d, actual dump out: %d, batch %" PRId64 "\n", + errorPrint("%s() LN%d, table rows is zero. actual dump out: %d, batch %" PRId64 " dbName=%s tbName=%s\n", __func__, __LINE__, - countInBatch, limit); + countInBatch, limit, dbName, tbName); } taos_free_result(res); printDotOrX(offset, &printDot); From 6b26c34b4cfb349aaeea74624924e3d89fed5b5c Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 10 Oct 2023 20:56:13 +0800 Subject: [PATCH 05/12] feat: support now write data --- src/benchInsert.c | 8 ++++++-- src/benchJsonOpt.c | 5 +++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index 6d0cc253..bfe6a7ab 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -1544,8 +1544,12 @@ static void *syncWriteInterlace(void *sarg) { int64_t disorderTs = getDisorderTs(stbInfo, &disorderRange); char time_string[BIGINT_BUFF_LEN]; - snprintf(time_string, BIGINT_BUFF_LEN, "%"PRId64"", - disorderTs?disorderTs:timestamp); + if(stbInfo->useNow) { + snprintf(time_string, BIGINT_BUFF_LEN, "now"); + } else { + snprintf(time_string, BIGINT_BUFF_LEN, "%"PRId64"", + disorderTs?disorderTs:timestamp); + } ds_add_strs(&pThreadInfo->buffer, 5, "(", time_string, diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 8c83bf4a..9d50e726 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -890,6 +890,11 @@ static int getStableInfo(tools_cJSON *dbinfos, int index) { superTable->interlaceRows = (uint32_t)stbInterlaceRows->valueint; } + // force set + if(superTable->useNow) { + superTable->interlaceRows = 1; + } + // disorder tools_cJSON *disorderRatio = tools_cJSON_GetObjectItem(stbInfo, "disorder_ratio"); From 9fea21d4bf5aa1ba923e6eee9332297a97ca4e54 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 11 Oct 2023 10:20:52 +0800 Subject: [PATCH 06/12] feat: add insertNow.json --- example/insertNow.json | 76 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 example/insertNow.json diff --git a/example/insertNow.json b/example/insertNow.json new file mode 100644 index 00000000..dddb9f95 --- /dev/null +++ b/example/insertNow.json @@ -0,0 +1,76 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "thread_count": 4, + "create_table_thread_count": 7, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 10000, + "prepared_rand": 10000, + "chinese": "no", + "escape_character": "yes", + "databases": [ + { + "dbinfo": { + "name": "test", + "drop": "yes", + "precision": "ms" + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 10000, + "childtable_prefix": "d", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "rand", + "insert_mode": "taosc", + "non_stop_mode": "no", + "line_protocol": "line", + "insert_rows": 900000000000000, + "interlace_rows": 1, + "insert_interval": 0, + "start_timestamp": "now", + "sample_format": "csv", + "sample_file": "./sample.csv", + "use_sample_ts": "no", + "tags_file": "", + "columns": [ + { + "type": "FLOAT", + "name": "current", + "count": 1, + "max": 12, + "min": 8 + }, + { "type": "INT", "name": "voltage", "max": 225, "min": 215 }, + { "type": "FLOAT", "name": "phase", "max": 1, "min": 0 } + ], + "tags": [ + { + "type": "TINYINT", + "name": "groupid", + "max": 10, + "min": 1 + }, + { + "name": "location", + "type": "BINARY", + "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} From a8a2a0dfe94b08d2a16032da641e6d437c95903c Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 11 Oct 2023 10:39:23 +0800 Subject: [PATCH 07/12] feat: adjust json config --- example/insertNow.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/example/insertNow.json b/example/insertNow.json index dddb9f95..5b805551 100644 --- a/example/insertNow.json +++ b/example/insertNow.json @@ -20,7 +20,8 @@ "dbinfo": { "name": "test", "drop": "yes", - "precision": "ms" + "precision": "ms", + "wal_retention_period": "60" }, "super_tables": [ { @@ -36,7 +37,7 @@ "line_protocol": "line", "insert_rows": 900000000000000, "interlace_rows": 1, - "insert_interval": 0, + "insert_interval": 1000, "start_timestamp": "now", "sample_format": "csv", "sample_file": "./sample.csv", From 7af901c102f6f43252d1c538d80eea3ba3a821e0 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 11 Oct 2023 12:49:29 +0800 Subject: [PATCH 08/12] feat: adjust json config1 --- example/insertNow.json | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/example/insertNow.json b/example/insertNow.json index 5b805551..38a7b61b 100644 --- a/example/insertNow.json +++ b/example/insertNow.json @@ -6,22 +6,30 @@ "user": "root", "password": "taosdata", "connection_pool_size": 8, - "thread_count": 4, + "thread_count": 5, "create_table_thread_count": 7, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, - "num_of_records_per_req": 10000, + "num_of_records_per_req": 2000, "prepared_rand": 10000, "chinese": "no", "escape_character": "yes", "databases": [ { "dbinfo": { - "name": "test", + "name": "alive", "drop": "yes", - "precision": "ms", - "wal_retention_period": "60" + "vgroups": 4, + "duration": 3650, + "pages": 512, + "minrows":5000, + "maxrows":10000, + "stt_trigger":10, + "wal_retention_period": 10, + "wal_retention_size": 100, + "cachemodel": "'both'", + "precision": "ms" }, "super_tables": [ { @@ -37,7 +45,7 @@ "line_protocol": "line", "insert_rows": 900000000000000, "interlace_rows": 1, - "insert_interval": 1000, + "insert_interval": 10000, "start_timestamp": "now", "sample_format": "csv", "sample_file": "./sample.csv", From 8e4de081d825f40e4a136cb90c2e2adea8c214c9 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 11 Oct 2023 12:51:22 +0800 Subject: [PATCH 09/12] feat: adjust json config1 --- example/insertNow.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/insertNow.json b/example/insertNow.json index 38a7b61b..f8c69148 100644 --- a/example/insertNow.json +++ b/example/insertNow.json @@ -25,7 +25,7 @@ "pages": 512, "minrows":5000, "maxrows":10000, - "stt_trigger":10, + "stt_trigger":1, "wal_retention_period": 10, "wal_retention_size": 100, "cachemodel": "'both'", From baab0525b8ecd2fe7c8ba2e33cd236c2e16da105 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 11 Oct 2023 12:56:13 +0800 Subject: [PATCH 10/12] feat: adjust json keep duration --- example/insertNow.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/example/insertNow.json b/example/insertNow.json index f8c69148..8032654d 100644 --- a/example/insertNow.json +++ b/example/insertNow.json @@ -18,10 +18,11 @@ "databases": [ { "dbinfo": { - "name": "alive", + "name": "smart", "drop": "yes", "vgroups": 4, - "duration": 3650, + "duration": "10d", + "keep": "100d", "pages": 512, "minrows":5000, "maxrows":10000, @@ -45,7 +46,7 @@ "line_protocol": "line", "insert_rows": 900000000000000, "interlace_rows": 1, - "insert_interval": 10000, + "insert_interval": 1000, "start_timestamp": "now", "sample_format": "csv", "sample_file": "./sample.csv", From cfb7be6ac4e39afaae3e199fb4a1454367331d37 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 13 Oct 2023 14:48:00 +0800 Subject: [PATCH 11/12] fix: timestampstart use now with interlace 1 --- src/benchInsert.c | 2 +- src/benchJsonOpt.c | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index bfe6a7ab..f5d871fd 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -1544,7 +1544,7 @@ static void *syncWriteInterlace(void *sarg) { int64_t disorderTs = getDisorderTs(stbInfo, &disorderRange); char time_string[BIGINT_BUFF_LEN]; - if(stbInfo->useNow) { + if(stbInfo->useNow && stbInfo->interlaceRows == 1) { snprintf(time_string, BIGINT_BUFF_LEN, "now"); } else { snprintf(time_string, BIGINT_BUFF_LEN, "%"PRId64"", diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 9d50e726..8c83bf4a 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -890,11 +890,6 @@ static int getStableInfo(tools_cJSON *dbinfos, int index) { superTable->interlaceRows = (uint32_t)stbInterlaceRows->valueint; } - // force set - if(superTable->useNow) { - superTable->interlaceRows = 1; - } - // disorder tools_cJSON *disorderRatio = tools_cJSON_GetObjectItem(stbInfo, "disorder_ratio"); From 9e2a2c28a2dc781b00b2e6c8120493dcf62ea11e Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Wed, 8 Nov 2023 10:50:28 +0800 Subject: [PATCH 12/12] fix: release build on windows --- deps/CMakeLists.txt | 3 ++- src/CMakeLists.txt | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index 6f135fe3..84492fa2 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -96,6 +96,7 @@ ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES "Windows") COMMAND cmake -E copy ${PROJECT_SOURCE_DIR}/deps/jansson-value.c ${PROJECT_SOURCE_DIR}/deps/jansson/src/value.c ) + MESSAGE(STATUS "deps' cmake detected BUILD_TYPE: ${CMAKE_BUILD_TYPE}") ExternalProject_Add( deps-libargp PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libargp/ @@ -114,7 +115,7 @@ ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES "Windows") deps-snappy PREFIX ${CMAKE_CURRENT_BINARY_DIR}/snappy SOURCE_DIR ${PROJECT_SOURCE_DIR}/deps/snappy - CONFIGURE_COMMAND cmake -G "NMake Makefiles JOM" -DCMAKE_MAKE_PROGRAM=jom -DBUILD_SHARED_LIBS=OFF -DSNAPPY_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/build/ -S ${PROJECT_SOURCE_DIR}/deps/snappy + CONFIGURE_COMMAND cmake -G "NMake Makefiles JOM" -DCMAKE_MAKE_PROGRAM=jom -DBUILD_SHARED_LIBS=OFF -DSNAPPY_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/build/ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -S ${PROJECT_SOURCE_DIR}/deps/snappy ) IF (${TOOLS_BUILD_TYPE} MATCHES "Debug") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cb2a20f1..821db7bb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -426,6 +426,7 @@ ELSE () ADD_DEFINITIONS(-DWINDOWS) SET(CMAKE_C_STANDARD 11) SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /utf-8") + SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /utf-8") IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0") ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchDataGeometry.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsString.c toolsSys.c toolsString.c) ELSE ()