From d17b018b1a087987477cae3a43942c389fdfef3c Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 21 Sep 2023 13:04:05 +0800 Subject: [PATCH 01/11] fix: fillchildtable report waring when error --- src/benchInsert.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index 5c3f0f24..a3a565d1 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -3530,8 +3530,8 @@ int insertTestProcess() { } // check fill child table count valid if(fillChildTblName(database, stbInfo) <= 0) { - errorPrint(" fill child table is zero, please check parameters in json is correct. database:%s stb: %s \n", database->dbName, stbInfo->stbName); - return -1; + infoPrint(" warning fill child table is zero, please check parameters in json is correct. database:%s stb: %s \n", database->dbName, stbInfo->stbName); + return 0; } if (0 != prepareSampleData(database, stbInfo)) { return -1; From 3ce29b2dc9e501d02e1b170dd15bd0e31c596b30 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 21 Sep 2023 14:18:10 +0800 Subject: [PATCH 02/11] fix: do not return 1 --- src/benchInsert.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index a3a565d1..51fd3431 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -3530,8 +3530,7 @@ int insertTestProcess() { } // check fill child table count valid if(fillChildTblName(database, stbInfo) <= 0) { - infoPrint(" warning fill child table is zero, please check parameters in json is correct. database:%s stb: %s \n", database->dbName, stbInfo->stbName); - return 0; + infoPrint(" warning fill childs table count is zero, please check parameters in json is correct. database:%s stb: %s \n", database->dbName, stbInfo->stbName); } if (0 != prepareSampleData(database, stbInfo)) { return -1; From 655652ad1312b0b88b91da970e43d7171a0e7299 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 21 Sep 2023 18:46:04 +0800 Subject: [PATCH 03/11] fix: set tbname for every one --- src/taosdump.c | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/taosdump.c b/src/taosdump.c index 97f5ed99..cc2387e3 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -6969,8 +6969,10 @@ static int64_t dumpInAvroDataImpl( avro_value_get_current_branch(&tbname_value, &tbname_branch); size_t tbname_size; + char * avroName = NULL; avro_value_get_string(&tbname_branch, - (const char **)&tbName, &tbname_size); + (const char **)&avroName, &tbname_size); + tbName = strdup(avroName); } else { tbName = malloc(TSDB_TABLE_NAME_LEN+1); ASSERT(tbName); @@ -7040,7 +7042,7 @@ static int64_t dumpInAvroDataImpl( "reason: %s\n", escapedTbName, taos_stmt_errstr(stmt)); free(escapedTbName); - freeTbNameIfLooseMode(tbName); + free(tbName); tbName = NULL; continue; } @@ -7061,8 +7063,28 @@ static int64_t dumpInAvroDataImpl( #ifdef WEBSOCKET } #endif - } + } } // tbName +#ifndef TD_VER_COMPATIBLE_3_0_0_0 + else { + // 2.6 need call taos_stmt_set_tbname every loop + const int escapedTbNameLen = TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN + 3; + char *escapedTbName = calloc(1, escapedTbNameLen); + snprintf(escapedTbName, escapedTbNameLen, "%s%s%s", + g_escapeChar, tbName, g_escapeChar); + + if (0 != taos_stmt_set_tbname(stmt, escapedTbName)) { + errorPrint("Failed to execute taos_stmt_set_tbname(%s)." + "reason: %s\n", + escapedTbName, taos_stmt_errstr(stmt)); + free(escapedTbName); + freeTbNameIfLooseMode(tbName); + tbName = NULL; + continue; + } + free(escapedTbName); + } +#endif debugPrint("%s() LN%d, count: %"PRId64"\n", __func__, __LINE__, count); @@ -7343,7 +7365,7 @@ static int64_t dumpInAvroDataImpl( } } - freeTbNameIfLooseMode(tbName); + free(tbName); avro_value_decref(&value); avro_value_iface_decref(value_class); tfree(bindArray); From 471886c0aea0e73a9ca5505fb551f9321649e001 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 22 Sep 2023 15:38:56 +0800 Subject: [PATCH 04/11] fix: add dbname tbname output in error information --- src/taosdump.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/taosdump.c b/src/taosdump.c index cc2387e3..e0fc0ecb 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -5203,8 +5203,8 @@ static int64_t writeResultToAvroNative( if (0 != avro_value_get_by_name( &record, "tbname", &avro_value, NULL)) { errorPrint("%s() LN%d, avro_value_get_by_name(tbname) " - "failed\n", - __func__, __LINE__); + "failed dbName=%s tbName=%s\n", + __func__, __LINE__, dbName, tbName); break; } avro_value_set_branch(&avro_value, 1, &branch); @@ -5224,9 +5224,9 @@ static int64_t writeResultToAvroNative( if (0 != avro_file_writer_append_value(db, &record)) { errorPrint("%s() LN%d, " - "Unable to write record to file. Message: %s\n", + "Unable to write record to file. Message: %s dbName=%s tbName=%s\n", __func__, __LINE__, - avro_strerror()); + avro_strerror(), dbName, tbName); failed--; } else { success++; @@ -5237,9 +5237,9 @@ static int64_t writeResultToAvroNative( } if (countInBatch != limit) { - errorPrint("%s() LN%d, actual dump out: %d, batch %" PRId64 "\n", + errorPrint("%s() LN%d, table rows is zero. actual dump out: %d, batch %" PRId64 " dbName=%s tbName=%s\n", __func__, __LINE__, - countInBatch, limit); + countInBatch, limit, dbName, tbName); } taos_free_result(res); printDotOrX(offset, &printDot); From dae6b5dc1dca3139fa5492f378f951dded08460b Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 10 Oct 2023 20:56:13 +0800 Subject: [PATCH 05/11] feat: support now write data --- src/benchInsert.c | 8 ++++++-- src/benchJsonOpt.c | 5 +++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index 51fd3431..bad5e85c 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -1532,8 +1532,12 @@ static void *syncWriteInterlace(void *sarg) { int64_t disorderTs = getDisorderTs(stbInfo, &disorderRange); char time_string[BIGINT_BUFF_LEN]; - snprintf(time_string, BIGINT_BUFF_LEN, "%"PRId64"", - disorderTs?disorderTs:timestamp); + if(stbInfo->useNow) { + snprintf(time_string, BIGINT_BUFF_LEN, "now"); + } else { + snprintf(time_string, BIGINT_BUFF_LEN, "%"PRId64"", + disorderTs?disorderTs:timestamp); + } ds_add_strs(&pThreadInfo->buffer, 5, "(", time_string, diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 535d960b..dfc60352 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -889,6 +889,11 @@ static int getStableInfo(tools_cJSON *dbinfos, int index) { superTable->interlaceRows = (uint32_t)stbInterlaceRows->valueint; } + // force set + if(superTable->useNow) { + superTable->interlaceRows = 1; + } + // disorder tools_cJSON *disorderRatio = tools_cJSON_GetObjectItem(stbInfo, "disorder_ratio"); From eb59be851570e3efb08156903c6ce8b11e5f0f91 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 11 Oct 2023 10:20:52 +0800 Subject: [PATCH 06/11] feat: add insertNow.json --- example/insertNow.json | 76 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 example/insertNow.json diff --git a/example/insertNow.json b/example/insertNow.json new file mode 100644 index 00000000..dddb9f95 --- /dev/null +++ b/example/insertNow.json @@ -0,0 +1,76 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "thread_count": 4, + "create_table_thread_count": 7, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 10000, + "prepared_rand": 10000, + "chinese": "no", + "escape_character": "yes", + "databases": [ + { + "dbinfo": { + "name": "test", + "drop": "yes", + "precision": "ms" + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 10000, + "childtable_prefix": "d", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "rand", + "insert_mode": "taosc", + "non_stop_mode": "no", + "line_protocol": "line", + "insert_rows": 900000000000000, + "interlace_rows": 1, + "insert_interval": 0, + "start_timestamp": "now", + "sample_format": "csv", + "sample_file": "./sample.csv", + "use_sample_ts": "no", + "tags_file": "", + "columns": [ + { + "type": "FLOAT", + "name": "current", + "count": 1, + "max": 12, + "min": 8 + }, + { "type": "INT", "name": "voltage", "max": 225, "min": 215 }, + { "type": "FLOAT", "name": "phase", "max": 1, "min": 0 } + ], + "tags": [ + { + "type": "TINYINT", + "name": "groupid", + "max": 10, + "min": 1 + }, + { + "name": "location", + "type": "BINARY", + "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} From 60b2a90701be672dce88f45ff4fc0f723a98e310 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 11 Oct 2023 10:39:23 +0800 Subject: [PATCH 07/11] feat: adjust json config --- example/insertNow.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/example/insertNow.json b/example/insertNow.json index dddb9f95..5b805551 100644 --- a/example/insertNow.json +++ b/example/insertNow.json @@ -20,7 +20,8 @@ "dbinfo": { "name": "test", "drop": "yes", - "precision": "ms" + "precision": "ms", + "wal_retention_period": "60" }, "super_tables": [ { @@ -36,7 +37,7 @@ "line_protocol": "line", "insert_rows": 900000000000000, "interlace_rows": 1, - "insert_interval": 0, + "insert_interval": 1000, "start_timestamp": "now", "sample_format": "csv", "sample_file": "./sample.csv", From a37ee989ab010266138558c1e302de92930b72a1 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 11 Oct 2023 12:49:29 +0800 Subject: [PATCH 08/11] feat: adjust json config1 --- example/insertNow.json | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/example/insertNow.json b/example/insertNow.json index 5b805551..38a7b61b 100644 --- a/example/insertNow.json +++ b/example/insertNow.json @@ -6,22 +6,30 @@ "user": "root", "password": "taosdata", "connection_pool_size": 8, - "thread_count": 4, + "thread_count": 5, "create_table_thread_count": 7, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, - "num_of_records_per_req": 10000, + "num_of_records_per_req": 2000, "prepared_rand": 10000, "chinese": "no", "escape_character": "yes", "databases": [ { "dbinfo": { - "name": "test", + "name": "alive", "drop": "yes", - "precision": "ms", - "wal_retention_period": "60" + "vgroups": 4, + "duration": 3650, + "pages": 512, + "minrows":5000, + "maxrows":10000, + "stt_trigger":10, + "wal_retention_period": 10, + "wal_retention_size": 100, + "cachemodel": "'both'", + "precision": "ms" }, "super_tables": [ { @@ -37,7 +45,7 @@ "line_protocol": "line", "insert_rows": 900000000000000, "interlace_rows": 1, - "insert_interval": 1000, + "insert_interval": 10000, "start_timestamp": "now", "sample_format": "csv", "sample_file": "./sample.csv", From 3056d92b047695ac81b634ab0a3d8f705e29ec9e Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 11 Oct 2023 12:51:22 +0800 Subject: [PATCH 09/11] feat: adjust json config1 --- example/insertNow.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/insertNow.json b/example/insertNow.json index 38a7b61b..f8c69148 100644 --- a/example/insertNow.json +++ b/example/insertNow.json @@ -25,7 +25,7 @@ "pages": 512, "minrows":5000, "maxrows":10000, - "stt_trigger":10, + "stt_trigger":1, "wal_retention_period": 10, "wal_retention_size": 100, "cachemodel": "'both'", From 35dc194d6e61df7aeb0fed4a7707140569ba8e5f Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 11 Oct 2023 12:56:13 +0800 Subject: [PATCH 10/11] feat: adjust json keep duration --- example/insertNow.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/example/insertNow.json b/example/insertNow.json index f8c69148..8032654d 100644 --- a/example/insertNow.json +++ b/example/insertNow.json @@ -18,10 +18,11 @@ "databases": [ { "dbinfo": { - "name": "alive", + "name": "smart", "drop": "yes", "vgroups": 4, - "duration": 3650, + "duration": "10d", + "keep": "100d", "pages": 512, "minrows":5000, "maxrows":10000, @@ -45,7 +46,7 @@ "line_protocol": "line", "insert_rows": 900000000000000, "interlace_rows": 1, - "insert_interval": 10000, + "insert_interval": 1000, "start_timestamp": "now", "sample_format": "csv", "sample_file": "./sample.csv", From 6e089c61305f15be2c4ffd84b0e89c271f46d6d7 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 13 Oct 2023 14:48:00 +0800 Subject: [PATCH 11/11] fix: timestampstart use now with interlace 1 --- src/benchInsert.c | 2 +- src/benchJsonOpt.c | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index bad5e85c..06440d30 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -1532,7 +1532,7 @@ static void *syncWriteInterlace(void *sarg) { int64_t disorderTs = getDisorderTs(stbInfo, &disorderRange); char time_string[BIGINT_BUFF_LEN]; - if(stbInfo->useNow) { + if(stbInfo->useNow && stbInfo->interlaceRows == 1) { snprintf(time_string, BIGINT_BUFF_LEN, "now"); } else { snprintf(time_string, BIGINT_BUFF_LEN, "%"PRId64"", diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index dfc60352..535d960b 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -889,11 +889,6 @@ static int getStableInfo(tools_cJSON *dbinfos, int index) { superTable->interlaceRows = (uint32_t)stbInterlaceRows->valueint; } - // force set - if(superTable->useNow) { - superTable->interlaceRows = 1; - } - // disorder tools_cJSON *disorderRatio = tools_cJSON_GetObjectItem(stbInfo, "disorder_ratio");