diff --git a/VERSION b/VERSION index 5754c92f..2d9d10b2 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ taosbenchmark-3.2.3 -taosdump-2.5.3 -2.5.3 +taosdump-2.5.4 +2.5.4 diff --git a/src/taosdump.c b/src/taosdump.c index e1de127a..97f5ed99 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -437,7 +437,7 @@ static struct argp_option options[] = { "Server host from which to dump data. Default is localhost.", 0}, {"user", 'u', "USER", 0, "User name used to connect to server. Default is root.", 0}, - {"password", 'p', 0, 0, + {"password", 'p', 0, 0, "User password to connect to server. Default is taosdata.", 0}, {"port", 'P', "PORT", 0, "Port to connect", 0}, // input/output file @@ -494,12 +494,21 @@ static struct argp_option options[] = { #endif {"debug", 'g', 0, 0, "Print debug info.", 15}, {"dot-replace", 'Q', 0, 0, "Repalce dot character with underline character in the table name.", 10}, + {"rename", 'W', "RENAME-LIST", 0, "Rename database name with new name during importing data. RENAME-LIST: \"db1=newDB1|db2=newDB2\" means rename db1 to newDB1 and rename db2 to newDB2", 10}, {0} }; #define HUMAN_TIME_LEN 28 #define DUMP_DIR_LEN (MAX_DIR_LEN - (TSDB_DB_NAME_LEN + 10)) +// rename db +struct SRenameDB; +typedef struct SRenameDB { + char* old; + char* new; + void* next; +}SRenameDB; + /* Used by main to communicate with parse_opt. */ typedef struct arguments { // connection option @@ -545,9 +554,7 @@ typedef struct arguments { bool verbose_print; bool performance_print; bool dotReplace; - int dumpDbCount; - #ifdef WEBSOCKET bool restful; bool cloud; @@ -557,6 +564,10 @@ typedef struct arguments { int cloudPort; char cloudHost[MAX_HOSTNAME_LEN]; #endif + + // put rename db string + char * renameBuf; + SRenameDB * renameHead; } SArguments; static resultStatistics g_resultStatistics = {0}; @@ -611,6 +622,7 @@ struct arguments g_args = { false, // performance_print false, // dotRepalce 0, // dumpDbCount + #ifdef WEBSOCKET false, // restful false, // cloud @@ -620,6 +632,9 @@ struct arguments g_args = { 0, // cloudPort {0}, // cloudHost #endif // WEBSOCKET + + NULL, // renameBuf + NULL // renameHead }; @@ -781,6 +796,158 @@ int64_t getEndTime(int precision) { return end_time; } +SRenameDB* newNode(char* first, SRenameDB* prev) { + SRenameDB* node = (SRenameDB*) malloc(sizeof(SRenameDB)); + memset(node, 0, sizeof(SRenameDB)); + node->old = first; + // link to list + if(prev) { + prev->next = node; + } + + return node; +} + +void setRenameDbs(char* arg) { + if (arg == NULL) return ; + // malloc new + int len = strlen(arg); + if(len <= 2) { + return ; + } + len += 1; // include \0 + + // malloc + char* p = malloc(len); + int j = 0; // j is p pos + for (int i = 0; i < len; i++) { + if (arg[i] == ' ') { + // do nothing + } else if (arg[i] == '=' || arg[i] == '|') { + // set zero + p[j++] = 0; + } else { + // copy + p[j++] = arg[i]; + } + } + + // splite + SRenameDB* node = newNode(p, NULL); + g_args.renameHead = node; + for (int k = 0; k < j; k++) { + if(p[k] == 0 && k + 1 != j && k > 0) { + // string end and not last end + char* name = &p[k] + 1; + if (node->new == NULL) { + node->new = name; + } else { + node = newNode(name, node); + } + } + } + + // end + g_args.renameBuf = p; +} + +// find newName +char* findNewName(char* oldName) { + SRenameDB* node = g_args.renameHead; + while(node) { + if (strcmp(node->old, oldName) == 0) { + return node->new; + } + node = (SRenameDB* )node->next; + } + return NULL; +} + +bool replaceCopy(char *des, char *src) { + size_t len = strlen(src); + bool replace = false; + for (size_t i = 0; i <= len; i++) { + if (src[i] == '.') { + des[i] = '_'; + replace = true; + } else { + des[i] = src[i]; + } + } + + return replace; +} + +// repalce old name with new +char * replaceNewName(char* cmd, int len) { + // database name left char and right char + int nLeftSql = len; + char left = cmd[len]; + char right = '.'; + if(left == '`') { + right = left; + nLeftSql += 1; + } + + // get old database name + char oldName[TSDB_DB_NAME_LEN]; + char* s = &cmd[nLeftSql]; + char* e = strchr(s, right); + char* e1 = strchr(s, ' '); + if(e == NULL && e1 == NULL) { + return NULL; + } else if(e == NULL && e1) { + e = e1; + } else if(e && e1 ) { + if (e > e1) { + e = e1; + } + } + + int oldLen = e - s; + if(oldLen + 1 > TSDB_DB_NAME_LEN) { + return NULL; + } + memcpy(oldName, s, oldLen); + oldName[oldLen] = 0; + + // macth new database + char* newName = findNewName(oldName); + if(newName == NULL){ + return NULL; + } + + // malloc new buff put new sql with new name + int newLen = strlen(cmd) + (strlen(newName) - oldLen) + 1; + char* newCmd = (char *)malloc(newLen); + memset(newCmd, 0, newLen); + + // copy left + newName + right from cmd + memcpy(newCmd, cmd, nLeftSql); // left sql + strcat(newCmd, newName); // newName + strcat(newCmd, e); // right sql + + return newCmd; +} + +// if have database name rename, return new sql with new database name +// retrn value need call free() to free memory +char * afterRenameSql(char *cmd) { + // match pattern + const char* CREATE_DB = "CREATE DATABASE IF NOT EXISTS "; + const char* CREATE_TB = "CREATE TABLE IF NOT EXISTS "; + + const char* pres[] = {CREATE_DB, CREATE_TB}; + for (int i = 0; i < sizeof(pres); i++ ) { + int len = strlen(pres[i]); + if (strncmp(cmd, pres[i], len) == 0) { + // found + return replaceNewName(cmd, len); + } + } + return NULL; +} + /* Parse a single option. */ static error_t parse_opt(int key, char *arg, struct argp_state *state) { /* Get the input argument from argp_parse, which we @@ -967,6 +1134,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { } state->next = state->argc; break; + case 'W': + setRenameDbs(arg); + break; default: return ARGP_ERR_UNKNOWN; @@ -1805,21 +1975,6 @@ static int getDumpDbCount() { return count; } -bool replaceCopy(char *des, char *src) { - size_t len = strlen(src); - bool replace = false; - for (size_t i = 0; i <= len; i++) { - if (src[i] == '.') { - des[i] = '_'; - replace = true; - } else { - des[i] = src[i]; - } - } - - return replace; -} - static int dumpCreateMTableClause( const char* dbName, const char *stable, @@ -6007,6 +6162,13 @@ static int64_t dumpInAvroNtbImpl( __func__, __LINE__); continue; } + + char* newBuf = afterRenameSql(buf); + if(newBuf) { + infoPrint(" rename database name for create normal table sql: \n old=%s\n new=%s\n", buf, newBuf); + buf = newBuf; + } + #ifdef WEBSOCKET if (g_args.cloud || g_args.restful) { WS_RES *ws_res = ws_query_timeout(taos, buf, g_args.ws_timeout); @@ -6026,6 +6188,9 @@ static int64_t dumpInAvroNtbImpl( } else { #endif TAOS_RES *res = taos_query(taos, buf); + if(newBuf) { + free(newBuf); + } int code = taos_errno(res); if (0 != code) { errorPrint("%s() LN%d," @@ -7320,6 +7485,13 @@ static int64_t dumpInOneAvroFile( } const char *namespace = avro_schema_namespace((const avro_schema_t)schema); + if(g_args.renameHead) { + char* newDbName = findNewName((char *)namespace); + if(newDbName) { + infoPrint(" ------- rename DB Name %s to %s ------\n", namespace, newDbName); + namespace = newDbName; + } + } debugPrint("%s() LN%d, Namespace: %s\n", __func__, __LINE__, namespace); @@ -10083,14 +10255,24 @@ static int64_t dumpInOneDebugFile( } int ret; + char *newSql = NULL; + + if(g_args.renameHead) { + // have rename database options + newSql = afterRenameSql(cmd); + } debugPrint("%s() LN%d, cmd: %s\n", __func__, __LINE__, cmd); #ifdef WEBSOCKET if (g_args.cloud || g_args.restful) { - ret = queryDbImplWS(taos, cmd); + ret = queryDbImplWS(taos, newSql?newSql:cmd); } else { #endif - ret = queryDbImplNative(taos, cmd); + ret = queryDbImplNative(taos, newSql?newSql:cmd); + if(newSql) { + free(newSql); + } + #ifdef WEBSOCKET } #endif @@ -13127,6 +13309,24 @@ int main(int argc, char *argv[]) { } else { ret = dumpEntry(); } + + // free buf + if (g_args.renameBuf) { + free(g_args.renameBuf); + g_args.renameBuf = NULL; + } + + // free node + SRenameDB* node = g_args.renameHead; + g_args.renameHead = NULL; + while(node) { + SRenameDB* next = (SRenameDB*)node->next; + free(node); + node = next; + } + + + return ret; } diff --git a/tests/taosdump/native/taosdumpDbNtb.py b/tests/taosdump/native/taosdumpDbNtb.py index 99b2922a..d9f149f3 100644 --- a/tests/taosdump/native/taosdumpDbNtb.py +++ b/tests/taosdump/native/taosdumpDbNtb.py @@ -107,7 +107,7 @@ def run(self): tdSql.execute("drop database db") # sys.exit(1) - os.system("%s -i %s -T 1" % (binPath, self.tmpdir)) + os.system("%s -i %s -T 1 -W db=newdb" % (binPath, self.tmpdir)) tdSql.query("show databases") dbresult = tdSql.queryResult @@ -115,13 +115,13 @@ def run(self): found = False for i in range(len(dbresult)): print("Found db: %s" % dbresult[i][0]) - if dbresult[i][0] == "db": + if dbresult[i][0] == "newdb": found = True break assert found == True - tdSql.execute("use db") + tdSql.execute("use newdb") tdSql.query("show stables") tdSql.checkRows(1) tdSql.checkData(0, 0, "st") diff --git a/tests/taosdump/native/taosdumpDbStb.py b/tests/taosdump/native/taosdumpDbStb.py index 025a1a2c..bdc3f89d 100644 --- a/tests/taosdump/native/taosdumpDbStb.py +++ b/tests/taosdump/native/taosdumpDbStb.py @@ -107,7 +107,7 @@ def run(self): tdSql.execute("drop database db") # sys.exit(1) - os.system("%s -i %s -T 1" % (binPath, self.tmpdir)) + os.system("%s -i %s -T 1 -W db=newdb" % (binPath, self.tmpdir)) tdSql.query("show databases") dbresult = tdSql.queryResult @@ -115,13 +115,13 @@ def run(self): found = False for i in range(len(dbresult)): print("Found db: %s" % dbresult[i][0]) - if dbresult[i][0] == "db": + if dbresult[i][0] == "newdb": found = True break assert found == True - tdSql.execute("use db") + tdSql.execute("use newdb") tdSql.query("show stables") tdSql.checkRows(1) tdSql.checkData(0, 0, "st") diff --git a/tests/taosdump/native/taosdumpDbWithNonRoot.py b/tests/taosdump/native/taosdumpDbWithNonRoot.py index bdc74611..6bb7eb1d 100644 --- a/tests/taosdump/native/taosdumpDbWithNonRoot.py +++ b/tests/taosdump/native/taosdumpDbWithNonRoot.py @@ -109,7 +109,7 @@ def run(self): tdSql.execute("drop database db") # sys.exit(1) - os.system("%s -i %s -T 1" % (binPath, self.tmpdir)) + os.system("%s -i %s -T 1 -W db=newdb" % (binPath, self.tmpdir)) tdSql.query("show databases") dbresult = tdSql.queryResult @@ -117,13 +117,13 @@ def run(self): found = False for i in range(len(dbresult)): print("Found db: %s" % dbresult[i][0]) - if dbresult[i][0] == "db": + if dbresult[i][0] == "newdb": found = True break assert found == True - tdSql.execute("use db") + tdSql.execute("use newdb") tdSql.query("show stables") tdSql.checkRows(1) tdSql.checkData(0, 0, "st") diff --git a/tests/taosdump/native/taosdumpEscapedDb.py b/tests/taosdump/native/taosdumpEscapedDb.py index 19fb8df7..77a17f76 100644 --- a/tests/taosdump/native/taosdumpEscapedDb.py +++ b/tests/taosdump/native/taosdumpEscapedDb.py @@ -93,7 +93,7 @@ def run(self): tdSql.execute("drop database `Db`") # sys.exit(1) - os.system("%s -e -i %s -T 1" % (binPath, self.tmpdir)) + os.system("%s -e -i %s -T 1 -W Db=NewDb" % (binPath, self.tmpdir)) tdSql.query("show databases") dbresult = tdSql.queryResult @@ -101,18 +101,18 @@ def run(self): found = False for i in range(len(dbresult)): print("Found db: %s" % dbresult[i][0]) - if dbresult[i][0] == "Db": + if dbresult[i][0] == "NewDb": found = True break assert found == True - tdSql.execute("use `Db`") + tdSql.execute("use `NewDb`") tdSql.query("show stables") tdSql.checkRows(1) tdSql.checkData(0, 0, "st") - tdSql.query("select count(*) from `Db`.st") + tdSql.query("select count(*) from `NewDb`.st") tdSql.checkData(0, 0, 1) def stop(self): diff --git a/tests/taosdump/old/taosdumpTest.py b/tests/taosdump/old/taosdumpTest.py index 3f4dad35..0d2f8f22 100644 --- a/tests/taosdump/old/taosdumpTest.py +++ b/tests/taosdump/old/taosdumpTest.py @@ -107,10 +107,10 @@ def run(self): tdSql.query("select * from information_schema.ins_databases") tdSql.checkRows(2) - os.system("%s -i ./taosdumptest/tmp1" % binPath) - os.system("%s -i ./taosdumptest/tmp2" % binPath) + os.system("%s -W db=newdb -i ./taosdumptest/tmp1" % binPath) + os.system("%s -W \"db=newdb|db1=newdb1\" -i ./taosdumptest/tmp2" % binPath) - tdSql.execute("use db") + tdSql.execute("use newdb") tdSql.query("select * from information_schema.ins_databases") tdSql.checkRows(4) dbresult = tdSql.queryResult @@ -119,7 +119,7 @@ def run(self): isCommunity = self.checkCommunity() print("iscommunity: %d" % isCommunity) for i in range(len(dbresult)): - if dbresult[i][0] == "db": + if dbresult[i][0] == "newdb": print(dbresult[i]) print(type(dbresult[i][6])) print(type(dbresult[i][7])) @@ -127,7 +127,7 @@ def run(self): assert dbresult[i][6] == "15840m" print((dbresult[i][7])) assert dbresult[i][7] == "5254560m,5254560m,5254560m" - if dbresult[i][0] == "db1": + if dbresult[i][0] == "newdb1": print((dbresult[i][6])) assert dbresult[i][6] == "17280m" print((dbresult[i][7])) @@ -158,8 +158,8 @@ def run(self): # drop all databases，boundary value testing. # length(databasename)<=32;length(tablesname)<=192 - tdSql.execute("drop database db") - tdSql.execute("drop database db1") + tdSql.execute("drop database newdb") + tdSql.execute("drop database newdb1") os.system("rm -rf ./taosdumptest/tmp1") os.system("rm -rf ./taosdumptest/tmp2") os.makedirs("./taosdumptest/tmp1") @@ -194,8 +194,8 @@ def run(self): % binPath ) tdSql.execute("drop database db12312313231231321312312312_323") - os.system("%s -i ./taosdumptest/tmp1" % binPath) - tdSql.execute("use db12312313231231321312312312_323") + os.system("%s -W db12312313231231321312312312_323=db12312313231231321312312312_323abc -i ./taosdumptest/tmp1" % binPath) + tdSql.execute("use db12312313231231321312312312_323abc") tdSql.query("show stables") tdSql.checkRows(2) os.system("rm -rf ./taosdumptest/tmp1")