Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: taos-tools for 3.0 branch #754

Merged
merged 10 commits into from
Jun 15, 2024
Merged

fix: taos-tools for 3.0 branch #754

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
d5dd5e6
fix: change files
DuanKuanJun Jun 12, 2024
2f2aa16
fix: add geometry and varbinary
DuanKuanJun Jun 13, 2024
d912320
fix: fix csv read line
DuanKuanJun Jun 13, 2024
5ef449b
fix: free command twice
DuanKuanJun Jun 13, 2024
134f111
fix: build error
DuanKuanJun Jun 13, 2024
5ebe168
remove -V blank
DuanKuanJun Jun 13, 2024
5e2929a
fix: tmpGeometry format
DuanKuanJun Jun 13, 2024
6521354
fix: tmpGeometry format1
DuanKuanJun Jun 13, 2024
1310afc
fix: tmpGeometry format2
DuanKuanJun Jun 13, 2024
1278ac0
fix: build error
DuanKuanJun Jun 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion deps/avro/lang/c/examples/quickstop.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ int print_person(avro_file_reader_t db, avro_schema_t reader_schema)
if (rval == 0) {
int64_t id;
int32_t age;
int32_t *p;
char *p = NULL;
size_t size;
avro_value_t id_value;
avro_value_t first_value;
Expand Down
1 change: 1 addition & 0 deletions inc/bench.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1180,5 +1180,6 @@ uint64_t tmpUint64Impl(Field *field, int32_t angle, int64_t k);
float tmpFloatImpl(Field *field, int i, int32_t angle, int32_t k);
double tmpDoubleImpl(Field *field, int32_t angle, int32_t k);
int tmpStr(char *tmp, int iface, Field *field, int64_t k);
int tmpGeometry(char *tmp, int iface, Field *field, int64_t k);

#endif // INC_BENCH_H_
7 changes: 2 additions & 5 deletions src/benchCommandOpt.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,6 @@ extern char g_configDir[MAX_PATH_LEN];
#define TAOSBENCHMARK_STATUS "unknown"
#endif

#ifndef TD_PRODUCT_NAME
#define TD_PRODUCT_NAME "TDengine"
#endif

// libtaos.so
extern char buildinfo[];
Expand All @@ -52,9 +49,9 @@ void printVersion() {
char taosBenchmark_status[] = TAOSBENCHMARK_STATUS;

// version
printf("%s\ntaosBenchmark version: %s\ngit: %s\n", TD_PRODUCT_NAME, taosBenchmark_ver, taosBenchmark_commit);
printf("taosBenchmark version: %s\ngit: %s\n", taosBenchmark_ver, taosBenchmark_commit);
#ifdef LINUX
printf("build: %s\n ", buildinfo);
printf("build: %s\n", buildinfo);
#endif
if (strlen(taosBenchmark_status) > 0) {
printf("status: %s\n", taosBenchmark_status);
Expand Down
141 changes: 135 additions & 6 deletions src/benchData.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -323,8 +323,17 @@
continue;
}

if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
line[--readLen] = 0;
int32_t pos = (int32_t)(readLen - 1);
if(pos > 0) {
if (('\r' == line[pos]) || ('\n' == line[pos])) {
line[pos] = 0;
}
}
pos = (int32_t)(readLen - 2);
if(pos > 0) {
if (('\r' == line[pos]) || ('\n' == line[pos])) {
line[pos] = 0;
}
}

if (readLen == 0) {
Expand All @@ -339,7 +348,7 @@
continue;
}

memcpy(buffer + getRows * length, line, readLen);
memcpy(buffer + getRows * length, line, readLen + 1);
getRows++;

if (getRows == size) {
Expand Down Expand Up @@ -391,7 +400,6 @@
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_GEOMETRY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_GEOMETRY:
len += field->length + 3;
break;
case TSDB_DATA_TYPE_INT:
Expand Down Expand Up @@ -483,6 +491,45 @@
return 0;
}

int tmpGeometry(char *tmp, int iface, Field *field, int64_t k) {
// values
if (field->values) {
int arraySize = tools_cJSON_GetArraySize(field->values);
if (arraySize) {
tools_cJSON *buf = tools_cJSON_GetArrayItem(
field->values,
taosRandom() % arraySize);
snprintf(tmp, field->length,
"%s", buf->valuestring);
} else {
errorPrint("%s() cannot read correct value "
"from json file. array size: %d\n",
__func__, arraySize);
return -1;
}
return 0;
}

// gen point count
int32_t cnt = field->length / 24;
if(cnt == 0) {
snprintf(tmp, field->length, "POINT(%d %d)", tmpUint16(field), tmpUint16(field));
return 0;
}

int32_t pos = snprintf(tmp, field->length, "LINESTRING(");
char * format = "%d %d,";
for(int32_t i = 0; i < cnt; i++) {
if (i == cnt - 1) {
format = "%d %d";
}
pos += snprintf(tmp + pos, field->length - pos, format, tmpUint16(field), tmpUint16(field));
Dismissed Show dismissed Hide dismissed
}
strcat(tmp, ")");

return 0;
}

bool tmpBool(Field *field) {
bool boolTmp;
if (field->min == field->max) {
Expand Down Expand Up @@ -800,7 +847,7 @@
break;
}
case TSDB_DATA_TYPE_GEOMETRY: {
int bufferSize = geoCalcBufferSize(field->length);
int bufferSize = field->length + 1;
char *tmp = benchCalloc(1, bufferSize, false);
if (0 != tmpGeometry(tmp, stbInfo->iface, field, i)) {
free(tmp);
Expand Down Expand Up @@ -1001,6 +1048,28 @@
tmfree(tmp);
break;
}
case TSDB_DATA_TYPE_GEOMETRY: {
char *tmp = benchCalloc(1, field->length + 1, false);
if (0 != tmpGeometry(tmp, stbInfo->iface, field, k)) {
tmfree(tmp);
return -1;
}
if (childCol) {
snprintf((char *)childCol->stmtData.data
+ k * field->length,
field->length,
"%s", tmp);
} else {
snprintf((char *)field->stmtData.data
+ k * field->length,
field->length,
"%s", tmp);
}
n = snprintf(sampleDataBuf + pos, bufLen - pos,
"'%s',", tmp);
tmfree(tmp);
break;
}
case TSDB_DATA_TYPE_JSON: {
pos += tmpJson(sampleDataBuf, bufLen, pos,
fieldsSize, field);
Expand Down Expand Up @@ -1231,7 +1300,7 @@
case TSDB_DATA_TYPE_NCHAR: {
char *tmp = benchCalloc(1, field->length + 1, false);
if (0 != tmpStr(tmp, stbInfo->iface, field, k)) {
free(tmp);
tmfree(tmp);
return -1;
}
if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY) {
Expand Down Expand Up @@ -1272,6 +1341,31 @@
tmfree(tmp);
break;
}
case TSDB_DATA_TYPE_GEOMETRY: {
char *tmp = benchCalloc(1, field->length + 1, false);
if (0 != tmpGeometry(tmp, stbInfo->iface, field, k)) {
tmfree(tmp);
return -1;
}
if (tag) {
n = snprintf(sampleDataBuf + pos, bufLen - pos,
"%s=L\"%s\" ",
field->name, tmp);
} else {
n = snprintf(sampleDataBuf + pos, bufLen - pos,
"\"%s\" ", tmp);
}
if (n < 0 || n >= bufLen - pos) {
errorPrint("%s() LN%d snprintf overflow\n",
__func__, __LINE__);
tmfree(tmp);
return -1;
} else {
pos += n;
}
tmfree(tmp);
break;
}
case TSDB_DATA_TYPE_JSON: {
pos += tmpJson(sampleDataBuf, bufLen, pos,
fieldsSize, field);
Expand Down Expand Up @@ -1534,6 +1628,18 @@
tmfree(tmp);
break;
}
case TSDB_DATA_TYPE_GEOMETRY: {
char *tmp = benchCalloc(1, field->length + 1, false);
if (0 != tmpGeometry(tmp, stbInfo->iface, field, k)) {
tmfree(tmp);
return -1;
}
n = snprintf(sampleDataBuf + pos, bufLen - pos,
"%s=\"%s\",",
field->name, tmp);
tmfree(tmp);
break;
}
case TSDB_DATA_TYPE_JSON: {
n = tmpJson(sampleDataBuf, bufLen, pos,
fieldsSize, field);
Expand Down Expand Up @@ -2075,6 +2181,15 @@
tmfree(buf);
break;
}
case TSDB_DATA_TYPE_GEOMETRY: {
char *buf = (char *)benchCalloc(col->length + 1, 1, false);
tmpGeometry(buf, stbInfo->iface, col, 0);
value_buf = benchCalloc(len_key + col->length + 3, 1, true);
snprintf(value_buf, len_key + col->length + 3,
"\"value\":\"%s\",", buf);
tmfree(buf);
break;
}
default: {
value_buf = benchCalloc(len_key + 20, 1, true);
double doubleTmp = tmpDouble(col);
Expand Down Expand Up @@ -2120,6 +2235,13 @@
tmfree(buf);
break;
}
case TSDB_DATA_TYPE_GEOMETRY: {
char *buf = (char *)benchCalloc(col->length + 1, 1, false);
tmpGeometry(buf, stbInfo->iface, col, 0);
tools_cJSON_AddStringToObject(record, "value", buf);
tmfree(buf);
break;
}
default: {
double doubleTmp = tmpDouble(col);
tools_cJSON_AddNumberToObject(record, "value", doubleTmp);
Expand Down Expand Up @@ -2180,6 +2302,13 @@
tmfree(buf);
break;
}
case TSDB_DATA_TYPE_GEOMETRY: {
char *buf = (char *)benchCalloc(col->length + 1, 1, false);
tmpGeometry(buf, stbInfo->iface, col, 0);
tools_cJSON_AddStringToObject(value, "value", buf);
tools_cJSON_AddStringToObject(value, "type", "geometry");
tmfree(buf);
}
default: {
double dblTmp = (double)col->min;
if (col->max != col->min) {
Expand Down
3 changes: 3 additions & 0 deletions src/benchDataMix.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,9 @@ uint32_t dataGenByField(Field* fd, char* pstr, uint32_t len, char* prefix, int64
format = ",\'%s\'";
tmpStr(val, 0, fd, *k);
break;
case TSDB_DATA_TYPE_GEOMETRY:
tmpGeometry(val, 0, fd, 0);
break;
default:
break;
}
Expand Down
5 changes: 5 additions & 0 deletions src/benchInsert.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,7 @@ static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) {
int n;
if (col->type == TSDB_DATA_TYPE_BINARY ||
col->type == TSDB_DATA_TYPE_NCHAR ||
col->type == TSDB_DATA_TYPE_VARBINARY ||
col->type == TSDB_DATA_TYPE_GEOMETRY) {
n = snprintf(colsBuf + len, col_buffer_len - len,
",%s %s(%d)", col->name,
Expand Down Expand Up @@ -318,6 +319,7 @@ static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) {
Field *tag = benchArrayGet(stbInfo->tags, tagIndex);
if (tag->type == TSDB_DATA_TYPE_BINARY ||
tag->type == TSDB_DATA_TYPE_NCHAR ||
tag->type == TSDB_DATA_TYPE_VARBINARY ||
tag->type == TSDB_DATA_TYPE_GEOMETRY) {
n = snprintf(tagsBuf + len, tag_buffer_len - len,
"%s %s(%d),", tag->name,
Expand Down Expand Up @@ -2834,6 +2836,8 @@ static int initStmtDataValue(SSuperTable *stbInfo, SChildTable *childTbl) {
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_GEOMETRY:
{
size_t tmpLen = strlen(tmpStr);
debugPrint("%s() LN%d, index: %d, "
Expand Down Expand Up @@ -2926,6 +2930,7 @@ static void initStmtData(char dataType, void **data, uint32_t length) {

case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_GEOMETRY:
tmpP = calloc(1, g_arguments->prepared_rand * length);
assert(tmpP);
tmfree(*data);
Expand Down
2 changes: 2 additions & 0 deletions src/benchJsonOpt.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,8 @@ static int getColumnAndTagTypeFromInsertJsonFile(
} else {
if (type == TSDB_DATA_TYPE_BINARY
|| type == TSDB_DATA_TYPE_JSON
|| type == TSDB_DATA_TYPE_VARBINARY
|| type == TSDB_DATA_TYPE_GEOMETRY
|| type == TSDB_DATA_TYPE_NCHAR) {
length = g_arguments->binwidth;
} else {
Expand Down
5 changes: 5 additions & 0 deletions src/benchUtil.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1006,6 +1006,8 @@ int convertStringToDatatype(char *type, int length) {
return TSDB_DATA_TYPE_BINARY;
} else if (0 == strcasecmp(type, "varbinary")) {
return TSDB_DATA_TYPE_VARBINARY;
} else if (0 == strcasecmp(type, "geometry")) {
return TSDB_DATA_TYPE_GEOMETRY;
} else {
errorPrint("unknown data type: %s\n", type);
exit(EXIT_FAILURE);
Expand Down Expand Up @@ -1045,13 +1047,16 @@ int convertStringToDatatype(char *type, int length) {
return TSDB_DATA_TYPE_BINARY;
} else if (0 == strncasecmp(type, "varbinary", length)) {
return TSDB_DATA_TYPE_VARBINARY;
} else if (0 == strncasecmp(type, "geometry", length)) {
return TSDB_DATA_TYPE_GEOMETRY;
} else {
errorPrint("unknown data type: %s\n", type);
exit(EXIT_FAILURE);
}
}
}


int compare(const void *a, const void *b) {
return *(int64_t *)a - *(int64_t *)b;
}
Expand Down
9 changes: 2 additions & 7 deletions src/taosdump.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,6 @@
#define TAOSDUMP_STATUS "unknown"
#endif

#ifndef TD_PRODUCT_NAME
#define TD_PRODUCT_NAME "TDengine"
#endif

// use 256 as normal buffer length
#define BUFFER_LEN 256

Expand Down Expand Up @@ -673,7 +669,7 @@ static void printVersion(FILE *file) {

char taosdump_commit[] = TAOSDUMP_COMMIT_SHA1;

fprintf(file,"%s\ntaosdump version: %s\ngit: %s\n", TD_PRODUCT_NAME, taostools_ver, taosdump_commit);
fprintf(file,"ntaosdump version: %s\ngit: %s\n", taostools_ver, taosdump_commit);
#ifdef LINUX
printf("build: %s\n ", buildinfo);
#endif
Expand Down Expand Up @@ -2181,7 +2177,6 @@ static int64_t getTbCountOfStbNative(const char *dbName, const char *stbName) {
int32_t code = taos_errno(res);
if (code != 0) {
cleanIfQueryFailed(__func__, __LINE__, command, res);
free(command);
taos_close(taos);
return -1;
}
Expand Down Expand Up @@ -9980,7 +9975,7 @@ int readNextTableDesWS(void* ws_res, TableDes* tbDes, int *idx, int *cnt) {
// read next table tags to tbDes
int readNextTableDesNative(void* res, TableDes* tbDes) {
// tbname, tagName , tagValue
TAOS_ROW row;
TAOS_ROW row = NULL;
int index = 0;
while( index < tbDes->tags && NULL != (row = taos_fetch_row(res))) {
// tbname changed check
Expand Down
Loading