Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: add geometry support for main #740

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

feat: add geometry support for main #740

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e2843bc
feat: support geometry (not include schemaless)
LiShunGang Sep 15, 2023
b48f1a9
feat: support for random geometry
LiShunGang Sep 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions inc/benchData.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,10 @@
/***** Global variables ******/
/***** Declare functions *****/
void rand_string(char *str, int size, bool chinese);
int64_t getTSRandTail(int64_t timeStampStep, int32_t seq, int disorderRatio,
int disorderRange);
void rand_geometry(char *str, int fieldLen, int maxType);
int geoCalcBufferSize(int fieldLen);
int getGeoMaxType(int fieldLen);
int64_t getTSRandTail(int64_t timeStampStep, int32_t seq, int disorderRatio, int disorderRange);
int generateRandData(SSuperTable *stbInfo, char *sampleDataBuf,
int64_t bufLen,
int lenOfOneRow, BArray * fields, int64_t loop,
Expand Down
12 changes: 6 additions & 6 deletions src/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,9 +235,9 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
ADD_DEPENDENCIES(taosdump deps-jansson)
ADD_DEPENDENCIES(taosdump deps-snappy)
IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0")
ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchDataGeometry.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ELSE()
ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchDataGeometry.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ENDIF()
ELSE ()
INCLUDE_DIRECTORIES(/usr/local/include)
Expand All @@ -246,9 +246,9 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
SET(OS_ID "Darwin")

IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0")
ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchDataGeometry.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ELSE()
ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchDataGeometry.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ENDIF()
ENDIF ()

Expand Down Expand Up @@ -441,9 +441,9 @@ ELSE ()
SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /utf-8")
SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /utf-8")
IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0")
ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsString.c toolsSys.c toolsString.c)
ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchDataGeometry.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsString.c toolsSys.c toolsString.c)
ELSE ()
ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchDataGeometry.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ENDIF ()

ADD_EXECUTABLE(taosdump taosdump.c toolsSys.c toolstime.c toolsDir.c toolsString.c)
Expand Down
44 changes: 39 additions & 5 deletions src/benchData.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ int32_t funTriAngle(int32_t min, int32_t max, int32_t period, int32_t loop) {
int32_t range = abs(max - min);
int32_t change = (loop/period) % 2;
int32_t step = range / period;
int32_t cnt = 0;
int32_t cnt = 0;
if(change)
cnt = period - loop % period;
else
Expand Down Expand Up @@ -99,7 +99,7 @@ float funValueFloat(Field *field, int32_t angle, int32_t loop) {

if(field->multiple != 0)
funVal *= field->multiple;

if ( field->addend !=0 && field->random > 0 ) {
float rate = taosRandom() % field->random;
funVal += field->addend * (rate/100);
Expand Down Expand Up @@ -131,7 +131,7 @@ int32_t funValueInt32(Field *field, int32_t angle, int32_t loop) {

if(field->multiple != 0)
funVal *= field->multiple;

if ( field->addend !=0 && field->random > 0 ) {
float rate = taosRandom() % field->random;
funVal += field->addend * (rate/100);
Expand Down Expand Up @@ -385,6 +385,7 @@ uint32_t accumulateRowLen(BArray *fields, int iface) {
switch (field->type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_GEOMETRY:
len += field->length + 3;
break;
case TSDB_DATA_TYPE_INT:
Expand Down Expand Up @@ -476,6 +477,28 @@ int tmpStr(char *tmp, int iface, Field *field, int64_t k) {
return 0;
}

static int tmpGeometry(char *tmp, int iface, Field *field, int i) {
if (g_arguments->demo_mode) {
// TODO
} else if (field->values) {
int arraySize = tools_cJSON_GetArraySize(field->values);
if (arraySize) {
tools_cJSON *buf = tools_cJSON_GetArrayItem(field->values, taosRandom() % arraySize);
snprintf(tmp, field->length, "%s", buf->valuestring);
} else {
errorPrint(
"%s() cannot read correct value "
"from json file. array size: %d\n",
__func__, arraySize);
return -1;
}
} else {
int maxType = getGeoMaxType(field->length);
rand_geometry(tmp, field->length, maxType);
}
return 0;
}

bool tmpBool(Field *field) {
bool boolTmp;
if (field->min == field->max) {
Expand Down Expand Up @@ -791,6 +814,17 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf,
tmfree(tmp);
break;
}
case TSDB_DATA_TYPE_GEOMETRY: {
int bufferSize = geoCalcBufferSize(field->length);
char *tmp = benchCalloc(1, bufferSize, false);
if (0 != tmpGeometry(tmp, stbInfo->iface, field, i)) {
free(tmp);
return -1;
}
n = snprintf(sampleDataBuf + pos, bufLen - pos, "'%s',", tmp);
tmfree(tmp);
break;
}
case TSDB_DATA_TYPE_JSON: {
pos += tmpJson(sampleDataBuf, bufLen, pos,
fieldsSize, field);
Expand Down Expand Up @@ -1401,7 +1435,7 @@ static int generateRandDataSmlLine(SSuperTable *stbInfo, char *sampleDataBuf,
int bufLen,
int lenOfOneRow, BArray * fields, int64_t loop,
bool tag) {
int angle = stbInfo->startTimestamp % 360; // 0 ~ 360
int angle = stbInfo->startTimestamp % 360; // 0 ~ 360
for (int64_t k = 0; k < loop; ++k) {
int64_t pos = k * lenOfOneRow;
int n = 0;
Expand Down Expand Up @@ -1653,7 +1687,7 @@ int prepareSampleData(SDataBase* database, SSuperTable* stbInfo) {
pos += n;
}
}

// first part set noen
for (uint32_t i = 0; i < stbInfo->partialColFrom; ++i) {
Field * col = benchArrayGet(stbInfo->cols, i);
Expand Down
158 changes: 158 additions & 0 deletions src/benchDataGeometry.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <[email protected]>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the MIT license as published by the Free Software
* Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*/

#include <bench.h>
#include <benchData.h>

typedef struct {
double x;
double y;
} SGeoCoord2D;

typedef enum { GEO_SUB_TYPE_POINT = 0, GEO_SUB_TYPE_LINESTRING, GEO_SUB_TYPE_POLYGON, GEO_SUB_TYPE_COUNT } EGeoSubType;

const int CCoordSize = 16;

typedef struct {
const char *preifx;
const char *suffix;
int baseLen;
int minCoordNum;
int maxCoordNum;
bool isClosed; // if true, first coord and last coord must be the same
} SGeoSubTypeInfo;

// should be ordered by (baseLen + minCoordNum * CCoordSize), ASC
const SGeoSubTypeInfo GeoInfo[] = {
{"POINT(", ")", 5, 1, 1, false}, {"LINESTRING(", ")", 9, 2, 4094, false}, {"POLYGON((", "))", 13, 3, 4094, true}};

typedef struct {
EGeoSubType type;
BArray *coordArray; // element: SGeoCoord2D
} SGeoObj2D;

static SGeoObj2D *geoObject2DInit(EGeoSubType subType, BArray *coordArray);
static SGeoObj2D *geoObject2DRandInit(EGeoSubType subType, int coordNum);
static void geoObject2DDestory(SGeoObj2D *pObj);

static int geoCoord2DToStr(char *buffer, SGeoCoord2D *pCoord);
static int geoCoord2DArrayToStr(char *buffer, BArray *coordArray);
static int geoObject2DToStr(char *buffer, SGeoObj2D *object);

static BArray *randCoordArray(int count);

/*--- init & destory ---*/
static SGeoObj2D *geoObject2DInit(EGeoSubType subType, BArray *coordArray) {
SGeoObj2D *pObj = (SGeoObj2D *)benchCalloc(1, sizeof(SGeoObj2D), true);
pObj->type = subType;
pObj->coordArray = coordArray;
return pObj;
}

static SGeoObj2D *geoObject2DRandInit(EGeoSubType subType, int coordNum) {
const SGeoSubTypeInfo *info = &GeoInfo[subType];

if (info->isClosed) {
coordNum = coordNum - 1;
}
BArray *array = randCoordArray(coordNum);
if (info->isClosed) {
SGeoCoord2D *pCoord = (SGeoCoord2D *)benchCalloc(1, sizeof(SGeoCoord2D), true);
SGeoCoord2D *pFirstCoord= benchArrayGet(array, 0);
memcpy(pCoord, pFirstCoord, sizeof(SGeoCoord2D));
benchArrayPush(array, pCoord);
}
return geoObject2DInit(subType, array);
}

static void geoObject2DDestory(SGeoObj2D *pObj) {
if (!pObj) return;
benchArrayDestroy(pObj->coordArray);
tmfree(pObj);
}

/*--- string formatters ---*/
static int geoCoord2DToStr(char *buffer, SGeoCoord2D *pCoord) { return sprintf(buffer, "%10.6lf %10.6lf", pCoord->x, pCoord->y); }

static int geoCoord2DArrayToStr(char *buffer, BArray *coordArray) {
int pos = 0;
bool firstCoord = true;
for (int i = 0; i < coordArray->size; i++) {
int size = 0;
if (firstCoord) {
firstCoord = false;
} else {
size = sprintf(buffer + pos, "%s", ", ");
pos += size;
}
size = geoCoord2DToStr(buffer + pos, benchArrayGet(coordArray, i));
pos += size;
}
return pos;
}

static int geoObject2DToStr(char *buffer, SGeoObj2D *object) {
int pos = sprintf(buffer, "%s", GeoInfo[object->type].preifx);
pos += geoCoord2DArrayToStr(buffer + pos, object->coordArray);
pos += sprintf(buffer + pos, "%s", GeoInfo[object->type].suffix);
return pos;
}

static BArray *randCoordArray(int count) {
BArray *array = benchArrayInit(8, sizeof(SGeoCoord2D));
int minVal = -1000, maxVal = 1000;
for (int i = 0; i < count; i++) {
double x = minVal + 1.0 * taosRandom() / RAND_MAX * (maxVal - minVal);
double y = minVal + 1.0 * taosRandom() / RAND_MAX * (maxVal - minVal);
SGeoCoord2D *pCoord = (SGeoCoord2D *)benchCalloc(1, sizeof(SGeoCoord2D), true);
pCoord->x = x;
pCoord->y = y;
benchArrayPush(array, pCoord);
}
return array;
}

int geoCalcBufferSize(int fieldLen) {
// not accurate, but enough
return fieldLen + 20;
}

int getGeoMaxType(int fieldLen) {
int maxType = -1;
for (int type = GEO_SUB_TYPE_COUNT - 1; type >= 0; type--) {
const SGeoSubTypeInfo *info = &GeoInfo[type];

int minLen = info->baseLen + info->minCoordNum * CCoordSize;
if (fieldLen >= minLen) {
maxType = type;
break;
}
}
return maxType;
}

void rand_geometry(char *str, int fieldLen, int maxType) {
EGeoSubType type = taosRandom() % (maxType + 1);
const SGeoSubTypeInfo *info = &GeoInfo[type];

int maxCoordNum = (fieldLen - info->baseLen) / CCoordSize;
maxCoordNum = min(maxCoordNum, info->maxCoordNum);

int coordNum = info->minCoordNum;
if (maxCoordNum > info->minCoordNum) {
coordNum = info->minCoordNum + taosRandom() % (maxCoordNum - info->minCoordNum);
}

SGeoObj2D *pObj = geoObject2DRandInit(type, coordNum);
geoObject2DToStr(str, pObj);
geoObject2DDestory(pObj);
}
18 changes: 15 additions & 3 deletions src/benchInsert.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,10 +246,16 @@ static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) {
Field * col = benchArrayGet(stbInfo->cols, colIndex);
int n;
if (col->type == TSDB_DATA_TYPE_BINARY ||
col->type == TSDB_DATA_TYPE_NCHAR) {
col->type == TSDB_DATA_TYPE_NCHAR ||
col->type == TSDB_DATA_TYPE_GEOMETRY) {
n = snprintf(colsBuf + len, col_buffer_len - len,
",%s %s(%d)", col->name,
convertDatatypeToString(col->type), col->length);
if (col->type == TSDB_DATA_TYPE_GEOMETRY && col->length < 21) {
errorPrint("%s() LN%d, geometry filed len must be greater than 21 on %d\n", __func__, __LINE__,
colIndex);
return -1;
}
} else {
n = snprintf(colsBuf + len, col_buffer_len - len,
",%s %s", col->name,
Expand Down Expand Up @@ -311,10 +317,16 @@ static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) {
for (tagIndex = 0; tagIndex < stbInfo->tags->size; tagIndex++) {
Field *tag = benchArrayGet(stbInfo->tags, tagIndex);
if (tag->type == TSDB_DATA_TYPE_BINARY ||
tag->type == TSDB_DATA_TYPE_NCHAR) {
tag->type == TSDB_DATA_TYPE_NCHAR ||
tag->type == TSDB_DATA_TYPE_GEOMETRY) {
n = snprintf(tagsBuf + len, tag_buffer_len - len,
"%s %s(%d),", tag->name,
convertDatatypeToString(tag->type), tag->length);
if (tag->type == TSDB_DATA_TYPE_GEOMETRY && tag->length < 21) {
errorPrint("%s() LN%d, geometry filed len must be greater than 21 on %d\n", __func__, __LINE__,
tagIndex);
return -1;
}
} else if (tag->type == TSDB_DATA_TYPE_JSON) {
n = snprintf(tagsBuf + len, tag_buffer_len - len,
"%s json", tag->name);
Expand Down Expand Up @@ -1873,7 +1885,7 @@ static void *syncWriteInterlace(void *sarg) {
infoPrint(
"thread[%d] has currently inserted rows: %" PRIu64
", peroid insert rate: %.3f rows/s \n",
pThreadInfo->threadID, pThreadInfo->totalInsertRows,
pThreadInfo->threadID, pThreadInfo->totalInsertRows,
(double)(pThreadInfo->totalInsertRows - lastTotalInsertRows) * 1000.0/(currentPrintTime - lastPrintTime));
lastPrintTime = currentPrintTime;
lastTotalInsertRows = pThreadInfo->totalInsertRows;
Expand Down
5 changes: 3 additions & 2 deletions src/benchJsonOpt.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,8 @@ static int getColumnAndTagTypeFromInsertJsonFile(
} else {
if (type == TSDB_DATA_TYPE_BINARY
|| type == TSDB_DATA_TYPE_JSON
|| type == TSDB_DATA_TYPE_NCHAR) {
|| type == TSDB_DATA_TYPE_NCHAR
|| type == TSDB_DATA_TYPE_GEOMETRY) {
length = g_arguments->binwidth;
} else {
length = convertTypeToLength(type);
Expand Down Expand Up @@ -2070,7 +2071,7 @@ static int getMetaFromTmqJsonFile(tools_cJSON *json) {
if (tools_cJSON_IsString(groupMode)) {
g_tmqInfo.consumerInfo.groupMode = groupMode->valuestring;
}


tools_cJSON *pollDelay = tools_cJSON_GetObjectItem(tmqInfo, "poll_delay");
if (tools_cJSON_IsNumber(pollDelay)) {
Expand Down
Loading
Loading