Skip to content

Commit

Permalink
Merge pull request #690 from taosdata/feat/math
Browse files Browse the repository at this point in the history 
support to generate column data with function like sin cos
  • Loading branch information
@plum-lihui
plum-lihui authored Jul 23, 2023
2 parents 76ad294 + 044523a commit 561daca
Show file tree
Hide file tree
Showing 5 changed files with 256 additions and 30 deletions.
77 changes: 77 additions & 0 deletions example/insert_math.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"connection_pool_size": 8,
"thread_count": 4,
"create_table_thread_count": 7,
"result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"interlace_rows": 100,
"num_of_records_per_req": 100,
"prepared_rand": 3600,
"chinese": "no",
"escape_character": "yes",
"databases": [
{
"dbinfo": {
"name": "test",
"drop": "yes",
"precision": "ms"
},
"super_tables": [
{
"name": "meters",
"child_table_exists": "no",
"childtable_count": 1000,
"childtable_prefix": "d",
"auto_create_table": "no",
"batch_create_tbl_num": 5,
"data_source": "rand",
"insert_mode": "taosc",
"non_stop_mode": "no",
"line_protocol": "line",
"insert_rows": 1000,
"childtable_limit": 10,
"childtable_offset": 100,
"interlace_rows": 0,
"insert_interval": 0,
"partial_col_num": 0,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 10,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"use_sample_ts": "no",
"tags_file": "",
"columns": [
{ "type": "FLOAT", "name": "current", "fun": "1*sin(x)+10*random(15)"},
{ "type": "INT", "name": "voltage", "fun": "40*sin(x)+200*random(10)"},
{ "type": "FLOAT", "name": "phase", "fun": "1*sin(x)+1*random(3)"}
],
"tags": [
{
"type": "TINYINT",
"name": "groupid",
"max": 10,
"min": 1
},
{
"name": "location",
"type": "BINARY",
"len": 16,
"values": ["San Francisco", "Los Angles", "San Diego",
"San Jose", "Palo Alto", "Campbell", "Mountain View",
"Sunnyvale", "Santa Clara", "Cupertino"]
}
]
}
]
}
]
}
16 changes: 16 additions & 0 deletions inc/bench.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -553,6 +553,15 @@ typedef struct SChildField {
StmtData stmtData;
} ChildField;

#define PI 3.141592654
#define ATOR(x) (x*3.141592654/180)

#define FUNTYPE_NONE 0
#define FUNTYPE_SIN 1
#define FUNTYPE_COS 2

#define FUNTYPE_CNT 2

typedef struct SField {
uint8_t type;
char name[TSDB_COL_NAME_LEN + 1];
Expand All @@ -563,6 +572,13 @@ typedef struct SField {
int64_t max;
int64_t min;
tools_cJSON * values;

// fun
uint8_t funType;
float multiple;
int32_t addend;
int32_t random;

bool sma;
} Field;

Expand Down
16 changes: 8 additions & 8 deletions src/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,8 +308,8 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
FIND_LIBRARY(LIBZ_LIBRARY z)
MESSAGE(${ARGP_LIBRARY})

TARGET_LINK_LIBRARIES(taosBenchmark taos pthread toolscJson $<$<BOOL:${LIBZ_LIBRARY}>:${LIBZ_LIBRARY}> $<$<BOOL:${ARGP_LIBRARY}>:${ARGP_LIBRARY}> ${WEBSOCKET_LINK_FLAGS})
TARGET_LINK_LIBRARIES(taosdump taos avro jansson atomic pthread argp $<$<BOOL:${LIBZ_LIBRARY}>:${LIBZ_LIBRARY}> $<$<BOOL:${ARGP_LIBRARY}>:${ARGP_LIBRARY}> ${WEBSOCKET_LINK_FLAGS})
TARGET_LINK_LIBRARIES(taosBenchmark taos pthread m toolscJson $<$<BOOL:${LIBZ_LIBRARY}>:${LIBZ_LIBRARY}> $<$<BOOL:${ARGP_LIBRARY}>:${ARGP_LIBRARY}> ${WEBSOCKET_LINK_FLAGS})
TARGET_LINK_LIBRARIES(taosdump taos avro jansson atomic pthread m argp $<$<BOOL:${LIBZ_LIBRARY}>:${LIBZ_LIBRARY}> $<$<BOOL:${ARGP_LIBRARY}>:${ARGP_LIBRARY}> ${WEBSOCKET_LINK_FLAGS})
ELSEIF(${OS_ID} MATCHES "Darwin")
ADD_LIBRARY(argp STATIC IMPORTED)
IF (CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64")
Expand All @@ -319,11 +319,11 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
SET_PROPERTY(TARGET argp PROPERTY IMPORTED_LOCATION "/usr/local/lib/libargp.a")
INCLUDE_DIRECTORIES(/usr/local/include/)
ENDIF ()
TARGET_LINK_LIBRARIES(taosBenchmark taos pthread toolscJson argp ${WEBSOCKET_LINK_FLAGS})
TARGET_LINK_LIBRARIES(taosBenchmark taos pthread m toolscJson argp ${WEBSOCKET_LINK_FLAGS})
ElSE ()
MESSAGE("${Yellow} DEBUG mode use shared avro library to link for debug ${ColourReset}")
TARGET_LINK_LIBRARIES(taosdump taos avro jansson atomic pthread ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS})
TARGET_LINK_LIBRARIES(taosBenchmark taos pthread toolscJson ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS})
TARGET_LINK_LIBRARIES(taosdump taos avro jansson atomic pthread m ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS})
TARGET_LINK_LIBRARIES(taosBenchmark taos pthread m toolscJson ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS})
ENDIF()

ELSE ()
Expand Down Expand Up @@ -362,7 +362,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
INCLUDE_DIRECTORIES(/usr/local/include/)
ENDIF ()

TARGET_LINK_LIBRARIES(taosBenchmark taos pthread toolscJson argp ${WEBSOCKET_LINK_FLAGS})
TARGET_LINK_LIBRARIES(taosBenchmark taos pthread m toolscJson argp ${WEBSOCKET_LINK_FLAGS})
ELSE ()
EXECUTE_PROCESS (
COMMAND sh -c "awk -F= '/^ID=/{print $2}' /etc/os-release |tr -d '\n' | tr -d '\"'"
Expand Down Expand Up @@ -414,10 +414,10 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
MESSAGE(${LIBZ_LIBRARY})

TARGET_LINK_LIBRARIES(taosdump taos avro jansson snappy stdc++ lzma atomic pthread $<$<BOOL:${LIBZ_LIBRARY}>:${LIBZ_LIBRARY}> $<$<BOOL:${ARGP_LIBRARY}>:${ARGP_LIBRARY}> ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS})
TARGET_LINK_LIBRARIES(taosBenchmark taos pthread toolscJson $<$<BOOL:${LIBZ_LIBRARY}>:${LIBZ_LIBRARY}> $<$<BOOL:${ARGP_LIBRARY}>:${ARGP_LIBRARY}> ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS})
TARGET_LINK_LIBRARIES(taosBenchmark taos pthread m toolscJson $<$<BOOL:${LIBZ_LIBRARY}>:${LIBZ_LIBRARY}> $<$<BOOL:${ARGP_LIBRARY}>:${ARGP_LIBRARY}> ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS})
ELSE()
TARGET_LINK_LIBRARIES(taosdump taos avro jansson snappy stdc++ lzma libz-static atomic pthread ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS})
TARGET_LINK_LIBRARIES(taosBenchmark taos pthread toolscJson ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS})
TARGET_LINK_LIBRARIES(taosBenchmark taos pthread m toolscJson ${WEBSOCKET_LINK_FLAGS} ${GCC_COVERAGE_LINK_FLAGS})
ENDIF()
ENDIF ()

Expand Down
100 changes: 78 additions & 22 deletions src/benchData.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
*/

#include <bench.h>
#include <math.h>
#include <benchData.h>

const char charset[] =
Expand Down Expand Up @@ -41,6 +42,25 @@ const char* locations_sml[] = {
#include "benchLocations.h"
#endif

// calc expression value like 10*sin(x) + 100
float calc_expr_value(Field *field, int32_t angle) {
float radian = ATOR(angle);
float funVal = 0;
if (field->funType == FUNTYPE_SIN)
funVal = sin(radian);
else if (field->funType == FUNTYPE_COS)
funVal = cos(radian);

float val = field->multiple * funVal + field->addend;
if (field->random >0) {
float rate = taosRandom() % field->random;
val += field->addend * (rate/100);
}

return val;
}


static int usc2utf8(char *p, int unic) {
int ret = 0;
if (unic <= 0x0000007F) {
Expand Down Expand Up @@ -367,16 +387,24 @@ static int tmpStr(char *tmp, int iface, Field *field, int i) {
return 0;
}

FORCE_INLINE double tmpDouble(Field *field) {
FORCE_INLINE double tmpDoubleImpl(Field *field, int32_t angle) {
double doubleTmp = (double)(field->min);
if (field->max != field->min) {

if(field->funType != FUNTYPE_NONE) {
doubleTmp = calc_expr_value(field, angle);
} else if (field->max != field->min) {
doubleTmp += ((taosRandom() %
(field->max - field->min)) +
taosRandom() % 1000000 / 1000000.0);
}
return doubleTmp;
}

FORCE_INLINE double tmpDouble(Field *field) {
return tmpDoubleImpl(field, 0);
}


FORCE_INLINE uint64_t tmpUint64(Field *field) {
uint64_t ubigintTmp = field->min;
if (field->max != field->min) {
Expand Down Expand Up @@ -435,14 +463,20 @@ FORCE_INLINE uint16_t tmpUint16(Field *field) {
return usmallintTmp;
}

FORCE_INLINE int64_t tmpInt64(Field *field) {
FORCE_INLINE int64_t tmpInt64Impl(Field *field, int32_t angle) {
int64_t bigintTmp = field->min;
if (field->min != field->max) {
if(field->funType != FUNTYPE_NONE) {
bigintTmp = calc_expr_value(field, angle);
} else if (field->min != field->max) {
bigintTmp += (taosRandom() % (field->max - field->min));
}
return bigintTmp;
}

FORCE_INLINE int64_t tmpInt64(Field *field) {
return tmpInt64Impl(field, 0);
}

FORCE_INLINE float tmpFloat(Field *field) {
float floatTmp = field->min;
if (field->max != field->min) {
Expand All @@ -452,30 +486,41 @@ FORCE_INLINE float tmpFloat(Field *field) {
return floatTmp;
}

static float tmpFloatI(Field *field, int i) {
static float tmpFloatImpl(Field *field, int i, int32_t angle) {
float floatTmp = (float)field->min;
if (field->max != field->min) {
floatTmp += ((taosRandom() %
(field->max - field->min))
+ (taosRandom() % 1000) / 1000.0);
}
if (g_arguments->demo_mode && i == 0) {
floatTmp = (float)(9.8 + 0.04 * (taosRandom() % 10)
+ floatTmp / 1000000000);
} else if (g_arguments->demo_mode && i == 2) {
floatTmp = (float)((105 + taosRandom() % 10
+ floatTmp / 1000000000) / 360);
if(field->funType != FUNTYPE_NONE) {
floatTmp = calc_expr_value(field, angle);
} else {
if (field->max != field->min) {
floatTmp += ((taosRandom() %
(field->max - field->min))
+ (taosRandom() % 1000) / 1000.0);
}
if (g_arguments->demo_mode && i == 0) {
floatTmp = (float)(9.8 + 0.04 * (taosRandom() % 10)
+ floatTmp / 1000000000);
} else if (g_arguments->demo_mode && i == 2) {
floatTmp = (float)((105 + taosRandom() % 10
+ floatTmp / 1000000000) / 360);
}
}
return floatTmp;
}

static int tmpInt32(Field *field, int i) {
static float tmpFloatI(Field *field, int i) {
return tmpFloatImpl(field, i, 0);
}

static int tmpInt32Impl(Field *field, int i, int angle) {
int intTmp;
if ((g_arguments->demo_mode) && (i == 0)) {
unsigned int tmpRand = taosRandom();
intTmp = tmpRand % 10 + 1;
} else if ((g_arguments->demo_mode) && (i == 1)) {
intTmp = 105 + taosRandom() % 10;
} else if (field->funType != FUNTYPE_NONE) {
// calc from function
intTmp = calc_expr_value(field, angle);
} else {
if (field->min < (-1 * (RAND_MAX >> 1))) {
field->min = -1 * (RAND_MAX >> 1);
Expand All @@ -491,6 +536,10 @@ static int tmpInt32(Field *field, int i) {
return intTmp;
}

static int tmpInt32(Field *field, int i) {
return tmpInt32Impl(field, i, 0);
}

static int tmpJson(char *sampleDataBuf,
int bufLen, int64_t pos,
int fieldsSize, Field *field) {
Expand Down Expand Up @@ -545,6 +594,8 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf,
int64_t bufLen,
int lenOfOneRow, BArray * fields, int64_t loop,
bool tag) {

int angle = 0; // 0 ~ 360
for (int64_t k = 0; k < loop; ++k) {
int64_t pos = k * lenOfOneRow;
int fieldsSize = fields->size;
Expand Down Expand Up @@ -609,13 +660,13 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf,
break;
}
case TSDB_DATA_TYPE_INT: {
int32_t intTmp = tmpInt32(field, i);
int32_t intTmp = tmpInt32Impl(field, i, angle);
n = snprintf(sampleDataBuf + pos, bufLen - pos,
"%d,", intTmp);
break;
}
case TSDB_DATA_TYPE_BIGINT: {
int64_t bigintTmp = tmpInt64(field);
int64_t bigintTmp = tmpInt64Impl(field, angle);
n = snprintf(sampleDataBuf + pos,
bufLen - pos, "%"PRId64",", bigintTmp);
break;
Expand All @@ -634,13 +685,13 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf,
break;
}
case TSDB_DATA_TYPE_FLOAT: {
float floatTmp = tmpFloatI(field, i);
float floatTmp = tmpFloatImpl(field, i, angle);
n = snprintf(sampleDataBuf + pos, bufLen - pos,
"%f,", floatTmp);
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
double double_ = tmpDouble(field);
double double_ = tmpDoubleImpl(field, angle);
n = snprintf(sampleDataBuf + pos, bufLen - pos,
"%f,", double_);
break;
Expand Down Expand Up @@ -675,7 +726,12 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf,
}
skip_sql:
*(sampleDataBuf + pos - 1) = 0;
}
angle += 1;
if (angle > 360) {
// 360 is a circle
angle = 0;
}
}

return 0;
}
Expand Down
Loading

0 comments on commit 561daca

Please sign in to comment.