Skip to content

Commit

Permalink
Merge pull request #739 from taosdata/feat/TD-29835
Browse files Browse the repository at this point in the history 
support encode compress and level on creating stable
  • Loading branch information
@DuanKuanJun
DuanKuanJun committed Apr 27, 2024
2 parents 2bc4598 + bd52c64 commit 876cf83
Show file tree
Hide file tree
Showing 4 changed files with 142 additions and 2 deletions.
67 changes: 67 additions & 0 deletions case/insertCompress.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"connection_pool_size": 8,
"num_of_records_per_req": 2000,
"thread_count": 3,
"create_table_thread_count": 10,
"prepare_rand": 1000,
"confirm_parameter_prompt": "no",
"databases": [
{
"dbinfo": {
"name": "test",
"drop": "yes",
"vgroups": 3,
"replica": 1,
"precision": "ms"
},
"super_tables": [
{
"name": "meters",
"child_table_exists": "no",
"childtable_count": 10,
"insert_rows": 1000000,
"childtable_prefix": "d",
"insert_mode": "taosc",
"insert_interval": 0,
"timestamp_step": 10,
"start_timestamp":1600000000000,
"disorder_ratio": 10,
"update_ratio": 5,
"delete_ratio": 1,
"disorder_fill_interval": 300,
"update_fill_interval": 25,
"generate_row_rule": 2,
"columns": [
{ "type": "bool", "name": "bc", "encode":"bit-packing", "compress":"zstd", "level":"high" },
{ "type": "float", "name": "fc", "encode":"delta-d", "compress":"zlib", "level":"medium", "max": 100, "min": 0 },
{ "type": "double", "name": "dc", "encode":"delta-d", "compress":"xz", "level":"low", "max": 100, "min": 0 },
{ "type": "tinyint", "name": "ti", "encode":"delta-i", "compress":"zstd", "level":"high", "max": 100, "min": 0 },
{ "type": "smallint", "name": "si", "max": 100, "min": 0, "compress":"zlib" },
{ "type": "int", "name": "ic", "max": 100, "min": 0, "compress":"zstd" },
{ "type": "bigint", "name": "bi", "max": 100, "min": 0, "encode":"delta-i" },
{ "type": "utinyint", "name": "uti", "max": 100, "min": 0, "level":"high" },
{ "type": "usmallint", "name": "usi", "max": 100, "min": 0, "level":"medium" },
{ "type": "uint", "name": "ui", "max": 100, "min": 0, "level":"low" },
{ "type": "ubigint", "name": "ubi", "max": 100, "min": 0, "compress":"xz", "level":"medium" },
{ "type": "binary", "name": "bin", "len": 32, "compress":"zstd"},
{ "type": "nchar", "name": "nch", "len": 64, "compress":"xz"}
],
"tags": [
{"type": "tinyint", "name": "groupid","max": 10,"min": 1},
{"type": "binary", "name": "location", "len": 16,
"values": ["San Francisco", "Los Angles", "San Diego",
"San Jose", "Palo Alto", "Campbell", "Mountain View",
"Sunnyvale", "Santa Clara", "Cupertino"]
}
]
}
]
}
]
}
6 changes: 6 additions & 0 deletions inc/bench.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -587,6 +587,7 @@ typedef struct SChildField {
#define tmpFloat(field) tmpFloatImpl (field,0,0,0)
#define tmpDouble(field) tmpDoubleImpl(field,0,0)

#define COMP_NAME_LEN 32

typedef struct SField {
uint8_t type;
Expand Down Expand Up @@ -614,6 +615,11 @@ typedef struct SField {
bool fillNull;
uint8_t gen; // see GEN_ define

// compress
char encode[COMP_NAME_LEN];
char compress[COMP_NAME_LEN];
char level[COMP_NAME_LEN];

} Field;

typedef struct STSMA {
Expand Down
24 changes: 23 additions & 1 deletion src/benchInsert.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,12 +217,27 @@ static void dropSuperTable(SDataBase* database, SSuperTable* stbInfo) {
}
#endif // WEBSOCKET

int getCompressStr(Field* col, char* buf) {
int pos = 0;
if(strlen(col->encode) > 0) {
pos +=sprintf(buf + pos, "encode \'%s\' ", col->encode);
}
if(strlen(col->compress) > 0) {
pos +=sprintf(buf + pos, "compress \'%s\' ", col->compress);
}
if(strlen(col->level) > 0) {
pos +=sprintf(buf + pos, "level \'%s\' ", col->level);
}

return pos;
}

static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) {
if (g_arguments->supplementInsert) {
return 0;
}

uint32_t col_buffer_len = (TSDB_COL_NAME_LEN + 15) * stbInfo->cols->size;
uint32_t col_buffer_len = (TSDB_COL_NAME_LEN + 15 + COMP_NAME_LEN*3) * stbInfo->cols->size;
char *colsBuf = benchCalloc(1, col_buffer_len, false);
char* command = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false);
int len = 0;
Expand All @@ -247,6 +262,13 @@ static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) {
n = snprintf(colsBuf + len, col_buffer_len - len, " %s", PRIMARY_KEY);
}

// compress key
char keys[COMP_NAME_LEN*3] = "";
if (getCompressStr(col, keys) > 0) {
len += n;
n = snprintf(colsBuf + len, col_buffer_len - len, " %s", keys);
}

if (n < 0 || n >= col_buffer_len - len) {
errorPrint("%s() LN%d, snprintf overflow on %d\n",
__func__, __LINE__, colIndex);
Expand Down
47 changes: 46 additions & 1 deletion src/benchJsonOpt.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,9 @@ static int getColumnAndTagTypeFromInsertJsonFile(
int32_t offset = 0;
uint8_t gen = GEN_RANDOM;
bool fillNull = true;
char* encode = NULL;
char* compress = NULL;
char* level = NULL;

tools_cJSON *column = tools_cJSON_GetArrayItem(columnsObj, k);
if (!tools_cJSON_IsObject(column)) {
Expand Down Expand Up @@ -255,14 +258,30 @@ static int getColumnAndTagTypeFromInsertJsonFile(
gen = GEN_ORDER;
}
}
// gen
// fillNull
tools_cJSON *dataNull = tools_cJSON_GetObjectItem(column, "fillNull");
if (tools_cJSON_IsString(dataNull)) {
if (strcasecmp(dataNull->valuestring, "false") == 0) {
fillNull = false;
}
}

// encode
tools_cJSON *dataEncode = tools_cJSON_GetObjectItem(column, "encode");
if (tools_cJSON_IsString(dataEncode)) {
encode = dataEncode->valuestring;
}
// compress
tools_cJSON *dataCompress = tools_cJSON_GetObjectItem(column, "compress");
if (tools_cJSON_IsString(dataCompress)) {
compress = dataCompress->valuestring;
}
// level
tools_cJSON *dataLevel = tools_cJSON_GetObjectItem(column, "level");
if (tools_cJSON_IsString(dataLevel)) {
level = dataLevel->valuestring;
}

// fun
tools_cJSON *fun = tools_cJSON_GetObjectItem(column, "fun");
if (tools_cJSON_IsString(fun)) {
Expand Down Expand Up @@ -328,6 +347,32 @@ static int getColumnAndTagTypeFromInsertJsonFile(
} else {
snprintf(col->name, TSDB_COL_NAME_LEN, "c%d", index);
}

// encode
if(encode) {
if (strlen(encode) < COMP_NAME_LEN) {
strcpy(col->encode, encode);
} else {
errorPrint("encode name length over (%d) bytes, ignore. name=%s", COMP_NAME_LEN, encode);
}
}
// compress
if(compress) {
if (strlen(compress) < COMP_NAME_LEN) {
strcpy(col->compress, compress);
} else {
errorPrint("compress name length over (%d) bytes, ignore. name=%s", COMP_NAME_LEN, compress);
}
}
// level
if(level) {
if (strlen(level) < COMP_NAME_LEN) {
strcpy(col->level, level);
} else {
errorPrint("level name length over (%d) bytes, ignore. name=%s", COMP_NAME_LEN, level);
}
}

index++;
}
}
Expand Down

0 comments on commit 876cf83

Please sign in to comment.