Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

support encode compress and level on creating stable #739

Merged
merged 4 commits into from
Apr 27, 2024
Merged

support encode compress and level on creating stable #739

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
63c9959
fix: add support compress
DuanKuanJun Apr 27, 2024
ce6c202
fix: add support compress
DuanKuanJun Apr 27, 2024
e1cc27f
fix: modify Field encode name
DuanKuanJun Apr 27, 2024
bd52c64
fix: add all column compress
DuanKuanJun Apr 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 67 additions & 0 deletions case/insertCompress.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"connection_pool_size": 8,
"num_of_records_per_req": 2000,
"thread_count": 3,
"create_table_thread_count": 10,
"prepare_rand": 1000,
"confirm_parameter_prompt": "no",
"databases": [
{
"dbinfo": {
"name": "test",
"drop": "yes",
"vgroups": 3,
"replica": 1,
"precision": "ms"
},
"super_tables": [
{
"name": "meters",
"child_table_exists": "no",
"childtable_count": 10,
"insert_rows": 1000000,
"childtable_prefix": "d",
"insert_mode": "taosc",
"insert_interval": 0,
"timestamp_step": 10,
"start_timestamp":1600000000000,
"disorder_ratio": 10,
"update_ratio": 5,
"delete_ratio": 1,
"disorder_fill_interval": 300,
"update_fill_interval": 25,
"generate_row_rule": 2,
"columns": [
{ "type": "bool", "name": "bc", "encode":"bit-packing", "compress":"zstd", "level":"high" },
{ "type": "float", "name": "fc", "encode":"delta-d", "compress":"zlib", "level":"medium", "max": 100, "min": 0 },
{ "type": "double", "name": "dc", "encode":"delta-d", "compress":"xz", "level":"low", "max": 100, "min": 0 },
{ "type": "tinyint", "name": "ti", "encode":"delta-i", "compress":"zstd", "level":"high", "max": 100, "min": 0 },
{ "type": "smallint", "name": "si", "max": 100, "min": 0, "compress":"zlib" },
{ "type": "int", "name": "ic", "max": 100, "min": 0, "compress":"zstd" },
{ "type": "bigint", "name": "bi", "max": 100, "min": 0, "encode":"delta-i" },
{ "type": "utinyint", "name": "uti", "max": 100, "min": 0, "level":"high" },
{ "type": "usmallint", "name": "usi", "max": 100, "min": 0, "level":"medium" },
{ "type": "uint", "name": "ui", "max": 100, "min": 0, "level":"low" },
{ "type": "ubigint", "name": "ubi", "max": 100, "min": 0, "compress":"xz", "level":"medium" },
{ "type": "binary", "name": "bin", "len": 32, "compress":"zstd"},
{ "type": "nchar", "name": "nch", "len": 64, "compress":"xz"}
],
"tags": [
{"type": "tinyint", "name": "groupid","max": 10,"min": 1},
{"type": "binary", "name": "location", "len": 16,
"values": ["San Francisco", "Los Angles", "San Diego",
"San Jose", "Palo Alto", "Campbell", "Mountain View",
"Sunnyvale", "Santa Clara", "Cupertino"]
}
]
}
]
}
]
}
6 changes: 6 additions & 0 deletions inc/bench.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -587,6 +587,7 @@ typedef struct SChildField {
#define tmpFloat(field) tmpFloatImpl (field,0,0,0)
#define tmpDouble(field) tmpDoubleImpl(field,0,0)

#define COMP_NAME_LEN 32

typedef struct SField {
uint8_t type;
Expand Down Expand Up @@ -614,6 +615,11 @@ typedef struct SField {
bool fillNull;
uint8_t gen; // see GEN_ define

// compress
char encode[COMP_NAME_LEN];
char compress[COMP_NAME_LEN];
char level[COMP_NAME_LEN];

} Field;

typedef struct STSMA {
Expand Down
24 changes: 23 additions & 1 deletion src/benchInsert.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,12 +217,27 @@ static void dropSuperTable(SDataBase* database, SSuperTable* stbInfo) {
}
#endif // WEBSOCKET

int getCompressStr(Field* col, char* buf) {
int pos = 0;
if(strlen(col->encode) > 0) {
pos +=sprintf(buf + pos, "encode \'%s\' ", col->encode);
}
if(strlen(col->compress) > 0) {
pos +=sprintf(buf + pos, "compress \'%s\' ", col->compress);
}
if(strlen(col->level) > 0) {
pos +=sprintf(buf + pos, "level \'%s\' ", col->level);
}

return pos;
}

static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) {
if (g_arguments->supplementInsert) {
return 0;
}

uint32_t col_buffer_len = (TSDB_COL_NAME_LEN + 15) * stbInfo->cols->size;
uint32_t col_buffer_len = (TSDB_COL_NAME_LEN + 15 + COMP_NAME_LEN*3) * stbInfo->cols->size;
char *colsBuf = benchCalloc(1, col_buffer_len, false);
char* command = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false);
int len = 0;
Expand All @@ -247,6 +262,13 @@ static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) {
n = snprintf(colsBuf + len, col_buffer_len - len, " %s", PRIMARY_KEY);
}

// compress key
char keys[COMP_NAME_LEN*3] = "";
if (getCompressStr(col, keys) > 0) {
len += n;
n = snprintf(colsBuf + len, col_buffer_len - len, " %s", keys);
}

if (n < 0 || n >= col_buffer_len - len) {
errorPrint("%s() LN%d, snprintf overflow on %d\n",
__func__, __LINE__, colIndex);
Expand Down
47 changes: 46 additions & 1 deletion src/benchJsonOpt.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,9 @@ static int getColumnAndTagTypeFromInsertJsonFile(
int32_t offset = 0;
uint8_t gen = GEN_RANDOM;
bool fillNull = true;
char* encode = NULL;
char* compress = NULL;
char* level = NULL;

tools_cJSON *column = tools_cJSON_GetArrayItem(columnsObj, k);
if (!tools_cJSON_IsObject(column)) {
Expand Down Expand Up @@ -255,14 +258,30 @@ static int getColumnAndTagTypeFromInsertJsonFile(
gen = GEN_ORDER;
}
}
// gen
// fillNull
tools_cJSON *dataNull = tools_cJSON_GetObjectItem(column, "fillNull");
if (tools_cJSON_IsString(dataNull)) {
if (strcasecmp(dataNull->valuestring, "false") == 0) {
fillNull = false;
}
}

// encode
tools_cJSON *dataEncode = tools_cJSON_GetObjectItem(column, "encode");
if (tools_cJSON_IsString(dataEncode)) {
encode = dataEncode->valuestring;
}
// compress
tools_cJSON *dataCompress = tools_cJSON_GetObjectItem(column, "compress");
if (tools_cJSON_IsString(dataCompress)) {
compress = dataCompress->valuestring;
}
// level
tools_cJSON *dataLevel = tools_cJSON_GetObjectItem(column, "level");
if (tools_cJSON_IsString(dataLevel)) {
level = dataLevel->valuestring;
}

// fun
tools_cJSON *fun = tools_cJSON_GetObjectItem(column, "fun");
if (tools_cJSON_IsString(fun)) {
Expand Down Expand Up @@ -328,6 +347,32 @@ static int getColumnAndTagTypeFromInsertJsonFile(
} else {
snprintf(col->name, TSDB_COL_NAME_LEN, "c%d", index);
}

// encode
if(encode) {
if (strlen(encode) < COMP_NAME_LEN) {
strcpy(col->encode, encode);
} else {
errorPrint("encode name length over (%d) bytes, ignore. name=%s", COMP_NAME_LEN, encode);
}
}
// compress
if(compress) {
if (strlen(compress) < COMP_NAME_LEN) {
strcpy(col->compress, compress);
} else {
errorPrint("compress name length over (%d) bytes, ignore. name=%s", COMP_NAME_LEN, compress);
}
}
// level
if(level) {
if (strlen(level) < COMP_NAME_LEN) {
strcpy(col->level, level);
} else {
errorPrint("level name length over (%d) bytes, ignore. name=%s", COMP_NAME_LEN, level);
}
}

index++;
}
}
Expand Down
Loading