Skip to content

Commit 63e4887

Browse files
jakeloojakeloo
authored
Tune async insert buffer (#284)
1 parent 2914e3f commit 63e4887

File tree

2 files changed

+10
-7
lines changed

2 files changed

+10
-7
lines changed

‎cmd/root.go

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -80,13 +80,13 @@ func init() {
8080
rootCmd.PersistentFlags().String("storage-orchestrator-clickhouse-host", "", "Clickhouse host for orchestrator storage")
8181
rootCmd.PersistentFlags().String("storage-orchestrator-clickhouse-username", "", "Clickhouse username for orchestrator storage")
8282
rootCmd.PersistentFlags().String("storage-orchestrator-clickhouse-password", "", "Clickhouse password for orchestrator storage")
83-
rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-asyncInsert", false, "Clickhouse async insert for orchestrator storage")
83+
rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-asyncInsert", true, "Clickhouse async insert for orchestrator storage")
8484
rootCmd.PersistentFlags().Int("storage-orchestrator-clickhouse-maxRowsPerInsert", 100000, "Clickhouse max rows per insert for orchestrator storage")
8585
rootCmd.PersistentFlags().Int("storage-orchestrator-clickhouse-maxOpenConns", 30, "Clickhouse max open connections for orchestrator storage")
8686
rootCmd.PersistentFlags().Int("storage-orchestrator-clickhouse-maxIdleConns", 30, "Clickhouse max idle connections for orchestrator storage")
8787
rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-disableTLS", false, "Clickhouse disableTLS for orchestrator storage")
8888
rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-enableParallelViewProcessing", false, "Clickhouse enableParallelViewProcessing for orchestrator storage")
89-
rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-enableCompression", false, "Clickhouse enableCompression for orchestrator storage")
89+
rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-enableCompression", true, "Clickhouse enableCompression for orchestrator storage")
9090
rootCmd.PersistentFlags().Int("storage-orchestrator-clickhouse-maxQueryTime", 60, "Clickhouse max query time for orchestrator storage")
9191
rootCmd.PersistentFlags().Int("storage-orchestrator-clickhouse-maxMemoryUsage", 1000000000, "Clickhouse max memory usage in bytes for orchestrator storage")
9292
rootCmd.PersistentFlags().String("storage-orchestrator-postgres-host", "", "PostgreSQL host for orchestrator storage")
@@ -103,24 +103,24 @@ func init() {
103103
rootCmd.PersistentFlags().String("storage-main-clickhouse-host", "", "Clickhouse host for main storage")
104104
rootCmd.PersistentFlags().String("storage-main-clickhouse-username", "", "Clickhouse username for main storage")
105105
rootCmd.PersistentFlags().String("storage-main-clickhouse-password", "", "Clickhouse password for main storage")
106-
rootCmd.PersistentFlags().Bool("storage-main-clickhouse-asyncInsert", false, "Clickhouse async insert for main storage")
106+
rootCmd.PersistentFlags().Bool("storage-main-clickhouse-asyncInsert", true, "Clickhouse async insert for main storage")
107107
rootCmd.PersistentFlags().Int("storage-main-clickhouse-maxRowsPerInsert", 100000, "Clickhouse max rows per insert for main storage")
108108
rootCmd.PersistentFlags().Int("storage-main-clickhouse-maxOpenConns", 30, "Clickhouse max open connections for main storage")
109109
rootCmd.PersistentFlags().Int("storage-main-clickhouse-maxIdleConns", 30, "Clickhouse max idle connections for main storage")
110110
rootCmd.PersistentFlags().Bool("storage-main-clickhouse-disableTLS", false, "Clickhouse disableTLS for main storage")
111111
rootCmd.PersistentFlags().Bool("storage-main-clickhouse-enableParallelViewProcessing", false, "Clickhouse enableParallelViewProcessing for main storage")
112-
rootCmd.PersistentFlags().Bool("storage-main-clickhouse-enableCompression", false, "Clickhouse enableCompression for main storage")
112+
rootCmd.PersistentFlags().Bool("storage-main-clickhouse-enableCompression", true, "Clickhouse enableCompression for main storage")
113113
rootCmd.PersistentFlags().Int("storage-main-clickhouse-maxQueryTime", 60, "Clickhouse max query time for main storage")
114114
rootCmd.PersistentFlags().Int("storage-main-clickhouse-maxMemoryUsage", 1000000000, "Clickhouse max memory usage in bytes for main storage")
115115
rootCmd.PersistentFlags().String("storage-staging-clickhouse-username", "", "Clickhouse username for staging storage")
116116
rootCmd.PersistentFlags().String("storage-staging-clickhouse-password", "", "Clickhouse password for staging storage")
117-
rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-asyncInsert", false, "Clickhouse async insert for staging storage")
117+
rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-asyncInsert", true, "Clickhouse async insert for staging storage")
118118
rootCmd.PersistentFlags().Int("storage-staging-clickhouse-maxRowsPerInsert", 100000, "Clickhouse max rows per insert for staging storage")
119119
rootCmd.PersistentFlags().Int("storage-staging-clickhouse-maxOpenConns", 30, "Clickhouse max open connections for staging storage")
120120
rootCmd.PersistentFlags().Int("storage-staging-clickhouse-maxIdleConns", 30, "Clickhouse max idle connections for staging storage")
121121
rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-disableTLS", false, "Clickhouse disableTLS for staging storage")
122122
rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-enableParallelViewProcessing", false, "Clickhouse enableParallelViewProcessing for staging storage")
123-
rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-enableCompression", false, "Clickhouse enableCompression for staging storage")
123+
rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-enableCompression", true, "Clickhouse enableCompression for staging storage")
124124
rootCmd.PersistentFlags().Int("storage-staging-clickhouse-maxQueryTime", 60, "Clickhouse max query time for staging storage")
125125
rootCmd.PersistentFlags().Int("storage-staging-clickhouse-maxMemoryUsage", 1000000000, "Clickhouse max memory usage in bytes for staging storage")
126126
rootCmd.PersistentFlags().String("storage-staging-postgres-host", "", "PostgreSQL host for staging storage")
@@ -198,7 +198,7 @@ func init() {
198198
rootCmd.PersistentFlags().String("migrator-destination-clickhouse-password", "", "Clickhouse password for migrator destination")
199199
rootCmd.PersistentFlags().String("migrator-destination-clickhouse-database", "", "Clickhouse database for migrator destination")
200200
rootCmd.PersistentFlags().Bool("migrator-destination-clickhouse-disableTLS", false, "Clickhouse disableTLS for migrator destination")
201-
rootCmd.PersistentFlags().Bool("migrator-destination-clickhouse-asyncInsert", false, "Clickhouse async insert for migrator destination")
201+
rootCmd.PersistentFlags().Bool("migrator-destination-clickhouse-asyncInsert", true, "Clickhouse async insert for migrator destination")
202202
rootCmd.PersistentFlags().Int("migrator-destination-clickhouse-maxRowsPerInsert", 100000, "Clickhouse max rows per insert for migrator destination")
203203
rootCmd.PersistentFlags().Int("migrator-destination-clickhouse-maxOpenConns", 30, "Clickhouse max open connections for migrator destination")
204204
rootCmd.PersistentFlags().Int("migrator-destination-clickhouse-maxIdleConns", 30, "Clickhouse max idle connections for migrator destination")

‎internal/storage/clickhouse.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,9 @@ func connectDB(cfg *config.ClickhouseConfig) (clickhouse.Conn, error) {
127127
if cfg.AsyncInsert {
128128
settings["async_insert"] = "1"
129129
settings["wait_for_async_insert"] = "1"
130+
settings["async_insert_busy_timeout_ms"] = "3000" // 3 seconds
131+
settings["async_insert_max_data_size"] = "104857600" // 100MB
132+
settings["async_insert_max_query_number"] = "100000" // 100k
130133
}
131134
return settings
132135
}(),

0 commit comments

Comments
 (0)