Skip to content

Commit 05d1071

Browse files
iuwqyiriuwqyir
committed
make ch parallel view processing configurable
1 parent 03325a3 commit 05d1071

File tree

3 files changed

+21
-12
lines changed

3 files changed

+21
-12
lines changed

‎cmd/root.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ func init() {
8282
rootCmd.PersistentFlags().Int("storage-orchestrator-clickhouse-maxOpenConns", 30, "Clickhouse max open connections for orchestrator storage")
8383
rootCmd.PersistentFlags().Int("storage-orchestrator-clickhouse-maxIdleConns", 30, "Clickhouse max idle connections for orchestrator storage")
8484
rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-disableTLS", false, "Clickhouse disableTLS for orchestrator storage")
85+
rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-enableParallelViewProcessing", false, "Clickhouse enableParallelViewProcessing for orchestrator storage")
8586
rootCmd.PersistentFlags().String("storage-staging-clickhouse-host", "", "Clickhouse host for staging storage")
8687
rootCmd.PersistentFlags().String("storage-main-clickhouse-host", "", "Clickhouse host for main storage")
8788
rootCmd.PersistentFlags().String("storage-main-clickhouse-username", "", "Clickhouse username for main storage")
@@ -91,13 +92,15 @@ func init() {
9192
rootCmd.PersistentFlags().Int("storage-main-clickhouse-maxOpenConns", 30, "Clickhouse max open connections for main storage")
9293
rootCmd.PersistentFlags().Int("storage-main-clickhouse-maxIdleConns", 30, "Clickhouse max idle connections for main storage")
9394
rootCmd.PersistentFlags().Bool("storage-main-clickhouse-disableTLS", false, "Clickhouse disableTLS for main storage")
95+
rootCmd.PersistentFlags().Bool("storage-main-clickhouse-enableParallelViewProcessing", false, "Clickhouse enableParallelViewProcessing for main storage")
9496
rootCmd.PersistentFlags().String("storage-staging-clickhouse-username", "", "Clickhouse username for staging storage")
9597
rootCmd.PersistentFlags().String("storage-staging-clickhouse-password", "", "Clickhouse password for staging storage")
9698
rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-asyncInsert", false, "Clickhouse async insert for staging storage")
9799
rootCmd.PersistentFlags().Int("storage-staging-clickhouse-maxRowsPerInsert", 100000, "Clickhouse max rows per insert for staging storage")
98100
rootCmd.PersistentFlags().Int("storage-staging-clickhouse-maxOpenConns", 30, "Clickhouse max open connections for staging storage")
99101
rootCmd.PersistentFlags().Int("storage-staging-clickhouse-maxIdleConns", 30, "Clickhouse max idle connections for staging storage")
100102
rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-disableTLS", false, "Clickhouse disableTLS for staging storage")
103+
rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-enableParallelViewProcessing", false, "Clickhouse enableParallelViewProcessing for staging storage")
101104
rootCmd.PersistentFlags().String("api-host", "localhost:3000", "API host")
102105
rootCmd.PersistentFlags().String("api-basicAuth-username", "", "API basic auth username")
103106
rootCmd.PersistentFlags().String("api-basicAuth-password", "", "API basic auth password")
@@ -164,6 +167,7 @@ func init() {
164167
viper.BindPFlag("storage.staging.clickhouse.maxOpenConns", rootCmd.PersistentFlags().Lookup("storage-staging-clickhouse-maxOpenConns"))
165168
viper.BindPFlag("storage.staging.clickhouse.maxIdleConns", rootCmd.PersistentFlags().Lookup("storage-staging-clickhouse-maxIdleConns"))
166169
viper.BindPFlag("storage.staging.clickhouse.disableTLS", rootCmd.PersistentFlags().Lookup("storage-staging-clickhouse-disableTLS"))
170+
viper.BindPFlag("storage.staging.clickhouse.enableParallelViewProcessing", rootCmd.PersistentFlags().Lookup("storage-staging-clickhouse-enableParallelViewProcessing"))
167171
viper.BindPFlag("storage.main.clickhouse.database", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-database"))
168172
viper.BindPFlag("storage.main.clickhouse.host", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-host"))
169173
viper.BindPFlag("storage.main.clickhouse.port", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-port"))
@@ -174,6 +178,7 @@ func init() {
174178
viper.BindPFlag("storage.main.clickhouse.maxOpenConns", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-maxOpenConns"))
175179
viper.BindPFlag("storage.main.clickhouse.maxIdleConns", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-maxIdleConns"))
176180
viper.BindPFlag("storage.main.clickhouse.disableTLS", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-disableTLS"))
181+
viper.BindPFlag("storage.main.clickhouse.enableParallelViewProcessing", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-enableParallelViewProcessing"))
177182
viper.BindPFlag("storage.orchestrator.clickhouse.database", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-database"))
178183
viper.BindPFlag("storage.orchestrator.clickhouse.host", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-host"))
179184
viper.BindPFlag("storage.orchestrator.clickhouse.port", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-port"))
@@ -184,6 +189,7 @@ func init() {
184189
viper.BindPFlag("storage.orchestrator.clickhouse.maxOpenConns", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-maxOpenConns"))
185190
viper.BindPFlag("storage.orchestrator.clickhouse.maxIdleConns", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-maxIdleConns"))
186191
viper.BindPFlag("storage.orchestrator.clickhouse.disableTLS", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-disableTLS"))
192+
viper.BindPFlag("storage.orchestrator.clickhouse.enableParallelViewProcessing", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-enableParallelViewProcessing"))
187193
viper.BindPFlag("api.host", rootCmd.PersistentFlags().Lookup("api-host"))
188194
viper.BindPFlag("api.basicAuth.username", rootCmd.PersistentFlags().Lookup("api-basicAuth-username"))
189195
viper.BindPFlag("api.basicAuth.password", rootCmd.PersistentFlags().Lookup("api-basicAuth-password"))

‎configs/config.go

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -71,17 +71,18 @@ type TableConfig struct {
7171
type TableOverrideConfig map[string]TableConfig
7272

7373
type ClickhouseConfig struct {
74-
Host string `mapstructure:"host"`
75-
Port int `mapstructure:"port"`
76-
Username string `mapstructure:"username"`
77-
Password string `mapstructure:"password"`
78-
Database string `mapstructure:"database"`
79-
DisableTLS bool `mapstructure:"disableTLS"`
80-
AsyncInsert bool `mapstructure:"asyncInsert"`
81-
MaxRowsPerInsert int `mapstructure:"maxRowsPerInsert"`
82-
MaxOpenConns int `mapstructure:"maxOpenConns"`
83-
MaxIdleConns int `mapstructure:"maxIdleConns"`
84-
ChainBasedConfig map[string]TableOverrideConfig `mapstructure:"chainBasedConfig"`
74+
Host string `mapstructure:"host"`
75+
Port int `mapstructure:"port"`
76+
Username string `mapstructure:"username"`
77+
Password string `mapstructure:"password"`
78+
Database string `mapstructure:"database"`
79+
DisableTLS bool `mapstructure:"disableTLS"`
80+
AsyncInsert bool `mapstructure:"asyncInsert"`
81+
MaxRowsPerInsert int `mapstructure:"maxRowsPerInsert"`
82+
MaxOpenConns int `mapstructure:"maxOpenConns"`
83+
MaxIdleConns int `mapstructure:"maxIdleConns"`
84+
ChainBasedConfig map[string]TableOverrideConfig `mapstructure:"chainBasedConfig"`
85+
EnableParallelViewProcessing bool `mapstructure:"enableParallelViewProcessing"`
8586
}
8687

8788
type RPCBatchRequestConfig struct {

‎internal/storage/clickhouse.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,9 @@ func connectDB(cfg *config.ClickhouseConfig) (clickhouse.Conn, error) {
103103
"do_not_merge_across_partitions_select_final": "1",
104104
"use_skip_indexes_if_final": "1",
105105
"optimize_move_to_prewhere_if_final": "1",
106-
"parallel_view_processing": "1",
106+
}
107+
if cfg.EnableParallelViewProcessing {
108+
settings["parallel_view_processing"] = "1"
107109
}
108110
if cfg.AsyncInsert {
109111
settings["async_insert"] = "1"

0 commit comments

Comments
 (0)