Add keep-alive in splunk&elastic output plugins (#673)

* Add keep-alive in splunk&elastic output plugins * Fix doc
ozontech · Sep 11, 2024 · cfa0fd2 · cfa0fd2
1 parent cb0b804
commit cfa0fd2
Show file tree

Hide file tree

Showing 4 changed files with 78 additions and 3 deletions.
diff --git a/plugin/output/elasticsearch/README.md b/plugin/output/elasticsearch/README.md
@@ -44,6 +44,19 @@ Path or content of a PEM-encoded CA file.
 
 <br>
 
+**`keep_alive`** *`KeepAliveConfig`* 
+
+Keep-alive config.
+
+`KeepAliveConfig` params:
+* `max_idle_conn_duration` - idle keep-alive connections are closed after this duration.
+By default idle connections are closed after `10s`.
+* `max_conn_duration` - keep-alive connections are closed after this duration.
+If set to `0` - connection duration is unlimited.
+By default connection duration is `5m`.
+
+<br>
+
 **`index_format`** *`string`* *`default=file-d-%`* 
 
 It defines the pattern of elasticsearch index name. Use `%` character as a placeholder. Use `index_values` to define values for the replacement.

diff --git a/plugin/output/elasticsearch/elasticsearch.go b/plugin/output/elasticsearch/elasticsearch.go
@@ -124,6 +124,18 @@ type Config struct {
 	// > Path or content of a PEM-encoded CA file.
 	CACert string `json:"ca_cert"` // *
 
+	// > @3@4@5@6
+	// >
+	// > Keep-alive config.
+	// >
+	// > `KeepAliveConfig` params:
+	// > * `max_idle_conn_duration` - idle keep-alive connections are closed after this duration.
+	// > By default idle connections are closed after `10s`.
+	// > * `max_conn_duration` - keep-alive connections are closed after this duration.
+	// > If set to `0` - connection duration is unlimited.
+	// > By default connection duration is `5m`.
+	KeepAlive KeepAliveConfig `json:"keep_alive" child:"true"` // *
+
 	// > @3@4@5@6
 	// >
 	// > It defines the pattern of elasticsearch index name. Use `%` character as a placeholder. Use `index_values` to define values for the replacement.
@@ -206,6 +218,16 @@ type Config struct {
 	RetentionExponentMultiplier int `json:"retention_exponentially_multiplier" default:"2"` // *
 }
 
+type KeepAliveConfig struct {
+	// Idle keep-alive connections are closed after this duration.
+	MaxIdleConnDuration  cfg.Duration `json:"max_idle_conn_duration" parse:"duration" default:"10s"`
+	MaxIdleConnDuration_ time.Duration
+
+	// Keep-alive connections are closed after this duration.
+	MaxConnDuration  cfg.Duration `json:"max_conn_duration" parse:"duration" default:"5m"`
+	MaxConnDuration_ time.Duration
+}
+
 type data struct {
 	outBuf []byte
 }
@@ -301,9 +323,11 @@ func (p *Plugin) registerMetrics(ctl *metric.Ctl) {
 
 func (p *Plugin) prepareClient() {
 	p.client = &fasthttp.Client{
-		ReadTimeout:     p.config.ConnectionTimeout_ * 2,
-		WriteTimeout:    p.config.ConnectionTimeout_ * 2,
-		MaxConnDuration: time.Minute * 5,
+		ReadTimeout:  p.config.ConnectionTimeout_ * 2,
+		WriteTimeout: p.config.ConnectionTimeout_ * 2,
+
+		MaxIdleConnDuration: p.config.KeepAlive.MaxIdleConnDuration_,
+		MaxConnDuration:     p.config.KeepAlive.MaxConnDuration_,
 	}
 	if p.config.CACert != "" {
 		b := xtls.NewConfigBuilder()

diff --git a/plugin/output/splunk/README.md b/plugin/output/splunk/README.md
@@ -72,6 +72,19 @@ Token for an authentication for a HEC endpoint.
 
 <br>
 
+**`keep_alive`** *`KeepAliveConfig`* 
+
+Keep-alive config.
+
+`KeepAliveConfig` params:
+* `max_idle_conn_duration` - idle keep-alive connections are closed after this duration.
+By default idle connections are closed after `10s`.
+* `max_conn_duration` - keep-alive connections are closed after this duration.
+If set to `0` - connection duration is unlimited.
+By default connection duration is unlimited.
+
+<br>
+
 **`workers_count`** *`cfg.Expression`* *`default=gomaxprocs*4`* 
 
 How many workers will be instantiated to send batches.

diff --git a/plugin/output/splunk/splunk.go b/plugin/output/splunk/splunk.go
@@ -157,6 +157,18 @@ type Config struct {
 	// > Token for an authentication for a HEC endpoint.
 	Token string `json:"token" required:"true"` // *
 
+	// > @3@4@5@6
+	// >
+	// > Keep-alive config.
+	// >
+	// > `KeepAliveConfig` params:
+	// > * `max_idle_conn_duration` - idle keep-alive connections are closed after this duration.
+	// > By default idle connections are closed after `10s`.
+	// > * `max_conn_duration` - keep-alive connections are closed after this duration.
+	// > If set to `0` - connection duration is unlimited.
+	// > By default connection duration is unlimited.
+	KeepAlive KeepAliveConfig `json:"keep_alive" child:"true"` // *
+
 	// > @3@4@5@6
 	// >
 	// > How many workers will be instantiated to send batches.
@@ -221,6 +233,16 @@ type Config struct {
 	CopyFields []CopyField `json:"copy_fields" slice:"true"` // *
 }
 
+type KeepAliveConfig struct {
+	// Idle keep-alive connections are closed after this duration.
+	MaxIdleConnDuration  cfg.Duration `json:"max_idle_conn_duration" parse:"duration" default:"10s"`
+	MaxIdleConnDuration_ time.Duration
+
+	// Keep-alive connections are closed after this duration.
+	MaxConnDuration  cfg.Duration `json:"max_conn_duration" parse:"duration" default:"0"`
+	MaxConnDuration_ time.Duration
+}
+
 type data struct {
 	outBuf []byte
 }
@@ -316,6 +338,9 @@ func (p *Plugin) prepareClient() {
 		ReadTimeout:  p.config.RequestTimeout_,
 		WriteTimeout: p.config.RequestTimeout_,
 
+		MaxIdleConnDuration: p.config.KeepAlive.MaxIdleConnDuration_,
+		MaxConnDuration:     p.config.KeepAlive.MaxConnDuration_,
+
 		TLSConfig: &tls.Config{
 			// TODO: make this configuration option and false by default
 			InsecureSkipVerify: true,