Provide QueryLimit options for Third Party Logs API (Loki)

This adds a new config to provide query limit for Third Party Logging
API.

config/base/env/config

@@ -42,5 +42,6 @@ LOGGING_PLUGIN_TOKEN_PATH=/var/run/secrets/kubernetes.io/serviceaccount/token
 LOGGING_PLUGIN_NAMESPACE_KEY=kubernetes_namespace_name
 LOGGING_PLUGIN_STATIC_LABELS='log_type=application'
 LOGGING_PLUGIN_CA_CERT=
+LOGGING_PLUGIN_QUERY_LIMIT=1700
 LOGGING_PLUGIN_TLS_VERIFICATION_DISABLE=
 LOGGING_PLUGIN_FORWARDER_DELAY_DURATION=10

docs/logging-support.md

@@ -28,3 +28,4 @@ These are the common configuration options for all third party logging APIs.
 - `LOGGING_PLUGIN_CA_CERT`: The CA certificate to use for the third party logging API. This should ideally be passed as environment variable in the deployment spec of the results-api pod. (optional)
 - `LOGGING_PLUGIN_TLS_VERIFICATION_DISABLE`: Set to `true` to disable TLS verification for the third party logging API. (optional)
 - `LOGGING_PLUGIN_FORWARDER_DELAY_DURATION`: This is the max duration in minutes taken by third party logging system to forward and store the logs after completion of taskrun and pipelinerun. This is used to search between start time of runs and completion plus buffer duration.
+- `LOGGING_PLUGIN_QUERY_LIMIT`: Sets the query limit for Third Party Logging API if logging backend has a limit on number of log lines returned.

pkg/api/server/config/config.go

@@ -58,6 +58,7 @@ type Config struct {
 	LOGGING_PLUGIN_TOKEN_PATH               string `mapstructure:"LOGGING_PLUGIN_TOKEN_PATH"`
 	LOGGING_PLUGIN_PROXY_PATH               string `mapstructure:"LOGGING_PLUGIN_PROXY_PATH"`
 	LOGGING_PLUGIN_CA_CERT                  string `mapstructure:"LOGGING_PLUGIN_CA_CERT"`
+	LOGGING_PLUGIN_QUERY_LIMIT              uint   `mapstructure:"LOGGING_PLUGIN_QUERY_LIMIT"`
 	LOGGING_PLUGIN_TLS_VERIFICATION_DISABLE bool   `mapstructure:"LOGGING_PLUGIN_TLS_VERIFICATION_DISABLE"`
 	LOGGING_PLUGIN_FORWARDER_DELAY_DURATION uint   `mapstructure:"LOGGING_PLUGIN_FORWARDER_DELAY_DURATION"`
 }

pkg/api/server/v1alpha2/plugin_logs.go

@@ -145,6 +145,7 @@ func (s *LogPluginServer) getLokiLogs(writer *logs.BufferedLog, parent string, r
 	parameters.Add("query", `{ `+s.staticLabels+s.config.LOGGING_PLUGIN_NAMESPACE_KEY+`="`+parent+`" }|json uid="`+uidKey+`", message="message" |uid="`+rec.Name+`"| line_format "{{.message}}"`)
 	parameters.Add("end", endTime)
 	parameters.Add("start", startTime)
+	parameters.Add("limit", strconv.Itoa(int(s.queryLimit)))
 
 	URL.RawQuery = parameters.Encode()
 	s.logger.Debugf("loki request url:%s", URL.String())

pkg/api/server/v1alpha2/plugin_logs_test.go

@@ -65,6 +65,7 @@ func TestLogPluginServer_GetLog(t *testing.T) {
 		LOGGING_PLUGIN_TLS_VERIFICATION_DISABLE: true,
 		LOGGING_PLUGIN_STATIC_LABELS:            "namespace=\"foo\"",
 		LOGGING_PLUGIN_NAMESPACE_KEY:            "namespace",
+		LOGGING_PLUGIN_QUERY_LIMIT:              1500,
 	}, logger.Get("info"), test.NewDB(t))
 	if err != nil {
 		t.Fatalf("failed to create server: %v", err)

pkg/api/server/v1alpha2/server.go

@@ -83,6 +83,8 @@ type LogPluginServer struct {
 
 	forwarderDelayDuration time.Duration
 
+	queryLimit uint
+
 	// TODO: In future add support for non Oauth support
 	tokenSource oauth2.TokenSource
 }
@@ -208,6 +210,7 @@ func (s *Server) createLogPluginServer() error {
 	s.LogPluginServer.forwarderDelayDuration = time.Duration(s.config.LOGGING_PLUGIN_FORWARDER_DELAY_DURATION) * time.Minute
 
 	s.LogPluginServer.tokenSource = transport.NewCachedFileTokenSource(s.config.LOGGING_PLUGIN_TOKEN_PATH)
+	s.LogPluginServer.queryLimit = s.config.LOGGING_PLUGIN_QUERY_LIMIT
 
 	return nil
 }