Merge main branch

.drone/drone.jsonnet

@@ -160,14 +160,14 @@ local promtail_win() = pipeline('promtail-windows') {
   steps: [
     {
       name: 'identify-runner',
-      image: 'golang:1.19-windowsservercore-1809',
+      image: 'golang:1.21.3-windowsservercore-1809',
       commands: [
         'Write-Output $env:DRONE_RUNNER_NAME',
       ],
     },
     {
       name: 'test',
-      image: 'golang:1.19-windowsservercore-1809',
+      image: 'golang:1.21.3-windowsservercore-1809',
       commands: [
         'go test .\\clients\\pkg\\promtail\\targets\\windows\\... -v',
       ],

.drone/drone.yml

@@ -1666,11 +1666,11 @@ platform:
 steps:
 - commands:
   - Write-Output $env:DRONE_RUNNER_NAME
-  image: golang:1.19-windowsservercore-1809
+  image: golang:1.21.3-windowsservercore-1809
   name: identify-runner
 - commands:
   - go test .\clients\pkg\promtail\targets\windows\... -v
-  image: golang:1.19-windowsservercore-1809
+  image: golang:1.21.3-windowsservercore-1809
   name: test
 trigger:
   ref:
@@ -2106,6 +2106,6 @@ kind: secret
 name: gpg_private_key
 ---
 kind: signature
-hmac: caf375427f92f78711f801f56341357b67737330e906346ee908a796c61dd314
+hmac: 8ae9cff1a379503d0b568f727d9c12bcb486a5e8d1fc3271deea32f07939baf1
 
 ...

.github/workflows/snyk-pr-comment.yml

@@ -36,7 +36,11 @@ jobs:
         continue-on-error: true
         id: should-comment
         run: |
-          if [[ $(wc -l < snyk.txt) -gt 1 ]]; then exit 0; fi
+          if [[ $(wc -l < snyk.txt) -gt 1 ]]; then
+            echo "\nTo see more details on these vulnerabilities, and how/where to fix them, please run `make scan-vulnerabilities` on your branch. If these were not introduced by your PR, please considering fixing them in `main` via a subsequent PR. Thanks!" >> snyk.txt
+            exit 0;
+          fi
+
           exit 1
 
       - name: Comment on PR with Snyk scan results

.github/workflows/trivy-pr-comment.yml

@@ -19,12 +19,21 @@ jobs:
           make loki-image
           echo "IMAGE_TAG=${IMAGE_TAG}" >> $GITHUB_ENV
 
-      - name: Run Trivy vulnerability scanner
+      - name: Run Trivy image scanner
         uses: aquasecurity/trivy-action@master
         with:
           image-ref: "docker.io/grafana/loki:${{ env.IMAGE_TAG }}"
           format: "json"
-          output: "trivy.json"
+          output: "trivy-image.json"
+          severity: "CRITICAL,HIGH"
+
+      - name: Run Trivy fs scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: "fs"
+          scan-ref: "go.mod"
+          format: "json"
+          output: "trivy-fs.json"
           severity: "CRITICAL,HIGH"
 
       - name: Prepare Trivy Message
@@ -35,13 +44,19 @@ jobs:
         uses: sergeysova/jq-action@v2
         continue-on-error: true
         with:
-          cmd: jq -r '.Results[] | select(.Vulnerabilities != null) | .Vulnerabilities[] | "* **\(.Severity)** [\(.Title)](\(.PrimaryURL)) in `\(.PkgName)` v\(.InstalledVersion). Fixed in v\(.FixedVersion)"' trivy.json >> trivy.txt
+          cmd: |
+            jq -r '.Results[] | select(.Vulnerabilities != null) | .Target as $target | .Type as $type | .Vulnerabilities[] | "* **\(.Severity)**, Target: \($target), Type: \($type) [\(.Title)](\(.PrimaryURL)) in `\(.PkgName)` v\(.InstalledVersion). Fixed in v\(.FixedVersion)"' trivy-image.json >> trivy.txt
+            jq -r '.Results[] | select(.Vulnerabilities != null) | .Target as $target | .Type as $type | .Vulnerabilities[] | "* **\(.Severity)**, Target: \($target), Type: \($type) [\(.Title)](\(.PrimaryURL)) in `\(.PkgName)` v\(.InstalledVersion). Fixed in v\(.FixedVersion)"' trivy-fs.json >> trivy.text
 
       - name: Determine whether to comment
         continue-on-error: true
         id: should-comment
         run: |
-          if [[ $(wc -l < trivy.txt) -gt 1 ]]; then exit 0; fi
+          if [[ $(wc -l < trivy.txt) -gt 1 ]]; then
+            echo "\nTo see more details on these vulnerabilities, and how/where to fix them, please run `make scan-vulnerabilities` on your branch. If these were not introduced by your PR, please considering fixing them in `main` via a subsequent PR. Thanks!" >> trivy.txt
+            exit 0;
+          fi
+
           exit 1
 
       - name: Comment on PR with Trivy scan results

CHANGELOG.md

@@ -6,6 +6,10 @@
 
 ##### Enhancements
 
+* [11363](https://github.com/grafana/loki/pull/11477) **MichelHollands**: support GET for /ingester/shutdown
+* [11363](https://github.com/grafana/loki/pull/11363) **kavirajk**: bugfix(memcached): Make memcached batch fetch truely context aware.
+* [11319](https://github.com/grafana/loki/pull/11319) **someStrangerFromTheAbyss**: Helm: Add extraContainers to the write pods.
+* [11243](https://github.com/grafana/loki/pull/11243) **kavirajk**: Inflight-logging: Add extra metadata to inflight requests logging.
 * [11110](https://github.com/grafana/loki/pull/11003) **MichelHollands**: Change the default of the `metrics-namespace` flag to 'loki'.
 * [11086](https://github.com/grafana/loki/pull/11086) **kandrew5**: Helm: Allow topologySpreadConstraints
 * [11003](https://github.com/grafana/loki/pull/11003) **MichelHollands**: Add the `metrics-namespace` flag to change the namespace of metrics currently using cortex as namespace.
@@ -36,18 +40,22 @@
 * [10727](https://github.com/grafana/loki/pull/10727) **sandeepsukhani** Native otlp ingestion support
 * [11051](https://github.com/grafana/loki/pull/11051) Refactor to not use global logger in modules
 * [10956](https://github.com/grafana/loki/pull/10956) **jeschkies** do not wrap requests but send pure Protobuf from frontend v2 via scheduler to querier when `-frontend.encoding=protobuf`.
+* [10417](https://github.com/grafana/loki/pull/10417) **jeschkies** shard `quantile_over_time` range queries using probabilistic data structures.
+* [11284](https://github.com/grafana/loki/pull/11284) **ashwanthgoli** Config: Adds `frontend.max-query-capacity` to tune per-tenant query capacity.
 
 ##### Fixes
 * [11074](https://github.com/grafana/loki/pull/11074) **hainenber** Fix panic in lambda-promtail due to mishandling of empty DROP_LABELS env var.
 * [11195](https://github.com/grafana/loki/pull/11195) **canuteson** Generate tsdb_shipper storage_config even if using_boltdb_shipper is false
 
 ##### Changes
 
+* [11490](https://github.com/grafana/loki/pull/11490) **andresperezl**: Helm: Use `/ingester/shutdown` for `preStop` hook in write pods.
 * [10366](https://github.com/grafana/loki/pull/10366) **shantanualsi** Upgrade thanos objstore, dskit and other modules
 * [10451](https://github.com/grafana/loki/pull/10451) **shantanualsi** Upgrade thanos `objstore`
 * [10814](https://github.com/grafana/loki/pull/10814) **shantanualsi,kaviraj** Upgrade prometheus to v0.47.1 and dskit
 * [10959](https://github.com/grafana/loki/pull/10959) **slim-bean** introduce a backoff wait on subquery retries.
 * [11121](https://github.com/grafana/loki/pull/11121) **periklis** Ensure all lifecycler cfgs ref a valid IPv6 addr and port combination
+* [10650](https://github.com/grafana/loki/pull/10650) **matthewpi** Ensure the frontend uses a valid IPv6 addr and port combination
 
 #### Promtail
 
@@ -74,12 +82,16 @@
 
 #### Mixins
 
+* [11087](https://github.com/grafana/loki/pull/11087) **JoaoBraveCoding**: Adds structured metadata panels for ingested data
+
 #### Fixes
 
 #### FluentD
 
 #### Jsonnet
 
+* [11312](https://github.com/grafana/loki/pull/11312) **sentoz**: Loki ksonnet: Do not generate configMap for consul if you are using memberlist
+
 * [11020](https://github.com/grafana/loki/pull/11020) **ashwanthgoli**: Loki ksonnet: Do not generate table-manager manifests if shipper store is in-use.
 
 * [10784](https://github.com/grafana/loki/pull/10894) **slim-bean** Update index gateway client to use a headless service.

CONTRIBUTING.md

@@ -149,4 +149,11 @@ To get a local preview of the documentation:
 3. Run the command `make docs`. This uses the `grafana/docs` image which internally uses Hugo to generate the static site.
 4. Open http://localhost:3002/docs/loki/latest/ to review your changes.
 
+**Remember:** If running `make docs` command gave you the following error.
+
+   - `path /tmp/make-docs.Dcq is not shared from the host and is not known to Docker.`
+   - `You can configure shared paths from Docker -> Preferences... -> Resources -> File Sharing.`
+
+Then you can go to Docker Desktop settings and open the resources, add the temporary directory path `/tmp`.
+
 > Note that `make docs` uses a lot of memory. If it crashes, increase the memory allocated to Docker and try again.

Makefile

@@ -42,6 +42,8 @@ BUILD_IMAGE_VERSION ?= 0.31.2
 # Docker image info
 IMAGE_PREFIX ?= grafana
 
+BUILD_IMAGE_PREFIX ?= grafana
+
 IMAGE_TAG ?= $(shell ./tools/image-tag)
 
 # Version info for binaries
@@ -102,7 +104,7 @@ RM := --rm
 TTY := --tty
 
 DOCKER_BUILDKIT ?= 1
-BUILD_IMAGE = BUILD_IMAGE=$(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION)
+BUILD_IMAGE = BUILD_IMAGE=$(BUILD_IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION)
 PUSH_OCI=docker push
 TAG_OCI=docker tag
 ifeq ($(CI), true)
@@ -425,7 +427,7 @@ PLUGIN_ARCH ?=
 define build-rootfs
 	rm -rf clients/cmd/docker-driver/rootfs || true
 	mkdir clients/cmd/docker-driver/rootfs
-	docker build -t rootfsimage -f clients/cmd/docker-driver/Dockerfile .
+	docker build --build-arg $(BUILD_IMAGE) -t rootfsimage -f clients/cmd/docker-driver/Dockerfile .
 
 	ID=$$(docker create rootfsimage true) && \
 	(docker export $$ID | tar -x -C clients/cmd/docker-driver/rootfs) && \
@@ -836,6 +838,7 @@ dev-k3d-down:
 .PHONY: trivy
 trivy: loki-image
 	trivy i $(IMAGE_PREFIX)/loki:$(IMAGE_TAG)
+	trivy fs go.mod
 
 # Synk is also used to scan for vulnerabilities, and detects things that trivy might miss
 .PHONY: snyk

clients/cmd/docker-driver/Dockerfile

@@ -1,4 +1,4 @@
-ARG BUILD_IMAGE=grafana/loki-build-image:0.29.3
+ARG BUILD_IMAGE=grafana/loki-build-image:0.31.2
 # Directories in this file are referenced from the root of the project not this folder
 # This file is intended to be called from the root like so:
 # docker build -t grafana/loki -f cmd/loki/Dockerfile .
@@ -9,7 +9,7 @@ COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && make BUILD_IN_CONTAINER=false clients/cmd/docker-driver/docker-driver
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 RUN apk add --update --no-cache ca-certificates tzdata
 COPY --from=build /src/loki/clients/cmd/docker-driver/docker-driver /bin/docker-driver
 WORKDIR /bin/

clients/cmd/fluent-bit/loki_test.go

@@ -1,3 +1,5 @@
+//go:build cgo
+
 package main
 
 import (

clients/cmd/fluent-bit/out_grafana_loki.go

@@ -63,7 +63,7 @@ func FLBPluginInit(ctx unsafe.Pointer) int {
 
 	level.Info(logger).Log("[flb-go]", "Starting fluent-bit-go-loki", "version", version.Info())
 	paramLogger := log.With(logger, "[flb-go]", "provided parameter")
-	level.Info(paramLogger).Log("URL", conf.clientConfig.URL)
+	level.Info(paramLogger).Log("URL", conf.clientConfig.URL.Redacted())
 	level.Info(paramLogger).Log("TenantID", conf.clientConfig.TenantID)
 	level.Info(paramLogger).Log("BatchWait", fmt.Sprintf("%.3fs", conf.clientConfig.BatchWait.Seconds()))
 	level.Info(paramLogger).Log("BatchSize", conf.clientConfig.BatchSize)

clients/cmd/promtail/Dockerfile.debug

@@ -9,7 +9,7 @@ WORKDIR /src/loki
 RUN make clean && make BUILD_IN_CONTAINER=false PROMTAIL_JOURNAL_ENABLED=true promtail-debug
 
 
-FROM       alpine:3.18.3
+FROM       alpine:3.18.4
 RUN        apk add --update --no-cache ca-certificates tzdata
 COPY       --from=build /src/loki/clients/cmd/promtail/promtail-debug /usr/bin/promtail-debug
 COPY       --from=build /usr/bin/dlv /usr/bin/dlv

clients/pkg/logentry/logql/parser.go

@@ -1,13 +1,14 @@
 package logql
 
 import (
-	"errors"
 	"fmt"
 	"strconv"
 	"strings"
 	"text/scanner"
 
 	"github.com/prometheus/prometheus/model/labels"
+
+	"github.com/grafana/loki/pkg/logqlmodel"
 )
 
 func init() {
@@ -44,7 +45,7 @@ func ParseMatchers(input string) ([]*labels.Matcher, error) {
 	}
 	matcherExpr, ok := expr.(*matchersExpr)
 	if !ok {
-		return nil, errors.New("only label matchers is supported")
+		return nil, logqlmodel.ErrParseMatchers
 	}
 	return matcherExpr.matchers, nil
 }

clients/pkg/promtail/discovery/consulagent/consul.go

@@ -8,7 +8,6 @@ package consulagent
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"net"
 	"net/http"
 	"strconv"
@@ -527,9 +526,9 @@ func (srv *consulService) watch(ctx context.Context, ch chan<- []*targetgroup.Gr
 		// since the service may be registered remotely through a different node.
 		var addr string
 		if srvCheck.Service.Address != "" {
-			addr = net.JoinHostPort(srvCheck.Service.Address, fmt.Sprintf("%d", srvCheck.Service.Port))
+			addr = net.JoinHostPort(srvCheck.Service.Address, strconv.Itoa(srvCheck.Service.Port))
 		} else {
-			addr = net.JoinHostPort(member.Addr, fmt.Sprintf("%d", srvCheck.Service.Port))
+			addr = net.JoinHostPort(member.Addr, strconv.Itoa(srvCheck.Service.Port))
 		}
 
 		labels := model.LabelSet{
@@ -560,7 +559,7 @@ func (srv *consulService) watch(ctx context.Context, ch chan<- []*targetgroup.Gr
 		// Add all key/value pairs from the service's tagged addresses as their own labels.
 		for k, v := range srvCheck.Service.TaggedAddresses {
 			name := strutil.SanitizeLabelName(k)
-			address := fmt.Sprintf("%s:%d", v.Address, v.Port)
+			address := net.JoinHostPort(v.Address, strconv.Itoa(v.Port))
 			labels[taggedAddressesLabel+model.LabelName(name)] = model.LabelValue(address)
 		}
 

cmd/logcli/Dockerfile

@@ -4,7 +4,7 @@ COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && make BUILD_IN_CONTAINER=false logcli
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 
 RUN apk add --no-cache ca-certificates
 

cmd/logql-analyzer/Dockerfile

@@ -4,7 +4,7 @@ COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && CGO_ENABLED=0 go build ./cmd/logql-analyzer/
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 
 RUN apk add --no-cache ca-certificates
 

cmd/loki-canary-boringcrypto/Dockerfile

@@ -5,7 +5,7 @@ WORKDIR /src/loki
 RUN go env GOARCH > /goarch
 RUN make clean && make GOARCH=$(cat /goarch) BUILD_IN_CONTAINER=true GOEXPERIMENT=boringcrypto loki-canary-boringcrypto
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 RUN apk add --update --no-cache ca-certificates
 RUN apk add --no-cache libc6-compat
 COPY --from=build /src/loki/cmd/loki-canary-boringcrypto/loki-canary-boringcrypto /usr/bin/loki-canary

cmd/loki-canary/Dockerfile

@@ -4,7 +4,7 @@ COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && make BUILD_IN_CONTAINER=false loki-canary
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 RUN apk add --update --no-cache ca-certificates
 COPY --from=build /src/loki/cmd/loki-canary/loki-canary /usr/bin/loki-canary
 ENTRYPOINT [ "/usr/bin/loki-canary" ]

cmd/loki-canary/Dockerfile.cross

@@ -12,7 +12,7 @@ COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false loki-canary
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 RUN apk add --update --no-cache ca-certificates
 COPY --from=build /src/loki/cmd/loki-canary/loki-canary /usr/bin/loki-canary
 ENTRYPOINT [ "/usr/bin/loki-canary" ]

cmd/loki/Dockerfile

@@ -4,7 +4,7 @@ COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && make BUILD_IN_CONTAINER=false loki
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 
 RUN apk add --no-cache ca-certificates libcap
 

cmd/loki/Dockerfile.cross

@@ -12,7 +12,7 @@ COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false loki
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 
 RUN apk add --no-cache ca-certificates
 

cmd/loki/Dockerfile.debug

@@ -15,7 +15,7 @@ WORKDIR /src/loki
 RUN make clean && \
     GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false loki-debug
 
-FROM       alpine:3.18.3
+FROM       alpine:3.18.4
 RUN        apk add --update --no-cache ca-certificates
 COPY       --from=build /src/loki/cmd/loki/loki-debug /usr/bin/loki-debug
 COPY       --from=goenv /go/bin/dlv /usr/bin/dlv

cmd/migrate/Dockerfile

@@ -3,7 +3,7 @@ COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && make BUILD_IN_CONTAINER=false migrate
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 RUN apk add --update --no-cache ca-certificates
 COPY --from=build /src/loki/cmd/migrate/migrate /usr/bin/migrate
 #ENTRYPOINT [ "/usr/bin/migrate" ]

cmd/querytee/Dockerfile

@@ -4,7 +4,7 @@ COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && make BUILD_IN_CONTAINER=false loki-querytee
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 RUN apk add --update --no-cache ca-certificates
 COPY --from=build /src/loki/cmd/querytee/querytee /usr/bin/querytee
 ENTRYPOINT [ "/usr/bin/querytee" ]

cmd/querytee/Dockerfile.cross

@@ -12,7 +12,7 @@ COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false loki-querytee
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 RUN apk add --update --no-cache ca-certificates
 COPY --from=build /src/loki/cmd/querytee/querytee /usr/bin/querytee
 ENTRYPOINT [ "/usr/bin/querytee" ]