Retry tests 100

.github/workflows-doc.md

@@ -69,7 +69,7 @@ As of October 2020, GitHub Actions do not persist between different jobs in the
 - name: Compressing Images
         run: tar -zcvf images.tar.gz /tmp/images
       - name: Cache Images
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: Docker Images
           path: ./images.tar.gz

.github/workflows/build-image.yml

@@ -0,0 +1,68 @@
+name: Build Image
+
+on:
+  push:
+    branches: [ master ]
+    paths:
+      - 'build-image/**'
+      - '.github/workflows/build-image.yml'
+  pull_request:
+    branches: [ master ]
+    paths:
+      - 'build-image/**'
+      - '.github/workflows/build-image.yml'
+
+jobs:
+  build:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+        name: Checkout
+        with:
+          fetch-depth: 0
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Save image
+        run: make save-multiarch-build-image
+
+      - name: Upload Docker Images Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: build-image
+          path: |
+            ./build-image-amd64.tar
+            ./build-image-arm64.tar
+          if-no-files-found: error
+
+  push:
+    needs: build
+    if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex'
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+        name: Checkout
+        with:
+          fetch-depth: 0
+
+      - name: Download Docker Images Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: build-image
+
+      - name: Load image
+        run: make load-multiarch-build-image
+
+      - name: Login to Quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{secrets.QUAY_REGISTRY_USER}}
+          password: ${{secrets.QUAY_REGISTRY_PASSWORD}}
+
+      - name: Push image
+        run: make push-multiarch-build-image

.github/workflows/test-build-deploy.yml

@@ -5,13 +5,19 @@ on:
     branches: [master]
     tags:
       - v[0-9]+.[0-9]+.[0-9]+** # Tag filters not as strict due to different regex system on Github Actions
+    paths-ignore:
+      - 'build-image/**'
+      - '.github/workflows/build-image.yml'
   pull_request:
+    paths-ignore:
+      - 'build-image/**'
+      - '.github/workflows/build-image.yml'
 
 jobs:
   lint:
     runs-on: ubuntu-20.04
     container:
-      image: quay.io/cortexproject/build-image:node_version_upgrade-60582e680
+      image: quay.io/cortexproject/build-image:master-6dd64fcce
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v2
@@ -40,7 +46,7 @@ jobs:
   test:
     runs-on: ubuntu-20.04
     container:
-      image: quay.io/cortexproject/build-image:node_version_upgrade-60582e680
+      image: quay.io/cortexproject/build-image:master-6dd64fcce
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v2
@@ -54,7 +60,10 @@ jobs:
           mkdir -p /go/src/github.com/cortexproject/cortex
           ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
       - name: Run Tests
-        run: make BUILD_IN_CONTAINER=false test
+        run: |
+          for run in $(seq 1 100); do
+            make BUILD_IN_CONTAINER=false test || true
+          done
 
   security:
     name: CodeQL
@@ -83,7 +92,7 @@ jobs:
   build:
     runs-on: ubuntu-20.04
     container:
-      image: quay.io/cortexproject/build-image:node_version_upgrade-60582e680
+      image: quay.io/cortexproject/build-image:master-6dd64fcce
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v2
@@ -107,7 +116,7 @@ jobs:
           touch build-image/.uptodate
           make BUILD_IN_CONTAINER=false web-build
       - name: Upload Website Artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: website public
           path: website/public/
@@ -119,7 +128,7 @@ jobs:
       - name: Create Docker Images Archive
         run: tar -cvf images.tar /tmp/images
       - name: Upload Docker Images Artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: Docker Images
           path: ./images.tar
@@ -152,7 +161,7 @@ jobs:
           sudo mkdir -p /go/src/github.com/cortexproject/cortex
           sudo ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
       - name: Download Docker Images Artifacts
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: Docker Images
       - name: Extract Docker Images Archive
@@ -194,7 +203,9 @@ jobs:
           export CORTEX_IMAGE="${CORTEX_IMAGE_PREFIX}cortex:$IMAGE_TAG-amd64"
           export CORTEX_CHECKOUT_DIR="/go/src/github.com/cortexproject/cortex"
           echo "Running integration tests with image: $CORTEX_IMAGE"
-          go test -tags=integration,${{ matrix.tags }} -timeout 2400s -v -count=1 ./integration/...
+          for run in $(seq 1 100); do
+            go test -tags=integration,${{ matrix.tags }} -timeout 2400s -v -count=1 -failfast ./integration/... || true
+          done
         env:
           IMAGE_PREFIX: ${{ secrets.IMAGE_PREFIX }}
 
@@ -207,7 +218,7 @@ jobs:
       - name: Install Docker Client
         run: sudo ./.github/workflows/scripts/install-docker.sh
       - name: Download Docker Images Artifact
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: Docker Images
       - name: Extract Docker Images Archive
@@ -217,14 +228,14 @@ jobs:
         run: |
           touch build-image/.uptodate
           MIGRATIONS_DIR=$(pwd)/cmd/cortex/migrations
-          make BUILD_IMAGE=quay.io/cortexproject/build-image:node_version_upgrade-60582e680 TTY='' configs-integration-test
+          make BUILD_IMAGE=quay.io/cortexproject/build-image:master-6dd64fcce TTY='' configs-integration-test
 
   deploy_website:
     needs: [build, test]
     if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex'
     runs-on: ubuntu-20.04
     container:
-      image: quay.io/cortexproject/build-image:node_version_upgrade-60582e680
+      image: quay.io/cortexproject/build-image:master-6dd64fcce
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v2
@@ -241,7 +252,7 @@ jobs:
           mkdir -p /go/src/github.com/cortexproject/cortex
           ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
       - name: Download Website Artifact
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: website public
           path: website/public
@@ -266,7 +277,7 @@ jobs:
     if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex'
     runs-on: ubuntu-20.04
     container:
-      image: quay.io/cortexproject/build-image:node_version_upgrade-60582e680
+      image: quay.io/cortexproject/build-image:master-6dd64fcce
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v2
@@ -282,7 +293,7 @@ jobs:
           mkdir -p /go/src/github.com/cortexproject/cortex
           ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
       - name: Download Docker Images Artifact
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: Docker Images
       - name: Extract Docker Images Archive

.gitignore

@@ -27,3 +27,6 @@ Makefile.local
 .vscode
 compose
 compose-simple
+
+/build-image-arm64.tar
+/build-image-amd64.tar

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## master / unreleased
 * [CHANGE] Upgrade Dockerfile Node version from 14x to 18x. #5906
 * [CHANGE] Ingester: Remove `-querier.query-store-for-labels-enabled` flag. Querying long-term store for labels is always enabled. #5984
+* [FEATURE] Ingester: Experimental: Enable native histogram ingestion via `-blocks-storage.tsdb.enable-native-histograms` flag. #5986
 * [ENHANCEMENT] rulers: Add support to persist tokens in rulers. #5987
 * [ENHANCEMENT] Query Frontend/Querier: Added store gateway postings touched count and touched size in Querier stats and log in Query Frontend. #5892
 * [ENHANCEMENT] Query Frontend/Querier: Returns `warnings` on prometheus query responses. #5916
@@ -11,7 +12,7 @@
 * [ENHANCEMENT] Distributor/Querier: Clean stale per-ingester metrics after ingester restarts. #5930
 * [ENHANCEMENT] Distributor/Ring: Allow disabling detailed ring metrics by ring member. #5931
 * [ENHANCEMENT] KV: Etcd Added etcd.ping-without-stream-allowed parameter to disable/enable  PermitWithoutStream #5933
-* [ENHANCEMENT] Ingester: Add a new `max_series_per_label_set` limit. This limit functions similarly to `max_series_per_metric`, but allowing users to define the maximum number of series per LabelSet. #5950
+* [ENHANCEMENT] Ingester: Add a new `limits_per_label_set` limit. This limit functions similarly to `max_series_per_metric`, but allowing users to define the maximum number of series per LabelSet. #5950 #5993
 * [ENHANCEMENT] Store Gateway: Log gRPC requests together with headers configured in `http_request_headers_to_log`. #5958
 * [BUGFIX] Configsdb: Fix endline issue in db password. #5920
 * [BUGFIX] Ingester: Fix `user` and `type` labels for the `cortex_ingester_tsdb_head_samples_appended_total` TSDB metric. #5952

Makefile

@@ -53,14 +53,7 @@ fetch-build-image:
 	docker tag $(BUILD_IMAGE):$(LATEST_BUILD_IMAGE_TAG) $(BUILD_IMAGE):latest
 	touch build-image/.uptodate
 
-push-multiarch-build-image:
-	@echo
-	# Build image for each platform separately... it tends to generate fewer errors.
-	$(SUDO) docker buildx build --platform linux/amd64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) build-image/
-	$(SUDO) docker buildx build --platform linux/arm64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) build-image/
-	# This command will run the same build as above, but it will reuse existing platform-specific images,
-	# put them together and push to registry.
-	$(SUDO) docker buildx build -o type=registry --platform linux/amd64,linux/arm64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)build-image:$(IMAGE_TAG) build-image/
+-include build-image/Makefile
 
 # We don't want find to scan inside a bunch of directories, to accelerate the
 # 'make: Entering directory '/go/src/github.com/cortexproject/cortex' phase.
@@ -122,7 +115,7 @@ build-image/$(UPTODATE): build-image/*
 SUDO := $(shell docker info >/dev/null 2>&1 || echo "sudo -E")
 BUILD_IN_CONTAINER := true
 BUILD_IMAGE ?= $(IMAGE_PREFIX)build-image
-LATEST_BUILD_IMAGE_TAG ?= node_version_upgrade-60582e680
+LATEST_BUILD_IMAGE_TAG ?= master-6dd64fcce
 
 # TTY is parameterized to allow Google Cloud Builder to run builds,
 # as it currently disallows TTY devices. This value needs to be overridden
@@ -223,7 +216,7 @@ lint:
 		./pkg/ruler/...
 
 test:
-	go test -tags netgo -timeout 30m -race -count 1 ./...
+	go test -tags netgo -timeout 30m -race -failfast -count 1 ./...
 
 cover:
 	$(eval COVERDIR := $(shell mktemp -d coverage.XXXXXXXXXX))

build-image/Makefile

@@ -0,0 +1,17 @@
+save-multiarch-build-image:
+	@echo
+	# Build image for each platform separately... it tends to generate fewer errors.
+	$(SUDO) docker buildx build --platform linux/amd64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)build-image:$(IMAGE_TAG)-amd64 --output type=docker,dest=./build-image-amd64.tar build-image/
+	$(SUDO) docker buildx build --platform linux/arm64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)build-image:$(IMAGE_TAG)-arm64 --output type=docker,dest=./build-image-arm64.tar build-image/
+
+load-multiarch-build-image:
+	$(SUDO) docker load -i build-image-amd64.tar
+	$(SUDO) docker load -i build-image-arm64.tar
+
+push-multiarch-build-image:
+	# This command will run the same build as multiarch-build-image, but it will reuse existing platform-specific images,
+	# put them together and push to registry.
+	$(SUDO) docker push $(IMAGE_PREFIX)build-image:${IMAGE_TAG}-amd64
+	$(SUDO) docker push $(IMAGE_PREFIX)build-image:${IMAGE_TAG}-arm64
+	$(SUDO) docker manifest create $(IMAGE_PREFIX)build-image:$(IMAGE_TAG) --amend $(IMAGE_PREFIX)build-image:${IMAGE_TAG}-amd64 --amend $(IMAGE_PREFIX)build-image:${IMAGE_TAG}-arm64
+	$(SUDO) docker manifest push $(IMAGE_PREFIX)build-image:$(IMAGE_TAG)

docs/blocks-storage/querier.md

@@ -1441,4 +1441,8 @@ blocks_storage:
     # be out-of-order.
     # CLI flag: -blocks-storage.tsdb.out-of-order-cap-max
     [out_of_order_cap_max: <int> | default = 32]
+
+    # [EXPERIMENTAL] True to enable native histogram.
+    # CLI flag: -blocks-storage.tsdb.enable-native-histograms
+    [enable_native_histograms: <boolean> | default = false]
 ```

docs/blocks-storage/store-gateway.md

@@ -1566,4 +1566,8 @@ blocks_storage:
     # be out-of-order.
     # CLI flag: -blocks-storage.tsdb.out-of-order-cap-max
     [out_of_order_cap_max: <int> | default = 32]
+
+    # [EXPERIMENTAL] True to enable native histogram.
+    # CLI flag: -blocks-storage.tsdb.enable-native-histograms
+    [enable_native_histograms: <boolean> | default = false]
 ```

docs/configuration/config-file-reference.md

@@ -1998,6 +1998,10 @@ tsdb:
   # be out-of-order.
   # CLI flag: -blocks-storage.tsdb.out-of-order-cap-max
   [out_of_order_cap_max: <int> | default = 32]
+
+  # [EXPERIMENTAL] True to enable native histogram.
+  # CLI flag: -blocks-storage.tsdb.enable-native-histograms
+  [enable_native_histograms: <boolean> | default = false]
 ```
 
 ### `compactor_config`
@@ -3172,9 +3176,9 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s
 # CLI flag: -ingester.max-global-series-per-metric
 [max_global_series_per_metric: <int> | default = 0]
 
-# [Experimental] The maximum number of active series per LabelSet, across the
-# cluster before replication. Empty list to disable.
-[max_series_per_label_set: <list of MaxSeriesPerLabelSet> | default = []]
+# [Experimental] Enable limits per LabelSet. Supported limits per labelSet:
+# [max_series]
+[limits_per_label_set: <list of LimitsPerLabelSet> | default = []]
 
 # The maximum number of active metrics with metadata per user, per ingester. 0
 # to disable.
@@ -5314,11 +5318,14 @@ otel:
     [tls_insecure_skip_verify: <boolean> | default = false]
 ```
 
-### `MaxSeriesPerLabelSet`
+### `LimitsPerLabelSet`
 
 ```yaml
-# The maximum number of active series per LabelSet before replication.
-[limit: <int> | default = ]
+limits:
+  # The maximum number of active series per LabelSet, across the cluster before
+  # replication. Setting the value 0 will enable the monitoring (metrics) but
+  # would not enforce any limits.
+  [max_series: <int> | default = ]
 
 # LabelSet which the limit should be applied.
 [label_set: <map of string (labelName) to string (labelValue)> | default = []]

docs/configuration/v1-guarantees.md

@@ -111,3 +111,5 @@ Currently experimental features are:
 - OTLP Receiver
 - Persistent tokens in the Ruler Ring:
   - `-ruler.ring.tokens-file-path` (path) CLI flag
+- Native Histograms
+  - Ingestion can be enabled by setting `-blocks-storage.tsdb.enable-native-histograms=true` on Ingester.

docs/contributing/how-to-update-the-build-image.md

@@ -5,12 +5,10 @@ weight: 5
 slug: how-to-update-the-build-image
 ---
 
-The build image currently can only be updated by a Cortex maintainer. If you're not a maintainer you can still open a PR with the changes, asking a maintainer to assist you publishing the updated image. The procedure is:
+The procedure is:
 
 1. Update `build-image/Dockerfile`
-1. Run `go env` and make sure `GOPROXY=https://proxy.golang.org,direct` (Go's default). Some environment may required `GOPROXY=direct`, and if you push a build image with this, build workflow on GitHub will take a lot longer to download modules.
-1. `docker login quay.io`. Note that pushing to `quay.io/cortexproject/build-image` repository can only be done by a maintainer.
-1. Build the and publish the image by using `make push-multiarch-build-image`. This will build and push multi-platform docker image (for linux/amd64 and linux/arm64).  Running this step successfully requires [Docker Buildx](https://docs.docker.com/buildx/working-with-buildx/), but does not require a specific platform.
-1. Replace the image tag in `.github/workflows/*` (_there may be multiple references_) and Makefile (variable `LATEST_BUILD_IMAGE_TAG`).
+1. Create a PR to master with that changed, after the PR is merged to master, the new build image is available in the quay.io repository. Check github action logs [here](https://github.com/cortexproject/cortex/actions/workflows/build-image.yml) for to find the image tag.
+1. Create another PR to replace the image tag in `.github/workflows/*` (_there may be multiple references_) and Makefile (variable `LATEST_BUILD_IMAGE_TAG`).
 1. If you are updating Go's runtime version be sure to change `actions/setup-go`'s `go-version` in ``.github/workflows/*`.
 1. Open a PR and make sure the CI with new build-image passes