Add stupidly simple storage retention for parca

hack/cmd/janitor/main.go

@@ -0,0 +1,98 @@
+/*
+Copyright 2023 Tim Ebert.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
+)
+
+func main() {
+	var (
+		maxAge time.Duration
+	)
+
+	cmd := &cobra.Command{
+		Use:   "janitor path",
+		Short: "janitor is a stupidly simple storage retention tool",
+		Long:  `janitor deletes everything in the given directory that has a modification timestamp older than a given retention time.`,
+
+		SilenceErrors: true,
+		Args:          cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			path := args[0]
+			if path == "" {
+				return fmt.Errorf("path must not be empty")
+			}
+			if maxAge <= 0 {
+				return fmt.Errorf("max-age must be greater than 0")
+			}
+
+			cmd.SilenceUsage = true
+
+			return run(path, time.Now().Add(-maxAge))
+		},
+	}
+
+	cmd.Flags().DurationVar(&maxAge, "max-age", 0, "Maximum age of files and directories to keep. Elements with an older modification timestamp are deleted.")
+
+	if err := cmd.ExecuteContext(signals.SetupSignalHandler()); err != nil {
+		log.Fatalln(err)
+	}
+}
+
+func run(dir string, cleanBefore time.Time) error {
+	performedCleanup := false
+
+	elements, err := os.ReadDir(dir)
+	if err != nil {
+		return err
+	}
+
+	for _, element := range elements {
+		path := filepath.Join(dir, element.Name())
+
+		info, err := element.Info()
+		if err != nil {
+			return fmt.Errorf("failed to get info for %q: %w", path, err)
+		}
+
+		modTime := info.ModTime()
+		if !modTime.Before(cleanBefore) {
+			continue
+		}
+
+		log.Printf("Cleaning %s (last modified: %s)", path, modTime.UTC().Format(time.RFC3339))
+
+		if err := os.RemoveAll(path); err != nil {
+			return fmt.Errorf("failed to remove %q: %w", path, err)
+		}
+		performedCleanup = true
+	}
+
+	if !performedCleanup {
+		log.Println("Nothing to clean")
+	}
+
+	return nil
+}

hack/config/profiling/janitor_cronjob.yaml

@@ -0,0 +1,51 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: janitor
+  namespace: parca
+spec:
+  concurrencyPolicy: Forbid
+  schedule: "*/10 * * * *"
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/component: janitor
+            app.kubernetes.io/instance: parca
+            app.kubernetes.io/name: parca
+        spec:
+          restartPolicy: Never
+          containers:
+          - name: janitor
+            image: janitor
+            args:
+            - /var/lib/parca/blocks/parca/stacktraces
+            - --max-age=48h
+            volumeMounts:
+            - mountPath: /var/lib/parca
+              name: parca
+          securityContext:
+            fsGroupChangePolicy: OnRootMismatch
+            fsGroup: 65534
+            runAsNonRoot: true
+            runAsUser: 65534
+            seccompProfile:
+              type: RuntimeDefault
+            supplementalGroups:
+            - 65534
+          volumes:
+          - name: parca
+            persistentVolumeClaim:
+              claimName: parca
+          # Require janitor to be scheduled to the same node as parca itself because the volume is RWO and cannot be
+          # attached to multiple nodes simultaneously.
+          affinity:
+            podAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+              - topologyKey: kubernetes.io/hostname
+                labelSelector:
+                  matchLabels:
+                    app.kubernetes.io/component: observability
+                    app.kubernetes.io/instance: parca
+                    app.kubernetes.io/name: parca

hack/config/profiling/kustomization.yaml

@@ -8,6 +8,12 @@ resources:
 # grant parca running in namespace "parca" permissions required for service discovery in namespace
 # "sharding-system" and scrape the pprof endpoints of sharder
 - rbac_sharder.yaml
+- janitor_cronjob.yaml
+
+images:
+- name: janitor
+  newName: ghcr.io/timebertt/kubernetes-controller-sharding/janitor
+  newTag: latest
 
 generatorOptions:
   disableNameSuffixHash: true

hack/config/profiling/parca_pvc.yaml

@@ -12,4 +12,4 @@ spec:
   - ReadWriteOnce
   resources:
     requests:
-      storage: 50Gi
+      storage: 150Gi

hack/config/skaffold.yaml

@@ -197,7 +197,7 @@ profiles:
         dependencies:
           paths:
           - go.mod
-          - '**/*.go'
+          - './hack/cmd/shard/**/*.go'
         main: ./hack/cmd/shard
     local:
       concurrency: 0
@@ -278,6 +278,17 @@ apiVersion: skaffold/v4beta7
 kind: Config
 metadata:
   name: profiling
+build:
+  artifacts:
+  - image: ghcr.io/timebertt/kubernetes-controller-sharding/janitor
+    ko:
+      dependencies:
+        paths:
+        - go.mod
+        - './hack/cmd/janitor/**/*.go'
+      main: ./hack/cmd/janitor
+  local:
+    concurrency: 0
 manifests:
   kustomize:
     paths: