From 44ff81cea0fc5f471ce3b9c13af39623f2d1a05d Mon Sep 17 00:00:00 2001 From: Kushal Shukla Date: Sun, 3 Nov 2024 13:09:48 +0530 Subject: [PATCH] 1. Added key.yml 2. changed upload-download function to extract key 3. updated bash.sh file Signed-off-by: Kushal Shukla --- .../benchmark/2a_prometheus-secret.yaml | 16 ++ .../3b_prometheus-test_deployment.yaml | 67 +++-- tools/block-sync/README.md | 4 +- tools/block-sync/upload_download.go | 17 +- tools/load-generator/main.go | 246 ------------------ tools/prometheus-builder/build.sh | 5 +- 6 files changed, 83 insertions(+), 272 deletions(-) create mode 100644 prombench/manifests/prombench/benchmark/2a_prometheus-secret.yaml delete mode 100644 tools/load-generator/main.go diff --git a/prombench/manifests/prombench/benchmark/2a_prometheus-secret.yaml b/prombench/manifests/prombench/benchmark/2a_prometheus-secret.yaml new file mode 100644 index 000000000..5dd4dc132 --- /dev/null +++ b/prombench/manifests/prombench/benchmark/2a_prometheus-secret.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Secret +metadata: + name: s3-secret + namespace: prombench-{{ .PR_NUMBER }} # Replace with your actual namespace +type: Opaque +stringData: + object-config.yml: | + type: S3 + config: + bucket: {{ .BUCKET_KEY }} + endpoint: minio:9000 + access_key: {{ .ACCESS_KEY }} + secret_key: {{ .SECRET_KEY }} + insecure: true + diff --git a/prombench/manifests/prombench/benchmark/3b_prometheus-test_deployment.yaml b/prombench/manifests/prombench/benchmark/3b_prometheus-test_deployment.yaml index 36a7eb7bb..7e76f94f2 100644 --- a/prombench/manifests/prombench/benchmark/3b_prometheus-test_deployment.yaml +++ b/prombench/manifests/prombench/benchmark/3b_prometheus-test_deployment.yaml @@ -44,25 +44,34 @@ spec: - name: GITHUB_ORG value: "{{ .GITHUB_ORG }}" - name: GITHUB_REPO - value: "{{ .GITHUB_REPO }}" + value: "{{ .GITHUB_REPO }}" volumeMounts: - name: prometheus-executable mountPath: /prometheus-builder - - name: config - mountPath: /config + - name: key + mountPath: /config - name: data-downloader image: kushalshukla/writer - imagePullPolicy: Always + imagePullPolicy: Always + env: + - name: ACCESS_KEY + value: "{{ .ACCESS_KEY }}" + - name: SECRET_KEY + value: "{{ .SECRET_KEY }}" + - name: BUCKET_KEY + value: "{{ .BUCKET_KEY }}" volumeMounts: - name: instance-ssd mountPath: /data - - name: config - mountPath: /mnc + - name: s3-config + mountPath: /config + - name: key + mountPath: /key args: [ "download", "--tsdb-path=/data", - "--objstore.config-file=/mnc/objectconfig.yml", - "--key=gendata" + "--objstore.config-file=/config/object-config.yml", + "--key=/key/key.yml" ] containers: - name: prometheus @@ -104,9 +113,11 @@ spec: path: /mnt/disks/ssd0 #gke ssds - name: prometheus-executable emptyDir: {} - - name: config - hostPath: - path: /config-file + - name: s3-config # Define the Secret volume + secret: + secretName: s3-secret + - name: key + emptyDir: {} terminationGracePeriodSeconds: 300 nodeSelector: node-name: prometheus-{{ .PR_NUMBER }} @@ -175,30 +186,41 @@ spec: cd /repo1 && \ git fetch origin pull/{{ .PR_NUMBER }}/head:pr-branch && \ git checkout pr-branch && \ - cp objectconfig.yml /mnt/repo/objectconfig.yml && \ + cp key.yml /config/key.yml && \ rm -rf /repo1 volumeMounts: - - name: config - mountPath: /mnt/repo + - name: key + mountPath: /config - name: data-downloader image: kushalshukla/writer - imagePullPolicy: Always + imagePullPolicy: Always + env: + - name: ACCESS_KEY + value: "{{ .ACCESS_KEY }}" + - name: SECRET_KEY + value: "{{ .SECRET_KEY }}" + - name: BUCKET_KEY + value: "{{ .BUCKET_KEY }}" volumeMounts: - name: instance-ssd mountPath: /data - name: config - mountPath: /mnc + mountPath: /mnc + - name: s3-config + mountPath: /config + - name: key + mountPath: /key args: [ "download", "--tsdb-path=/data", - "--objstore.config-file=/mnc/objectconfig.yml", - "--key=gendata" + "--objstore.config-file=/config/object-config.yml", + "--key=/key/key.yml" ] containers: - name: prometheus image: quay.io/prometheus/prometheus:{{ .RELEASE }} imagePullPolicy: Always - command: [ "/bin/prometheus"] + command: [ "/bin/prometheus" ] args: [ "--web.external-url=http://{{ .DOMAIN_NAME }}/{{ .PR_NUMBER }}/prometheus-release", "--storage.tsdb.path=/prometheus", @@ -224,7 +246,12 @@ spec: path: /mnt/disks/ssd0 - name: config hostPath: - path: /object-config + path: /object-config + - name: s3-config # Define the Secret volume + secret: + secretName: s3-secret + - name: key + emptyDir: {} terminationGracePeriodSeconds: 300 nodeSelector: node-name: prometheus-{{ .PR_NUMBER }} diff --git a/tools/block-sync/README.md b/tools/block-sync/README.md index 945b042e0..51c3e2a57 100644 --- a/tools/block-sync/README.md +++ b/tools/block-sync/README.md @@ -41,10 +41,10 @@ The `download` command allows you to retrieve TSDB data from an object storage b The configuration file is essential for connecting to your object storage solution. Below are basic templates for different object storage systems. ```yaml -type: s3 +type: s3, GCS , AZURE , etc. config: bucket: your-bucket-name - endpoint: https://your-minio-endpoint.com + endpoint: https://your-endpoint access_key: your-access-key secret_key: your-secret-key insecure: false # Set to true if using HTTP instead of HTTPS diff --git a/tools/block-sync/upload_download.go b/tools/block-sync/upload_download.go index 3830ab9b4..a9fbd04e7 100644 --- a/tools/block-sync/upload_download.go +++ b/tools/block-sync/upload_download.go @@ -18,6 +18,7 @@ import ( "fmt" "log/slog" "os" + "strings" "github.com/go-kit/log" "github.com/thanos-io/objstore" @@ -42,10 +43,24 @@ func newStore(tsdbPath, objectConfig, objectKey string, logger *slog.Logger) (*S if err != nil { return nil, fmt.Errorf("failed to create bucket existence:%w", err) } + key, err := os.ReadFile(objectKey) + if err != nil { + return nil, fmt.Errorf("failed to read objectKey file: %w", err) + } + + content := strings.TrimSpace(string(key)) + var value string + if strings.HasPrefix(content, "key:") { + value = strings.TrimSpace(strings.TrimPrefix(content, "key:")) + fmt.Println(value) + } else { + fmt.Println("Expected 'key:' prefix not found") + } + return &Store{ bucket: bucket, tsdbpath: tsdbPath, - objectkey: objectKey, + objectkey: value, objectconfig: objectConfig, bucketlogger: logger, }, nil diff --git a/tools/load-generator/main.go b/tools/load-generator/main.go deleted file mode 100644 index fa40950a9..000000000 --- a/tools/load-generator/main.go +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright 2024 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "fmt" - "io" - "log" - "net/http" - "os" - "sync" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/prometheus/common/model" - "gopkg.in/yaml.v2" -) - -// Global variables and Prometheus metrics - -const max404Errors = 30 - -var ( - domainName = os.Getenv("DOMAIN_NAME") - - queryDuration = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: "loadgen", - Name: "query_duration_seconds", - Help: "Query duration", - Buckets: prometheus.LinearBuckets(0.05, 0.1, 20), - }, - []string{"prometheus", "group", "expr", "type"}, - ) - queryCount = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "loadgen", - Name: "queries_total", - Help: "Total amount of queries", - }, - []string{"prometheus", "group", "expr", "type"}, - ) - queryFailCount = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "loadgen", - Name: "failed_queries_total", - Help: "Amount of failed queries", - }, - []string{"prometheus", "group", "expr", "type"}, - ) -) - -// Querier struct and methods -type Querier struct { - target string - name string - groupID int - numberOfErrors int - - interval time.Duration - queries []Query - qtype string - start time.Duration - end time.Duration - step string - url string -} - -type Query struct { - Expr string `yaml:"expr"` -} - -type QueryGroup struct { - Name string `yaml:"name"` - Interval string `yaml:"interval"` - Queries []Query `yaml:"queries"` - Type string `yaml:"type,omitempty"` - Start string `yaml:"start,omitempty"` - End string `yaml:"end,omitempty"` - Step string `yaml:"step,omitempty"` -} - -func NewQuerier(groupID int, target, prNumber string, qg QueryGroup) *Querier { - qtype := qg.Type - if qtype == "" { - qtype = "instant" - } - - start := durationSeconds(qg.Start) - end := durationSeconds(qg.End) - fmt.Println(qtype, "HI from query ka type range or instant") - nodePort := 30198 - url := fmt.Sprintf("http://%s:%d/%s/prometheus-%s/api/v1/query", domainName, nodePort, prNumber, target) - if qtype == "range" { - url = fmt.Sprintf("http://%s:%d/%s/prometheus-%s/api/v1/query_range", domainName, nodePort, prNumber, target) - } - - return &Querier{ - target: target, - name: qg.Name, - groupID: groupID, - interval: durationSeconds(qg.Interval), - queries: qg.Queries, - qtype: qtype, - start: start, - end: end, - step: qg.Step, - url: url, - } -} - -func (q *Querier) run(wg *sync.WaitGroup, baseTime *time.Time) { - defer wg.Done() - fmt.Printf("Running querier %s %s for %s\n", q.target, q.name, q.url) - time.Sleep(20 * time.Second) - for { - start := time.Now() - - for _, query := range q.queries { - q.query(query.Expr, baseTime) - } - *baseTime = baseTime.Add(2 * time.Minute) - wait := q.interval - time.Since(start) - if wait > 0 { - time.Sleep(wait) - } - } -} - -func (q *Querier) query(expr string, baseTime *time.Time) { - queryCount.WithLabelValues(q.target, q.name, expr, q.qtype).Inc() - - req, err := http.NewRequest("GET", q.url, nil) - if err != nil { - log.Printf("Error creating request: %v", err) - queryFailCount.WithLabelValues(q.target, q.name, expr, q.qtype).Inc() - return - } - - qParams := req.URL.Query() - qParams.Set("query", expr) - if q.qtype == "range" { - qParams.Set("start", fmt.Sprintf("%d", int64(baseTime.Add(-q.start).Unix()))) - qParams.Set("end", fmt.Sprintf("%d", int64(baseTime.Add(-q.end).Unix()))) - qParams.Set("step", q.step) - } - fmt.Println(qParams["start"], "Hi from start Time") - fmt.Println(qParams["end"], "Hi from end End Time") - req.URL.RawQuery = qParams.Encode() - - resp, err := http.DefaultClient.Do(req) - if err != nil { - log.Printf("Error querying Prometheus: %v", err) - queryFailCount.WithLabelValues(q.target, q.name, expr, q.qtype).Inc() - return - } - // body, err := io.ReadAll(resp.Body) - // if err != nil { - // fmt.Println("Error reading response body:", err) - // return - // } - // fmt.Println(baseTime, "Hi from baseTime") - // fmt.Println("Response Body:", string(body)) - defer resp.Body.Close() - - duration := time.Since(*baseTime) - queryDuration.WithLabelValues(q.target, q.name, expr, q.qtype).Observe(duration.Seconds()) - if resp.StatusCode == http.StatusNotFound { - log.Printf("WARNING: GroupID#%d: Querier returned 404 for Prometheus instance %s.", q.groupID, q.url) - q.numberOfErrors++ - if q.numberOfErrors >= max404Errors { - log.Fatalf("ERROR: GroupID#%d: Querier returned 404 for Prometheus instance %s %d times.", q.groupID, q.url, max404Errors) - } - } else if resp.StatusCode != http.StatusOK { - log.Printf("WARNING: GroupID#%d: Querier returned %d for Prometheus instance %s.", q.groupID, resp.StatusCode, q.url) - } else { - body, _ := io.ReadAll(resp.Body) - log.Printf("GroupID#%d: query %s %s, status=%d, size=%d, duration=%.3f", q.groupID, q.target, expr, resp.StatusCode, len(body), duration.Seconds()) - } -} - -func durationSeconds(s string) time.Duration { - if s == "" { - return 0 - } - value, err := model.ParseDuration(s) - if err != nil { - log.Fatalf("%s", err.Error()) - } - return time.Duration(value) -} - -func main() { - if len(os.Args) < 3 { - fmt.Println("unexpected arguments") - fmt.Println("usage: ") - os.Exit(2) - } - prNumber := os.Args[2] - - configFile, err := os.ReadFile("/etc/loadgen/config.yaml") - if err != nil { - log.Fatalf("Failed to load config: %v", err) - } - - var config struct { - Querier struct { - Groups []QueryGroup `yaml:"groups"` - } `yaml:"querier"` - } - if err := yaml.Unmarshal(configFile, &config); err != nil { - log.Fatalf("Failed to parse config: %v", err) - } - - fmt.Println("Loaded configuration") - - var wg sync.WaitGroup - baseTime := time.Date(2024, time.October, 23, 15, 34, 32, 541000000, time.UTC) - for i, group := range config.Querier.Groups { - wg.Add(1) - go NewQuerier(i, "pr", prNumber, group).run(&wg, &baseTime) - wg.Add(1) - go NewQuerier(i, "release", prNumber, group).run(&wg, &baseTime) - } - - prometheus.MustRegister(queryDuration, queryCount, queryFailCount) - http.Handle("/metrics", promhttp.Handler()) - go func() { - log.Println("Starting HTTP server on :8080") - log.Fatal(http.ListenAndServe(":8080", nil)) - }() - - wg.Wait() -} diff --git a/tools/prometheus-builder/build.sh b/tools/prometheus-builder/build.sh index a37e040e4..0d11212fb 100755 --- a/tools/prometheus-builder/build.sh +++ b/tools/prometheus-builder/build.sh @@ -5,8 +5,7 @@ DIR="/go/src/github.com/prometheus/prometheus" if [[ -z $PR_NUMBER || -z $VOLUME_DIR || -z $GITHUB_ORG || -z $GITHUB_REPO ]]; then echo "ERROR:: environment variables not set correctly" exit 1; -fi - +fi # Clone the repository with a shallow clone echo ">> Cloning repository $GITHUB_ORG/$GITHUB_REPO (shallow clone)" if ! git clone --depth 1 https://github.com/$GITHUB_ORG/$GITHUB_REPO.git $DIR; then @@ -25,7 +24,7 @@ fi git checkout pr-branch MKDIR="/config" -cp "$DIR/objectconfig.yml" "$MKDIR/objectconfig.yml" +cp "$DIR/key.yml" "$MKDIR/key.yml" echo ">> Creating prometheus binaries" if ! make build PROMU_BINARIES="prometheus"; then