diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b4092f9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,30 @@ +prometheus-rds-exporter +metrics + +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out +coverage.txt +coverage.xml +lint-report.xml + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work + +# Build files +dist diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..c5e1379 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,94 @@ +run: + concurrency: 4 + deadline: 2m + issues-exit-code: 1 + tests: true + +output: + format: colored-line-number + print-issued-lines: true + print-linter-name: true + +linters: + enable-all: false + disable-all: false + enable: + - errcheck + - gosimple + - govet + - ineffassign + - staticcheck + - typecheck + - unused + - asciicheck + - bodyclose + - dogsled + - durationcheck + - errorlint + - exhaustive + - exportloopref + - forcetypeassert + - gochecknoinits + - goconst + - gocritic + - gocyclo + - godox + - goerr113 + - gofmt + - gofumpt + - goimports + - gomnd + - gomodguard + - goprintffuncname + - gosec + - importas + - makezero + - misspell + - nakedret + - nestif + - nilerr + - nlreturn + - noctx + - nolintlint + - prealloc + - predeclared + - revive + - rowserrcheck + - sqlclosecheck + - stylecheck + - testpackage + - thelper + - tparallel + - unconvert + - unparam + - wastedassign + - whitespace + - wrapcheck + - wsl + +linters-settings: + gocyclo: + min-complexity: 35 + + revive: + rules: + - name: exported + disabled: true + + lll: + line-length: 120 + +issues: + exclude-use-default: false + max-per-linter: 1024 + max-same: 1024 + + exclude-rules: + # Exclude some linters from running on test files + - path: _test\.go + linters: + # bodyclose reports some false-positives when using a test request recorder + - bodyclose + # It's overkill to use `NewRequestWithContext` in tests + - noctx + diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..98c220a --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,143 @@ +# yaml-language-server: $schema=https://goreleaser.com/static/schema.json +env: + - BUILD_INFO_PACKAGE_PATH=github.com/qonto/prometheus-rds-exporter/internal/infra/build + - DOCKER_REGISTRY=public.ecr.aws/g1r8z6f4 + - DOCKER_IMAGE_NAME=prometheus-rds-exporter + +builds: + - env: + - CGO_ENABLED=0 + ldflags: + - '-s' + - '-w' + - '-X "{{ .Env.BUILD_INFO_PACKAGE_PATH }}.Version={{.Version}}"' + - '-X "{{ .Env.BUILD_INFO_PACKAGE_PATH }}.Commit={{.Commit}}"' + - '-X "{{ .Env.BUILD_INFO_PACKAGE_PATH }}.Date={{.Date}}"' + goos: + - linux + - darwin + goarch: + - amd64 + - arm64 + +archives: + - format: tar.gz + # this name template makes the OS and Arch compatible with the results of uname. + name_template: >- + {{ .ProjectName }}_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else }}{{ .Arch }}{{ end }} + {{- if .Arm }}v{{ .Arm }}{{ end }} + # use zip for windows archives + format_overrides: + - goos: windows + format: zip + +checksum: + name_template: 'checksums.txt' + +snapshot: + name_template: "{{ incpatch .Version }}-next" + +# changelog: +# sort: asc +# use: github +# filters: +# exclude: +# - "^test:" +# - "^chore" +# - "merge conflict" +# - Merge pull request +# - Merge remote-tracking branch +# - Merge branch +# - go mod tidy +# groups: +# - title: Dependency updates +# regexp: '^.*?(feat|fix)\(deps\)!?:.+$' +# order: 300 +# - title: "New Features" +# regexp: '^.*?feat(\([[:word:]]+\))??!?:.+$' +# order: 100 +# - title: "Security updates" +# regexp: '^.*?sec(\([[:word:]]+\))??!?:.+$' +# order: 150 +# - title: "Bug fixes" +# regexp: '^.*?fix(\([[:word:]]+\))??!?:.+$' +# order: 200 +# - title: "Documentation updates" +# regexp: ^.*?doc(\([[:word:]]+\))??!?:.+$ +# order: 400 +# - title: "Build process updates" +# regexp: ^.*?build(\([[:word:]]+\))??!?:.+$ +# order: 400 +# - title: Other work +# order: 9999 + +dockers: +- image_templates: + - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:{{ .Tag }}-amd64" + - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:v{{ .Major }}-amd64" + - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:v{{ .Major }}.{{ .Minor }}-amd64" + - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:latest-amd64" + dockerfile: configs/goreleaser/Dockerfile + build_flag_templates: + - --label=org.opencontainers.image.title={{ .ProjectName }} + - --label=org.opencontainers.image.description={{ .ProjectName }} + - --label=org.opencontainers.image.url=https://github.com/goreleaser/example + - --label=org.opencontainers.image.source=https://github.com/goreleaser/example + - --label=org.opencontainers.image.version={{ .Version }} + - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} + - --label=org.opencontainers.image.revision={{ .FullCommit }} + - --label=org.opencontainers.image.licenses=MIT + - "--pull" + - "--platform=linux/amd64" + - "--builder=desktop-linux" # TODO remove before public release + use: buildx +- image_templates: + - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:{{ .Tag }}-arm64" + - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:v{{ .Major }}-arm64" + - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:v{{ .Major }}.{{ .Minor }}-arm64" + - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:latest-arm64" + dockerfile: configs/goreleaser/Dockerfile + build_flag_templates: + - --label=org.opencontainers.image.title={{ .ProjectName }} + - --label=org.opencontainers.image.description={{ .ProjectName }} + - --label=org.opencontainers.image.url=https://github.com/goreleaser/example + - --label=org.opencontainers.image.source=https://github.com/goreleaser/example + - --label=org.opencontainers.image.version={{ .Version }} + - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} + - --label=org.opencontainers.image.revision={{ .FullCommit }} + - --label=org.opencontainers.image.licenses=MIT + - "--pull" + - "--platform=linux/arm64" + - "--builder=desktop-linux" # TODO remove before public release + goarch: arm64 + +docker_manifests: +- name_template: '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:{{ .Tag }}' + image_templates: + - '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:{{ .Tag }}-amd64' + - '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:{{ .Tag }}-arm64' +- name_template: '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:v{{ .Major }}' + image_templates: + - '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:v{{ .Major }}-amd64' + - '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:v{{ .Major }}-arm64' +- name_template: '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:v{{ .Major }}.{{ .Minor }}' + image_templates: + - '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:v{{ .Major }}.{{ .Minor }}-amd64' + - '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:v{{ .Major }}.{{ .Minor }}-arm64' +- name_template: '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:latest' + image_templates: + - '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:latest-amd64' + - '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:latest-arm64' + +release: + disable: true + github: + owner: qonto + name: prometheus-rds-exporter + name_template: "v{{.Version}}" + footer: | + **Full Changelog**: https://github.com/qonto/poc-goreleaser/compare/{{ .PreviousTag }}...{{ .Tag }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..c617b7b --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,67 @@ +# Contributing + +Prometheus RDS exporter uses GitHub to manage reviews of pull requests. + +* If you are a new contributor see: [Steps to Contribute](#steps-to-contribute) + +* If you have a trivial fix or improvement, go ahead and create a pull request + +* Relevant coding style guidelines are the [Go Code Review + Comments](https://code.google.com/p/go-wiki/wiki/CodeReviewComments) + and the _Formatting and style_ section of Peter Bourgon's [Go: Best + Practices for Production + Environments](https://peter.bourgon.org/go-in-production/#formatting-and-style). + +* Be sure to enabled [signed commits](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits) + +## Steps to Contribute + +Should you wish to work on an issue, please claim it first by commenting on the GitHub issue that you want to work on it. This is to prevent duplicated efforts from contributors on the same issue. + +For quickly compiling and testing your changes do: + +```bash +# For building. +make build +./rds_exporter + +# For testing. +make test # Make sure all the tests pass before you commit and push :) +``` + +We use [`golangci-lint`](https://github.com/golangci/golangci-lint) for linting the code. If it reports an issue and you think that the warning needs to be disregarded or is a false-positive, you can add a special comment `//nolint:linter1[,linter2,...]` before the offending line. Use this sparingly though, fixing the code to comply with the linter's recommendation is in general the preferred course of action. + +All our issues are regularly tagged so that you can also filter down the issues involving the components you want to work on. + +## Pull Request Checklist + +* Branch from the `main` branch and, if needed, rebase to the current main branch before submitting your pull request. If it doesn't merge cleanly with main you may be asked to rebase your changes. + +* Commits should be as small as possible, while ensuring that each commit is correct independently (i.e., each commit should compile and pass tests). + +* Add tests relevant to the fixed bug or new feature. + +* New Prometheus metrics must follow the [metric and label naming guidelines](https://prometheus.io/docs/practices/naming/) and be added in the `README.md` + +## Dependency management + +Project uses [Go modules](https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more) to manage dependencies on external packages. + +To add or update a new dependency, use the `go get` command: + +```bash +# Pick the latest tagged release. +go get example.com/some/module/pkg@latest + +# Pick a specific version. +go get example.com/some/module/pkg@vX.Y.Z +``` + +Tidy up the `go.mod` and `go.sum` files: + +```bash +# The GO111MODULE variable can be omitted when the code isn't located in GOPATH. +GO111MODULE=on go mod tidy +``` + +You have to commit the changes to `go.mod` and `go.sum` before submitting the pull request. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..871fd35 --- /dev/null +++ b/Makefile @@ -0,0 +1,50 @@ +kubeconform_command := kubeconform -kubernetes-version $${KUBERNETES_VERSION-1.25.0} -cache $${KUBECONFORM_CACHE_DIRECTORY-/tmp} -summary -exit-on-error --strict -schema-location default -schema-location 'kubeconform/{{ .ResourceKind }}{{ .KindSuffix }}.json' -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' +BUILD_INFO_PACKAGE_PATH=github.com/qonto/prometheus-rds-exporter/internal/infra/build +BUILD_DATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ') +GIT_COMMIT_SHA=$(shell git rev-parse HEAD) +AWS_ECR_PUBLIC_ORGANIZATION=g1r8z6f4 + +all: build + +.PHONY: format +format: + gofumpt -l -w . + +.PHONY: build +build: + CGO_ENABLED=0 go build -v -ldflags="-X '$(BUILD_INFO_PACKAGE_PATH).Version=development' -X '$(BUILD_INFO_PACKAGE_PATH).CommitSHA=$(GIT_COMMIT_SHA)' -X '$(BUILD_INFO_PACKAGE_PATH).Date=$(BUILD_DATE)'" -o prometheus-rds-exporter + +.PHONY: run +run: + ./prometheus-rds-exporter $(argument) + +.PHONY: test +test: + go test -race -v ./... -coverprofile=coverage.txt -covermode atomic + go install github.com/boumenot/gocover-cobertura@latest + go run github.com/boumenot/gocover-cobertura@latest < coverage.txt > coverage.xml + +.PHONY: lint +lint: + golangci-lint run --verbose --timeout 2m --out-format junit-xml:lint-report.xml,colored-line-number + +.PHONY: helm-test +helm-test: + helm unittest configs/helm + +.PHONY: helm-release +helm-release: + ./scripts/helm-release.sh prometheus-rds-exporter-chart $(RELEASE_VERSION) $(AWS_ECR_PUBLIC_ORGANIZATION) + +.PHONY: kubeconform +kubeconform: + helm template configs/helm | $(kubeconform_command) + +.PHONY: metrics-list +metrics-list: + echo "| Name | Description |" > metrics + echo "| ------ | ----------- |" >> metrics + curl -s localhost:9043/metrics | grep -E '^# HELP' | awk '{metric = $$3; $$1=$$2=$$3=""; print "| " metric " | " $$0 " | "}' | sed -e's/ */ /g' >> metrics + +.PHONY: all-tests +all-tests: test kubeconform helm-test \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..417cc36 --- /dev/null +++ b/README.md @@ -0,0 +1,241 @@ +# RDS exporter + +Opiniated Prometheus exporter for AWS RDS that provides A cost-efficient advanced RDS metrics collector. + +When combined with [prometheus-community/postgres_exporter](https://github.com/prometheus-community/postgres_exporter), it provides a production ready monitoring framework for RDS PostgreSQL. + +## Metrics + +| Name | Description | +| ------ | ----------- | +| rds_allocated_storage_bytes | Allocated storage | +| rds_api_call_total | Number of call to AWS API | +| rds_backup_retention_period_seconds | Automatic DB snapshots retention period | +| rds_cpu_usage_percent_average | Instance CPU used | +| rds_database_connections_average | The number of client network connections to the database instance | +| rds_dbload_average | Number of active sessions for the DB engine | +| rds_dbload_cpu_average | Number of active sessions where the wait event type is CPU | +| rds_dbload_noncpu_average | Number of active sessions where the wait event type is not CPU | +| rds_exporter_build_info | A metric with constant '1' value labeled by version from which exporter was built | +| rds_exporter_errors_total | Total number of errors encountered by the exporter | +| rds_free_storage_bytes | Free storage on the instance | +| rds_freeable_memory_bytes | Amount of available random access memory. For MariaDB, MySQL, Oracle, and PostgreSQL DB instances, this metric reports the value of the MemAvailable field of /proc/meminfo | +| rds_instance_info | RDS instance information | +| rds_instance_log_files_size_bytes | Total of log files on the instance | +| rds_instance_maxIops_average | Maximum IOPS of underlying EC2 instance | +| rds_instance_max_throughput_bytes | Maximum throughput of underlying EC2 instance | +| rds_instance_memory_bytes | Instance memory | +| rds_instance_status | Instance status (1: ok, 0: can't scrap metrics) | +| rds_instance_vcpu_average | Total vCPU for this isntance class | +| rds_max_allocated_storage_bytes | Upper limit in gibibytes to which Amazon RDS can automatically scale the storage of the DB instance | +| rds_max_disk_iops_average | Max IOPS for the instance | +| rds_max_storage_throughput_bytes | Max storage throughput | +| rds_maximum_used_transaction_ids_average | Maximum transaction IDs that have been used. Applies to only PostgreSQL | +| rds_quota_max_dbinstances_average | Maximum number of RDS instances allowed in the AWS account | +| rds_quota_maximum_db_instance_snapshots_average | Maximum number of manual DB instance snapshots | +| rds_quota_total_storage_bytes | Maximum total storage for all DB instances | +| rds_read_iops_average | Average number of disk read I/O operations per second | +| rds_read_throughput_bytes | Average number of bytes read from disk per second | +| rds_replica_lag_seconds | For read replica configurations, the amount of time a read replica DB instance lags behind the source DB instance. Applies to MariaDB, Microsoft SQL Server, MySQL, Oracle, and PostgreSQL read replicas | +| rds_replication_slot_disk_usage_average | Disk space used by replication slot files. Applies to PostgreSQL | +| rds_swap_usage_bytes | Amount of swap space used on the DB instance. This metric is not available for SQL Server | +| rds_usage_allocated_storage_average | Total storage used by AWS RDS instances | +| rds_usage_db_instances_average | AWS RDS instance count | +| rds_usage_manual_snapshots_average | Manual snapshots count | +| rds_write_iops_average | Average number of disk write I/O operations per second | +| rds_write_throughput_bytes | Average number of bytes written to disk per second | +| up | Was the last scrape of RDS successful | + + +
+ Standard Go and Prometheus metrics are also available + +| Name | Description | +| ------ | ----------- | +| go_gc_duration_seconds | A summary of the pause duration of garbage collection cycles. | +| go_goroutines | Number of goroutines that currently exist. | +| go_info | Information about the Go environment. | +| go_memstats_alloc_bytes | Number of bytes allocated and still in use. | +| go_memstats_alloc_bytes_total | Total number of bytes allocated, even if freed. | +| go_memstats_buck_hash_sys_bytes | Number of bytes used by the profiling bucket hash table. | +| go_memstats_frees_total | Total number of frees. | +| go_memstats_gc_sys_bytes | Number of bytes used for garbage collection system metadata. | +| go_memstats_heap_alloc_bytes | Number of heap bytes allocated and still in use. | +| go_memstats_heap_idle_bytes | Number of heap bytes waiting to be used. | +| go_memstats_heap_inuse_bytes | Number of heap bytes that are in use. | +| go_memstats_heap_objects | Number of allocated objects. | +| go_memstats_heap_released_bytes | Number of heap bytes released to OS. | +| go_memstats_heap_sys_bytes | Number of heap bytes obtained from system. | +| go_memstats_last_gc_time_seconds | Number of seconds since 1970 of last garbage collection. | +| go_memstats_lookups_total | Total number of pointer lookups. | +| go_memstats_mallocs_total | Total number of mallocs. | +| go_memstats_mcache_inuse_bytes | Number of bytes in use by mcache structures. | +| go_memstats_mcache_sys_bytes | Number of bytes used for mcache structures obtained from system. | +| go_memstats_mspan_inuse_bytes | Number of bytes in use by mspan structures. | +| go_memstats_mspan_sys_bytes | Number of bytes used for mspan structures obtained from system. | +| go_memstats_next_gc_bytes | Number of heap bytes when next garbage collection will take place. | +| go_memstats_other_sys_bytes | Number of bytes used for other system allocations. | +| go_memstats_stack_inuse_bytes | Number of bytes in use by the stack allocator. | +| go_memstats_stack_sys_bytes | Number of bytes obtained from system for stack allocator. | +| go_memstats_sys_bytes | Number of bytes obtained from system. | +| go_threads | Number of OS threads created. | +| promhttp_metric_handler_requests_in_flight | Current number of scrapes being served. | +| promhttp_metric_handler_requests_total | Total number of scrapes by HTTP status code. | + +
+ +## Configuration + +Configuration could be defined in `.prometheus-rds-exporter.yaml` or environment variables (format `PROMETHEUS_RDS_EXPORTER_`). + +| Parameter | Description | Default | +| --- | --- | --- | +| aws-assume-role-arn | AWS IAM ARN role to assume to fetch metrics | | +| aws-assume-role-session | AWS assume role session name | prometheus-rds-exporter | +| debug | Enable debug mode | | +| listen-address | Address to listen on for web interface | :9043 | +| log-format | Log format (`text` or `json`) | text | +| metrics-path | Path under which to expose metrics | /metrics | + +Configuration parameters priorities: + +1. `$HOME/.prometheus-rds-exporter.yaml` file +2. `.prometheus-rds-exporter.yaml` file +3. Environment variables +4. Command line flags + +### AWS authentication + +Prometheus RDS exporter needs read only AWS IAM permissions to fetch metrics from AWS RDS, CloudWatch, EC2 and ServiceQuota AWS APIs. + +Standard AWS authentication methods (AWS credentials, SSO and assume role), see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html. + +If you are running on [AWS EKS](), we strongly recommend to use [IRSA](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) + +Minimal required IAM permissions: + +```yaml +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowFetchingRDSMetrics", + "Effect": "Allow", + "Action": [ + "cloudwatch:GetMetricData", + "ec2:DescribeInstanceTypes", + "rds:DescribeAccountAttributes", + "rds:DescribeDBInstances", + "rds:DescribeDBLogFiles", + "rds:DescribePendingMaintenanceActions", + "servicequotas:GetServiceQuota" + ], + "Resource": "*" + } + ] +} +``` + +## Installation + +### Locally + +1. Connect on AWS with any method + + ```bash + aws configure + ``` + +2. Start application + + ```bash + prometheus-rds-exporter + ``` + +### Docker + +1. Connect on AWS with any method + +2. Start application + + ```bash + docker run -p 9043:9043 -e AWS_PROFILE=${AWS_PROFILE} -v $HOME/.aws:/app/.aws public.ecr.aws/g1r8z6f4/prometheus-rds-exporter:latest + ``` + +### EKS (using IRSA and Helm) + +1. Create an IAM policy + + ```bash + IAM_POLICY_NAME=prometheus-rds-exporter + aws iam create-policy --policy-name ${IAM_POLICY_NAME} --policy-document file://configs/aws/policy.json + ``` + +2. Create IAM role and EKS service account + + ```bash + IAM_ROLE_NAME=prometheus-rds-exporter + EKS_CLUSTER_NAME=default # Replace with your EKS cluster name + KUBERNETES_NAMESPACE=default # Replace with the namespace that you want to use + KUBERNETES_SERVICE_ACCOUNT_NAME=prometheus-rds-exporter + AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text) # Replace with your AWS ACCOUNT ID + eksctl create iamserviceaccount --cluster ${EKS_CLUSTER_NAME} --namespace ${KUBERNETES_NAMESPACE} --name ${KUBERNETES_SERVICE_ACCOUNT_NAME} --role-name ${IAM_ROLE_NAME} --attach-policy-arn arn:aws:iam::${AWS_ACCOUNT_ID}:policy/${IAM_POLICY_NAME} --approve + ``` + +3. Deploy chart with service account annotation + + ```bash + helm install prometheus-rds-exporter oci://public.ecr.aws/g1r8z6f4/prometheus-rds-exporter-chart --namespace ${KUBERNETES_NAMESPACE} --set serviceAccount.annotations."eks\.amazonaws\.com\/role-arn"="arn:aws:iam::${AWS_ACCOUNT_ID}:role/${IAM_ROLE_NAME}" + ``` + +### Terraform + +You can take example on Terraform code in `configs/terraform/`. + +## Alternative + +[percona/rds_exporter](https://github.com/percona/rds_exporter) and [mtanda/rds_enhanced_monitoring_exporter](https://github.com/mtanda/rds_enhanced_monitoring_exporter) provides are great alternatives. + +[prometheus/cloudwatch_exporter](https://github.com/prometheus/cloudwatch_exporter) could be used to collect additional CloudWatch metrics. + +## Contribute + +See CONTRIBUTING.md + +## Development + +### Running the tests + +Execute golang tests: + +```bash +make test +``` + +Execute Helm chart tests: + +```bash +make helm-test # Helm unit test +make kubeconform # Kubernetes manifest validation +``` + +### Development environment + +You can start a simple development environment using the Docker compose configuration in `/scripts/prometheus`. + +It will start and configure Grafana, Prometheus, and the RDS exporter: + +1. Connect on AWS using the AWS CLI + +1. Launch development stack + + ```bash + cd scripts/prometheus + docker compose up --build + ``` + +1. Connect on the services + + - Grafana: http://localhost:3000 (credential: admin/hackme) + - Prometheus: http://localhost:9090 + - Prometheus RDS exporter: http://localhost:9043 diff --git a/cmd/helper.go b/cmd/helper.go new file mode 100644 index 0000000..5b5079c --- /dev/null +++ b/cmd/helper.go @@ -0,0 +1,31 @@ +package cmd + +import ( + "context" + "fmt" + "log/slog" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/sts" +) + +func getAWSConfiguration(logger *slog.Logger, roleArn string, sessionName string) (aws.Config, error) { + cfg, err := config.LoadDefaultConfig(context.TODO()) + if err != nil { + return aws.Config{}, fmt.Errorf("can't create AWS session: %w", err) + } + + if roleArn != "" { + logger.Debug("Assume role", "role", roleArn) + + client := sts.NewFromConfig(cfg) + creds := stscreds.NewAssumeRoleProvider(client, roleArn, func(o *stscreds.AssumeRoleOptions) { + o.RoleSessionName = sessionName + }) + cfg.Credentials = aws.NewCredentialsCache(creds) + } + + return cfg, nil +} diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..7b34869 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,170 @@ +// Package cmd implements command to start the RDS exporter +package cmd + +import ( + "fmt" + "os" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/rds" + "github.com/aws/aws-sdk-go-v2/service/servicequotas" + "github.com/prometheus/client_golang/prometheus" + "github.com/qonto/prometheus-rds-exporter/internal/app/exporter" + "github.com/qonto/prometheus-rds-exporter/internal/infra/build" + "github.com/qonto/prometheus-rds-exporter/internal/infra/http" + "github.com/qonto/prometheus-rds-exporter/internal/infra/logger" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +const ( + configErrorExitCode = 1 + httpErrorExitCode = 2 + exporterErrorExitCode = 3 + awsErrorExitCode = 4 +) + +var cfgFile string + +type exporterConfig struct { + Debug bool `mapstructure:"debug"` + LogFormat string `mapstructure:"log-format"` + MetricPath string `mapstructure:"metrics-path"` + ListenAddress string `mapstructure:"listen-address"` + AWSAssumeRoleSession string `mapstructure:"aws-assume-role-session"` + AWSAssumeRoleArn string `mapstructure:"aws-assume-role-arn"` +} + +func run(configuration exporterConfig) { + logger, err := logger.New(configuration.Debug, configuration.LogFormat) + if err != nil { + fmt.Println("ERROR: Fail to initialize logger: %w", err) + panic(err) + } + + cfg, err := getAWSConfiguration(logger, configuration.AWSAssumeRoleArn, configuration.AWSAssumeRoleSession) + if err != nil { + logger.Error("can't initialize AWS configuration", "reason", err) + os.Exit(awsErrorExitCode) + } + + rdsClient := rds.NewFromConfig(cfg) + ec2Client := ec2.NewFromConfig(cfg) + cloudWatchClient := cloudwatch.NewFromConfig(cfg) + servicequotasClient := servicequotas.NewFromConfig(cfg) + collector := exporter.NewCollector(*logger, rdsClient, ec2Client, cloudWatchClient, servicequotasClient) + + prometheus.MustRegister(collector) + + server := http.New(*logger, configuration.ListenAddress, configuration.MetricPath) + + err = server.Start() + if err != nil { + logger.Error("web server error", "reason", err) + os.Exit(httpErrorExitCode) + } +} + +func NewRootCommand() (*cobra.Command, error) { + cmd := &cobra.Command{ + Use: "rds-exporter", + Version: fmt.Sprintf("%s, commit %s, built at %s", build.Version, build.CommitSHA, build.Date), + Short: "Prometheus exporter for AWS RDS", + Long: `Collect AWS RDS key metrics from AWS APIs + and expose them as Prometheus metrics.`, + Run: func(cmd *cobra.Command, args []string) { + var c exporterConfig + err := viper.Unmarshal(&c) + if err != nil { + fmt.Println("ERROR: Unable to decode configuration, %w", err) + + return + } + run(c) + }, + } + + cobra.OnInitialize(initConfig) + + cmd.Flags().StringVarP(&cfgFile, "config", "c", "", "config file (default is $HOME/.prometheus-rds-exporter.yaml)") + cmd.Flags().BoolP("debug", "d", false, "Enable debug mode") + cmd.Flags().StringP("log-format", "l", "text", "Log format (text or json)") + cmd.Flags().StringP("metrics-path", "", "/metrics", "Path under which to expose metrics") + cmd.Flags().StringP("listen-address", "", ":9043", "Address to listen on for web interface") + cmd.Flags().StringP("aws-assume-role-arn", "", "", "AWS IAM ARN role to assume to fetch metrics") + cmd.Flags().StringP("aws-assume-role-session", "", "prometheus-rds-exporter", "AWS assume role session name") + + err := viper.BindPFlag("debug", cmd.Flags().Lookup("debug")) + if err != nil { + return cmd, fmt.Errorf("failed to bind 'debug' parameter: %w", err) + } + + err = viper.BindPFlag("log-format", cmd.Flags().Lookup("log-format")) + if err != nil { + return cmd, fmt.Errorf("failed to bind 'log-format' parameter: %w", err) + } + + err = viper.BindPFlag("metrics-path", cmd.Flags().Lookup("metrics-path")) + if err != nil { + return cmd, fmt.Errorf("failed to bind 'metrics-path' parameter: %w", err) + } + + err = viper.BindPFlag("listen-address", cmd.Flags().Lookup("listen-address")) + if err != nil { + return cmd, fmt.Errorf("failed to bind 'listen-address' parameter: %w", err) + } + + err = viper.BindPFlag("aws-assume-role-arn", cmd.Flags().Lookup("aws-assume-role-arn")) + if err != nil { + return cmd, fmt.Errorf("failed to bind 'aws-assume-role-arn' parameter: %w", err) + } + + err = viper.BindPFlag("aws-assume-role-session", cmd.Flags().Lookup("aws-assume-role-session")) + if err != nil { + return cmd, fmt.Errorf("failed to bind 'aws-assume-role-session' parameter: %w", err) + } + + return cmd, nil +} + +func Execute() { + cmd, err := NewRootCommand() + if err != nil { + fmt.Println("ERROR: Failed to load configuration: %w", err) + os.Exit(configErrorExitCode) + } + + err = cmd.Execute() + if err != nil { + fmt.Println("ERROR: Failed to execute exporter: %w", err) + os.Exit(exporterErrorExitCode) + } +} + +// initConfig reads in config file and ENV variables if set. +func initConfig() { + if cfgFile != "" { + // Use config file from the flag. + viper.SetConfigFile(cfgFile) + } else { + // Find home directory. + home, err := os.UserHomeDir() + cobra.CheckErr(err) + + // Search config in home directory or current directory with name ".rds-exporter" (without extension). + viper.AddConfigPath(".") + viper.AddConfigPath(home) + viper.SetConfigType("yaml") + viper.SetConfigName(".prometheus-rds-exporter") + } + + if err := viper.ReadInConfig(); err == nil { + fmt.Fprintln(os.Stderr, "Using config file:", viper.ConfigFileUsed()) + } + + viper.SetEnvPrefix("prometheus_rds_exporter") // will be uppercased automatically + viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + viper.AutomaticEnv() +} diff --git a/configs/aws/policy.json b/configs/aws/policy.json new file mode 100644 index 0000000..7c43524 --- /dev/null +++ b/configs/aws/policy.json @@ -0,0 +1,19 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowFetchingRDSMetrics", + "Effect": "Allow", + "Action": [ + "cloudwatch:GetMetricData", + "ec2:DescribeInstanceTypes", + "rds:DescribeAccountAttributes", + "rds:DescribeDBInstances", + "rds:DescribeDBLogFiles", + "rds:DescribePendingMaintenanceActions", + "servicequotas:GetServiceQuota" + ], + "Resource": "*" + } + ] +} \ No newline at end of file diff --git a/configs/goreleaser/Dockerfile b/configs/goreleaser/Dockerfile new file mode 100644 index 0000000..0940bf8 --- /dev/null +++ b/configs/goreleaser/Dockerfile @@ -0,0 +1,17 @@ +FROM alpine:3 + +ARG USER=exporter +ARG HOME=/app + +RUN addgroup -g 1001 -S exporter \ + && adduser --home /app -u 1001 -S exporter -G exporter \ + && mkdir -p /app \ + && chown exporter:exporter -R /app + +WORKDIR $HOME +USER $USER + +EXPOSE 9043 + +COPY prometheus-rds-exporter /app/ +ENTRYPOINT ["/app/prometheus-rds-exporter"] diff --git a/configs/helm/.helmignore b/configs/helm/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/configs/helm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/configs/helm/Chart.yaml b/configs/helm/Chart.yaml new file mode 100644 index 0000000..3af4eca --- /dev/null +++ b/configs/helm/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v2 +type: application + +# version and appVersion values are overrided during build step +version: 0.0.0 +appVersion: 0.0.0 + +name: prometheus-rds-exporter-chart +description: Prometheus RDS exporter +home: https://github.com/qonto/prometheus-rds-exporter +sources: + - https://github.com/qonto/prometheus-rds-exporter +annotations: + artifacthub.io/category: database + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts/blob/main/charts/prometheus-statsd-exporter +keywords: + - aws + - rds + - prometheus + - exporter + - postgresql diff --git a/configs/helm/LICENSE b/configs/helm/LICENSE new file mode 120000 index 0000000..30cff74 --- /dev/null +++ b/configs/helm/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/configs/helm/templates/NOTES.txt b/configs/helm/templates/NOTES.txt new file mode 100644 index 0000000..f7ceb88 --- /dev/null +++ b/configs/helm/templates/NOTES.txt @@ -0,0 +1 @@ +RDS Prometheus exporter is deployed. diff --git a/configs/helm/templates/_helpers.tpl b/configs/helm/templates/_helpers.tpl new file mode 100644 index 0000000..dd750ae --- /dev/null +++ b/configs/helm/templates/_helpers.tpl @@ -0,0 +1,72 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "prometheus-rds-exporter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "prometheus-rds-exporter.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus-rds-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "prometheus-rds-exporter.labels" -}} +helm.sh/chart: {{ include "prometheus-rds-exporter.chart" . }} +{{ include "prometheus-rds-exporter.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- range $key, $value := .Values.additionalLabels }} +{{ $key | quote }}: {{ $value | quote }} +{{- end -}} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "prometheus-rds-exporter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "prometheus-rds-exporter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "prometheus-rds-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "prometheus-rds-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{- define "prometheus-rds-exporter.env" -}} +{{- range $key, $value := .Values.env }} +- name: {{ $key | quote }} + value: {{ $value | quote }} +{{- end -}} +{{- end -}} diff --git a/configs/helm/templates/deployment.yaml b/configs/helm/templates/deployment.yaml new file mode 100644 index 0000000..9336e09 --- /dev/null +++ b/configs/helm/templates/deployment.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "prometheus-rds-exporter.fullname" . }} + labels: + {{- include "prometheus-rds-exporter.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "prometheus-rds-exporter.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "prometheus-rds-exporter.selectorLabels" . | nindent 8 }} + {{- include "prometheus-rds-exporter.labels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "prometheus-rds-exporter.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + {{- include "prometheus-rds-exporter.env" . | nindent 12 }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + readinessProbe: + httpGet: + path: / + port: http + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/configs/helm/templates/ingress.yaml b/configs/helm/templates/ingress.yaml new file mode 100644 index 0000000..137b655 --- /dev/null +++ b/configs/helm/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "prometheus-rds-exporter.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "prometheus-rds-exporter.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/configs/helm/templates/service.yaml b/configs/helm/templates/service.yaml new file mode 100644 index 0000000..bea891b --- /dev/null +++ b/configs/helm/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "prometheus-rds-exporter.fullname" . }} + labels: + {{- include "prometheus-rds-exporter.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "prometheus-rds-exporter.selectorLabels" . | nindent 4 }} diff --git a/configs/helm/templates/serviceaccount.yaml b/configs/helm/templates/serviceaccount.yaml new file mode 100644 index 0000000..38660bc --- /dev/null +++ b/configs/helm/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "prometheus-rds-exporter.serviceAccountName" . }} + labels: + {{- include "prometheus-rds-exporter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/configs/helm/templates/servicemonitor.yaml b/configs/helm/templates/servicemonitor.yaml new file mode 100644 index 0000000..6145845 --- /dev/null +++ b/configs/helm/templates/servicemonitor.yaml @@ -0,0 +1,25 @@ +{{- if .Values.serviceMonitor.enabled -}} +# CRD documentation available on https://github.com/prometheus-operator/prometheus-operator/blob/v0.68.0/Documentation/api.md#servicemonitor +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "prometheus-rds-exporter.fullname" . }} + labels: + {{- include "prometheus-rds-exporter.labels" . | nindent 4 }} + {{- with .Values.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: app.kubernetes.io/name # Ensure prometheus job name match application's name + endpoints: + - path: /metrics + port: http + interval: {{ .Values.serviceMonitor.interval }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + {{- include "prometheus-rds-exporter.selectorLabels" . | nindent 6 }} +{{- end }} + diff --git a/configs/helm/tests/deployment_test.yaml b/configs/helm/tests/deployment_test.yaml new file mode 100644 index 0000000..8fe52a4 --- /dev/null +++ b/configs/helm/tests/deployment_test.yaml @@ -0,0 +1,51 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/helm-unittest/helm-unittest/main/schema/helm-testsuite.json +suite: deployment tests +templates: + - deployment.yaml +tests: + - it: render default deployment + asserts: + - isKind: + of: Deployment + - equal: + path: metadata.name + value: RELEASE-NAME-prometheus-rds-exporter-chart + - equal: + path: spec.template.spec.containers[0].image + value: public.ecr.aws/g1r8z6f4/prometheus-rds-exporter:0.0.0 + - equal: + path: spec.replicas + value: 1 + - it: render deployment with image tag + values: + - ./values/with_image_tag.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: public.ecr.aws/g1r8z6f4/prometheus-rds-exporter:1.0 + - it: render deployment with environment variables + values: + - ./values/with_environment_variables.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].env[0].name + value: variable1 + - equal: + path: spec.template.spec.containers[0].env[0].value + value: value1 + - equal: + path: spec.template.spec.containers[0].env[1].name + value: variable2 + - equal: + path: spec.template.spec.containers[0].env[1].value + value: value2 + - it: render with additional labels + values: + - ./values/with_additional_labels.yaml + asserts: + - equal: + path: metadata.labels.label1 + value: value1 + - equal: + path: metadata.labels.label2 + value: value2 diff --git a/configs/helm/tests/serviceaccount_test.yaml b/configs/helm/tests/serviceaccount_test.yaml new file mode 100644 index 0000000..b815ff9 --- /dev/null +++ b/configs/helm/tests/serviceaccount_test.yaml @@ -0,0 +1,20 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/helm-unittest/helm-unittest/main/schema/helm-testsuite.json +suite: service account tests +templates: + - serviceaccount.yaml +tests: + - it: render default service account + asserts: + - isKind: + of: ServiceAccount + - equal: + path: metadata.name + value: RELEASE-NAME-prometheus-rds-exporter-chart + - it: render service account with annotations + values: + - ./values/with_serviceaccount.yaml + asserts: + - equal: + path: metadata.annotations + value: + "eks.amazonaws.com/role-arn": "arn:aws:iam::012345678901:role/prometheus-rds-exporter" diff --git a/configs/helm/tests/servicemonitor_test.yaml b/configs/helm/tests/servicemonitor_test.yaml new file mode 100644 index 0000000..0e2ea44 --- /dev/null +++ b/configs/helm/tests/servicemonitor_test.yaml @@ -0,0 +1,25 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/helm-unittest/helm-unittest/main/schema/helm-testsuite.json +suite: service monitor tests +templates: + - servicemonitor.yaml +tests: + - it: render defaul service monitor + asserts: + - isKind: + of: ServiceMonitor + - equal: + path: metadata.name + value: RELEASE-NAME-prometheus-rds-exporter-chart + - equal: + path: spec.endpoints[0].interval + value: 60s + - it: render custom service monitor + values: + - ./values/with_servicemonitor.yaml + asserts: + - equal: + path: spec.endpoints[0].interval + value: 120s + - equal: + path: metadata.labels.additionalLabel1 + value: value1 diff --git a/configs/helm/tests/values/with_additional_labels.yaml b/configs/helm/tests/values/with_additional_labels.yaml new file mode 100644 index 0000000..3aa48fa --- /dev/null +++ b/configs/helm/tests/values/with_additional_labels.yaml @@ -0,0 +1,3 @@ +additionalLabels: + label1: value1 + label2: value2 diff --git a/configs/helm/tests/values/with_environment_variables.yaml b/configs/helm/tests/values/with_environment_variables.yaml new file mode 100644 index 0000000..72b14b0 --- /dev/null +++ b/configs/helm/tests/values/with_environment_variables.yaml @@ -0,0 +1,3 @@ +env: + variable1: value1 + variable2: value2 diff --git a/configs/helm/tests/values/with_image_tag.yaml b/configs/helm/tests/values/with_image_tag.yaml new file mode 100644 index 0000000..410f024 --- /dev/null +++ b/configs/helm/tests/values/with_image_tag.yaml @@ -0,0 +1,2 @@ +image: + tag: "1.0" diff --git a/configs/helm/tests/values/with_serviceaccount.yaml b/configs/helm/tests/values/with_serviceaccount.yaml new file mode 100644 index 0000000..d5a2842 --- /dev/null +++ b/configs/helm/tests/values/with_serviceaccount.yaml @@ -0,0 +1,3 @@ +serviceAccount: + annotations: + "eks.amazonaws.com/role-arn": "arn:aws:iam::012345678901:role/prometheus-rds-exporter" diff --git a/configs/helm/tests/values/with_servicemonitor.yaml b/configs/helm/tests/values/with_servicemonitor.yaml new file mode 100644 index 0000000..255850f --- /dev/null +++ b/configs/helm/tests/values/with_servicemonitor.yaml @@ -0,0 +1,5 @@ +serviceMonitor: + enabled: true + interval: 120s + additionalLabels: + additionalLabel1: value1 diff --git a/configs/helm/values.yaml b/configs/helm/values.yaml new file mode 100644 index 0000000..9694d2f --- /dev/null +++ b/configs/helm/values.yaml @@ -0,0 +1,80 @@ +# Default values for prometheus-rds-exporter. + +replicaCount: 1 + +image: + repository: public.ecr.aws/g1r8z6f4/prometheus-rds-exporter + pullPolicy: IfNotPresent + tag: "" # Defined by chart appVersion parameter + +# Define environment variables +env: {} + +# Set additional labels on all resources +additionalLabels: {} + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsGroup: 1001 + runAsUser: 1001 + +service: + type: ClusterIP + port: 9043 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: prometheus-rds-exporter.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +serviceMonitor: + enabled: true + interval: 60s + additionalLabels: {} + +resources: + limits: + # cpu: 100m + memory: 64Mi + requests: + cpu: 100m + memory: 48Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/configs/terraform/main.tf b/configs/terraform/main.tf new file mode 100644 index 0000000..e01c490 --- /dev/null +++ b/configs/terraform/main.tf @@ -0,0 +1,39 @@ +terraform { + // Replace with your Terraform configuration (eg. S3 backend) +} + +provider "aws" { + // Replace with your AWS provider configuration +} + +resource "aws_iam_role" "default" { + name = "prometheus-rds-exporter" + assume_role_policy = data.aws_iam_policy_document.prometheus-rds-exporter-relationship.json +} + +data "aws_iam_policy_document" "prometheus-rds-exporter-relationship" { + // Replace with asumme role policy +} + +resource "aws_iam_role_policy" "prometheus-rds-exporter" { + name = "prometheus-rds-exporter" + role = aws_iam_role.default.name + policy = data.aws_iam_policy_document.prometheus-rds-exporter.json +} + +data "aws_iam_policy_document" "prometheus-rds-exporter" { + statement { + sid = "AllowFetchingRDSMetrics" + effect = "Allow" + actions = [ + "cloudwatch:GetMetricData", + "ec2:DescribeInstanceTypes", + "rds:DescribeAccountAttributes", + "rds:DescribeDBInstances", + "rds:DescribeDBLogFiles", + "rds:DescribePendingMaintenanceActions", + "servicequotas:GetServiceQuota", + ] + resources = ["*"] + } +} diff --git a/docs/manual-release.md b/docs/manual-release.md new file mode 100644 index 0000000..1f244d1 --- /dev/null +++ b/docs/manual-release.md @@ -0,0 +1,49 @@ +# Manual release + +This document explains how to manually release Prometheus RDS version. + +/!\ It's here only for troubleshooting since release process is fully automated using Github workflow. + +## Authenticate on AWS + +1. Grant `qonto-team-devops` in ECR policy (managed by Terraform in `opensource` stack) + +1. Use `opensource-production` AWS PROFILE + + ```bash + AWS_PROFILE=opensource-production + ``` + +1. Login to AWS ECR + + ```bash + aws ecr-public get-login-password --region us-east-1 | helm registry login --username AWS --password-stdin public.ecr.aws + ``` + +## Github release and ECR images + +1. Create git tag + + ```bash + git tag + ``` + +1. Release new version + + ```bash + goreleaser release --clean + ``` + +## Helm chart + +1. Build and publish the new release + + ```bash + make helm-publish + ``` + +1. Perform dry-run install + + ```bash + helm install prometheus-rds-exporter oci://public.ecr.aws/g1r8z6f4/test1-chart --dry-run + ``` diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..1b9593d --- /dev/null +++ b/go.mod @@ -0,0 +1,56 @@ +module github.com/qonto/prometheus-rds-exporter + +go 1.21 + +require ( + github.com/aws/aws-sdk-go-v2 v1.21.0 + github.com/aws/aws-sdk-go-v2/config v1.18.41 + github.com/aws/aws-sdk-go-v2/credentials v1.13.39 + github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.27.7 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.119.0 + github.com/aws/aws-sdk-go-v2/service/rds v1.54.0 + github.com/aws/aws-sdk-go-v2/service/servicequotas v1.16.0 + github.com/aws/aws-sdk-go-v2/service/sts v1.22.0 + github.com/prometheus/client_golang v1.16.0 + github.com/spf13/cobra v1.7.0 + github.com/spf13/viper v1.16.0 + github.com/stretchr/testify v1.8.3 + golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df +) + +require ( + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.14.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.0 // indirect + github.com/aws/smithy-go v1.14.2 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect + github.com/spf13/afero v1.9.5 // indirect + github.com/spf13/cast v1.5.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.4.2 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..cc5e8f1 --- /dev/null +++ b/go.sum @@ -0,0 +1,557 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= +github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= +github.com/aws/aws-sdk-go-v2/config v1.18.41 h1:Go7z97YDsBJVNAaL7pDPKB6LeHEsAkHmFe+CeK30fUQ= +github.com/aws/aws-sdk-go-v2/config v1.18.41/go.mod h1:+yR45+A0LIMKT8bWOKo90Hy9rSrovEmEKoPKLmmVec8= +github.com/aws/aws-sdk-go-v2/credentials v1.13.39 h1:UnwBXDIHKDaejSXaRzKR57IdGCizk+z1DEhnsFpus7Q= +github.com/aws/aws-sdk-go-v2/credentials v1.13.39/go.mod h1:OJ9P239A90TnglJEF3qofKiNeEM6PCV/m+aNGV5WC24= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 h1:uDZJF1hu0EVT/4bogChk8DyjSF6fof6uL/0Y26Ma7Fg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11/go.mod h1:TEPP4tENqBGO99KwVpV9MlOX4NSrSLP8u3KRy2CDwA8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 h1:GPUcE/Yq7Ur8YSUk6lVkoIMWnJNO0HT18GUzCWCgCI0= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42/go.mod h1:rzfdUlfA+jdgLDmPKjd3Chq9V7LVLYo1Nz++Wb91aRo= +github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.27.7 h1:qULF+ElcvjjSEO1+z5x+TmKE9d4yTej7PfpJQPVvexY= +github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.27.7/go.mod h1:1HKxVrj5wsKy/wb2v07vzTSd+YPV1sDsWxferwPK7PA= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.119.0 h1:oFrb1aQ07i+v63FOTywSG8xL/OYZbk+HmPE8FKSzkRk= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.119.0/go.mod h1:0FhI2Rzcv5BNM3dNnbcCx2qa2naFZoAidJi11cQgzL0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o= +github.com/aws/aws-sdk-go-v2/service/rds v1.54.0 h1:FmExQnV6PXPAwP2DT3nXlWyKtCJ30gCEQIu4MUOuESo= +github.com/aws/aws-sdk-go-v2/service/rds v1.54.0/go.mod h1:UNv1vk1fU1NJefzteykVpVLA88w4WxB05g3vp2kQhYM= +github.com/aws/aws-sdk-go-v2/service/servicequotas v1.16.0 h1:dhp7Do5oaybdDdYGcdUNyzYFPsM4sNCvuPqph7MG5X0= +github.com/aws/aws-sdk-go-v2/service/servicequotas v1.16.0/go.mod h1:7+H2efEiOCrUl5EEsDFhe5BeI4gHGLUlisCyAJAcSvs= +github.com/aws/aws-sdk-go-v2/service/sso v1.14.0 h1:AR/hlTsCyk1CwlyKnPFvIMvnONydRjDDRT9OGb0i+/g= +github.com/aws/aws-sdk-go-v2/service/sso v1.14.0/go.mod h1:fIAwKQKBFu90pBxx07BFOMJLpRUGu8VOzLJakeY+0K4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.0 h1:UniOmlPJelksyP5dGjfRoFTmLDy4/o0HH1lK2Op7zC8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.0/go.mod h1:yygr8ACQRY2PrEcy3xsUI357stq2AxnFM6DIsR9lij4= +github.com/aws/aws-sdk-go-v2/service/sts v1.22.0 h1:s4bioTgjSFRwOoyEFzAVCmFmoowBgjTR8gkrF/sQ4wk= +github.com/aws/aws-sdk-go-v2/service/sts v1.22.0/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU= +github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ= +github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= +github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= +github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/app/cloudwatch/rds.go b/internal/app/cloudwatch/rds.go new file mode 100644 index 0000000..3a48cf1 --- /dev/null +++ b/internal/app/cloudwatch/rds.go @@ -0,0 +1,251 @@ +// Package cloudwatch implements methods to retrieve AWS Cloudwatch information +package cloudwatch + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + aws_cloudwatch "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + aws_cloudwath_types "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" +) + +const ( + MaxQueriesPerCloudwatchRequest int = 500 + CloudwatchUsagePeriod int32 = 5 + Minute int32 = 60 +) + +var errUnknownMetric = errors.New("unknown metric") + +type CloudWatchMetrics struct { + Instances map[string]*RdsMetrics +} + +type RdsMetrics struct { + CPUUtilization float64 + DBLoad float64 + DBLoadCPU float64 + DBLoadNonCPU float64 + DatabaseConnections float64 + FreeStorageSpace float64 + FreeableMemory float64 + MaximumUsedTransactionIDs float64 + ReadIOPS float64 + ReadThroughput float64 + ReplicaLag float64 + ReplicationSlotDiskUsage float64 + SwapUsage float64 + WriteIOPS float64 + WriteThroughput float64 +} + +func (m *RdsMetrics) Update(field string, value float64) error { + switch field { + case "DBLoad": + m.DBLoad = value + case "DBLoadCPU": + m.DBLoadCPU = value + case "DBLoadNonCPU": + m.DBLoadNonCPU = value + case "CPUUtilization": + m.CPUUtilization = value + case "DatabaseConnections": + m.DatabaseConnections = value + case "FreeStorageSpace": + m.FreeStorageSpace = value + case "FreeableMemory": + m.FreeableMemory = value + case "SwapUsage": + m.SwapUsage = value + case "WriteIOPS": + m.WriteIOPS = value + case "ReadIOPS": + m.ReadIOPS = value + case "ReplicaLag": + m.ReplicaLag = value + case "ReplicationSlotDiskUsage": + m.ReplicationSlotDiskUsage = value + case "MaximumUsedTransactionIDs": + m.MaximumUsedTransactionIDs = value + case "ReadThroughput": + m.ReadThroughput = value + case "WriteThroughput": + m.WriteThroughput = value + default: + return fmt.Errorf("can't process '%s' metrics: %w", field, errUnknownMetric) + } + + return nil +} + +// getCloudWatchMetricsName returns names of Cloudwatch metrics to collect +func getCloudWatchMetricsName() [15]string { + return [15]string{ + "CPUUtilization", + "DBLoad", + "DBLoadCPU", + "DBLoadNonCPU", + "DatabaseConnections", + "FreeStorageSpace", + "FreeableMemory", + "MaximumUsedTransactionIDs", + "ReadIOPS", + "ReadThroughput", + "ReplicaLag", + "ReplicationSlotDiskUsage", + "SwapUsage", + "WriteIOPS", + "WriteThroughput", + } +} + +// generateCloudWatchQueryForInstance return the cloudwatch query for a specific instance's metric +func generateCloudWatchQueryForInstance(queryID *string, metricName string, dbIdentifier string) CloudWatchMetricRequest { + query := &aws_cloudwath_types.MetricDataQuery{ + Id: queryID, + MetricStat: &aws_cloudwath_types.MetricStat{ + Metric: &aws_cloudwath_types.Metric{ + Namespace: aws.String("AWS/RDS"), + MetricName: aws.String(metricName), + Dimensions: []aws_cloudwath_types.Dimension{ + { + Name: aws.String("DBInstanceIdentifier"), + Value: aws.String(dbIdentifier), + }, + }, + }, + Stat: aws.String("Average"), // Specify the statistic to retrieve + Period: aws.Int32(Minute), // Specify the period of the metric. Granularity - 1 minute + }, + } + + return CloudWatchMetricRequest{ + Dbidentifier: dbIdentifier, + MetricName: metricName, + Query: *query, + } +} + +// generateCloudWatchQueriesForInstances returns all cloudwatch queries for specified instances +func generateCloudWatchQueriesForInstances(dbIdentifiers []string) map[string]CloudWatchMetricRequest { + queries := make(map[string]CloudWatchMetricRequest) + + metrics := getCloudWatchMetricsName() + + for i, dbIdentifier := range dbIdentifiers { + for _, metricName := range metrics { + queryID := aws.String(fmt.Sprintf("%s_%d", strings.ToLower(metricName), i)) + + query := generateCloudWatchQueryForInstance(queryID, metricName, dbIdentifier) + + queries[*queryID] = query + } + } + + return queries +} + +func NewRDSFetcher(client CloudWatchClient, logger slog.Logger) *RdsFetcher { + return &RdsFetcher{ + client: client, + logger: &logger, + } +} + +type RdsFetcher struct { + client CloudWatchClient + statistics Statistics + logger *slog.Logger +} + +func (c *RdsFetcher) GetStatistics() *Statistics { + return &c.statistics +} + +func (c *RdsFetcher) updateMetricsWithCloudWatchQueriesResult(metrics map[string]*RdsMetrics, requests map[string]CloudWatchMetricRequest, startTime *time.Time, endTime *time.Time, chunk []string) error { + params := &aws_cloudwatch.GetMetricDataInput{ + StartTime: startTime, + EndTime: endTime, + ScanBy: "TimestampDescending", + MetricDataQueries: []aws_cloudwath_types.MetricDataQuery{}, + } + + for _, key := range chunk { + query := requests[key].Query + params.MetricDataQueries = append(params.MetricDataQueries, query) + } + + resp, err := c.client.GetMetricData(context.TODO(), params) + if err != nil { + return fmt.Errorf("error calling GetMetricData: %w", err) + } + + for _, m := range resp.MetricDataResults { + if m.Values == nil { + c.logger.Warn("cloudwatch value is empty", "metric", *m.Label) + + continue + } + + val := requests[*m.Id] + + _, instanceMetricExists := metrics[val.Dbidentifier] + if !instanceMetricExists { + metrics[val.Dbidentifier] = &RdsMetrics{} + } + + if len(m.Values) > 0 { + err := metrics[val.Dbidentifier].Update(val.MetricName, m.Values[0]) + if err != nil { + return fmt.Errorf("failed to process metrics %s: %w", val.MetricName, err) + } + } + } + + return nil +} + +func (c *RdsFetcher) GetRDSInstanceMetrics(dbIdentifiers []string) (CloudWatchMetrics, error) { + metrics := make(map[string]*RdsMetrics) + + cloudWatchQueries := generateCloudWatchQueriesForInstances(dbIdentifiers) + startTime := aws.Time(time.Now().Add(-3 * time.Minute)) // Start time - 1 hour ago + endTime := aws.Time(time.Now()) // End time - now + chunkSize := MaxQueriesPerCloudwatchRequest + + cloudWatchAPICalls := float64(0) + chunk := make([]string, 0, chunkSize) + + for query := range cloudWatchQueries { + chunk = append(chunk, query) + + if len(chunk) == chunkSize { + err := c.updateMetricsWithCloudWatchQueriesResult(metrics, cloudWatchQueries, startTime, endTime, chunk) + if err != nil { + return CloudWatchMetrics{}, fmt.Errorf("can't fetch Cloudwatch metrics: %w", err) + } + + chunk = nil + cloudWatchAPICalls += 1 + } + } + + // Process last, potentially incomplete batch + if len(chunk) > 0 { + err := c.updateMetricsWithCloudWatchQueriesResult(metrics, cloudWatchQueries, startTime, endTime, chunk) + if err != nil { + return CloudWatchMetrics{}, fmt.Errorf("can't fetch Cloudwatch metrics: %w", err) + } + + c.statistics.CloudWatchAPICall++ + } + + return CloudWatchMetrics{ + Instances: metrics, + }, nil +} diff --git a/internal/app/cloudwatch/rds_test.go b/internal/app/cloudwatch/rds_test.go new file mode 100644 index 0000000..8ca68ee --- /dev/null +++ b/internal/app/cloudwatch/rds_test.go @@ -0,0 +1,179 @@ +package cloudwatch_test + +import ( + "fmt" + "log/slog" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + aws_cloudwatch_types "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" + "github.com/qonto/prometheus-rds-exporter/internal/app/cloudwatch" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var db1ExpecteRdsMetrics = cloudwatch.RdsMetrics{ + CPUUtilization: 10, + DBLoad: 1, + DBLoadCPU: 2, + DBLoadNonCPU: 4, + DatabaseConnections: 42, + FreeStorageSpace: 5, + FreeableMemory: 10, + MaximumUsedTransactionIDs: 1000000, + ReadIOPS: 100, + ReadThroughput: 101, + ReplicaLag: 42, + ReplicationSlotDiskUsage: 100, + SwapUsage: 10, + WriteIOPS: 11, + WriteThroughput: 12, +} + +var db2ExpecteRdsMetrics = cloudwatch.RdsMetrics{ + CPUUtilization: 40, + DBLoad: 2, + DBLoadCPU: 8, + DBLoadNonCPU: 1, + DatabaseConnections: 1000, + FreeStorageSpace: 10, + FreeableMemory: 10, + MaximumUsedTransactionIDs: 1000000, + ReadIOPS: 100, + ReadThroughput: 101, + ReplicaLag: 42, + ReplicationSlotDiskUsage: 100, + SwapUsage: 10, + WriteIOPS: 11, + WriteThroughput: 12, +} + +// generateMockedMetricsForInstance returns cloudwatch API output for the instance +func generateMockedMetricsForInstance(id int, m cloudwatch.RdsMetrics) []aws_cloudwatch_types.MetricDataResult { + metrics := []aws_cloudwatch_types.MetricDataResult{ + { + Id: aws.String(fmt.Sprintf("cpuutilization_%d", id)), + Label: aws.String("CPUUtilization"), + Values: []float64{m.CPUUtilization}, + }, + { + Id: aws.String(fmt.Sprintf("dbload_%d", id)), + Label: aws.String("DBLoad"), + Values: []float64{m.DBLoad}, + }, + { + Id: aws.String(fmt.Sprintf("dbloadcpu_%d", id)), + Label: aws.String("DBLoadCPU"), + Values: []float64{m.DBLoadCPU}, + }, + { + Id: aws.String(fmt.Sprintf("dbloadnoncpu_%d", id)), + Label: aws.String("DBLoadNonCPU"), + Values: []float64{m.DBLoadNonCPU}, + }, + { + Id: aws.String(fmt.Sprintf("databaseconnections_%d", id)), + Label: aws.String("DatabaseConnections"), + Values: []float64{m.DatabaseConnections}, + }, + { + Id: aws.String(fmt.Sprintf("freestoragespace_%d", id)), + Label: aws.String("FreeStorageSpace"), + Values: []float64{m.FreeStorageSpace}, + }, + { + Id: aws.String(fmt.Sprintf("freeablememory_%d", id)), + Label: aws.String("FreeableMemory"), + Values: []float64{m.FreeableMemory}, + }, + { + Id: aws.String(fmt.Sprintf("maximumusedtransactionids_%d", id)), + Label: aws.String("MaximumUsedTransactionIDs"), + Values: []float64{m.MaximumUsedTransactionIDs}, + }, + { + Id: aws.String(fmt.Sprintf("readiops_%d", id)), + Label: aws.String("ReadIOPS"), + Values: []float64{m.ReadIOPS}, + }, + { + Id: aws.String(fmt.Sprintf("readthroughput_%d", id)), + Label: aws.String("ReadThroughput"), + Values: []float64{m.ReadThroughput}, + }, + { + Id: aws.String(fmt.Sprintf("replicalag_%d", id)), + Label: aws.String("ReplicaLag"), + Values: []float64{m.ReplicaLag}, + }, + { + Id: aws.String(fmt.Sprintf("replicationslotdiskusage_%d", id)), + Label: aws.String("ReplicationSlotDiskUsage"), + Values: []float64{m.ReplicationSlotDiskUsage}, + }, + { + Id: aws.String(fmt.Sprintf("swapusage_%d", id)), + Label: aws.String("SwapUsage"), + Values: []float64{m.SwapUsage}, + }, + { + Id: aws.String(fmt.Sprintf("writeiops_%d", id)), + Label: aws.String("WriteIOPS"), + Values: []float64{m.WriteIOPS}, + }, + { + Id: aws.String(fmt.Sprintf("writethroughput_%d", id)), + Label: aws.String("WriteThroughput"), + Values: []float64{m.WriteThroughput}, + }, + } + + return metrics +} + +func TestGetDBInstanceTypeInformation(t *testing.T) { + instancesName := []string{} + data := []aws_cloudwatch_types.MetricDataResult{} + + // Generate instances metrics + instances := make(map[string]cloudwatch.RdsMetrics) + instances["db1"] = db1ExpecteRdsMetrics + instances["db2"] = db2ExpecteRdsMetrics + + // Generate Cloudwatch API output metrics + i := 0 + + for id := range instances { + instancesName = append(instancesName, id) + instancesMetrics := generateMockedMetricsForInstance(i, instances[id]) + + data = append(data, instancesMetrics...) + i++ + } + + mock := mockCloudwatchClient{metrics: data} + client := cloudwatch.NewRDSFetcher(mock, slog.Logger{}) + result, err := client.GetRDSInstanceMetrics(instancesName) + + require.NoError(t, err, "GetRDSInstanceMetrics must succeed") + assert.Equal(t, float64(1), client.GetStatistics().CloudWatchAPICall, "One call to Cloudwatch API") + + for id, value := range instances { + assert.Equal(t, value.DatabaseConnections, result.Instances[id].DatabaseConnections, "DatabaseConnections mismatch") + assert.Equal(t, value.CPUUtilization, result.Instances[id].CPUUtilization, "CPU utilization mismatch") + assert.Equal(t, value.DBLoad, result.Instances[id].DBLoad, "DBLoad mismatch") + assert.Equal(t, value.DBLoadCPU, result.Instances[id].DBLoadCPU, "DBLoadCPU mismatch") + assert.Equal(t, value.DBLoadNonCPU, result.Instances[id].DBLoadNonCPU, "DBLoadNonCPU mismatch") + assert.Equal(t, value.DatabaseConnections, result.Instances[id].DatabaseConnections, "DatabaseConnections mismatch") + assert.Equal(t, value.FreeStorageSpace, result.Instances[id].FreeStorageSpace, "FreeStorageSpace mismatch") + assert.Equal(t, value.FreeableMemory, result.Instances[id].FreeableMemory, "FreeableMemory mismatch") + assert.Equal(t, value.MaximumUsedTransactionIDs, result.Instances[id].MaximumUsedTransactionIDs, "MaximumUsedTransactionIDs mismatch") + assert.Equal(t, value.ReadIOPS, result.Instances[id].ReadIOPS, "ReadIOPS mismatch") + assert.Equal(t, value.ReadThroughput, result.Instances[id].ReadThroughput, "ReadThroughput mismatch") + assert.Equal(t, value.ReplicaLag, result.Instances[id].ReplicaLag, "ReplicaLag mismatch") + assert.Equal(t, value.ReplicationSlotDiskUsage, result.Instances[id].ReplicationSlotDiskUsage, "ReplicationSlotDiskUsage mismatch") + assert.Equal(t, value.SwapUsage, result.Instances[id].SwapUsage, "SwapUsage mismatch") + assert.Equal(t, value.WriteIOPS, result.Instances[id].WriteIOPS, "WriteIOPS mismatch") + assert.Equal(t, value.WriteThroughput, result.Instances[id].WriteThroughput, "WriteThroughput mismatch") + } +} diff --git a/internal/app/cloudwatch/type.go b/internal/app/cloudwatch/type.go new file mode 100644 index 0000000..951b339 --- /dev/null +++ b/internal/app/cloudwatch/type.go @@ -0,0 +1,22 @@ +package cloudwatch + +import ( + "context" + + aws_cloudwatch "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + aws_cloudwatch_types "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" +) + +type Statistics struct { + CloudWatchAPICall float64 +} + +type CloudWatchMetricRequest struct { + Query aws_cloudwatch_types.MetricDataQuery + Dbidentifier string + MetricName string +} + +type CloudWatchClient interface { + GetMetricData(context.Context, *aws_cloudwatch.GetMetricDataInput, ...func(*aws_cloudwatch.Options)) (*aws_cloudwatch.GetMetricDataOutput, error) +} diff --git a/internal/app/cloudwatch/type_test.go b/internal/app/cloudwatch/type_test.go new file mode 100644 index 0000000..c3a10d5 --- /dev/null +++ b/internal/app/cloudwatch/type_test.go @@ -0,0 +1,20 @@ +package cloudwatch_test + +import ( + "context" + + aws_cloudwatch "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + aws_cloudwatch_types "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" +) + +type mockCloudwatchClient struct { + metrics []aws_cloudwatch_types.MetricDataResult +} + +// GetMetricData returns custom metrics +func (m mockCloudwatchClient) GetMetricData(ctx context.Context, input *aws_cloudwatch.GetMetricDataInput, fn ...func(*aws_cloudwatch.Options)) (*aws_cloudwatch.GetMetricDataOutput, error) { + response := &aws_cloudwatch.GetMetricDataOutput{} + response.MetricDataResults = m.metrics + + return response, nil +} diff --git a/internal/app/cloudwatch/usage.go b/internal/app/cloudwatch/usage.go new file mode 100644 index 0000000..fe2e371 --- /dev/null +++ b/internal/app/cloudwatch/usage.go @@ -0,0 +1,160 @@ +package cloudwatch + +import ( + "context" + "fmt" + "log/slog" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + aws_cloudwatch "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + aws_cloudwatch_types "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" +) + +type UsageMetrics struct { + AllocatedStorage float64 + DBInstances float64 + ManualSnapshots float64 + ReservedDBInstances float64 +} + +func (u *UsageMetrics) Update(field string, value float64) error { + switch field { + case "AllocatedStorage": + u.AllocatedStorage = value + case "DBInstances": + u.DBInstances = value + case "ManualSnapshots": + u.ManualSnapshots = value + case "ReservedDBInstances": + u.ReservedDBInstances = value + default: + return fmt.Errorf("can't process %s metrics: %w", field, errUnknownMetric) + } + + return nil +} + +func generateCloudWatchDataQueriesForUsage(service string, metricsName []string) map[string]CloudWatchMetricRequest { + requests := make(map[string]CloudWatchMetricRequest) + + for i, metricName := range metricsName { + id := aws.String(fmt.Sprintf("%s_%d", strings.ToLower(metricName), i)) + query := &aws_cloudwatch_types.MetricDataQuery{ + Id: id, + MetricStat: &aws_cloudwatch_types.MetricStat{ + Metric: &aws_cloudwatch_types.Metric{ + Namespace: aws.String("AWS/Usage"), + MetricName: aws.String("ResourceCount"), + Dimensions: []aws_cloudwatch_types.Dimension{ + { + Name: aws.String("Service"), + Value: aws.String(service), + }, + { + Name: aws.String("Type"), + Value: aws.String("Resource"), + }, + { + Name: aws.String("Resource"), + Value: aws.String(metricName), + }, + { + Name: aws.String("Class"), + Value: aws.String("None"), + }, + }, + }, + Stat: aws.String("Average"), // Specify the statistic to retrieve + Period: aws.Int32(Minute), // Specify the period of the metric. Granularity - 1 minute + }, + } + + requests[*id] = CloudWatchMetricRequest{ + Dbidentifier: "", + MetricName: metricName, + Query: *query, + } + } + + return requests +} + +func generateCloudWatchQueriesForUsage() *aws_cloudwatch.GetMetricDataInput { + metricsName := []string{ + "AllocatedStorage", + "DBInstances", + "ManualSnapshots", + } + + cloudwatchDataQueries := []aws_cloudwatch_types.MetricDataQuery{} + queries := generateCloudWatchDataQueriesForUsage("RDS", metricsName) + + for _, usageQuery := range queries { + query := aws_cloudwatch_types.MetricDataQuery{ + Id: usageQuery.Query.Id, + MetricStat: &aws_cloudwatch_types.MetricStat{ + Metric: usageQuery.Query.MetricStat.Metric, + Stat: aws.String("Average"), + Period: aws.Int32(CloudwatchUsagePeriod * Minute), + }, + } + cloudwatchDataQueries = append(cloudwatchDataQueries, query) + } + + return &aws_cloudwatch.GetMetricDataInput{ + StartTime: aws.Time(time.Now().Add(-5 * time.Hour)), + EndTime: aws.Time(time.Now()), + ScanBy: "TimestampDescending", + MetricDataQueries: cloudwatchDataQueries, + } +} + +func NewUsageFetcher(client CloudWatchClient, logger slog.Logger) *usageFetcher { + return &usageFetcher{ + client: client, + logger: &logger, + } +} + +type usageFetcher struct { + client CloudWatchClient + statistics Statistics + logger *slog.Logger +} + +func (u *usageFetcher) GetStatistics() Statistics { + return u.statistics +} + +// GetUsageMetrics returns RDS service usages metrics +func (u *usageFetcher) GetUsageMetrics() (UsageMetrics, error) { + metrics := UsageMetrics{} + + query := generateCloudWatchQueriesForUsage() + + resp, err := u.client.GetMetricData(context.TODO(), query) + u.statistics.CloudWatchAPICall++ + + if err != nil { + return metrics, fmt.Errorf("error calling GetMetricData: %w", err) + } + + for _, m := range resp.MetricDataResults { + if m.Values == nil { + u.logger.Warn("cloudwatch value is empty", "metric", *m.Label) + + continue + } + + if len(m.Values) > 0 { + err = metrics.Update(*m.Label, m.Values[0]) + if err != nil { + return metrics, fmt.Errorf("can't update internal values: %w", err) + } + } + } + + return metrics, nil +} diff --git a/internal/app/cloudwatch/usage_test.go b/internal/app/cloudwatch/usage_test.go new file mode 100644 index 0000000..aa20831 --- /dev/null +++ b/internal/app/cloudwatch/usage_test.go @@ -0,0 +1,53 @@ +package cloudwatch_test + +import ( + "log/slog" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + aws_cloudwatch_types "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" + "github.com/qonto/prometheus-rds-exporter/internal/app/cloudwatch" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetUsageMetrics(t *testing.T) { + expected := cloudwatch.UsageMetrics{ + AllocatedStorage: 100, + DBInstances: 42, + ManualSnapshots: 10, + ReservedDBInstances: 3, + } + + mock := mockCloudwatchClient{ + metrics: []aws_cloudwatch_types.MetricDataResult{ + { + Label: aws.String("AllocatedStorage"), + Values: []float64{expected.AllocatedStorage}, + }, + { + Label: aws.String("DBInstances"), + Values: []float64{expected.DBInstances}, + }, + { + Label: aws.String("ManualSnapshots"), + Values: []float64{expected.ManualSnapshots}, + }, + { + Label: aws.String("ReservedDBInstances"), + Values: []float64{expected.ReservedDBInstances}, + }, + }, + } + + client := cloudwatch.NewUsageFetcher(mock, slog.Logger{}) + result, err := client.GetUsageMetrics() + + require.NoError(t, err, "GetUsageMetrics must succeed") + assert.Equal(t, expected.AllocatedStorage, result.AllocatedStorage, "Allocated storage mismatch") + assert.Equal(t, expected.DBInstances, result.DBInstances, "DB instances count mismatch") + assert.Equal(t, expected.ManualSnapshots, result.ManualSnapshots, "Manual snapshots mismatch") + assert.Equal(t, expected.ReservedDBInstances, result.ReservedDBInstances, "Reserved DB instances mismatch") + + assert.Equal(t, float64(1), client.GetStatistics().CloudWatchAPICall, "One call to Cloudwatch API") +} diff --git a/internal/app/ec2/ec2.go b/internal/app/ec2/ec2.go new file mode 100644 index 0000000..f3491ee --- /dev/null +++ b/internal/app/ec2/ec2.go @@ -0,0 +1,86 @@ +// Package ec2 implements methods to retrieve EC2 instance capabilities +package ec2 + +import ( + "context" + "fmt" + + aws_ec2 "github.com/aws/aws-sdk-go-v2/service/ec2" + aws_ec2_types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + converter "github.com/qonto/prometheus-rds-exporter/internal/app/unit" +) + +const ( + maxInstanceTypesPerEC2APIRequest int = 100 // Limit the number of instance types per request due to AWS API limits +) + +type EC2InstanceMetrics struct { + MaximumIops int32 + MaximumThroughput float64 + Memory int64 + Vcpu int32 +} + +type Metrics struct { + Instances map[string]EC2InstanceMetrics +} + +type Statistics struct { + EC2ApiCall float64 +} + +type EC2Client interface { + DescribeInstanceTypes(ctx context.Context, input *aws_ec2.DescribeInstanceTypesInput, fn ...func(*aws_ec2.Options)) (*aws_ec2.DescribeInstanceTypesOutput, error) +} + +func NewFetcher(client EC2Client) *EC2Fetcher { + return &EC2Fetcher{ + client: client, + } +} + +type EC2Fetcher struct { + client EC2Client + statistics Statistics +} + +func (e *EC2Fetcher) GetStatistics() Statistics { + return e.statistics +} + +// GetDBInstanceTypeInformation returns information about specified AWS EC2 instance types +// AWS RDS API use "db." prefix while AWS EC2 API don't so we must remove it to obtains instance type information +func (e *EC2Fetcher) GetDBInstanceTypeInformation(instanceTypes []string) (Metrics, error) { + metrics := make(map[string]EC2InstanceMetrics) + + for _, instances := range chunkBy(instanceTypes, maxInstanceTypesPerEC2APIRequest) { + // Remove "db." prefix from instance types + instanceTypesToFetch := make([]aws_ec2_types.InstanceType, len(instances)) + for i, instance := range instances { + instanceTypesToFetch[i] = (aws_ec2_types.InstanceType)(removeDBPrefix(instance)) + } + + input := &aws_ec2.DescribeInstanceTypesInput{InstanceTypes: instanceTypesToFetch} + + resp, err := e.client.DescribeInstanceTypes(context.TODO(), input) + if err != nil { + return Metrics{}, fmt.Errorf("can't fetch describe instance types: %w", err) + } + + e.statistics.EC2ApiCall++ + + for _, i := range resp.InstanceTypes { + instanceName := addDBPrefix(string(i.InstanceType)) + metrics[instanceName] = EC2InstanceMetrics{ + Vcpu: *i.VCpuInfo.DefaultVCpus, + MaximumIops: *i.EbsInfo.EbsOptimizedInfo.MaximumIops, + MaximumThroughput: converter.MegaBytesToBytes(*i.EbsInfo.EbsOptimizedInfo.MaximumThroughputInMBps), + Memory: converter.MegaBytesToBytes(*i.MemoryInfo.SizeInMiB), + } + } + } + + return Metrics{ + Instances: metrics, + }, nil +} diff --git a/internal/app/ec2/ec2_test.go b/internal/app/ec2/ec2_test.go new file mode 100644 index 0000000..9bcef56 --- /dev/null +++ b/internal/app/ec2/ec2_test.go @@ -0,0 +1,80 @@ +package ec2_test + +import ( + "context" + "testing" + + aws_ec2 "github.com/aws/aws-sdk-go-v2/service/ec2" + aws_ec2_types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/qonto/prometheus-rds-exporter/internal/app/ec2" + converter "github.com/qonto/prometheus-rds-exporter/internal/app/unit" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var t3Large = ec2.EC2InstanceMetrics{ + MaximumIops: 15700, + MaximumThroughput: 347.5, + Memory: 8, + Vcpu: 2, +} + +var t3Small = ec2.EC2InstanceMetrics{ + MaximumIops: 11800, + MaximumThroughput: 260.62, + Memory: 2, + Vcpu: 2, +} + +type mockEC2Client struct{} + +func (m mockEC2Client) DescribeInstanceTypes(ctx context.Context, input *aws_ec2.DescribeInstanceTypesInput, optFns ...func(*aws_ec2.Options)) (*aws_ec2.DescribeInstanceTypesOutput, error) { + var instances []aws_ec2_types.InstanceTypeInfo + + for _, instanceType := range input.InstanceTypes { + //nolint // Hide "missing cases in switch" alert because instanceType has many values. Mock with return empty result for unknown instances + switch instanceType { + case "t3.large": + instances = append(instances, aws_ec2_types.InstanceTypeInfo{ + InstanceType: instanceType, + VCpuInfo: &aws_ec2_types.VCpuInfo{DefaultVCpus: &t3Large.Vcpu}, + MemoryInfo: &aws_ec2_types.MemoryInfo{SizeInMiB: &t3Large.Memory}, + EbsInfo: &aws_ec2_types.EbsInfo{EbsOptimizedInfo: &aws_ec2_types.EbsOptimizedInfo{ + MaximumIops: &t3Large.MaximumIops, + MaximumThroughputInMBps: &t3Large.MaximumThroughput, + }}, + }) + case "t3.small": + instances = append(instances, aws_ec2_types.InstanceTypeInfo{ + InstanceType: instanceType, + VCpuInfo: &aws_ec2_types.VCpuInfo{DefaultVCpus: &t3Small.Vcpu}, + MemoryInfo: &aws_ec2_types.MemoryInfo{SizeInMiB: &t3Small.Memory}, + EbsInfo: &aws_ec2_types.EbsInfo{EbsOptimizedInfo: &aws_ec2_types.EbsOptimizedInfo{ + MaximumIops: &t3Small.MaximumIops, + MaximumThroughputInMBps: &t3Small.MaximumThroughput, + }}, + }) + } + } + + return &aws_ec2.DescribeInstanceTypesOutput{InstanceTypes: instances}, nil +} + +func TestGetDBInstanceTypeInformation(t *testing.T) { + mock := mockEC2Client{} + + instanceTypes := []string{"db.t3.large", "db.t3.small"} + client := ec2.NewFetcher(mock) + result, err := client.GetDBInstanceTypeInformation(instanceTypes) + + require.NoError(t, err, "GetDBInstanceTypeInformation must succeed") + assert.Equal(t, t3Large.Vcpu, result.Instances["db.t3.large"].Vcpu, "vCPU don't match") + assert.Equal(t, converter.MegaBytesToBytes(t3Large.Memory), result.Instances["db.t3.large"].Memory, "Memory don't match") + assert.Equal(t, t3Large.MaximumIops, result.Instances["db.t3.large"].MaximumIops, "MaximumThroughput don't match") + assert.Equal(t, converter.MegaBytesToBytes(t3Large.MaximumThroughput), result.Instances["db.t3.large"].MaximumThroughput, "MaximumThroughput don't match") + + assert.Equal(t, t3Small.Vcpu, result.Instances["db.t3.small"].Vcpu, "vCPU don't match") + assert.Equal(t, converter.MegaBytesToBytes(t3Small.Memory), result.Instances["db.t3.small"].Memory, "Memory don't match") + + assert.Equal(t, float64(1), client.GetStatistics().EC2ApiCall, "EC2 API call don't match") +} diff --git a/internal/app/ec2/helper.go b/internal/app/ec2/helper.go new file mode 100644 index 0000000..fcc132b --- /dev/null +++ b/internal/app/ec2/helper.go @@ -0,0 +1,21 @@ +package ec2 + +import "strings" + +func chunkBy[T any](items []T, chunkSize int) (chunks [][]T) { + for chunkSize < len(items) { + items, chunks = items[chunkSize:], append(chunks, items[0:chunkSize:chunkSize]) + } + + return append(chunks, items) +} + +// addDBPrefix adds "db." prefix for RDS instance type +func addDBPrefix(instance string) string { + return "db." + instance +} + +// removeDBPrefix removes "db." prefix for RDS instance type +func removeDBPrefix(instance string) string { + return strings.Trim(instance, "db.") +} diff --git a/internal/app/exporter/exporter.go b/internal/app/exporter/exporter.go new file mode 100644 index 0000000..742fd54 --- /dev/null +++ b/internal/app/exporter/exporter.go @@ -0,0 +1,438 @@ +// Package exporter implements Prometheus exporter +package exporter + +import ( + "fmt" + "log/slog" + "strconv" + "sync" + + "github.com/prometheus/client_golang/prometheus" + "github.com/qonto/prometheus-rds-exporter/internal/app/cloudwatch" + "github.com/qonto/prometheus-rds-exporter/internal/app/ec2" + "github.com/qonto/prometheus-rds-exporter/internal/app/rds" + "github.com/qonto/prometheus-rds-exporter/internal/app/servicequotas" + "github.com/qonto/prometheus-rds-exporter/internal/infra/build" +) + +const ( + exporterUpStatusCode float64 = 1 + exporterDownStatusCode float64 = 0 +) + +type counters struct { + cloudwatchAPICalls float64 + ec2APIcalls float64 + errors float64 + rdsAPIcalls float64 + serviceQuotasAPICalls float64 + usageAPIcalls float64 +} + +type metrics struct { + serviceQuota servicequotas.Metrics + rds rds.Metrics + ec2 ec2.Metrics + cloudwatchInstances cloudwatch.CloudWatchMetrics + cloudWatchUsage cloudwatch.UsageMetrics +} + +type rdsCollector struct { + wg sync.WaitGroup + logger slog.Logger + counters counters + metrics metrics + + rdsClient rdsClient + EC2Client EC2Client + servicequotasClient servicequotasClient + cloudWatchClient cloudWatchClient + + errors *prometheus.Desc + DBLoad *prometheus.Desc + dBLoadCPU *prometheus.Desc + dBLoadNonCPU *prometheus.Desc + allocatedStorage *prometheus.Desc + information *prometheus.Desc + instanceMaximumIops *prometheus.Desc + instanceMaximumThroughput *prometheus.Desc + instanceMemory *prometheus.Desc + instanceVCPU *prometheus.Desc + logFilesSize *prometheus.Desc + maxAllocatedStorage *prometheus.Desc + maxIops *prometheus.Desc + status *prometheus.Desc + storageThroughput *prometheus.Desc + up *prometheus.Desc + cpuUtilisation *prometheus.Desc + freeStorageSpace *prometheus.Desc + databaseConnections *prometheus.Desc + freeableMemory *prometheus.Desc + swapUsage *prometheus.Desc + writeIOPS *prometheus.Desc + readIOPS *prometheus.Desc + replicaLag *prometheus.Desc + replicationSlotDiskUsage *prometheus.Desc + maximumUsedTransactionIDs *prometheus.Desc + apiCall *prometheus.Desc + readThroughput *prometheus.Desc + writeThroughput *prometheus.Desc + backupRetentionPeriod *prometheus.Desc + quotaDBInstances *prometheus.Desc + quotaTotalStorage *prometheus.Desc + quotaMaxDBInstanceSnapshots *prometheus.Desc + usageAllocatedStorage *prometheus.Desc + usageDBInstances *prometheus.Desc + usageManualSnapshots *prometheus.Desc + exporterBuildInformation *prometheus.Desc +} + +func NewCollector(logger slog.Logger, rdsClient rdsClient, ec2Client EC2Client, cloudWatchClient cloudWatchClient, servicequotasClient servicequotasClient) *rdsCollector { + return &rdsCollector{ + logger: logger, + rdsClient: rdsClient, + servicequotasClient: servicequotasClient, + EC2Client: ec2Client, + cloudWatchClient: cloudWatchClient, + + exporterBuildInformation: prometheus.NewDesc("rds_exporter_build_info", + "A metric with constant '1' value labeled by version from which exporter was built", + []string{"version", "commit_sha", "build_date"}, nil, + ), + errors: prometheus.NewDesc("rds_exporter_errors_total", + "Total number of errors encountered by the exporter", + []string{}, nil, + ), + allocatedStorage: prometheus.NewDesc("rds_allocated_storage_bytes", + "Allocated storage", + []string{"dbidentifier"}, nil, + ), + information: prometheus.NewDesc("rds_instance_info", + "RDS instance information", + []string{"dbidentifier", "dbi_resource_id", "instance_class", "engine", "engine_version", "storage_type", "multi_az", "deletion_protection", "role", "source_dbidentifier", "pending_modified_values", "pending_maintenance"}, nil, + ), + maxAllocatedStorage: prometheus.NewDesc("rds_max_allocated_storage_bytes", + "Upper limit in gibibytes to which Amazon RDS can automatically scale the storage of the DB instance", + []string{"dbidentifier"}, nil, + ), + maxIops: prometheus.NewDesc("rds_max_disk_iops_average", + "Max IOPS for the instance", + []string{"dbidentifier"}, nil, + ), + storageThroughput: prometheus.NewDesc("rds_max_storage_throughput_bytes", + "Max storage throughput", + []string{"dbidentifier"}, nil, + ), + readThroughput: prometheus.NewDesc("rds_read_throughput_bytes", + "Average number of bytes read from disk per second", + []string{"dbidentifier"}, nil, + ), + writeThroughput: prometheus.NewDesc("rds_write_throughput_bytes", + "Average number of bytes written to disk per second", + []string{"dbidentifier"}, nil, + ), + status: prometheus.NewDesc("rds_instance_status", + fmt.Sprintf("Instance status (%d: ok, %d: can't scrap metrics)", int(exporterUpStatusCode), int(exporterDownStatusCode)), + []string{"dbidentifier"}, nil, + ), + logFilesSize: prometheus.NewDesc("rds_instance_log_files_size_bytes", + "Total of log files on the instance", + []string{"dbidentifier"}, nil, + ), + instanceVCPU: prometheus.NewDesc("rds_instance_vcpu_average", + "Total vCPU for this isntance class", + []string{"instance_class"}, nil, + ), + instanceMemory: prometheus.NewDesc("rds_instance_memory_bytes", + "Instance memory", + []string{"instance_class"}, nil, + ), + cpuUtilisation: prometheus.NewDesc("rds_cpu_usage_percent_average", + "Instance CPU used", + []string{"dbidentifier"}, nil, + ), + instanceMaximumThroughput: prometheus.NewDesc("rds_instance_max_throughput_bytes", + "Maximum throughput of underlying EC2 instance", + []string{"instance_class"}, nil, + ), + instanceMaximumIops: prometheus.NewDesc("rds_instance_maxIops_average", + "Maximum IOPS of underlying EC2 instance", + []string{"instance_class"}, nil, + ), + freeStorageSpace: prometheus.NewDesc("rds_free_storage_bytes", + "Free storage on the instance", + []string{"dbidentifier"}, nil, + ), + databaseConnections: prometheus.NewDesc("rds_database_connections_average", + "The number of client network connections to the database instance", + []string{"dbidentifier"}, nil, + ), + up: prometheus.NewDesc("up", + "Was the last scrape of RDS successful", + nil, nil, + ), + swapUsage: prometheus.NewDesc("rds_swap_usage_bytes", + "Amount of swap space used on the DB instance. This metric is not available for SQL Server", + []string{"dbidentifier"}, nil, + ), + writeIOPS: prometheus.NewDesc("rds_write_iops_average", + "Average number of disk write I/O operations per second", + []string{"dbidentifier"}, nil, + ), + readIOPS: prometheus.NewDesc("rds_read_iops_average", + "Average number of disk read I/O operations per second", + []string{"dbidentifier"}, nil, + ), + replicaLag: prometheus.NewDesc("rds_replica_lag_seconds", + "For read replica configurations, the amount of time a read replica DB instance lags behind the source DB instance. Applies to MariaDB, Microsoft SQL Server, MySQL, Oracle, and PostgreSQL read replicas", + []string{"dbidentifier"}, nil, + ), + replicationSlotDiskUsage: prometheus.NewDesc("rds_replication_slot_disk_usage_average", + "Disk space used by replication slot files. Applies to PostgreSQL", + []string{"dbidentifier"}, nil, + ), + maximumUsedTransactionIDs: prometheus.NewDesc("rds_maximum_used_transaction_ids_average", + "Maximum transaction IDs that have been used. Applies to only PostgreSQL", + []string{"dbidentifier"}, nil, + ), + freeableMemory: prometheus.NewDesc("rds_freeable_memory_bytes", + "Amount of available random access memory. For MariaDB, MySQL, Oracle, and PostgreSQL DB instances, this metric reports the value of the MemAvailable field of /proc/meminfo", + []string{"dbidentifier"}, nil, + ), + apiCall: prometheus.NewDesc("rds_api_call_total", + "Number of call to AWS API", + []string{"api"}, nil, + ), + backupRetentionPeriod: prometheus.NewDesc("rds_backup_retention_period_seconds", + "Automatic DB snapshots retention period", + []string{"dbidentifier"}, nil, + ), + DBLoad: prometheus.NewDesc("rds_dbload_average", + "Number of active sessions for the DB engine", + []string{"dbidentifier"}, nil, + ), + dBLoadCPU: prometheus.NewDesc("rds_dbload_cpu_average", + "Number of active sessions where the wait event type is CPU", + []string{"dbidentifier"}, nil, + ), + dBLoadNonCPU: prometheus.NewDesc("rds_dbload_noncpu_average", + "Number of active sessions where the wait event type is not CPU", + []string{"dbidentifier"}, nil, + ), + quotaDBInstances: prometheus.NewDesc("rds_quota_max_dbinstances_average", + "Maximum number of RDS instances allowed in the AWS account", + nil, nil, + ), + quotaTotalStorage: prometheus.NewDesc("rds_quota_total_storage_bytes", + "Maximum total storage for all DB instances", + nil, nil, + ), + quotaMaxDBInstanceSnapshots: prometheus.NewDesc("rds_quota_maximum_db_instance_snapshots_average", + "Maximum number of manual DB instance snapshots", + nil, nil, + ), + usageAllocatedStorage: prometheus.NewDesc("rds_usage_allocated_storage_average", + "Total storage used by AWS RDS instances", + nil, nil, + ), + usageDBInstances: prometheus.NewDesc("rds_usage_db_instances_average", + "AWS RDS instance count", + nil, nil, + ), + usageManualSnapshots: prometheus.NewDesc("rds_usage_manual_snapshots_average", + "Manual snapshots count", + nil, nil, + ), + } +} + +func (c *rdsCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.status + ch <- c.up +} + +// getMetrics collects and return all RDS metrics +func (c *rdsCollector) fetchMetrics() error { + c.logger.Debug("received query") + + // Fetch serviceQuotas metrics + go c.getQuotasMetrics(c.servicequotasClient) + c.wg.Add(1) + + // Fetch usages metrics + go c.getUsagesMetrics(c.cloudWatchClient) + c.wg.Add(1) + + // Fetch RDS instances metrics + c.logger.Info("get RDS metrics") + + rdsFetcher := rds.NewFetcher(c.rdsClient) + + rdsMetrics, err := rdsFetcher.GetInstancesMetrics() + if err != nil { + return fmt.Errorf("can't fetch RDS metrics: %w", err) + } + + c.metrics.rds = rdsMetrics + c.counters.rdsAPIcalls += rdsFetcher.GetStatistics().RdsAPICall + c.logger.Debug("RDS metrics fetched") + + // Compute uniq instances identifiers and instance types + instanceIdentifiers, instanceTypes := getUniqTypeAndIdentifiers(rdsMetrics.Instances) + + // Fetch EC2 Metrics for instance types + if len(instanceTypes) > 0 { + go c.getEC2Metrics(c.EC2Client, instanceTypes) + c.wg.Add(1) + } + + // Fetch Cloudwatch metrics for instances + go c.getCloudwatchMetrics(c.cloudWatchClient, instanceIdentifiers) + c.wg.Add(1) + + // Wait for all go routines to finish + c.wg.Wait() + + return nil +} + +func (c *rdsCollector) getCloudwatchMetrics(client cloudwatch.CloudWatchClient, instanceIdentifiers []string) { + defer c.wg.Done() + c.logger.Debug("fetch cloudwatch metrics") + + fetcher := cloudwatch.NewRDSFetcher(client, c.logger) + + metrics, err := fetcher.GetRDSInstanceMetrics(instanceIdentifiers) + if err != nil { + c.counters.errors++ + } + + c.counters.cloudwatchAPICalls += fetcher.GetStatistics().CloudWatchAPICall + c.metrics.cloudwatchInstances = metrics + + c.logger.Debug("cloudwatch metrics fetched", "metrics", metrics) +} + +func (c *rdsCollector) getUsagesMetrics(client cloudwatch.CloudWatchClient) { + defer c.wg.Done() + c.logger.Debug("fetch usage metrics") + + fetcher := cloudwatch.NewUsageFetcher(client, c.logger) + + metrics, err := fetcher.GetUsageMetrics() + if err != nil { + c.counters.errors++ + c.logger.Error(fmt.Sprintf("can't fetch usage metrics: %s", err)) + } + + c.counters.usageAPIcalls += fetcher.GetStatistics().CloudWatchAPICall + c.metrics.cloudWatchUsage = metrics + + c.logger.Debug("usage metrics fetched", "metrics", metrics) +} + +func (c *rdsCollector) getEC2Metrics(client ec2.EC2Client, instanceTypes []string) { + defer c.wg.Done() + c.logger.Debug("fetch EC2 metrics") + + fetcher := ec2.NewFetcher(client) + + metrics, err := fetcher.GetDBInstanceTypeInformation(instanceTypes) + if err != nil { + c.counters.errors++ + c.logger.Error(fmt.Sprintf("can't fetch EC2 metrics: %s", err)) + } + + c.counters.ec2APIcalls += fetcher.GetStatistics().EC2ApiCall + c.metrics.ec2 = metrics + + c.logger.Debug("EC2 metrics fetched", "metrics", metrics) +} + +func (c *rdsCollector) getQuotasMetrics(client servicequotas.ServiceQuotasClient) { + defer c.wg.Done() + c.logger.Debug("fetch quotas") + + fetcher := servicequotas.NewFetcher(client) + + metrics, err := fetcher.GetRDSQuotas() + if err != nil { + c.counters.errors++ + c.logger.Error(fmt.Sprintf("can't fetch service quota metrics: %s", err)) + } + + c.counters.serviceQuotasAPICalls += fetcher.GetStatistics().UsageAPICall + c.metrics.serviceQuota = metrics + + c.logger.Debug("quota metrics fetched", "metrics", metrics) +} + +func (c *rdsCollector) Collect(ch chan<- prometheus.Metric) { + ch <- prometheus.MustNewConstMetric(c.exporterBuildInformation, prometheus.GaugeValue, 1, build.Version, build.CommitSHA, build.Date) + ch <- prometheus.MustNewConstMetric(c.errors, prometheus.CounterValue, c.counters.errors) + + // Get all metrics + err := c.fetchMetrics() + if err != nil { + c.logger.Error(fmt.Sprintf("can't scrape metrics: %s", err)) + // Mark exporter as down + ch <- prometheus.MustNewConstMetric(c.up, prometheus.CounterValue, exporterDownStatusCode) + + return + } + ch <- prometheus.MustNewConstMetric(c.up, prometheus.CounterValue, exporterUpStatusCode) + + // RDS metrics + ch <- prometheus.MustNewConstMetric(c.apiCall, prometheus.CounterValue, c.counters.rdsAPIcalls, "rds") + for dbidentifier, instance := range c.metrics.rds.Instances { + ch <- prometheus.MustNewConstMetric(c.allocatedStorage, prometheus.GaugeValue, float64(instance.AllocatedStorage), dbidentifier) + ch <- prometheus.MustNewConstMetric(c.information, prometheus.GaugeValue, 1, dbidentifier, instance.DbiResourceID, instance.DBInstanceClass, instance.Engine, instance.EngineVersion, instance.StorageType, strconv.FormatBool(instance.MultiAZ), strconv.FormatBool(instance.DeletionProtection), instance.Role, instance.SourceDBInstanceIdentifier, strconv.FormatBool(instance.PendingModifiedValues), instance.PendingMaintenanceAction) + ch <- prometheus.MustNewConstMetric(c.logFilesSize, prometheus.GaugeValue, float64(instance.LogFilesSize), dbidentifier) + ch <- prometheus.MustNewConstMetric(c.maxAllocatedStorage, prometheus.GaugeValue, float64(instance.MaxAllocatedStorage), dbidentifier) + ch <- prometheus.MustNewConstMetric(c.maxIops, prometheus.GaugeValue, float64(instance.MaxIops), dbidentifier) + ch <- prometheus.MustNewConstMetric(c.status, prometheus.GaugeValue, float64(instance.Status), dbidentifier) + ch <- prometheus.MustNewConstMetric(c.storageThroughput, prometheus.GaugeValue, float64(instance.StorageThroughput), dbidentifier) + ch <- prometheus.MustNewConstMetric(c.backupRetentionPeriod, prometheus.GaugeValue, float64(instance.BackupRetentionPeriod), dbidentifier) + } + + // Cloudwatch metrics + ch <- prometheus.MustNewConstMetric(c.apiCall, prometheus.CounterValue, c.counters.cloudwatchAPICalls, "cloudwatch") + for dbidentifier, instance := range c.metrics.cloudwatchInstances.Instances { + ch <- prometheus.MustNewConstMetric(c.cpuUtilisation, prometheus.GaugeValue, instance.CPUUtilization, dbidentifier) + ch <- prometheus.MustNewConstMetric(c.databaseConnections, prometheus.GaugeValue, instance.DatabaseConnections, dbidentifier) + ch <- prometheus.MustNewConstMetric(c.freeStorageSpace, prometheus.GaugeValue, instance.FreeStorageSpace, dbidentifier) + ch <- prometheus.MustNewConstMetric(c.freeableMemory, prometheus.GaugeValue, instance.FreeableMemory, dbidentifier) + ch <- prometheus.MustNewConstMetric(c.maximumUsedTransactionIDs, prometheus.GaugeValue, instance.MaximumUsedTransactionIDs, dbidentifier) + ch <- prometheus.MustNewConstMetric(c.readIOPS, prometheus.GaugeValue, instance.ReadIOPS, dbidentifier) + ch <- prometheus.MustNewConstMetric(c.readThroughput, prometheus.GaugeValue, instance.ReadThroughput, dbidentifier) + ch <- prometheus.MustNewConstMetric(c.replicaLag, prometheus.GaugeValue, instance.ReplicaLag, dbidentifier) + ch <- prometheus.MustNewConstMetric(c.replicationSlotDiskUsage, prometheus.GaugeValue, instance.ReplicationSlotDiskUsage, dbidentifier) + ch <- prometheus.MustNewConstMetric(c.swapUsage, prometheus.GaugeValue, instance.SwapUsage, dbidentifier) + ch <- prometheus.MustNewConstMetric(c.writeIOPS, prometheus.GaugeValue, instance.WriteIOPS, dbidentifier) + ch <- prometheus.MustNewConstMetric(c.writeThroughput, prometheus.GaugeValue, instance.WriteThroughput, dbidentifier) + ch <- prometheus.MustNewConstMetric(c.DBLoad, prometheus.GaugeValue, instance.DBLoad, dbidentifier) + ch <- prometheus.MustNewConstMetric(c.dBLoadCPU, prometheus.GaugeValue, instance.DBLoadCPU, dbidentifier) + ch <- prometheus.MustNewConstMetric(c.dBLoadNonCPU, prometheus.GaugeValue, instance.DBLoadNonCPU, dbidentifier) + } + + // usage metrics + ch <- prometheus.MustNewConstMetric(c.apiCall, prometheus.CounterValue, c.counters.usageAPIcalls, "usage") + ch <- prometheus.MustNewConstMetric(c.usageAllocatedStorage, prometheus.GaugeValue, c.metrics.cloudWatchUsage.AllocatedStorage) + ch <- prometheus.MustNewConstMetric(c.usageDBInstances, prometheus.GaugeValue, c.metrics.cloudWatchUsage.DBInstances) + ch <- prometheus.MustNewConstMetric(c.usageManualSnapshots, prometheus.GaugeValue, c.metrics.cloudWatchUsage.ManualSnapshots) + + // EC2 metrics + ch <- prometheus.MustNewConstMetric(c.apiCall, prometheus.CounterValue, c.counters.ec2APIcalls, "ec2") + for instanceType, instance := range c.metrics.ec2.Instances { + ch <- prometheus.MustNewConstMetric(c.instanceMaximumIops, prometheus.GaugeValue, float64(instance.MaximumIops), instanceType) + ch <- prometheus.MustNewConstMetric(c.instanceMaximumThroughput, prometheus.GaugeValue, instance.MaximumThroughput, instanceType) + ch <- prometheus.MustNewConstMetric(c.instanceMemory, prometheus.GaugeValue, float64(instance.Memory), instanceType) + ch <- prometheus.MustNewConstMetric(c.instanceVCPU, prometheus.GaugeValue, float64(instance.Vcpu), instanceType) + } + + // serviceQuotas metrics + ch <- prometheus.MustNewConstMetric(c.apiCall, prometheus.CounterValue, c.counters.serviceQuotasAPICalls, "servicequotas") + ch <- prometheus.MustNewConstMetric(c.quotaDBInstances, prometheus.GaugeValue, c.metrics.serviceQuota.DBinstances) + ch <- prometheus.MustNewConstMetric(c.quotaTotalStorage, prometheus.GaugeValue, c.metrics.serviceQuota.TotalStorage) + ch <- prometheus.MustNewConstMetric(c.quotaMaxDBInstanceSnapshots, prometheus.GaugeValue, c.metrics.serviceQuota.ManualDBInstanceSnapshots) +} diff --git a/internal/app/exporter/helper.go b/internal/app/exporter/helper.go new file mode 100644 index 0000000..425abfe --- /dev/null +++ b/internal/app/exporter/helper.go @@ -0,0 +1,26 @@ +package exporter + +import ( + "github.com/qonto/prometheus-rds-exporter/internal/app/rds" + "golang.org/x/exp/slices" +) + +func getUniqTypeAndIdentifiers(instances map[string]rds.RdsInstanceMetrics) ([]string, []string) { + var ( + instanceTypes []string + instanceIdentifiers []string + ) + + for dbinstanceName := range instances { + instanceClass := instances[dbinstanceName].DBInstanceClass + if !slices.Contains(instanceTypes, instanceClass) { + instanceTypes = append(instanceTypes, instanceClass) + } + + if !slices.Contains(instanceIdentifiers, dbinstanceName) { + instanceIdentifiers = append(instanceIdentifiers, dbinstanceName) + } + } + + return instanceIdentifiers, instanceTypes +} diff --git a/internal/app/exporter/interface.go b/internal/app/exporter/interface.go new file mode 100644 index 0000000..682897a --- /dev/null +++ b/internal/app/exporter/interface.go @@ -0,0 +1,28 @@ +package exporter + +import ( + "context" + + aws_cloudwatch "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + aws_ec2 "github.com/aws/aws-sdk-go-v2/service/ec2" + aws_rds "github.com/aws/aws-sdk-go-v2/service/rds" + aws_servicequotas "github.com/aws/aws-sdk-go-v2/service/servicequotas" +) + +type rdsClient interface { + DescribeDBInstances(context.Context, *aws_rds.DescribeDBInstancesInput, ...func(*aws_rds.Options)) (*aws_rds.DescribeDBInstancesOutput, error) + DescribePendingMaintenanceActions(context.Context, *aws_rds.DescribePendingMaintenanceActionsInput, ...func(*aws_rds.Options)) (*aws_rds.DescribePendingMaintenanceActionsOutput, error) + DescribeDBLogFiles(context.Context, *aws_rds.DescribeDBLogFilesInput, ...func(*aws_rds.Options)) (*aws_rds.DescribeDBLogFilesOutput, error) +} + +type EC2Client interface { + DescribeInstanceTypes(context.Context, *aws_ec2.DescribeInstanceTypesInput, ...func(*aws_ec2.Options)) (*aws_ec2.DescribeInstanceTypesOutput, error) +} + +type cloudWatchClient interface { + GetMetricData(context.Context, *aws_cloudwatch.GetMetricDataInput, ...func(*aws_cloudwatch.Options)) (*aws_cloudwatch.GetMetricDataOutput, error) +} + +type servicequotasClient interface { + GetServiceQuota(context.Context, *aws_servicequotas.GetServiceQuotaInput, ...func(*aws_servicequotas.Options)) (*aws_servicequotas.GetServiceQuotaOutput, error) +} diff --git a/internal/app/rds/helper.go b/internal/app/rds/helper.go new file mode 100644 index 0000000..14a861b --- /dev/null +++ b/internal/app/rds/helper.go @@ -0,0 +1,110 @@ +package rds + +import ( + "strings" + + aws_rds_types "github.com/aws/aws-sdk-go-v2/service/rds/types" + converter "github.com/qonto/prometheus-rds-exporter/internal/app/unit" +) + +func ThresholdValue(min int64, value int64, max int64) int64 { + switch { + case value < min: + return min + case value > max: + return max + default: + return value + } +} + +// GetDBIdentifierFromARN returns instance identifier from its ARN +func GetDBIdentifierFromARN(arn string) string { + arnChunk := strings.Split(arn, ":") + + return arnChunk[len(arnChunk)-1] +} + +// GetDBInstanceStatusCode returns instance status numeric code +func GetDBInstanceStatusCode(status string) int { + var instanceStatus int + + instanceStatus, isFound := instanceStatuses[status] + if !isFound { + return InstanceStatusUnknown + } + + return instanceStatus +} + +// getStorageMetrics returns storage metrics following AWS rules +func getStorageMetrics(storageType string, allocatedStorage int64, rawIops int64, rawStorageThroughput int64) (int64, int64) { + // IOPS and throughput depends of the RDS storage class type and the allocated storage + // https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Storage.html + var iops, storageThroughput int64 + + switch storageType { + case "gp2": + /* + Baseline IOPS performance scales linearly between a minimum of 100 and a maximum of 16,000 at a rate of 3 IOPS per GiB of volume size. IOPS performance is provisioned as follows: + - Volumes 33.33 GiB and smaller are provisioned with the minimum of 100 IOPS. + - Volumes larger than 33.33 GiB are provisioned with 3 IOPS per GiB of volume size up to the maximum of 16,000 IOPS, which is reached at 5,334 GiB (3 X 5,334). + - Volumes 5,334 GiB and larger are provisioned with 16,000 IOPS. + https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/general-purpose.html#EBSVolumeTypes_gp2 + */ + iops = ThresholdValue(gp2IOPSMin, allocatedStorage*gp2IOPSPerGB, gp2IOPSMax) + + /* + Throughput performance is provisioned as follows: + - Volumes that are 170 GiB and smaller deliver a maximum throughput of 128 MiB/s. + - Volumes larger than 170 GiB but smaller than 334 GiB can burst to a maximum throughput of 250 MiB/s. + - Volumes that are 334 GiB and larger deliver 250 MiB/s. + https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/general-purpose.html#EBSVolumeTypes_gp2 + */ + if allocatedStorage >= gp2StorageThroughputVolumeThreshold { + storageThroughput = gp2StorageThroughputLargeVolume + } else { + storageThroughput = gp2StorageThroughputSmallVolume + } + case "gp3": + // iops and storageThroughput are returned by AWS RDS API for GP3 class type + iops = rawIops + storageThroughput = rawStorageThroughput + case "io1": + iops = rawIops + + /* + Volumes provisioned with more than 32,000 IOPS (up to the maximum of 64,000 IOPS) yield a linear increase in throughput at a rate of 16 KiB per provisioned IOPS. + https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/provisioned-iops.html#EBSVolumeTypes_piops + */ + switch { + case iops >= io1HighIOPSThroughputThreshold: + storageThroughput = io1HighIOPSThroughputValue + case iops >= io1LargeIOPSThroughputThreshold: + storageThroughput = converter.KiloByteToMegaBytes(iops * io1LargeIOPSThroughputValue) + case iops >= io1MediumIOPSThroughputThreshold: + storageThroughput = io1MediumIOPSThroughputValue + default: + storageThroughput = converter.KiloByteToMegaBytes(iops * io1DefaultIOPSThroughputValue) + } + } + + return iops, storageThroughput +} + +// getRoleInCluster returns role and source of the specified instance in the the cluster +func getRoleInCluster(instance *aws_rds_types.DBInstance) (string, string) { + var ( + role string + source string + ) + + if instance.ReadReplicaSourceDBInstanceIdentifier != nil { + source = *instance.ReadReplicaSourceDBInstanceIdentifier + role = replicaRole + } else { + role = primaryRole + } + + return role, source +} diff --git a/internal/app/rds/rds.go b/internal/app/rds/rds.go new file mode 100644 index 0000000..17425be --- /dev/null +++ b/internal/app/rds/rds.go @@ -0,0 +1,255 @@ +// Package rds implements methods to retrieve RDS information +package rds + +import ( + "context" + "fmt" + "reflect" + + aws_rds "github.com/aws/aws-sdk-go-v2/service/rds" + aws_rds_types "github.com/aws/aws-sdk-go-v2/service/rds/types" + converter "github.com/qonto/prometheus-rds-exporter/internal/app/unit" +) + +type Metrics struct { + Instances map[string]RdsInstanceMetrics +} + +type Statistics struct { + RdsAPICall float64 +} + +type RdsInstanceMetrics struct { + Engine string + EngineVersion string + DBInstanceClass string + DbiResourceID string + StorageType string + AllocatedStorage int64 + StorageThroughput int64 + MaxAllocatedStorage int64 + MaxIops int64 + LogFilesSize int64 + PendingMaintenanceAction string + PendingModifiedValues bool + BackupRetentionPeriod int32 + Status int + DeletionProtection bool + PubliclyAccessible bool + PerformanceInsightsEnabled bool + MultiAZ bool + IAMDatabaseAuthenticationEnabled bool + Role string + SourceDBInstanceIdentifier string +} + +const ( + InstanceStatusAvailable int = 1 + InstanceStatusStopped int = 0 + InstanceStatusUnknown int = -1 + InstanceStatusBackingUp int = 2 + NoPendingMaintenanceOperation string = "no" + UnscheduledPendingMaintenanceOperation string = "pending" + AutoAppliedPendingMaintenanceOperation string = "auto-applied" + ForcedPendingMaintenanceOperation string = "forced" + gp2IOPSMin int64 = 100 + gp2IOPSMax int64 = 16000 + gp2IOPSPerGB int64 = 3 + gp2StorageThroughputVolumeThreshold int64 = 334 + gp2StorageThroughputSmallVolume int64 = 128 + gp2StorageThroughputLargeVolume int64 = 250 + io1HighIOPSThroughputThreshold int64 = 64000 + io1HighIOPSThroughputValue int64 = 1000 + io1LargeIOPSThroughputThreshold int64 = 32000 + io1LargeIOPSThroughputValue int64 = 16 + io1MediumIOPSThroughputThreshold int64 = 2000 + io1MediumIOPSThroughputValue int64 = 500 + io1DefaultIOPSThroughputValue int64 = 256 + primaryRole string = "primary" + replicaRole string = "replica" +) + +var instanceStatuses = map[string]int{ + "available": InstanceStatusAvailable, + "stopped": InstanceStatusStopped, + "backing-up": InstanceStatusBackingUp, + "unknown": InstanceStatusUnknown, +} + +type RDSClient interface { + DescribeDBInstances(ctx context.Context, params *aws_rds.DescribeDBInstancesInput, optFns ...func(*aws_rds.Options)) (*aws_rds.DescribeDBInstancesOutput, error) + DescribePendingMaintenanceActions(context.Context, *aws_rds.DescribePendingMaintenanceActionsInput, ...func(*aws_rds.Options)) (*aws_rds.DescribePendingMaintenanceActionsOutput, error) + DescribeDBLogFiles(context.Context, *aws_rds.DescribeDBLogFilesInput, ...func(*aws_rds.Options)) (*aws_rds.DescribeDBLogFilesOutput, error) +} + +func NewFetcher(client RDSClient) RDSFetcher { + return RDSFetcher{ + client: client, + } +} + +type RDSFetcher struct { + client RDSClient + statistics Statistics +} + +func (r *RDSFetcher) GetStatistics() Statistics { + return r.statistics +} + +func (r *RDSFetcher) getPendingMaintenances() (map[string]string, error) { + instances := make(map[string]string) + + inputMaintenance := &aws_rds.DescribePendingMaintenanceActionsInput{} + + maintenances, err := r.client.DescribePendingMaintenanceActions(context.TODO(), inputMaintenance) + if err != nil { + return nil, fmt.Errorf("can't describe pending maintenance actions: %w", err) + } + r.statistics.RdsAPICall++ + + if maintenances == nil { + return nil, nil + } + + for _, maintenance := range maintenances.PendingMaintenanceActions { + var maintenanceMode string + + dbIdentifier := GetDBIdentifierFromARN(*maintenance.ResourceIdentifier) + + for _, action := range maintenance.PendingMaintenanceActionDetails { + switch { + case action.ForcedApplyDate != nil: + maintenanceMode = ForcedPendingMaintenanceOperation + case action.AutoAppliedAfterDate != nil && maintenanceMode != ForcedPendingMaintenanceOperation: + maintenanceMode = AutoAppliedPendingMaintenanceOperation + default: + maintenanceMode = UnscheduledPendingMaintenanceOperation + } + } + + instances[dbIdentifier] = maintenanceMode + } + + return instances, nil +} + +func (r *RDSFetcher) GetInstancesMetrics() (Metrics, error) { + metrics := make(map[string]RdsInstanceMetrics) + + instanceMaintenances, err := r.getPendingMaintenances() + if err != nil { + return Metrics{}, fmt.Errorf("can't get RDS maintenances: %w", err) + } + + input := &aws_rds.DescribeDBInstancesInput{} + + paginator := aws_rds.NewDescribeDBInstancesPaginator(r.client, input) + for paginator.HasMorePages() { + r.statistics.RdsAPICall++ + + output, err := paginator.NextPage(context.TODO()) + if err != nil { + return Metrics{}, fmt.Errorf("can't get instances: %w", err) + } + + for _, dbInstance := range output.DBInstances { + dbIdentifier := dbInstance.DBInstanceIdentifier + + instanceMetrics, err := r.computeInstanceMetrics(dbInstance, instanceMaintenances) + if err != nil { + return Metrics{}, fmt.Errorf("can't compute instance metrics for %s: %w", *dbIdentifier, err) + } + + metrics[*dbIdentifier] = instanceMetrics + } + } + + return Metrics{Instances: metrics}, nil +} + +// computeInstanceMetrics returns metrics about the specified instance +func (r *RDSFetcher) computeInstanceMetrics(dbInstance aws_rds_types.DBInstance, instanceMaintenances map[string]string) (RdsInstanceMetrics, error) { + dbIdentifier := dbInstance.DBInstanceIdentifier + + var iops int64 + if dbInstance.Iops != nil { + iops = int64(*dbInstance.Iops) + } + + var throughput int64 + if dbInstance.StorageThroughput != nil { + throughput = int64(*dbInstance.StorageThroughput) + } + + iops, storageThroughput := getStorageMetrics(*dbInstance.StorageType, int64(dbInstance.AllocatedStorage), iops, throughput) + + var maxAllocatedStorage int32 = 0 + if dbInstance.MaxAllocatedStorage != nil { + maxAllocatedStorage = *dbInstance.MaxAllocatedStorage + } + + pendingModifiedValues := false + if !reflect.DeepEqual(dbInstance.PendingModifiedValues, &aws_rds_types.PendingModifiedValues{}) { + pendingModifiedValues = true + } + + pendingMaintenanceAction := NoPendingMaintenanceOperation + if maintenanceMode, isFound := instanceMaintenances[*dbIdentifier]; isFound { + pendingMaintenanceAction = maintenanceMode + } + + logFilesSize, err := r.getLogFilesSize(*dbIdentifier) + if err != nil { + return RdsInstanceMetrics{}, fmt.Errorf("can't get log files size for %d: %w", dbIdentifier, err) + } + + role, sourceDBInstanceIdentifier := getRoleInCluster(&dbInstance) + + metrics := RdsInstanceMetrics{ + AllocatedStorage: converter.GigaBytesToBytes(dbInstance.AllocatedStorage), + BackupRetentionPeriod: converter.DaystoSeconds(dbInstance.BackupRetentionPeriod), + DBInstanceClass: *dbInstance.DBInstanceClass, + DbiResourceID: *dbInstance.DbiResourceId, + DeletionProtection: dbInstance.DeletionProtection, + Engine: *dbInstance.Engine, + EngineVersion: *dbInstance.EngineVersion, + LogFilesSize: logFilesSize, + MaxAllocatedStorage: converter.GigaBytesToBytes(maxAllocatedStorage), + MaxIops: iops, + MultiAZ: dbInstance.MultiAZ, + PendingMaintenanceAction: pendingMaintenanceAction, + PendingModifiedValues: pendingModifiedValues, + PerformanceInsightsEnabled: *dbInstance.PerformanceInsightsEnabled, + PubliclyAccessible: dbInstance.PubliclyAccessible, + Role: role, + SourceDBInstanceIdentifier: sourceDBInstanceIdentifier, + Status: GetDBInstanceStatusCode(*dbInstance.DBInstanceStatus), + StorageThroughput: converter.MegaBytesToBytes(storageThroughput), + StorageType: *dbInstance.StorageType, + } + + return metrics, nil +} + +// getLogFilesSize returns the size of all logs on the specified instance +func (r *RDSFetcher) getLogFilesSize(dbidentifier string) (int64, error) { + var filesSize int64 + + input := &aws_rds.DescribeDBLogFilesInput{DBInstanceIdentifier: &dbidentifier} + + result, err := r.client.DescribeDBLogFiles(context.TODO(), input) + if err != nil { + return 0, fmt.Errorf("can't describe db logs files for %s: %w", dbidentifier, err) + } + + r.statistics.RdsAPICall++ + + if result != nil { + for _, file := range result.DescribeDBLogFiles { + filesSize += file.Size + } + } + + return filesSize, nil +} diff --git a/internal/app/rds/rds_test.go b/internal/app/rds/rds_test.go new file mode 100644 index 0000000..d2ae696 --- /dev/null +++ b/internal/app/rds/rds_test.go @@ -0,0 +1,264 @@ +package rds_test + +import ( + "context" + "crypto/rand" + "fmt" + "reflect" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + aws_rds "github.com/aws/aws-sdk-go-v2/service/rds" + aws_rds_types "github.com/aws/aws-sdk-go-v2/service/rds/types" + "github.com/qonto/prometheus-rds-exporter/internal/app/rds" + converter "github.com/qonto/prometheus-rds-exporter/internal/app/unit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type mockRDSClient struct { + DescribeDBInstancesOutput *aws_rds.DescribeDBInstancesOutput + DescribePendingMaintenanceActionsOutput *aws_rds.DescribePendingMaintenanceActionsOutput + DescribeDBLogFilesOutput *aws_rds.DescribeDBLogFilesOutput + Error error +} + +func (m mockRDSClient) DescribeDBInstancesPages(input *aws_rds.DescribeDBInstancesInput, fn func(*aws_rds.DescribeDBInstancesOutput, bool) bool) error { + fn(m.DescribeDBInstancesOutput, false) + + return nil +} + +func (m mockRDSClient) DescribePendingMaintenanceActions(context.Context, *aws_rds.DescribePendingMaintenanceActionsInput, ...func(*aws_rds.Options)) (*aws_rds.DescribePendingMaintenanceActionsOutput, error) { + return m.DescribePendingMaintenanceActionsOutput, m.Error +} + +func (m mockRDSClient) DescribeDBLogFiles(context.Context, *aws_rds.DescribeDBLogFilesInput, ...func(*aws_rds.Options)) (*aws_rds.DescribeDBLogFilesOutput, error) { + return m.DescribeDBLogFilesOutput, nil +} + +func (m mockRDSClient) DescribeDBInstances(context.Context, *aws_rds.DescribeDBInstancesInput, ...func(*aws_rds.Options)) (*aws_rds.DescribeDBInstancesOutput, error) { + return m.DescribeDBInstancesOutput, nil +} + +// randomString returns a random alphanumeric string of the specified length +func randomString(length int) string { + buf := make([]byte, length) + + _, err := rand.Read(buf) + if err != nil { + panic(err) // out of randomness, should never happen + } + + return fmt.Sprintf("%x", buf) +} + +func newRdsInstance() *aws_rds_types.DBInstance { + DBInstanceIdentifier := randomString(10) + + return &aws_rds_types.DBInstance{ + AllocatedStorage: 5, + BackupRetentionPeriod: 7, + DBInstanceArn: aws.String("RandomDBInstanceArn"), + DBInstanceClass: aws.String("t3.large"), + DBInstanceIdentifier: aws.String(DBInstanceIdentifier), + DBInstanceStatus: aws.String("available"), + DbiResourceId: aws.String("resource1"), + DeletionProtection: true, + Engine: aws.String("postgres"), + EngineVersion: aws.String("14.9"), + Iops: aws.Int32(3000), + MaxAllocatedStorage: aws.Int32(10), + MultiAZ: true, + PerformanceInsightsEnabled: aws.Bool(true), + PubliclyAccessible: true, + StorageType: aws.String("gp3"), + } +} + +func TestGetMetrics(t *testing.T) { + rdsInstance := newRdsInstance() + mockDescribeDBInstancesOutput := &aws_rds.DescribeDBInstancesOutput{DBInstances: []aws_rds_types.DBInstance{*rdsInstance}} + + mock := mockRDSClient{DescribeDBInstancesOutput: mockDescribeDBInstancesOutput} + client := rds.NewFetcher(mock) + metrics, err := client.GetInstancesMetrics() + + require.NoError(t, err, "GetInstancesMetrics must succeed") + + m := metrics.Instances[*rdsInstance.DBInstanceIdentifier] + assert.Equal(t, rds.InstanceStatusAvailable, m.Status, "Instance is available") + assert.Equal(t, "primary", m.Role, "Should be primary node") + assert.Equal(t, int64(0), m.LogFilesSize, "Log file size mismatch") + + assert.Equal(t, converter.GigaBytesToBytes(rdsInstance.AllocatedStorage), m.AllocatedStorage, "Allocated storage mismatch") + assert.Equal(t, converter.GigaBytesToBytes(*rdsInstance.MaxAllocatedStorage), m.MaxAllocatedStorage, "Max allocated storage (aka autoscaling) mismatch") + assert.Equal(t, int64(*rdsInstance.Iops), m.MaxIops, "Max IOPS mismatch") + assert.Equal(t, converter.DaystoSeconds(rdsInstance.BackupRetentionPeriod), m.BackupRetentionPeriod, "Backup retention mismatch") + assert.Equal(t, rdsInstance.DeletionProtection, m.DeletionProtection, "Deletion protection mismatch") + assert.Equal(t, rdsInstance.MultiAZ, m.MultiAZ, "MultiAZ mismatch") + assert.Equal(t, *rdsInstance.Engine, m.Engine, "Engine mismatch") + assert.Equal(t, *rdsInstance.EngineVersion, m.EngineVersion, "Engine version mismatch") + assert.Equal(t, *rdsInstance.PerformanceInsightsEnabled, m.PerformanceInsightsEnabled, "PerformanceInsights enabled mismatch") + assert.Equal(t, rdsInstance.PubliclyAccessible, m.PubliclyAccessible, "PubliclyAccessible mismatch") + assert.Equal(t, *rdsInstance.DbiResourceId, m.DbiResourceID, "DbiResourceId mismatch") + assert.Equal(t, *rdsInstance.DBInstanceClass, m.DBInstanceClass, "DBInstanceIdentifier mismatch") + assert.Equal(t, *rdsInstance.DBInstanceClass, m.DBInstanceClass, "DBInstanceIdentifier mismatch") +} + +func TestGP2StorageType(t *testing.T) { + rdsInstanceWithSmallDisk := newRdsInstance() + rdsInstanceWithSmallDisk.StorageType = aws.String("gp2") + rdsInstanceWithSmallDisk.AllocatedStorage = 10 + + rdsInstanceWithMediumDisk := newRdsInstance() + rdsInstanceWithMediumDisk.StorageType = aws.String("gp2") + rdsInstanceWithMediumDisk.AllocatedStorage = 400 + + rdsInstanceWithLargeDisk := newRdsInstance() + rdsInstanceWithLargeDisk.StorageType = aws.String("gp2") + rdsInstanceWithLargeDisk.AllocatedStorage = 20000 + + mockDescribeDBInstancesOutput := &aws_rds.DescribeDBInstancesOutput{DBInstances: []aws_rds_types.DBInstance{*rdsInstanceWithSmallDisk, *rdsInstanceWithMediumDisk, *rdsInstanceWithLargeDisk}} + mock := mockRDSClient{DescribeDBInstancesOutput: mockDescribeDBInstancesOutput} + client := rds.NewFetcher(mock) + metrics, err := client.GetInstancesMetrics() + + require.NoError(t, err, "GetInstancesMetrics must succeed") + assert.Equal(t, int64(100), metrics.Instances[*rdsInstanceWithSmallDisk.DBInstanceIdentifier].MaxIops, "Minimum is 100 IOPS") + assert.Equal(t, converter.MegaBytesToBytes(int64(128)), metrics.Instances[*rdsInstanceWithSmallDisk.DBInstanceIdentifier].StorageThroughput, "Minimum is 128 MiB/s") + + assert.Equal(t, int64(1200), metrics.Instances[*rdsInstanceWithMediumDisk.DBInstanceIdentifier].MaxIops, "Should be 3 * disk size") + assert.Equal(t, converter.MegaBytesToBytes(int64(250)), metrics.Instances[*rdsInstanceWithMediumDisk.DBInstanceIdentifier].StorageThroughput, "Max 250 MiB/s") + + assert.Equal(t, int64(16000), metrics.Instances[*rdsInstanceWithLargeDisk.DBInstanceIdentifier].MaxIops, "Should be limited to 16K") + assert.Equal(t, converter.MegaBytesToBytes(int64(250)), metrics.Instances[*rdsInstanceWithLargeDisk.DBInstanceIdentifier].StorageThroughput, "Large volume are limited to 250 MiB/s") +} + +func TestGP3StorageType(t *testing.T) { + rdsInstanceWithSmallDisk := newRdsInstance() + rdsInstanceWithSmallDisk.StorageType = aws.String("gp3") + rdsInstanceWithSmallDisk.AllocatedStorage = 10 + rdsInstanceWithSmallDisk.Iops = aws.Int32(3000) + + rdsInstanceWithLargeDisk := newRdsInstance() + rdsInstanceWithLargeDisk.StorageType = aws.String("gp3") + rdsInstanceWithLargeDisk.AllocatedStorage = 500 + rdsInstanceWithLargeDisk.Iops = aws.Int32(12000) + + mockDescribeDBInstancesOutput := &aws_rds.DescribeDBInstancesOutput{DBInstances: []aws_rds_types.DBInstance{*rdsInstanceWithSmallDisk, *rdsInstanceWithLargeDisk}} + mock := mockRDSClient{DescribeDBInstancesOutput: mockDescribeDBInstancesOutput} + client := rds.NewFetcher(mock) + metrics, err := client.GetInstancesMetrics() + + require.NoError(t, err, "GetInstancesMetrics must succeed") + assert.Equal(t, int64(3000), metrics.Instances[*rdsInstanceWithSmallDisk.DBInstanceIdentifier].MaxIops, "IOPS should the same than RDS instance information") + assert.Equal(t, int64(12000), metrics.Instances[*rdsInstanceWithLargeDisk.DBInstanceIdentifier].MaxIops, "IOPS should the same than RDS instance information") +} + +func TestIO1StorageType(t *testing.T) { + rdsInstanceWithSmallIOPS := newRdsInstance() + rdsInstanceWithSmallIOPS.StorageType = aws.String("io1") + rdsInstanceWithSmallIOPS.Iops = aws.Int32(1000) + + rdsInstanceWithMediumIOPS := newRdsInstance() + rdsInstanceWithMediumIOPS.StorageType = aws.String("io1") + rdsInstanceWithMediumIOPS.Iops = aws.Int32(4000) + + rdsInstanceWithLargeIOPS := newRdsInstance() + rdsInstanceWithLargeIOPS.StorageType = aws.String("io1") + rdsInstanceWithLargeIOPS.Iops = aws.Int32(48000) + + rdsInstanceWithHighIOPS := newRdsInstance() + rdsInstanceWithHighIOPS.StorageType = aws.String("io1") + rdsInstanceWithHighIOPS.Iops = aws.Int32(64000) + + mockDescribeDBInstancesOutput := &aws_rds.DescribeDBInstancesOutput{DBInstances: []aws_rds_types.DBInstance{*rdsInstanceWithSmallIOPS, *rdsInstanceWithMediumIOPS, *rdsInstanceWithLargeIOPS, *rdsInstanceWithHighIOPS}} + mock := mockRDSClient{DescribeDBInstancesOutput: mockDescribeDBInstancesOutput} + client := rds.NewFetcher(mock) + metrics, err := client.GetInstancesMetrics() + + require.NoError(t, err, "GetInstancesMetrics must succeed") + assert.Equal(t, converter.MegaBytesToBytes(int64(250)), metrics.Instances[*rdsInstanceWithSmallIOPS.DBInstanceIdentifier].StorageThroughput, "Minimum is 256 MiB/s") + assert.Equal(t, converter.MegaBytesToBytes(int64(500)), metrics.Instances[*rdsInstanceWithMediumIOPS.DBInstanceIdentifier].StorageThroughput, "500 MiB/s for more than 2K IOPS") + assert.Equal(t, converter.MegaBytesToBytes(int64(750)), metrics.Instances[*rdsInstanceWithLargeIOPS.DBInstanceIdentifier].StorageThroughput, "16 * IOPS") + assert.Equal(t, converter.MegaBytesToBytes(int64(1000)), metrics.Instances[*rdsInstanceWithHighIOPS.DBInstanceIdentifier].StorageThroughput, "Max is 1 GiB/s") +} + +func TestLogSize(t *testing.T) { + // Mock RDS instance + rdsInstance := newRdsInstance() + mockDescribeDBInstancesOutput := &aws_rds.DescribeDBInstancesOutput{DBInstances: []aws_rds_types.DBInstance{*rdsInstance}} + + // Mock log files + logFileCount := int64(3) + logFileSize := int64(1024) + expectedLogFilesSize := logFileSize * logFileCount + + rdsLogFiles := []aws_rds_types.DescribeDBLogFilesDetails{} + for i := int64(0); i < logFileCount; i++ { + rdsLogFiles = append(rdsLogFiles, aws_rds_types.DescribeDBLogFilesDetails{Size: logFileSize}) + } + + mockDescribeDBLogFilesOutput := &aws_rds.DescribeDBLogFilesOutput{DescribeDBLogFiles: rdsLogFiles} + + mock := mockRDSClient{ + DescribeDBInstancesOutput: mockDescribeDBInstancesOutput, + DescribeDBLogFilesOutput: mockDescribeDBLogFilesOutput, + } + client := rds.NewFetcher(mock) + metrics, err := client.GetInstancesMetrics() + + require.NoError(t, err, "GetInstancesMetrics must succeed") + assert.Equal(t, expectedLogFilesSize, metrics.Instances[*rdsInstance.DBInstanceIdentifier].LogFilesSize, "Log files size mismatch") +} + +func TestReplicaNode(t *testing.T) { + primaryInstance := "primary-instance" + + // Mock RDS instance + rdsInstance := newRdsInstance() + rdsInstance.ReadReplicaSourceDBInstanceIdentifier = aws.String(primaryInstance) + mockDescribeDBInstancesOutput := &aws_rds.DescribeDBInstancesOutput{DBInstances: []aws_rds_types.DBInstance{*rdsInstance}} + + mock := mockRDSClient{DescribeDBInstancesOutput: mockDescribeDBInstancesOutput} + client := rds.NewFetcher(mock) + metrics, err := client.GetInstancesMetrics() + + require.NoError(t, err, "GetInstancesMetrics must succeed") + assert.Equal(t, "replica", metrics.Instances[*rdsInstance.DBInstanceIdentifier].Role, "Should be replica") + assert.Equal(t, primaryInstance, metrics.Instances[*rdsInstance.DBInstanceIdentifier].SourceDBInstanceIdentifier, "Should be replica") +} + +func TestThresholdValue(t *testing.T) { + assert.Equal(t, int64(100), rds.ThresholdValue(100, 42, 1000), "Should return minimum") + assert.Equal(t, int64(500), rds.ThresholdValue(100, 500, 1000), "Should return the value") + assert.Equal(t, int64(1000), rds.ThresholdValue(100, 999999, 1000), "Should return the maximum") +} + +func TestGetDBIdentifierFromARN(t *testing.T) { + assert.Equal(t, "pg1", rds.GetDBIdentifierFromARN("arn:aws:rds:eu-west-3:123456789012:db:pg1"), "Should return only the dbidentifier") +} + +func TestGetDBInstanceStatusCode(t *testing.T) { + type test struct { + input string + want int + } + + tests := []test{ + {input: "available", want: rds.InstanceStatusAvailable}, + {input: "stopped", want: rds.InstanceStatusStopped}, + {input: "backing-up", want: rds.InstanceStatusBackingUp}, + {input: "unknown", want: rds.InstanceStatusUnknown}, + {input: "future", want: rds.InstanceStatusUnknown}, + } + + for _, tc := range tests { + got := rds.GetDBInstanceStatusCode(tc.input) + if !reflect.DeepEqual(tc.want, got) { + t.Fatalf("expected: %v, got: %v", tc.want, got) + } + } +} diff --git a/internal/app/servicequotas/servicequotas.go b/internal/app/servicequotas/servicequotas.go new file mode 100644 index 0000000..5d3c904 --- /dev/null +++ b/internal/app/servicequotas/servicequotas.go @@ -0,0 +1,109 @@ +// Package servicequotas implements methods to retrieve AWS Service Quotas information +package servicequotas + +import ( + "context" + "errors" + "fmt" + + aws_servicequotas "github.com/aws/aws-sdk-go-v2/service/servicequotas" + converter "github.com/qonto/prometheus-rds-exporter/internal/app/unit" + "golang.org/x/exp/slog" +) + +var ( + errNoQuota = errors.New("no AWS quota with this code") + errQuotaError = errors.New("aws return error for this quota") +) + +const ( + RDSServiceCode = "rds" // AWS RDS service code in AWS quotas API + + // AWS RDS service quotas codes can be listed with "aws service-quotas list-service-quotas --service-code rds" + DBinstancesQuotacode = "L-7B6409FD" // DB instances + TotalStorageQuotaCode = "L-7ADDB58A" // Total storage for all DB instances + ManualDBInstanceSnapshotsQuotaCode = "L-272F1212" // Manual DB instance snapshots +) + +// Metrics contains the quotas to be monitored for the AWS RDS service +type Metrics struct { + DBinstances float64 + TotalStorage float64 + ManualDBInstanceSnapshots float64 +} + +type Statistics struct { + UsageAPICall float64 +} + +type ServiceQuotasClient interface { + GetServiceQuota(ctx context.Context, input *aws_servicequotas.GetServiceQuotaInput, optFns ...func(*aws_servicequotas.Options)) (*aws_servicequotas.GetServiceQuotaOutput, error) +} + +func NewFetcher(client ServiceQuotasClient) *serviceQuotaFetcher { + return &serviceQuotaFetcher{ + client: client, + } +} + +type serviceQuotaFetcher struct { + logger *slog.Logger + client ServiceQuotasClient + statistics Statistics +} + +func (s *serviceQuotaFetcher) GetStatistics() Statistics { + return s.statistics +} + +// GetQuota retrieves and returns the AWS quota value for the specified serviceCode and quotaCode +func (s *serviceQuotaFetcher) getQuota(serviceCode string, quotaCode string) (float64, error) { + params := &aws_servicequotas.GetServiceQuotaInput{ + ServiceCode: &serviceCode, + QuotaCode: "aCode, + } + + s.statistics.UsageAPICall++ + + result, err := s.client.GetServiceQuota(context.TODO(), params) + if err != nil { + return 0, fmt.Errorf("can't get %s/%s service quota: %w", serviceCode, quotaCode, err) + } + + // AWS response payload could contains errors (eg. missing permission) + if result.Quota.ErrorReason != nil { + s.logger.Error("AWS quota error: ", "errorCode", result.Quota.ErrorReason.ErrorCode, "message", *result.Quota.ErrorReason.ErrorMessage) + + return 0, errQuotaError + } + + if result.Quota == nil { + return 0, fmt.Errorf("no quota for %s/%s: %w", serviceCode, quotaCode, errNoQuota) + } + + return *result.Quota.Value, nil +} + +// GetRDSQuotas retrieves quotas for the AWS RDS service +func (s *serviceQuotaFetcher) GetRDSQuotas() (Metrics, error) { + DBinstances, err := s.getQuota(RDSServiceCode, DBinstancesQuotacode) + if err != nil { + return Metrics{}, fmt.Errorf("can't fetch DBinstance quota: %w", err) + } + + totalStorage, err := s.getQuota(RDSServiceCode, TotalStorageQuotaCode) + if err != nil { + return Metrics{}, fmt.Errorf("can't fetch total storage quota: %w", err) + } + + manualDBInstanceSnapshots, err := s.getQuota(RDSServiceCode, ManualDBInstanceSnapshotsQuotaCode) + if err != nil { + return Metrics{}, fmt.Errorf("can't fetch manual db instance snapshots quota: %w", err) + } + + return Metrics{ + DBinstances: DBinstances, + TotalStorage: float64(converter.GigaBytesToBytes(totalStorage)), + ManualDBInstanceSnapshots: manualDBInstanceSnapshots, + }, nil +} diff --git a/internal/app/servicequotas/servicequotas_test.go b/internal/app/servicequotas/servicequotas_test.go new file mode 100644 index 0000000..720d16f --- /dev/null +++ b/internal/app/servicequotas/servicequotas_test.go @@ -0,0 +1,54 @@ +package servicequotas_test + +import ( + "context" + "testing" + + aws_servicequotas "github.com/aws/aws-sdk-go-v2/service/servicequotas" + aws_servicequotas_types "github.com/aws/aws-sdk-go-v2/service/servicequotas/types" + "github.com/qonto/prometheus-rds-exporter/internal/app/servicequotas" + converter "github.com/qonto/prometheus-rds-exporter/internal/app/unit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Defines expected values for the mock and tests +const ( + UnknownServiceQuota = float64(42) + DBinstancesQuota = float64(10) + totalStorage = float64(10) + manualDBInstanceSnapshots = float64(42) +) + +// mockServiceQuotasClient defines a mock for the AWS service quotas +type mockServiceQuotasClient struct{} + +func (m mockServiceQuotasClient) GetServiceQuota(context context.Context, input *aws_servicequotas.GetServiceQuotaInput, optFns ...func(*aws_servicequotas.Options)) (*aws_servicequotas.GetServiceQuotaOutput, error) { + value := UnknownServiceQuota + + if *input.ServiceCode == servicequotas.RDSServiceCode { + switch *input.QuotaCode { + case servicequotas.DBinstancesQuotacode: + value = DBinstancesQuota + case servicequotas.TotalStorageQuotaCode: + value = totalStorage + case servicequotas.ManualDBInstanceSnapshotsQuotaCode: + value = manualDBInstanceSnapshots + } + } + + quota := &aws_servicequotas_types.ServiceQuota{Value: &value} + + return &aws_servicequotas.GetServiceQuotaOutput{Quota: quota}, nil +} + +func TestGetRDSQuotas(t *testing.T) { + mockClient := mockServiceQuotasClient{} + + result, err := servicequotas.NewFetcher(mockClient).GetRDSQuotas() + require.NoError(t, err, "GetRDSQuotas must succeed") + assert.Equal(t, DBinstancesQuota, result.DBinstances, "DbInstance quota is incorrect") + assert.Equal(t, float64(converter.GigaBytesToBytes(totalStorage)), result.TotalStorage, "Total storage quota is incorrect") + assert.Equal(t, manualDBInstanceSnapshots, result.ManualDBInstanceSnapshots, "Manual db instance snapshot quota is incorrect") +} diff --git a/internal/app/unit/converter.go b/internal/app/unit/converter.go new file mode 100644 index 0000000..6a7b8e6 --- /dev/null +++ b/internal/app/unit/converter.go @@ -0,0 +1,27 @@ +// Package converter implements function to convert units +package converter + +const ( + unit = 1024 + secondsPerDay = 86400 +) + +type Number interface { + int32 | int64 | float64 +} + +func GigaBytesToBytes[N Number](size N) int64 { + return int64(size) * unit * unit * unit +} + +func MegaBytesToBytes[N Number](size N) N { + return size * unit * unit +} + +func KiloByteToMegaBytes[N Number](size N) N { + return size / unit +} + +func DaystoSeconds[N Number](days N) N { + return days * secondsPerDay +} diff --git a/internal/app/unit/converter_test.go b/internal/app/unit/converter_test.go new file mode 100644 index 0000000..65f4a25 --- /dev/null +++ b/internal/app/unit/converter_test.go @@ -0,0 +1,28 @@ +package converter_test + +import ( + "testing" + + converter "github.com/qonto/prometheus-rds-exporter/internal/app/unit" + "github.com/stretchr/testify/assert" +) + +func TestGigaBytesToBytes(t *testing.T) { + assert.Equal(t, int64(1073741824), converter.GigaBytesToBytes(int32(1)), "1 GB conversion is not correct") + assert.Equal(t, int64(1073741824), converter.GigaBytesToBytes(int64(1)), "1 GB conversion is not correct") + assert.Equal(t, int64(1073741824), converter.GigaBytesToBytes(float64(1)), "1 GB conversion is not correct") +} + +func TestMegaBytesToBytes(t *testing.T) { + assert.Equal(t, int64(1048576), converter.MegaBytesToBytes(int64(1)), "1 MB conversion is not correct") + assert.Equal(t, float64(1048576), converter.MegaBytesToBytes(float64(1)), "1 MB conversion is not correct") +} + +func TestKiloByteToMegaBytes(t *testing.T) { + assert.Equal(t, int64(1), converter.KiloByteToMegaBytes(int64(1024)), "1 MB conversion is not correct") +} + +func TestDaystoSeconds(t *testing.T) { + assert.Equal(t, int32(86400), converter.DaystoSeconds(int32(1)), "1 day conversion is not correct") + assert.Equal(t, int32(604800), converter.DaystoSeconds(int32(7)), "7 days conversion is not correct") +} diff --git a/internal/infra/build/build.go b/internal/infra/build/build.go new file mode 100644 index 0000000..ce0aae1 --- /dev/null +++ b/internal/infra/build/build.go @@ -0,0 +1,8 @@ +// Package build store build information (version, date, ...) +package build + +var ( + Version = "unknown" + CommitSHA = "unknown" + Date = "unknown" +) diff --git a/internal/infra/http/index.go b/internal/infra/http/index.go new file mode 100644 index 0000000..2aec75d --- /dev/null +++ b/internal/infra/http/index.go @@ -0,0 +1,22 @@ +package http + +import ( + "fmt" + "net/http" + + "github.com/qonto/prometheus-rds-exporter/internal/infra/build" +) + +type helloWorldhandler struct{} + +func (h helloWorldhandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, ` + + Prometheus RDS Exporter + + +

Prometheus RDS Exporter (%s)

+

Metrics

+ + `, build.Version) +} diff --git a/internal/infra/http/server.go b/internal/infra/http/server.go new file mode 100644 index 0000000..713fb29 --- /dev/null +++ b/internal/infra/http/server.go @@ -0,0 +1,109 @@ +// Package http provides webserver functionnalities +package http + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +const ( + ReadTimeout = 1 + WriteTimeout = 120 + IdleTimeout = 30 + ReadHeaderTimeout = 2 + shutdownTimeout = 5 + httpErrorExitCode = 4 +) + +type Component struct { + config config + logger *slog.Logger + server *http.Server +} + +type config struct { + metricPath string + listenAddress string +} + +func New(logger slog.Logger, listenAddress string, metricPath string) (component Component) { + component = Component{ + logger: &logger, + config: config{ + metricPath: metricPath, + listenAddress: listenAddress, + }, + } + + return +} + +func (c *Component) Start() error { + c.logger.Info("starting the HTTP server component") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + c.server = &http.Server{ + Addr: c.config.listenAddress, + ReadTimeout: ReadTimeout * time.Second, + WriteTimeout: WriteTimeout * time.Second, + IdleTimeout: IdleTimeout * time.Second, + ReadHeaderTimeout: ReadHeaderTimeout * time.Second, + BaseContext: func(_ net.Listener) context.Context { return ctx }, + } + + http.Handle("/", helloWorldhandler{}) + http.Handle(c.config.metricPath, promhttp.Handler()) + + signalChan := make(chan os.Signal, 1) + signal.Notify( + signalChan, + syscall.SIGHUP, // kill -SIGHUP XXXX + syscall.SIGINT, // kill -SIGINT XXXX or Ctrl+c + syscall.SIGQUIT, // kill -SIGQUIT XXXX + ) + + go func() { + err := c.server.ListenAndServe() + if err != nil && !errors.Is(err, http.ErrServerClosed) { + c.logger.Error("can't start web server", "reason", err) + os.Exit(httpErrorExitCode) + } + }() + + <-signalChan // Wait until program received a stop signal + + err := c.Stop() + if err != nil { + return fmt.Errorf("can't stop websserver: %w", err) + } + + return nil +} + +func (c *Component) Stop() error { + c.logger.Info("stopping the HTTP server component") + + ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout*time.Second) + defer cancel() + + err := c.server.Shutdown(ctx) + if err != nil && !errors.Is(err, http.ErrServerClosed) { + return fmt.Errorf("can't stop webserver: %w", err) + } + + c.logger.Info("HTTP server stopped") + + return nil +} diff --git a/internal/infra/logger/logger.go b/internal/infra/logger/logger.go new file mode 100644 index 0000000..84704c3 --- /dev/null +++ b/internal/infra/logger/logger.go @@ -0,0 +1,25 @@ +// Package logger implements logging methods +package logger + +import ( + "log/slog" + "os" +) + +func New(debug bool, logFormat string) (*slog.Logger, error) { + logLevel := &slog.LevelVar{} + if debug { + logLevel.Set(slog.LevelDebug) + } + + opts := slog.HandlerOptions{ + Level: logLevel, + } + + logger := slog.New(slog.NewTextHandler(os.Stdout, &opts)) + if logFormat == "json" { + logger = slog.New(slog.NewJSONHandler(os.Stdout, &opts)) + } + + return logger, nil +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..1a0b149 --- /dev/null +++ b/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/qonto/prometheus-rds-exporter/cmd" + +func main() { + cmd.Execute() +} diff --git a/scripts/helm-release.sh b/scripts/helm-release.sh new file mode 100755 index 0000000..82c4fd8 --- /dev/null +++ b/scripts/helm-release.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# Build and release helm chart if the version does not already exists in the specified AWS ECR public repository + +CHART_NAME=$1 +CHART_DIRECTORY=$2 +RELEASE_VERSION=$3 +REPOSITORY=$4 + +usage() { + echo "Usage: $0 " + exit 1 +} + +check_parameters() { + if [ -z $CHART_NAME ]; + then + echo "ERROR: Chart name must be specified" + usage + fi + + if [ -z $CHART_DIRECTORY ]; + then + echo "ERROR: Chart directory must be specified" + usage + fi + + if [ -z $RELEASE_VERSION ]; + then + echo "ERROR: Release version must be specified" + usage + fi + + if [ -z $REPOSITORY ]; + then + echo "ERROR: Repository must be specified" + usage + fi +} + +check_version_exists() { + AWS_ERROR=$(aws ecr-public describe-images --region us-east-1 --repository-name ${CHART_NAME} --image-ids imageTag=${RELEASE_VERSION} --output json 2>&1 > /dev/null) + AWS_EXIT_CODE=$? + if [ $AWS_EXIT_CODE -eq 0 ]; + then + echo "Release ${RELEASE_VERSION} already exists in AWS ECR" + exit 0 + elif [ ! $AWS_EXIT_CODE -eq 254 ]; + then + echo "Unexpected error while checking if ${RELEASE_VERSION} version exists: exit code ${AWS_EXIT_CODE}" + echo ${AWS_ERROR} + exit 1 + fi +} + +build() { + helm package ${CHART_DIRECTORY} --app-version ${RELEASE_VERSION} --version ${RELEASE_VERSION} +} + +publish() { + helm push ${CHART_NAME}-${RELEASE_VERSION}.tgz oci://public.ecr.aws/${REPOSITORY} +} + +check_parameters +check_version_exists + +set -x + +build +publish \ No newline at end of file diff --git a/scripts/prometheus/.gitignore b/scripts/prometheus/.gitignore new file mode 100644 index 0000000..b5e2e68 --- /dev/null +++ b/scripts/prometheus/.gitignore @@ -0,0 +1,2 @@ +.grafana_data +.prometheus_data diff --git a/scripts/prometheus/Dockerfile b/scripts/prometheus/Dockerfile new file mode 100644 index 0000000..bb0a701 --- /dev/null +++ b/scripts/prometheus/Dockerfile @@ -0,0 +1,33 @@ +FROM golang:1.21 AS builder + +WORKDIR /build + +COPY go.mod go.sum ./ + +RUN go mod download + +COPY . . + +RUN make build + + +FROM alpine:3.18 + +RUN apk add aws-cli curl + +ARG USER=exporter +ARG HOME=/app + +RUN addgroup -g 1001 -S exporter \ + && adduser --home /app -u 1001 -S exporter -G exporter \ + && mkdir -p /app \ + && chown exporter:exporter -R /app + +WORKDIR $HOME +USER $USER + +COPY --from=builder /build/prometheus-rds-exporter $HOME/prometheus-rds-exporter + +EXPOSE 9043 + +ENTRYPOINT [ "/app/prometheus-rds-exporter" ] \ No newline at end of file diff --git a/scripts/prometheus/docker-compose.yml b/scripts/prometheus/docker-compose.yml new file mode 100644 index 0000000..e0bf84a --- /dev/null +++ b/scripts/prometheus/docker-compose.yml @@ -0,0 +1,45 @@ +version: '3.1' + +volumes: + prometheus_data: {} + grafana_data: {} + +services: + + prometheus: + image: prom/prometheus:v2.45.0 + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + - ./.prometheus_data:/prometheus + ports: + - 9090:9090 + + grafana: + image: grafana/grafana:10.0.1 + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=hackme + volumes: + - ./.grafana_data:/var/lib/grafana + ports: + - 3000:3000 + depends_on: + - prometheus + + rds-exporter: + build: + context: ../../ + dockerfile: scripts/prometheus/Dockerfile + environment: + - AWS_SDK_LOAD_CONFIG=1 + - AWS_PROFILE=${AWS_PROFILE-} + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID-} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY-} + volumes: + - $HOME/.aws:/app/.aws:ro + - ./../../rds_exporter:/mnt/rds_exporprometheus-rds-exporterter + command: ["./prometheus-rds-exporter", "--debug"] + ports: + - 9043:9043 + depends_on: + - prometheus diff --git a/scripts/prometheus/prometheus.yml b/scripts/prometheus/prometheus.yml new file mode 100644 index 0000000..488fa8e --- /dev/null +++ b/scripts/prometheus/prometheus.yml @@ -0,0 +1,10 @@ +global: + scrape_interval: 30s # Set the scrape interval. Default is every 1 minute. + evaluation_interval: 30s # Evaluate rules every. The default is every 1 minute. + scrape_timeout: 10s + +# Scrape configuration +scrape_configs: + - job_name: 'rds-exporter' + static_configs: + - targets: ['rds-exporter:9043'] \ No newline at end of file