diff --git a/Makefile b/Makefile index a4ab1c6cc6..8740a4fd13 100644 --- a/Makefile +++ b/Makefile @@ -131,7 +131,7 @@ test: ## Run tests docker-build: env CGO_ENABLED=0 GOOS=$(DOCKER_IMAGE_GOOS) GOARCH=$(DOCKER_IMAGE_GOARCH) \ go build -o ${BIN_FILENAME} - docker build -t ${GHCR_IMG} . + docker build --platform $(DOCKER_IMAGE_GOOS)/$(DOCKER_IMAGE_GOARCH) -t ${GHCR_IMG} . .PHONY: docker-push docker-push: docker-build ## Push docker image with the manager. diff --git a/cmd/slareport.go b/cmd/slareport.go new file mode 100644 index 0000000000..1eb77813bd --- /dev/null +++ b/cmd/slareport.go @@ -0,0 +1,129 @@ +package cmd + +import ( + "time" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/vshn/appcat/pkg/slareport" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +// SlareportCMD specifies the cobra command for triggering the maintenance. +var ( + SlareportCMD = newSlareportCMD() + customer string + date string + timeRange string + promURL string + awssecretaccesskey string + awsaccesskeyid string + endpointURL string + bucket string + mimirOrg string + previousMonth bool +) + +func newSlareportCMD() *cobra.Command { + + command := &cobra.Command{ + Use: "slareport", + Short: "Generate SLA reports for customers", + Long: ` +Run the sla report for services. This will query a prometheus endpoint, generate the SLA reports and +upload them to an S3 bucket.`, + RunE: c.runReport, + } + + now := time.Now() + + viper.AutomaticEnv() + viper.SetDefault("PROM_URL", "http://localhost:9090/prometheus") + viper.SetDefault("BUCKET_NAME", "slareports") + viper.SetDefault("AWS_SECRET_ACCESS_KEY", "minioadmin") + viper.SetDefault("AWS_ACCESS_KEY_ID", "minioadmin") + viper.SetDefault("ENDPOINT_URL", "http://localhost:9000") + + command.Flags().StringVar(&customer, "customer", "", "Set the customer name to run against a specific customer") + command.Flags().StringVar(&date, "date", now.Format(time.RFC3339), "Specify the date from which the SLA should be reported") + command.Flags().StringVar(&timeRange, "range", "30d", "Range during which the sla should be evaluated") + command.Flags().StringVar(&promURL, "promurl", viper.GetString("PROM_URL"), "Prometheus URL where the SLA metrics reside. ENV: PROM_URL") + command.Flags().StringVar(&awssecretaccesskey, "awssecretaccesskey", viper.GetString("AWS_SECRET_ACCESS_KEY"), "AWS secret access key for uploading the reports to S3. ENV: AWS_SECRET_ACCESS_KEY") + command.Flags().StringVar(&awsaccesskeyid, "awsaccesskeyid", viper.GetString("AWS_ACCESS_KEY_ID"), "AWS key id for uploading the reports to S3. ENV: AWS_ACCESS_KEY_ID") + command.Flags().StringVar(&endpointURL, "endpointurl", viper.GetString("ENDPOINT_URL"), "S3 endpoint url for uploading the reports. ENV: ENDPOINT_URL") + command.Flags().StringVar(&bucket, "bucket", viper.GetString("BUCKET_NAME"), "S3 bucketname for uploading the reports. ENV: BUCKET_NAME") + command.Flags().StringVar(&mimirOrg, "mimirorg", "", "Set the X-Scope-OrgID header for mimir queries") + command.Flags().BoolVar(&previousMonth, "previousmonth", false, "Run the report for the previous month. Sets the date to the last day of the previous month and range to 30d. This takes precedence over date and range.") + + return command +} + +func (c *controller) runReport(cmd *cobra.Command, _ []string) error { + + l := log.FromContext(cmd.Context()) + + setPreviousMonth() + + metrics, err := slareport.RunQuery(cmd.Context(), promURL, timeRange, date, mimirOrg) + if err != nil { + return err + } + + uploader, err := slareport.NewPDFUploader(cmd.Context(), + endpointURL, + bucket, + awsaccesskeyid, + awssecretaccesskey, + ) + if err != nil { + return err + } + + parsedDate, err := time.Parse(time.RFC3339, date) + if err != nil { + return err + } + + for customer, instanceMetrics := range metrics { + + l.Info("Rendering PDF for customer", "customer", customer) + + renderer := slareport.SLARenderer{ + Customer: customer, + SI: instanceMetrics, + Month: parsedDate.Month(), + Year: parsedDate.Year(), + ExceptionLink: "https://products.vshn.ch/service_levels.html#_exceptions_to_availability_guarantee", + } + + pdf, err := renderer.GeneratePDF() + if err != nil { + return err + } + + l.Info("Uploading PDF", "customer", customer, "endpoint", endpointURL, "bucket", bucket) + err = uploader.Upload(cmd.Context(), slareport.PDF{ + Customer: customer, + Date: parsedDate, + PDFData: pdf, + }) + + if err != nil { + return err + } + + } + + return err +} + +func setPreviousMonth() { + if !previousMonth { + return + } + + date = time.Date(time.Now().Year(), time.Now().Month(), 0, 23, 59, 59, 00, time.UTC).Format(time.RFC3339) + + timeRange = "30d" + +} diff --git a/config/sliexporter/rbac/role.yaml b/config/sliexporter/rbac/role.yaml index f9c12bb312..08a2ba8d7f 100644 --- a/config/sliexporter/rbac/role.yaml +++ b/config/sliexporter/rbac/role.yaml @@ -19,3 +19,12 @@ rules: - vshnpostgresqls/status verbs: - get +- apiGroups: + - "" + resources: + - namespaces + - secrets + verbs: + - get + - list + - watch diff --git a/config/sliexporter/rbac/role_binding.yaml b/config/sliexporter/rbac/role_binding.yaml index 38338f238a..76122d7c3a 100644 --- a/config/sliexporter/rbac/role_binding.yaml +++ b/config/sliexporter/rbac/role_binding.yaml @@ -1,12 +1,12 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: glrf-test-appcat-sli-probe + name: appcat-sli-exporter roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: cluster-admin + name: appcat-sli-exporter subjects: - kind: ServiceAccount name: controller-manager - namespace: glrf-test + namespace: system diff --git a/config/sliexporter/rbac/service_account.yaml b/config/sliexporter/rbac/service_account.yaml index e04a4c10e1..7cd6025bfc 100644 --- a/config/sliexporter/rbac/service_account.yaml +++ b/config/sliexporter/rbac/service_account.yaml @@ -2,4 +2,4 @@ apiVersion: v1 kind: ServiceAccount metadata: name: controller-manager - namespace: glrf-test + namespace: system diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index cf808c88b6..530ee30c2b 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -14,3 +14,4 @@ * xref:explanations/apiserver/boostrap.adoc[Boostrap] * xref:explanations/comp-functions/runtime.adoc[Runtime Library] * xref:explanations/comp-functions/vshn-postgres.adoc[VSHN Postgres Functions] +* xref:explanations/slareports.adoc[] diff --git a/docs/modules/ROOT/pages/explanations/slareports.adoc b/docs/modules/ROOT/pages/explanations/slareports.adoc new file mode 100644 index 0000000000..3d9f3089d2 --- /dev/null +++ b/docs/modules/ROOT/pages/explanations/slareports.adoc @@ -0,0 +1,16 @@ += SLA Reports + +== How do they work? + +This is a simple SLA reporting tool. +It connects to a Prometheus/Thanos/Mimir instance and queries them to gather the SLA information. + +It currently requires the information provided by the AppCat SLI exporter, as well as the `appuio.io/organization` label by joining the namespace information. +It also requires the `cluster_id` label, which is currently available on the `SLO Mimir` endpoint. + +It then creates an ASCIIdoc template and sends it to VSHN's docgen. +There it will be converted to a PDF and sent back to the slareporter. +The slareporter will then upload the PDFs to the configured S3 bucket. + +By default the slareporter will take the current time and a range of 30 days to query and calculate the SLA reports. +This can be adjusted, please consult the commands help for more information. diff --git a/go.mod b/go.mod index 03c38088af..d054f4fac5 100644 --- a/go.mod +++ b/go.mod @@ -17,9 +17,11 @@ require ( github.com/golang/protobuf v1.5.2 github.com/hashicorp/go-version v1.6.0 github.com/jackc/pgx/v5 v5.3.1 + github.com/minio/minio-go/v7 v7.0.56 github.com/ory/dockertest v3.3.5+incompatible github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.63.0 github.com/prometheus/client_golang v1.14.0 + github.com/prometheus/common v0.40.0 github.com/sethvargo/go-password v0.2.0 github.com/spf13/cobra v1.6.1 github.com/spf13/viper v1.15.0 @@ -65,6 +67,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect @@ -98,13 +101,17 @@ require ( github.com/jinzhu/copier v0.3.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/labstack/echo/v4 v4.9.1 // indirect + github.com/klauspost/compress v1.16.5 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/labstack/echo/v4 v4.10.2 // indirect github.com/labstack/gommon v0.4.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -119,11 +126,11 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.40.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect + github.com/rs/xid v1.5.0 // indirect github.com/rs/zerolog v1.29.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sirupsen/logrus v1.9.0 // indirect + github.com/sirupsen/logrus v1.9.2 // indirect github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect @@ -148,14 +155,14 @@ require ( go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect - golang.org/x/crypto v0.6.0 // indirect + golang.org/x/crypto v0.9.0 // indirect golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb // indirect golang.org/x/mod v0.10.0 // indirect - golang.org/x/net v0.9.0 // indirect + golang.org/x/net v0.10.0 // indirect golang.org/x/oauth2 v0.3.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/term v0.7.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/term v0.8.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.8.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect diff --git a/go.sum b/go.sum index e7767b7eab..93444aa2c7 100644 --- a/go.sum +++ b/go.sum @@ -132,8 +132,9 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -350,6 +351,11 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -364,8 +370,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= -github.com/labstack/echo/v4 v4.9.1 h1:GliPYSpzGKlyOhqIbG8nmHBo3i1saKWFOgh41AN3b+Y= -github.com/labstack/echo/v4 v4.9.1/go.mod h1:Pop5HLc+xoc4qhTZ1ip6C0RtP7Z+4VzRLWZZFKqbbjo= +github.com/labstack/echo/v4 v4.10.2 h1:n1jAhnq/elIFTHr1EYpiYtyKgx4RW9ccVgkqByZaN2M= +github.com/labstack/echo/v4 v4.10.2/go.mod h1:OEyqf2//K1DFdE57vw2DRgWY0M7s65IVQO2FzvI4J5k= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8= github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= @@ -405,6 +411,12 @@ github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.56 h1:pkZplIEHu8vinjkmhsexcXpWth2tjVLphrTZx6fBVZY= +github.com/minio/minio-go/v7 v7.0.56/go.mod h1:NUDy4A4oXPq1l2yK6LTSvCEzAMeIcoz9lcj5dbzSrRE= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -485,6 +497,8 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w= github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -498,8 +512,8 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= +github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sljeff/oapi-codegen v1.5.1-0.20211207091501-bc20f55d338a h1:bLe3hVqcJrse1CGVS2MtkPK139RaGVdhIGY3wNb0eFQ= github.com/sljeff/oapi-codegen v1.5.1-0.20211207091501-bc20f55d338a/go.mod h1:Vid3nS1BbbSPSqaGOuMCEU1Ml8R/8VLjcyZZGDCKkkY= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= @@ -635,8 +649,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -711,8 +725,8 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -800,15 +814,16 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211031064116-611d5d643895/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/main.go b/main.go index 005407a596..57764de2fe 100644 --- a/main.go +++ b/main.go @@ -15,7 +15,8 @@ func init() { cmd.APIServerCMD, cmd.SLIProberCMD, cmd.GrpcCMD, - cmd.MaintenanceCMD) + cmd.MaintenanceCMD, + cmd.SlareportCMD) } func main() { diff --git a/pkg/slareport/prom.go b/pkg/slareport/prom.go new file mode 100644 index 0000000000..717f996888 --- /dev/null +++ b/pkg/slareport/prom.go @@ -0,0 +1,185 @@ +package slareport + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/appuio/appuio-cloud-reporting/pkg/thanos" + "github.com/prometheus/client_golang/api" + apiv1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +var ( + metricQuery = `(1 - max without(prometheus_replica) (sum_over_time(slo:sli_error:ratio_rate5m{sloth_service=~"appcat-.+"}[{{DURATION}}]) +/ ignoring (sloth_window) count_over_time(slo:sli_error:ratio_rate5m{sloth_service=~"appcat-.+"}[{{DURATION}}]) +) >= 0) * 100` + slaQuery = `100*slo:objective:ratio{sloth_id="{{SLOTHID}}"}` + promClientFunc = getPrometheusAPIClient + getMetricsFunc = getSLAMetrics + getTargetSLAFunc = getTargetSLA +) + +func getPrometheusAPIClient(promURL string, thanosAllowPartialResponses bool, orgID string) (apiv1.API, error) { + rt := api.DefaultRoundTripper + rt = &thanos.PartialResponseRoundTripper{ + RoundTripper: rt, + Allow: thanosAllowPartialResponses, + } + + if orgID != "" { + rt = &thanos.AdditionalHeadersRoundTripper{ + RoundTripper: rt, + Headers: map[string][]string{ + "X-Scope-OrgID": {orgID}, + }, + } + } + + client, err := api.NewClient(api.Config{ + Address: promURL, + RoundTripper: rt, + }) + + return apiv1.NewAPI(client), err +} + +func RunQuery(ctx context.Context, promURL, timeRange, date, mimirOrg string) (map[string][]ServiceInstance, error) { + + l := log.FromContext(ctx) + + l.Info("Starting SLA queries", "date", date, "range", timeRange) + + client, err := promClientFunc(promURL, true, mimirOrg) + if err != nil { + return nil, err + } + + l.V(1).Info("Parsing timerange", "date", date, "range", timeRange) + startDate, endDate, err := parseRange(date, timeRange) + if err != nil { + return nil, fmt.Errorf("cannot parse date or timeRange: %w", err) + } + + l.V(1).Info("Querying prometheus", "promURL", promURL) + samples, err := getMetricsFunc(ctx, startDate, endDate, timeRange, client) + if err != nil { + return nil, fmt.Errorf("error during metrics prometheus query: %w", err) + } + + slaMetrics := map[string][]ServiceInstance{} + + for _, sample := range samples { + + org := string(sample.Metric["organization"]) + if org == "" { + org = "noOrganizationInfo" + } + + l.V(1).Info("Parsing metrics", "org", org) + + slothID := string(sample.Metric["sloth_id"]) + + targetSLA, err := getTargetSLAFunc(ctx, slothID, client, endDate) + if err != nil { + return nil, fmt.Errorf("error during SLA target query: %w", err) + } + + if len(sample.Values) == 0 { + return slaMetrics, errors.New("no values available for instance " + string(sample.Metric["name"])) + } + + outcomeSLA := float64(sample.Values[len(sample.Values)-1].Value) + color := "green" + if targetSLA > outcomeSLA { + l.V(1).Info("SLA not reached", "target", targetSLA, "outcome", outcomeSLA) + color = "red" + } + + slaMetrics[org] = append(slaMetrics[org], ServiceInstance{ + Instance: string(sample.Metric["name"]), + Namespace: string(sample.Metric["namespace"]), + OutcomeSLA: outcomeSLA, + TargetSLA: targetSLA, + Service: string(sample.Metric["service"]), + Cluster: string(sample.Metric["cluster_id"]), + Color: color, + }) + } + + return slaMetrics, nil +} + +func parseRange(date, duration string) (*time.Time, *time.Time, error) { + endDate, err := time.Parse(time.RFC3339, date) + if err != nil { + return nil, nil, err + } + + parsedRange, err := model.ParseDuration(duration) + if err != nil { + return nil, nil, err + } + + parsedDuration := time.Duration(parsedRange) + + startDate := endDate.Add(-1 * parsedDuration) + + return &startDate, &endDate, nil +} + +func getSLAMetrics(ctx context.Context, startDate, endDate *time.Time, timeRange string, client apiv1.API) (model.Matrix, error) { + queryRange := apiv1.Range{ + Start: *startDate, + End: *endDate, + Step: time.Minute * 5, + } + + rangedQuery := strings.Replace(metricQuery, "{{DURATION}}", timeRange, 2) + + value, warnings, err := client.QueryRange(ctx, rangedQuery, queryRange) + if err != nil { + return nil, err + } + + if len(warnings) != 0 { + warns := strings.Join(warnings, ",") + log.FromContext(ctx).Info("There were warnings during the prom query", "warnings", warns) + } + + samples, ok := value.(model.Matrix) + if !ok { + return nil, fmt.Errorf("expected prometheus query to return a model.Matrix, got %T", value) + } + + return samples, nil +} + +func getTargetSLA(ctx context.Context, slothID string, client apiv1.API, endDate *time.Time) (float64, error) { + query := strings.Replace(slaQuery, "{{SLOTHID}}", slothID, 1) + + res, warnings, err := client.Query(ctx, query, *endDate) + if err != nil { + return 0, err + } + + if len(warnings) != 0 { + warns := strings.Join(warnings, ",") + log.FromContext(ctx).Info("There were warnings during the prom query", "warnings", warns) + } + + samples, ok := res.(model.Vector) + if !ok { + return 0, fmt.Errorf("expected prometheus query to return a model.Vector, got %T", res) + } + + if len(samples) == 0 { + return 0, errors.New("no target SLA found in prometheus") + } + + return float64(samples[len(samples)-1].Value), nil +} diff --git a/pkg/slareport/prom_test.go b/pkg/slareport/prom_test.go new file mode 100644 index 0000000000..d3ab7059b6 --- /dev/null +++ b/pkg/slareport/prom_test.go @@ -0,0 +1,211 @@ +package slareport + +import ( + "context" + "testing" + "time" + + "github.com/prometheus/client_golang/api" + apiv1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/assert" +) + +func Test_parseRange(t *testing.T) { + type args struct { + date string + duration string + } + tests := []struct { + name string + args args + startDate time.Time + endDate time.Time + wantErr bool + }{ + { + name: "GivenValidDateAndDuration_ThenExpectOutput", + args: args{ + date: "2023-06-07T09:00:00Z", + duration: "30d", + }, + endDate: time.Date(2023, 6, 7, 9, 0, 0, 0, time.UTC), + startDate: time.Date(2023, 5, 8, 9, 0, 0, 0, time.UTC), + }, + { + name: "GivenInvalidDate_ThenExpectError", + args: args{ + date: "foo", + duration: "30d", + }, + wantErr: true, + }, + { + name: "GivenInvalidDuration_ThenExpectError", + args: args{ + date: "2023-06-07T09:00:00Z", + duration: "30tage", + }, + wantErr: true, + }, + { + name: "GivenNonDefaultDuration_ThenExpectOutput", + args: args{ + date: "2023-06-07T09:00:00Z", + duration: "5m", + }, + endDate: time.Date(2023, 6, 7, 9, 0, 0, 0, time.UTC), + startDate: time.Date(2023, 6, 7, 8, 55, 0, 0, time.UTC), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + startDate, endDate, err := parseRange(tt.args.date, tt.args.duration) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.Equal(t, startDate, &tt.startDate) + assert.Equal(t, endDate, &tt.endDate) + } + }) + } +} + +func TestRunQuery(t *testing.T) { + tests := []struct { + name string + metrics model.Matrix + want map[string][]ServiceInstance + wantErr bool + }{ + { + name: "GivenAvailableMetrics_ThenExpectOutput", + want: map[string][]ServiceInstance{ + "mycustomer": { + { + Namespace: "myns", + Instance: "test", + TargetSLA: 99.9, + OutcomeSLA: 99.99, + Cluster: "mycluster", + Service: "postgresql", + Color: "green", + }, + }, + }, + metrics: model.Matrix{ + { + Metric: model.Metric{ + "name": "test", + "namespace": "myns", + "service": "postgresql", + "cluster_id": "mycluster", + "organization": "mycustomer", + }, + Values: []model.SamplePair{ + { + Timestamp: model.Time(time.Now().UnixMilli()), + Value: model.SampleValue(99.99), + }, + { + Timestamp: model.Time(time.Now().UnixMilli()), + Value: model.SampleValue(99.99), + }, + }, + }, + }, + }, + { + name: "GivenUnreachedSLA_ThenExpectColorRed", + want: map[string][]ServiceInstance{ + "mycustomer": { + { + Namespace: "myns", + Instance: "test", + TargetSLA: 99.9, + OutcomeSLA: 99.8, + Cluster: "mycluster", + Service: "postgresql", + Color: "red", + }, + }, + }, + metrics: model.Matrix{ + { + Metric: model.Metric{ + "name": "test", + "namespace": "myns", + "service": "postgresql", + "cluster_id": "mycluster", + "organization": "mycustomer", + }, + Values: []model.SamplePair{ + { + Timestamp: model.Time(time.Now().UnixMilli()), + Value: model.SampleValue(99.8), + }, + { + Timestamp: model.Time(time.Now().UnixMilli()), + Value: model.SampleValue(99.8), + }, + }, + }, + }, + }, + { + name: "GivenNoMetrics_ThenExpectEmptyOutput", + want: map[string][]ServiceInstance{}, + metrics: model.Matrix{}, + }, + { + name: "GivenNoValues_ThenExpectAnError", + want: map[string][]ServiceInstance{}, + metrics: model.Matrix{ + { + Metric: model.Metric{ + "name": "test", + "namespace": "myns", + "service": "postgresql", + "cluster_id": "mycluster", + "organization": "mycustomer", + }, + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + + promClientFunc = getDummyPromClient + getTargetSLAFunc = getDummySLA + getMetricsFunc = func(ctx context.Context, startDate, endDate *time.Time, timeRange string, client apiv1.API) (model.Matrix, error) { + return tt.metrics, nil + } + + t.Run(tt.name, func(t *testing.T) { + got, err := RunQuery(context.TODO(), "dummy", "30d", "2023-06-07T09:00:00Z", "") + if tt.wantErr { + assert.Error(t, err) + } + assert.Equal(t, tt.want, got) + }) + } + + resetGlobalFunctions() +} + +func getDummyPromClient(promURL string, thanosAllowPartialResponses bool, orgID string) (apiv1.API, error) { + client, _ := api.NewClient(api.Config{}) + + return apiv1.NewAPI(client), nil +} + +func getDummySLA(ctx context.Context, slothID string, client apiv1.API, endDate *time.Time) (float64, error) { + return 99.9, nil +} + +func resetGlobalFunctions() { + promClientFunc = getPrometheusAPIClient + getMetricsFunc = getSLAMetrics + getTargetSLAFunc = getTargetSLA +} diff --git a/pkg/reporting/render.go b/pkg/slareport/render.go similarity index 71% rename from pkg/reporting/render.go rename to pkg/slareport/render.go index 10a02ffeb6..3fb4e7979a 100644 --- a/pkg/reporting/render.go +++ b/pkg/slareport/render.go @@ -1,10 +1,12 @@ -package reporting +package slareport import ( "bytes" "embed" "encoding/json" "fmt" + "io" + "net/http" "text/template" "time" ) @@ -27,15 +29,18 @@ var appcatSLAReport = "appcat-sla-report" type ServiceInstance struct { Namespace string Instance string - TargetSLA float32 - OutcomeSLA float32 + TargetSLA float64 + OutcomeSLA float64 + Cluster string + Service string + Color string } type SLARenderer struct { Customer string - Cluster string ExceptionLink string Month time.Month + Year int SI []ServiceInstance } @@ -75,3 +80,29 @@ func (s *SLARenderer) PrepareJSONPayload() ([]byte, error) { return payload, nil } + +// GeneratePDF sends a request to docgen an returns the rendered PDF. +func (s *SLARenderer) GeneratePDF() (io.ReadCloser, error) { + + reqJson, err := s.PrepareJSONPayload() + if err != nil { + return nil, err + } + + reader := bytes.NewReader(reqJson) + + req, err := http.NewRequest("POST", "https://docgen.vshn.net/api/pdf", reader) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", "application/json") + + client := http.Client{} + res, err := client.Do(req) + if err != nil { + return nil, err + } + + return res.Body, nil +} diff --git a/pkg/reporting/render_test.go b/pkg/slareport/render_test.go similarity index 59% rename from pkg/reporting/render_test.go rename to pkg/slareport/render_test.go index f16e7a037f..88ae6ff9ef 100644 --- a/pkg/reporting/render_test.go +++ b/pkg/slareport/render_test.go @@ -1,8 +1,9 @@ -package reporting +package slareport import ( - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" ) func TestSLARenderer_RenderAsciidoc(t *testing.T) { @@ -14,7 +15,6 @@ func TestSLARenderer_RenderAsciidoc(t *testing.T) { "WhenSLARender_ThenOutput": { renderer: SLARenderer{ Customer: "TestCustomer", - Cluster: "TestCluster", ExceptionLink: "https://vshn.ch", Month: 2, SI: []ServiceInstance{ @@ -23,14 +23,15 @@ func TestSLARenderer_RenderAsciidoc(t *testing.T) { Instance: "postgres-dev", TargetSLA: 99.9, OutcomeSLA: 99.1, + Color: "red", }, }, }, - expectedAsciidoc: "= SLA Report\n\nimage::vshn.png[VSHN Logo,100,54,id=vshn_logo]\n\n*Version of the document" + - ": May 2023* +\n\n---\n\n[big]#Customer: *TestCustomer* +\nMonth: *February* +\nCluster: *TestCluster*#\n" + - "\n---\n\n[cols=\"Namespace, Instance, SLA Target, SLA Outcome\"]\n|===\n| Namespace| Instance| SLA Target|" + - " SLA Outcome\n\n|dev-namespace|postgres-dev|99.9%|*99.1%*\n\n|===\n\nNOTE: [small]#The list of exceptions " + - "which are excluded from outcome can be viewed https://vshn.ch[at].#\n", + expectedAsciidoc: "= SLA Report\n\nimage::vshn.png[VSHN Logo,100,54,id=vshn_logo]\n\n---\n\n[big]#Customer: *TestCustomer* +\nMonth: *February* +\n" + + "Year: *0*#\n\n---\n\n[cols=\"Namespace, Instance, SLA Target, SLA Outcome\"]\n|===\n|Cluster| " + + "Service | Namespace| Instance| SLA Target| SLA Outcome\n\n|||dev-namespace|postgres-dev|99.9%|[red]*99." + + "10%*\n\n|===\n\nNOTE: [small]#The list of exceptions which are excluded from outcome can be viewed " + + "https://vshn.ch[at].#\n\n", err: nil, }, } @@ -60,7 +61,6 @@ func TestSLARenderer_PrepareJSONPayload(t *testing.T) { "WhenSLARender_ThenOutputPayload": { renderer: SLARenderer{ Customer: "TestCustomer", - Cluster: "TestCluster", ExceptionLink: "https://vshn.ch", Month: 2, SI: []ServiceInstance{ @@ -69,15 +69,16 @@ func TestSLARenderer_PrepareJSONPayload(t *testing.T) { Instance: "postgres-dev", TargetSLA: 99.9, OutcomeSLA: 99.1, + Color: "red", }, }, }, - expectedPayload: "{\"asciidoc\":\"= SLA Report\\n\\nimage::vshn.png[VSHN Logo,100,54,id=vshn_logo]\\n\\n*" + - "Version of the document: May 2023* +\\n\\n---\\n\\n[big]#Customer: *TestCustomer* +\\nMonth: *February*" + - " +\\nCluster: *TestCluster*#\\n\\n---\\n\\n[cols=\\\"Namespace, Instance, SLA Target, SLA Outcome\\\"]" + - "\\n|===\\n| Namespace| Instance| SLA Target| SLA Outcome\\n\\n|dev-namespace|postgres-dev|99.9%|*99.1%*" + - "\\n\\n|===\\n\\nNOTE: [small]#The list of exceptions which are excluded from outcome can be viewed " + - "https://vshn.ch[at].#\\n\",\"vshn_docgen_id\":\"appcat-sla-report\"}", + expectedPayload: `{"asciidoc":"= SLA Report\n\nimage::vshn.png[VSHN Logo,100,54,id=vshn_logo]\n` + + `\n---\n\n[big]#Customer: *TestCustomer* +\nMonth: ` + + `*February* +\nYear: *0*#\n\n---\n\n[cols=\"Namespace, Instance, SLA Target, SLA Outcome\"]\n` + + `|===\n|Cluster| Service | Namespace| Instance| SLA Target| SLA Outcome\n\n|||dev-namespace|postgres-dev` + + `|99.9%|[red]*99.10%*\n\n|===\n\nNOTE: [small]#The list of exceptions which are excluded from outcome ` + + `can be viewed https://vshn.ch[at].#\n\n","vshn_docgen_id":"appcat-sla-report"}`, err: nil, }, } @@ -93,7 +94,7 @@ func TestSLARenderer_PrepareJSONPayload(t *testing.T) { return } assert.NoError(t, err) - assert.Equal(t, []byte(tc.expectedPayload), payload) + assert.Equal(t, tc.expectedPayload, string(payload)) }) } } diff --git a/pkg/slareport/s3.go b/pkg/slareport/s3.go new file mode 100644 index 0000000000..a1b3a34530 --- /dev/null +++ b/pkg/slareport/s3.go @@ -0,0 +1,70 @@ +package slareport + +import ( + "bytes" + "context" + "fmt" + "io" + "net/url" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +// PDFUploader uploads pdfs to the configured S3 endpoint. +type PDFUploader struct { + client *minio.Client + bucket string +} + +// PDF is contains all metainformation to upload a PDF to S3. +type PDF struct { + Customer string + Date time.Time + PDFData io.ReadCloser +} + +// NewPDFUploader initializes an S3 client. +func NewPDFUploader(ctx context.Context, endpoint, bucket, keyID, secretKey string) (PDFUploader, error) { + + log.FromContext(ctx).V(1).Info("Logging into S3 endpoint", "endpointurl", endpoint) + + p := PDFUploader{} + + url, err := url.Parse(endpoint) + if err != nil { + return p, nil + } + + S3Client, err := minio.New(url.Host, &minio.Options{ + Creds: credentials.NewStaticV4(keyID, secretKey, ""), + Secure: url.Scheme == "https", + }) + if err != nil { + return p, err + } + + p.client = S3Client + p.bucket = bucket + return p, nil +} + +// Upload uploads the given PDF to the logged in S3 enspoint. +// It will create an object with the pattern `year/month/customer.pdf`. +func (p *PDFUploader) Upload(ctx context.Context, pdf PDF) error { + + obj := fmt.Sprintf("%d/%s/%s.pdf", pdf.Date.Year(), pdf.Date.Month(), pdf.Customer) + + log.FromContext(ctx).V(1).Info("Uploading PDF", "object", obj) + + buf := &bytes.Buffer{} + size, err := io.Copy(buf, pdf.PDFData) + if err != nil { + return err + } + + _, err = p.client.PutObject(ctx, p.bucket, obj, buf, size, minio.PutObjectOptions{}) + return err +} diff --git a/pkg/reporting/template/sla-report.txt b/pkg/slareport/template/sla-report.txt similarity index 62% rename from pkg/reporting/template/sla-report.txt rename to pkg/slareport/template/sla-report.txt index 6f6f47e4e8..a5e0fa2e30 100644 --- a/pkg/reporting/template/sla-report.txt +++ b/pkg/slareport/template/sla-report.txt @@ -2,22 +2,21 @@ image::vshn.png[VSHN Logo,100,54,id=vshn_logo] -*Version of the document: May 2023* + - --- [big]#Customer: *{{.Customer}}* + Month: *{{.Month}}* + -Cluster: *{{.Cluster}}*# +Year: *{{.Year}}*# --- [cols="Namespace, Instance, SLA Target, SLA Outcome"] |=== -| Namespace| Instance| SLA Target| SLA Outcome +|Cluster| Service | Namespace| Instance| SLA Target| SLA Outcome {{range .SI}} -|{{.Namespace}}|{{.Instance}}|{{.TargetSLA}}%|*{{.OutcomeSLA}}%* +|{{.Cluster}}|{{.Service}}|{{.Namespace}}|{{.Instance}}|{{.TargetSLA}}%|[{{.Color}}]*{{printf "%.2f" .OutcomeSLA}}%* {{end}} |=== NOTE: [small]#The list of exceptions which are excluded from outcome can be viewed {{.ExceptionLink}}[at].# + diff --git a/pkg/sliexporter/probes/manager.go b/pkg/sliexporter/probes/manager.go index 9e4a50b7d7..678609aa5c 100644 --- a/pkg/sliexporter/probes/manager.go +++ b/pkg/sliexporter/probes/manager.go @@ -27,9 +27,10 @@ type Prober interface { // ProbeInfo uniquely identifies a prober and in turn an AppCat service instance type ProbeInfo struct { - Service string - Name string - Namespace string + Service string + Name string + Namespace string + Organization string } var ErrTimeout = errors.New("probe timed out") @@ -39,7 +40,7 @@ func NewManager(l logr.Logger) Manager { Name: "appcat_probes_seconds", Help: "Latency of probes to appact services", Buckets: []float64{0.001, 0.002, 0.003, 0.004, 0.005, 0.01, 0.015, 0.02, 0.025, 0.05, 0.1, .5, 1}, - }, []string{"service", "namespace", "name", "reason"}) + }, []string{"service", "namespace", "name", "reason", "organization"}) return Manager{ probers: map[ProbeInfo]context.CancelFunc{}, @@ -97,9 +98,10 @@ func (m Manager) runProbe(ctx context.Context, p Prober) { func (m Manager) sendProbe(ctx context.Context, p Prober) { pi := p.GetInfo() o, err := m.hist.CurryWith(prometheus.Labels{ - "service": pi.Service, - "namespace": pi.Namespace, - "name": pi.Name, + "service": pi.Service, + "namespace": pi.Namespace, + "name": pi.Name, + "organization": pi.Organization, }) if err != nil { return diff --git a/pkg/sliexporter/probes/postgresql.go b/pkg/sliexporter/probes/postgresql.go index 1874a6ffe4..01feeab2ef 100644 --- a/pkg/sliexporter/probes/postgresql.go +++ b/pkg/sliexporter/probes/postgresql.go @@ -16,9 +16,10 @@ var _ Prober = PostgreSQL{} type PostgreSQL struct { db *pgxpool.Pool - Service string - Instance string - Namespace string + Service string + Instance string + Namespace string + Organization string } // Close closes open connections to the PostgreSQL server. @@ -32,9 +33,10 @@ func (p PostgreSQL) Close() error { // GetInfo returns the prober infos func (p PostgreSQL) GetInfo() ProbeInfo { return ProbeInfo{ - Service: p.Service, - Name: p.Instance, - Namespace: p.Namespace, + Service: p.Service, + Name: p.Instance, + Namespace: p.Namespace, + Organization: p.Organization, } } @@ -49,7 +51,7 @@ func (p PostgreSQL) Probe(ctx context.Context) error { } // NewPostgreSQL connects to the provided dsn and returns a prober -func NewPostgreSQL(service, name, namespace, dsn string, ops ...func(*pgxpool.Config) error) (*PostgreSQL, error) { +func NewPostgreSQL(service, name, namespace, dsn, organization string, ops ...func(*pgxpool.Config) error) (*PostgreSQL, error) { conf, err := pgxpool.ParseConfig(dsn) if err != nil { return nil, err @@ -70,10 +72,11 @@ func NewPostgreSQL(service, name, namespace, dsn string, ops ...func(*pgxpool.Co } return &PostgreSQL{ - db: db, - Service: service, - Instance: name, - Namespace: namespace, + db: db, + Service: service, + Instance: name, + Namespace: namespace, + Organization: organization, }, nil } diff --git a/pkg/sliexporter/probes/postgresql_test.go b/pkg/sliexporter/probes/postgresql_test.go index 73f2bdffe6..f0071a93c6 100644 --- a/pkg/sliexporter/probes/postgresql_test.go +++ b/pkg/sliexporter/probes/postgresql_test.go @@ -40,6 +40,7 @@ func TestPostgreSQL_Probe(t *testing.T) { res.GetPort("5432/tcp"), db, ), + "test", ) return err == nil }, 2*time.Second, 500*time.Millisecond) diff --git a/pkg/sliexporter/vshnpostgresql_controller.go b/pkg/sliexporter/vshnpostgresql_controller.go index a31c996a4a..ddb076597e 100644 --- a/pkg/sliexporter/vshnpostgresql_controller.go +++ b/pkg/sliexporter/vshnpostgresql_controller.go @@ -19,9 +19,10 @@ package sliexporter import ( "context" "fmt" - "github.com/vshn/appcat/pkg/sliexporter/probes" "time" + "github.com/vshn/appcat/pkg/sliexporter/probes" + "github.com/jackc/pgx/v5/pgxpool" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -43,7 +44,7 @@ type VSHNPostgreSQLReconciler struct { ProbeManager probeManager StartupGracePeriod time.Duration - PostgreDialer func(service, name, namespace, dsn string, ops ...func(*pgxpool.Config) error) (*probes.PostgreSQL, error) + PostgreDialer func(service, name, namespace, dsn, organization string, ops ...func(*pgxpool.Config) error) (*probes.PostgreSQL, error) } type probeManager interface { @@ -120,6 +121,14 @@ func (r VSHNPostgreSQLReconciler) fetchProberFor(ctx context.Context, inst *vshn return nil, err } + ns := &corev1.Namespace{} + err = r.Get(ctx, types.NamespacedName{Name: inst.Namespace}, ns) + if err != nil { + return nil, err + } + + org := ns.GetLabels()["appuio.io/organization"] + probe, err := r.PostgreDialer(vshnpostgresqlsServiceKey, inst.Name, inst.Namespace, fmt.Sprintf( "postgresql://%s:%s@%s:%s/%s?sslmode=verify-ca", @@ -128,7 +137,8 @@ func (r VSHNPostgreSQLReconciler) fetchProberFor(ctx context.Context, inst *vshn credSecret.Data["POSTGRESQL_HOST"], credSecret.Data["POSTGRESQL_PORT"], credSecret.Data["POSTGRESQL_DB"], - ), probes.PGWithCA(credSecret.Data["ca.crt"])) + ), org, + probes.PGWithCA(credSecret.Data["ca.crt"])) if err != nil { return nil, err } diff --git a/pkg/sliexporter/vshnpostgresql_controller_test.go b/pkg/sliexporter/vshnpostgresql_controller_test.go index bb3425d786..9cb4c90b0c 100644 --- a/pkg/sliexporter/vshnpostgresql_controller_test.go +++ b/pkg/sliexporter/vshnpostgresql_controller_test.go @@ -2,10 +2,11 @@ package sliexporter import ( "context" - "github.com/vshn/appcat/pkg/sliexporter/probes" "testing" "time" + "github.com/vshn/appcat/pkg/sliexporter/probes" + "github.com/jackc/pgx/v5/pgxpool" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -199,15 +200,24 @@ func TestVSHNPostgreSQL_PassCerdentials(t *testing.T) { r, manager, client := setupVSHNPostgreTest(t, db, cred, + &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Labels: map[string]string{ + "ogranization": "bar", + }, + }, + }, ) - r.PostgreDialer = func(service, name, namespace, dsn string, ops ...func(*pgxpool.Config) error) (*probes.PostgreSQL, error) { + r.PostgreDialer = func(service, name, namespace, dsn, organization string, ops ...func(*pgxpool.Config) error) (*probes.PostgreSQL, error) { assert.Equal(t, "VSHNPostgreSQL", service) assert.Equal(t, "foo", name) assert.Equal(t, "bar", namespace) assert.Equal(t, "postgresql://userfoo:password@foo.bar:5433/pg?sslmode=verify-ca", dsn) + assert.Equal(t, "bar", organization) - return fakePostgreDialer(service, name, namespace, dsn, ops...) + return fakePostgreDialer(service, name, namespace, dsn, organization, ops...) } req := ctrl.Request{ NamespacedName: types.NamespacedName{ @@ -231,11 +241,12 @@ func TestVSHNPostgreSQL_PassCerdentials(t *testing.T) { assert.False(t, manager.probers[pi]) } -func fakePostgreDialer(service string, name string, namespace string, dsn string, ops ...func(*pgxpool.Config) error) (*probes.PostgreSQL, error) { +func fakePostgreDialer(service string, name string, namespace string, dsn string, organization string, ops ...func(*pgxpool.Config) error) (*probes.PostgreSQL, error) { p := &probes.PostgreSQL{ - Service: service, - Instance: name, - Namespace: namespace, + Service: service, + Instance: name, + Namespace: namespace, + Organization: organization, } return p, nil }