diff --git a/apis/vshn/v1/dbaas_vshn_keycloak.go b/apis/vshn/v1/dbaas_vshn_keycloak.go index f75d5d2d6a..18c533fb1e 100644 --- a/apis/vshn/v1/dbaas_vshn_keycloak.go +++ b/apis/vshn/v1/dbaas_vshn_keycloak.go @@ -190,6 +190,10 @@ func (v *VSHNKeycloak) GetClaimNamespace() string { return v.GetLabels()["crossplane.io/claim-namespace"] } +func (v *VSHNKeycloak) GetClaimName() string { + return v.GetLabels()["crossplane.io/claim-name"] +} + func (v *VSHNKeycloak) GetInstanceNamespace() string { return fmt.Sprintf("vshn-keycloak-%s", v.GetName()) } @@ -341,3 +345,11 @@ func (v *VSHNKeycloak) GetWorkloadPodTemplateLabelsManager() PodTemplateLabelsMa func (v *VSHNKeycloak) GetWorkloadName() string { return v.GetName() + "-keycloakx" } + +func (v *VSHNKeycloak) GetBillingName() string { + return "appcat-" + v.GetServiceName() +} + +func (v *VSHNKeycloak) GetSLA() string { + return string(v.Spec.Parameters.Service.ServiceLevel) +} diff --git a/apis/vshn/v1/dbaas_vshn_mariadb.go b/apis/vshn/v1/dbaas_vshn_mariadb.go index dfe3d6a84e..66ca5e79fe 100644 --- a/apis/vshn/v1/dbaas_vshn_mariadb.go +++ b/apis/vshn/v1/dbaas_vshn_mariadb.go @@ -133,6 +133,10 @@ func (v *VSHNMariaDB) GetClaimNamespace() string { return v.GetLabels()["crossplane.io/claim-namespace"] } +func (v *VSHNMariaDB) GetClaimName() string { + return v.GetLabels()["crossplane.io/claim-name"] +} + func (v *VSHNMariaDB) GetInstanceNamespace() string { return fmt.Sprintf("vshn-mariadb-%s", v.GetName()) } @@ -280,3 +284,11 @@ func (v *VSHNMariaDB) GetWorkloadPodTemplateLabelsManager() PodTemplateLabelsMan func (v *VSHNMariaDB) GetWorkloadName() string { return v.GetName() } + +func (v *VSHNMariaDB) GetBillingName() string { + return "appcat-" + v.GetServiceName() +} + +func (v *VSHNMariaDB) GetSLA() string { + return string(v.Spec.Parameters.Service.ServiceLevel) +} diff --git a/apis/vshn/v1/dbaas_vshn_postgresql.go b/apis/vshn/v1/dbaas_vshn_postgresql.go index 285e997c36..b0b340a289 100644 --- a/apis/vshn/v1/dbaas_vshn_postgresql.go +++ b/apis/vshn/v1/dbaas_vshn_postgresql.go @@ -249,6 +249,10 @@ func (v *VSHNPostgreSQL) GetClaimNamespace() string { return v.GetLabels()["crossplane.io/claim-namespace"] } +func (v *VSHNPostgreSQL) GetClaimName() string { + return v.GetLabels()["crossplane.io/claim-name"] +} + // +kubebuilder:object:root=true // VSHNPostgreSQLList defines a list of VSHNPostgreSQL @@ -405,3 +409,11 @@ func (v *VSHNPostgreSQL) GetWorkloadPodTemplateLabelsManager() PodTemplateLabels func (v *VSHNPostgreSQL) GetWorkloadName() string { return v.GetName() } + +func (v *VSHNPostgreSQL) GetBillingName() string { + return "appcat-" + v.GetServiceName() +} + +func (v *VSHNPostgreSQL) GetSLA() string { + return string(v.Spec.Parameters.Service.ServiceLevel) +} diff --git a/apis/vshn/v1/dbaas_vshn_redis.go b/apis/vshn/v1/dbaas_vshn_redis.go index 3703fb2cb2..f8dafcd250 100644 --- a/apis/vshn/v1/dbaas_vshn_redis.go +++ b/apis/vshn/v1/dbaas_vshn_redis.go @@ -150,6 +150,10 @@ func (v *VSHNRedis) GetClaimNamespace() string { return v.GetLabels()["crossplane.io/claim-namespace"] } +func (v *VSHNRedis) GetClaimName() string { + return v.GetLabels()["crossplane.io/claim-name"] +} + // +kubebuilder:object:generate=true // +kubebuilder:object:root=true @@ -305,3 +309,11 @@ func (v *VSHNRedis) GetWorkloadPodTemplateLabelsManager() PodTemplateLabelsManag func (v *VSHNRedis) GetWorkloadName() string { return "redis-master" } + +func (v *VSHNRedis) GetBillingName() string { + return "appcat-" + v.GetServiceName() +} + +func (v *VSHNRedis) GetSLA() string { + return string(v.Spec.Parameters.Service.ServiceLevel) +} diff --git a/apis/vshn/v1/vshn_minio.go b/apis/vshn/v1/vshn_minio.go index e18726728b..881386ebc5 100644 --- a/apis/vshn/v1/vshn_minio.go +++ b/apis/vshn/v1/vshn_minio.go @@ -106,6 +106,10 @@ func (v *VSHNMinio) GetClaimNamespace() string { return v.GetLabels()["crossplane.io/claim-namespace"] } +func (v *VSHNMinio) GetClaimName() string { + return v.GetLabels()["crossplane.io/claim-name"] +} + func (v *VSHNMinio) GetInstanceNamespace() string { return fmt.Sprintf("vshn-minio-%s", v.GetName()) } @@ -257,3 +261,11 @@ func (v *VSHNMinio) GetWorkloadPodTemplateLabelsManager() PodTemplateLabelsManag func (v *VSHNMinio) GetWorkloadName() string { return v.GetName() } + +func (v *VSHNMinio) GetBillingName() string { + return "appcat-" + v.GetServiceName() +} + +func (v *VSHNMinio) GetSLA() string { + return string(BestEffort) +} diff --git a/apis/vshn/v1/vshn_nextcloud.go b/apis/vshn/v1/vshn_nextcloud.go index 4b846a49a0..6701022f2a 100644 --- a/apis/vshn/v1/vshn_nextcloud.go +++ b/apis/vshn/v1/vshn_nextcloud.go @@ -158,6 +158,10 @@ func (v *VSHNNextcloud) GetClaimNamespace() string { return v.GetLabels()["crossplane.io/claim-namespace"] } +func (v *VSHNNextcloud) GetClaimName() string { + return v.GetLabels()["crossplane.io/claim-name"] +} + func (v *VSHNNextcloud) GetInstanceNamespace() string { return fmt.Sprintf("vshn-nextcloud-%s", v.GetName()) } @@ -311,3 +315,11 @@ func (v *VSHNNextcloud) GetWorkloadPodTemplateLabelsManager() PodTemplateLabelsM func (v *VSHNNextcloud) GetWorkloadName() string { return v.GetName() } + +func (v *VSHNNextcloud) GetBillingName() string { + return "appcat-" + v.GetServiceName() +} + +func (v *VSHNNextcloud) GetSLA() string { + return string(v.Spec.Parameters.Service.ServiceLevel) +} diff --git a/pkg/comp-functions/functions/common/billing.go b/pkg/comp-functions/functions/common/billing.go index d4f5d74a79..0a6de133b4 100644 --- a/pkg/comp-functions/functions/common/billing.go +++ b/pkg/comp-functions/functions/common/billing.go @@ -1,53 +1,113 @@ package common import ( + "bytes" "context" + _ "embed" "fmt" - v12 "github.com/crossplane/crossplane-runtime/apis/common/v1" xfnproto "github.com/crossplane/function-sdk-go/proto/v1beta1" - xkube "github.com/vshn/appcat/v4/apis/kubernetes/v1alpha2" + v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "github.com/vshn/appcat/v4/pkg/comp-functions/runtime" - "reflect" + "k8s.io/apimachinery/pkg/util/intstr" controllerruntime "sigs.k8s.io/controller-runtime" - "strings" + "text/template" ) -const billingLabel = "appcat.io/billing" +var rawExpr = "vector({{.}})" -// InjectBillingLabelToService adds billing label to a service (StatefulSet or Deployment). -// It uses a kube Object to achieve post provisioning labelling -func InjectBillingLabelToService(ctx context.Context, svc *runtime.ServiceRuntime, comp InfoGetter) *xfnproto.Result { +// CreateBillingRecord creates a new prometheus rule per each instance namespace +// The rule is skipped for any secondary service such as postgresql instance for nextcloud +// The skipping is based on whether label appuio.io/billing-name is set or not on instance namespace +func CreateBillingRecord(ctx context.Context, svc *runtime.ServiceRuntime, comp InfoGetter) *xfnproto.Result { log := controllerruntime.LoggerFrom(ctx) log.Info("Enabling billing for service", "service", comp.GetName()) - s := comp.GetWorkloadPodTemplateLabelsManager() - s.SetName(comp.GetWorkloadName()) - s.SetNamespace(comp.GetInstanceNamespace()) - kubeName := comp.GetName() + "-" + getType(s) + expr, err := getExprFromTemplate(comp.GetInstances()) + if err != nil { + runtime.NewWarningResult(fmt.Sprintf("cannot add billing to service %s", comp.GetName())) + } + + org, err := getOrg(comp.GetName(), svc) + if err != nil { + log.Error(err, "billing not working, cannot get organization", "service", comp.GetName()) + return runtime.NewWarningResult(fmt.Sprintf("cannot add billing to service %s", comp.GetName())) + } + + controlNS, ok := svc.Config.Data["controlNamespace"] + if !ok { + log.Error(err, "billing not working, control namespace missing", "service", comp.GetName()) + return runtime.NewWarningResult(fmt.Sprintf("cannot add billing to service %s", comp.GetName())) + } + + disabled, err := isBillingDisabled(controlNS, comp.GetInstanceNamespace(), comp.GetName(), svc) + if err != nil { + log.Error(err, "billing not working, cannot determine if primary service", "service", comp.GetName()) + return runtime.NewWarningResult(fmt.Sprintf("cannot add billing to service %s", comp.GetName())) + } + + if disabled { + log.Info("secondary service, skipping billing", "service", comp.GetName()) + return runtime.NewNormalResult(fmt.Sprintf("billing disabled for instance %s", comp.GetName())) + } - _ = svc.GetObservedKubeObject(s, kubeName) - mp := v12.ManagementPolicies{v12.ManagementActionObserve} - labels := s.GetPodTemplateLabels() - _, exists := labels[billingLabel] - if !s.GetCreationTimestamp().Time.IsZero() { - if !exists { - labels[billingLabel] = "true" - s.SetPodTemplateLabels(labels) - mp = append(mp, v12.ManagementActionCreate, v12.ManagementActionUpdate) - } + p := &v1.PrometheusRule{ + Spec: v1.PrometheusRuleSpec{ + Groups: []v1.RuleGroup{ + { + Name: "appcat-metering-rules", + Rules: []v1.Rule{ + { + Record: "appcat:metering", + Expr: intstr.FromString(expr), + Labels: getLabels(org, comp, svc), + }, + }, + }, + }, + }, } + p.SetName(comp.GetName() + "-billing") + p.SetNamespace(comp.GetInstanceNamespace()) + kubeName := comp.GetName() + "-billing" - err := svc.SetDesiredKubeObject(s.GetObject(), kubeName, func(obj *xkube.Object) { - obj.Spec.ManagementPolicies = mp - }) + err = svc.SetDesiredKubeObject(p, kubeName) - if err != nil && !exists { - runtime.NewWarningResult(fmt.Sprintf("cannot add billing to service object %s", s.GetName())) + if err != nil { + log.Error(err, "cannot add billing to service, cannot set desired object", "service", comp.GetName()) + return runtime.NewWarningResult(fmt.Sprintf("cannot add billing to service %s", p.GetName())) } - return runtime.NewNormalResult("billing enabled") + return runtime.NewNormalResult(fmt.Sprintf("billing enabled for instance %s", comp.GetName())) } -func getType(myvar interface{}) (res string) { - return strings.ToLower(reflect.TypeOf(myvar).Elem().Field(0).Name) +func getLabels(org string, comp InfoGetter, svc *runtime.ServiceRuntime) map[string]string { + labels := map[string]string{ + "label_appcat_vshn_io_claim_name": comp.GetClaimName(), + "label_appcat_vshn_io_claim_namespace": comp.GetClaimNamespace(), + "label_appcat_vshn_io_sla": comp.GetSLA(), + "label_appuio_io_billing_name": comp.GetBillingName(), + "label_appuio_io_organization": org, + } + + so := svc.Config.Data["salesOrder"] + // if appuio managed then add the sales order + if so != "" { + labels["sales_order"] = so + } + return labels +} + +func getExprFromTemplate(i int) (string, error) { + var buf bytes.Buffer + tmpl, err := template.New("billing").Parse(rawExpr) + if err != nil { + return "", err + } + + err = tmpl.Execute(&buf, i) + if err != nil { + return "", err + } + + return buf.String(), err } diff --git a/pkg/comp-functions/functions/common/instance_namespace.go b/pkg/comp-functions/functions/common/instance_namespace.go index 5f01906237..84b9c75ba3 100644 --- a/pkg/comp-functions/functions/common/instance_namespace.go +++ b/pkg/comp-functions/functions/common/instance_namespace.go @@ -29,6 +29,7 @@ func BootstrapInstanceNs(ctx context.Context, comp Composite, serviceName, names compositionName := comp.GetName() instanceNs := comp.GetInstanceNamespace() claimName, ok := comp.GetLabels()[claimNameLabel] + billingName := comp.GetBillingName() if !ok { return errors.New("no claim name available in composite labels") } @@ -40,7 +41,7 @@ func BootstrapInstanceNs(ctx context.Context, comp Composite, serviceName, names } l.Info("Creating namespace for " + serviceName + " instance") - err = createInstanceNamespace(serviceName, compositionName, claimNs, instanceNs, namespaceResName, claimName, svc) + err = createInstanceNamespace(serviceName, compositionName, claimNs, instanceNs, namespaceResName, claimName, billingName, svc) if err != nil { return fmt.Errorf("cannot create %s namespace: %w", serviceName, err) } @@ -93,7 +94,7 @@ func createNamespaceObserver(claimNs string, instance string, svc *runtime.Servi } // Create the namespace for the service instance -func createInstanceNamespace(serviceName, compName, claimNamespace, instanceNamespace, namespaceResName, claimName string, svc *runtime.ServiceRuntime) error { +func createInstanceNamespace(serviceName, compName, claimNamespace, instanceNamespace, namespaceResName, claimName, billingName string, svc *runtime.ServiceRuntime) error { org, err := getOrg(compName, svc) if err != nil { @@ -111,12 +112,13 @@ func createInstanceNamespace(serviceName, compName, claimNamespace, instanceName ObjectMeta: metav1.ObjectMeta{ Name: instanceNamespace, Labels: map[string]string{ - "appcat.vshn.io/servicename": serviceName + "-" + mode, - "appcat.vshn.io/claim-namespace": claimNamespace, - "appcat.vshn.io/claim-name": claimName, - "appuio.io/no-rbac-creation": "true", - "appuio.io/billing-name": "appcat-" + serviceName, - "appuio.io/organization": org, + "appcat.vshn.io/servicename": serviceName + "-" + mode, + "appcat.vshn.io/claim-namespace": claimNamespace, + "appcat.vshn.io/claim-name": claimName, + "appuio.io/no-rbac-creation": "true", + "appuio.io/billing-name": billingName, + "appuio.io/organization": org, + "openshift.io/cluster-monitoring": "true", }, }, } diff --git a/pkg/comp-functions/functions/common/interfaces.go b/pkg/comp-functions/functions/common/interfaces.go index 31ba70c5f8..f4ff98b8ed 100644 --- a/pkg/comp-functions/functions/common/interfaces.go +++ b/pkg/comp-functions/functions/common/interfaces.go @@ -20,6 +20,9 @@ type InfoGetter interface { GetPDBLabels() map[string]string GetWorkloadPodTemplateLabelsManager() vshnv1.PodTemplateLabelsManager GetWorkloadName() string + GetClaimName() string + GetSLA() string + GetBillingName() string } // InstanceNamespaceInfo provides all the necessary information to create diff --git a/pkg/comp-functions/functions/vshnkeycloak/billing.go b/pkg/comp-functions/functions/vshnkeycloak/billing.go index 0d81ee5d79..b3594b4951 100644 --- a/pkg/comp-functions/functions/vshnkeycloak/billing.go +++ b/pkg/comp-functions/functions/vshnkeycloak/billing.go @@ -16,5 +16,5 @@ func AddServiceBillingLabel(ctx context.Context, comp *v1.VSHNKeycloak, svc *run return runtime.NewFatalResult(fmt.Errorf("can't get composite: %w", err)) } - return common.InjectBillingLabelToService(ctx, svc, comp) + return common.CreateBillingRecord(ctx, svc, comp) } diff --git a/pkg/comp-functions/functions/vshnmariadb/billing.go b/pkg/comp-functions/functions/vshnmariadb/billing.go index ed50df3866..96e8ad8a69 100644 --- a/pkg/comp-functions/functions/vshnmariadb/billing.go +++ b/pkg/comp-functions/functions/vshnmariadb/billing.go @@ -16,5 +16,5 @@ func AddServiceBillingLabel(ctx context.Context, comp *v1.VSHNMariaDB, svc *runt return runtime.NewFatalResult(fmt.Errorf("can't get composite: %w", err)) } - return common.InjectBillingLabelToService(ctx, svc, comp) + return common.CreateBillingRecord(ctx, svc, comp) } diff --git a/pkg/comp-functions/functions/vshnmariadb/mariadb_deploy.go b/pkg/comp-functions/functions/vshnmariadb/mariadb_deploy.go index 82d1adda92..fd200c0c0b 100644 --- a/pkg/comp-functions/functions/vshnmariadb/mariadb_deploy.go +++ b/pkg/comp-functions/functions/vshnmariadb/mariadb_deploy.go @@ -48,7 +48,7 @@ func DeployMariadb(ctx context.Context, comp *vshnv1.VSHNMariaDB, svc *runtime.S } l.Info("Bootstrapping instance namespace and rbac rules") - err = common.BootstrapInstanceNs(ctx, comp, "mariadb", comp.GetName()+"-instanceNs", svc) + err = common.BootstrapInstanceNs(ctx, comp, comp.GetServiceName(), comp.GetName()+"-instanceNs", svc) if err != nil { return runtime.NewWarningResult(fmt.Errorf("cannot bootstrap instance namespace: %w", err).Error()) } diff --git a/pkg/comp-functions/functions/vshnminio/billing.go b/pkg/comp-functions/functions/vshnminio/billing.go index feefc28e6a..78eeaacd00 100644 --- a/pkg/comp-functions/functions/vshnminio/billing.go +++ b/pkg/comp-functions/functions/vshnminio/billing.go @@ -16,5 +16,5 @@ func AddServiceBillingLabel(ctx context.Context, comp *v1.VSHNMinio, svc *runtim return runtime.NewFatalResult(fmt.Errorf("can't get composite: %w", err)) } - return common.InjectBillingLabelToService(ctx, svc, comp) + return common.CreateBillingRecord(ctx, svc, comp) } diff --git a/pkg/comp-functions/functions/vshnnextcloud/billing.go b/pkg/comp-functions/functions/vshnnextcloud/billing.go index 0fe1d72495..d81b382778 100644 --- a/pkg/comp-functions/functions/vshnnextcloud/billing.go +++ b/pkg/comp-functions/functions/vshnnextcloud/billing.go @@ -11,10 +11,11 @@ import ( // AddServiceBillingLabel adds billingLabel to all the objects of a services that must be billed func AddServiceBillingLabel(ctx context.Context, comp *v1.VSHNNextcloud, svc *runtime.ServiceRuntime) *xfnproto.Result { + err := svc.GetObservedComposite(comp) if err != nil { return runtime.NewFatalResult(fmt.Errorf("can't get composite: %w", err)) } - return common.InjectBillingLabelToService(ctx, svc, comp) + return common.CreateBillingRecord(ctx, svc, comp) } diff --git a/pkg/comp-functions/functions/vshnpostgres/billing.go b/pkg/comp-functions/functions/vshnpostgres/billing.go index 40efc47083..3b75630fd2 100644 --- a/pkg/comp-functions/functions/vshnpostgres/billing.go +++ b/pkg/comp-functions/functions/vshnpostgres/billing.go @@ -16,5 +16,5 @@ func AddServiceBillingLabel(ctx context.Context, comp *v1.VSHNPostgreSQL, svc *r return runtime.NewFatalResult(fmt.Errorf("can't get composite: %w", err)) } - return common.InjectBillingLabelToService(ctx, svc, comp) + return common.CreateBillingRecord(ctx, svc, comp) } diff --git a/pkg/comp-functions/functions/vshnredis/billing.go b/pkg/comp-functions/functions/vshnredis/billing.go index af57a46818..1ead72e563 100644 --- a/pkg/comp-functions/functions/vshnredis/billing.go +++ b/pkg/comp-functions/functions/vshnredis/billing.go @@ -16,5 +16,5 @@ func AddServiceBillingLabel(ctx context.Context, comp *v1.VSHNRedis, svc *runtim return runtime.NewFatalResult(fmt.Errorf("can't get composite: %w", err)) } - return common.InjectBillingLabelToService(ctx, svc, comp) + return common.CreateBillingRecord(ctx, svc, comp) }