Merge pull request #246 from vshn/change-billing

Refactor billing

apis/vshn/v1/dbaas_vshn_keycloak.go

@@ -190,6 +190,10 @@ func (v *VSHNKeycloak) GetClaimNamespace() string {
 	return v.GetLabels()["crossplane.io/claim-namespace"]
 }
 
+func (v *VSHNKeycloak) GetClaimName() string {
+	return v.GetLabels()["crossplane.io/claim-name"]
+}
+
 func (v *VSHNKeycloak) GetInstanceNamespace() string {
 	return fmt.Sprintf("vshn-keycloak-%s", v.GetName())
 }
@@ -341,3 +345,11 @@ func (v *VSHNKeycloak) GetWorkloadPodTemplateLabelsManager() PodTemplateLabelsMa
 func (v *VSHNKeycloak) GetWorkloadName() string {
 	return v.GetName() + "-keycloakx"
 }
+
+func (v *VSHNKeycloak) GetBillingName() string {
+	return "appcat-" + v.GetServiceName()
+}
+
+func (v *VSHNKeycloak) GetSLA() string {
+	return string(v.Spec.Parameters.Service.ServiceLevel)
+}

apis/vshn/v1/dbaas_vshn_mariadb.go

@@ -133,6 +133,10 @@ func (v *VSHNMariaDB) GetClaimNamespace() string {
 	return v.GetLabels()["crossplane.io/claim-namespace"]
 }
 
+func (v *VSHNMariaDB) GetClaimName() string {
+	return v.GetLabels()["crossplane.io/claim-name"]
+}
+
 func (v *VSHNMariaDB) GetInstanceNamespace() string {
 	return fmt.Sprintf("vshn-mariadb-%s", v.GetName())
 }
@@ -280,3 +284,11 @@ func (v *VSHNMariaDB) GetWorkloadPodTemplateLabelsManager() PodTemplateLabelsMan
 func (v *VSHNMariaDB) GetWorkloadName() string {
 	return v.GetName()
 }
+
+func (v *VSHNMariaDB) GetBillingName() string {
+	return "appcat-" + v.GetServiceName()
+}
+
+func (v *VSHNMariaDB) GetSLA() string {
+	return string(v.Spec.Parameters.Service.ServiceLevel)
+}

apis/vshn/v1/dbaas_vshn_postgresql.go

@@ -249,6 +249,10 @@ func (v *VSHNPostgreSQL) GetClaimNamespace() string {
 	return v.GetLabels()["crossplane.io/claim-namespace"]
 }
 
+func (v *VSHNPostgreSQL) GetClaimName() string {
+	return v.GetLabels()["crossplane.io/claim-name"]
+}
+
 // +kubebuilder:object:root=true
 
 // VSHNPostgreSQLList defines a list of VSHNPostgreSQL
@@ -405,3 +409,11 @@ func (v *VSHNPostgreSQL) GetWorkloadPodTemplateLabelsManager() PodTemplateLabels
 func (v *VSHNPostgreSQL) GetWorkloadName() string {
 	return v.GetName()
 }
+
+func (v *VSHNPostgreSQL) GetBillingName() string {
+	return "appcat-" + v.GetServiceName()
+}
+
+func (v *VSHNPostgreSQL) GetSLA() string {
+	return string(v.Spec.Parameters.Service.ServiceLevel)
+}

apis/vshn/v1/dbaas_vshn_redis.go

@@ -150,6 +150,10 @@ func (v *VSHNRedis) GetClaimNamespace() string {
 	return v.GetLabels()["crossplane.io/claim-namespace"]
 }
 
+func (v *VSHNRedis) GetClaimName() string {
+	return v.GetLabels()["crossplane.io/claim-name"]
+}
+
 // +kubebuilder:object:generate=true
 // +kubebuilder:object:root=true
 
@@ -305,3 +309,11 @@ func (v *VSHNRedis) GetWorkloadPodTemplateLabelsManager() PodTemplateLabelsManag
 func (v *VSHNRedis) GetWorkloadName() string {
 	return "redis-master"
 }
+
+func (v *VSHNRedis) GetBillingName() string {
+	return "appcat-" + v.GetServiceName()
+}
+
+func (v *VSHNRedis) GetSLA() string {
+	return string(v.Spec.Parameters.Service.ServiceLevel)
+}

apis/vshn/v1/vshn_minio.go

@@ -106,6 +106,10 @@ func (v *VSHNMinio) GetClaimNamespace() string {
 	return v.GetLabels()["crossplane.io/claim-namespace"]
 }
 
+func (v *VSHNMinio) GetClaimName() string {
+	return v.GetLabels()["crossplane.io/claim-name"]
+}
+
 func (v *VSHNMinio) GetInstanceNamespace() string {
 	return fmt.Sprintf("vshn-minio-%s", v.GetName())
 }
@@ -257,3 +261,11 @@ func (v *VSHNMinio) GetWorkloadPodTemplateLabelsManager() PodTemplateLabelsManag
 func (v *VSHNMinio) GetWorkloadName() string {
 	return v.GetName()
 }
+
+func (v *VSHNMinio) GetBillingName() string {
+	return "appcat-" + v.GetServiceName()
+}
+
+func (v *VSHNMinio) GetSLA() string {
+	return string(BestEffort)
+}

apis/vshn/v1/vshn_nextcloud.go

@@ -158,6 +158,10 @@ func (v *VSHNNextcloud) GetClaimNamespace() string {
 	return v.GetLabels()["crossplane.io/claim-namespace"]
 }
 
+func (v *VSHNNextcloud) GetClaimName() string {
+	return v.GetLabels()["crossplane.io/claim-name"]
+}
+
 func (v *VSHNNextcloud) GetInstanceNamespace() string {
 	return fmt.Sprintf("vshn-nextcloud-%s", v.GetName())
 }
@@ -311,3 +315,11 @@ func (v *VSHNNextcloud) GetWorkloadPodTemplateLabelsManager() PodTemplateLabelsM
 func (v *VSHNNextcloud) GetWorkloadName() string {
 	return v.GetName()
 }
+
+func (v *VSHNNextcloud) GetBillingName() string {
+	return "appcat-" + v.GetServiceName()
+}
+
+func (v *VSHNNextcloud) GetSLA() string {
+	return string(v.Spec.Parameters.Service.ServiceLevel)
+}

pkg/comp-functions/functions/common/billing.go

@@ -1,53 +1,113 @@
 package common
 
 import (
+	"bytes"
 	"context"
+	_ "embed"
 	"fmt"
-	v12 "github.com/crossplane/crossplane-runtime/apis/common/v1"
 	xfnproto "github.com/crossplane/function-sdk-go/proto/v1beta1"
-	xkube "github.com/vshn/appcat/v4/apis/kubernetes/v1alpha2"
+	v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	"github.com/vshn/appcat/v4/pkg/comp-functions/runtime"
-	"reflect"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	controllerruntime "sigs.k8s.io/controller-runtime"
-	"strings"
+	"text/template"
 )
 
-const billingLabel = "appcat.io/billing"
+var rawExpr = "vector({{.}})"
 
-// InjectBillingLabelToService adds billing label to a service (StatefulSet or Deployment).
-// It uses a kube Object to achieve post provisioning labelling
-func InjectBillingLabelToService(ctx context.Context, svc *runtime.ServiceRuntime, comp InfoGetter) *xfnproto.Result {
+// CreateBillingRecord creates a new prometheus rule per each instance namespace
+// The rule is skipped for any secondary service such as postgresql instance for nextcloud
+// The skipping is based on whether label appuio.io/billing-name is set or not on instance namespace
+func CreateBillingRecord(ctx context.Context, svc *runtime.ServiceRuntime, comp InfoGetter) *xfnproto.Result {
 	log := controllerruntime.LoggerFrom(ctx)
 	log.Info("Enabling billing for service", "service", comp.GetName())
 
-	s := comp.GetWorkloadPodTemplateLabelsManager()
-	s.SetName(comp.GetWorkloadName())
-	s.SetNamespace(comp.GetInstanceNamespace())
-	kubeName := comp.GetName() + "-" + getType(s)
+	expr, err := getExprFromTemplate(comp.GetInstances())
+	if err != nil {
+		runtime.NewWarningResult(fmt.Sprintf("cannot add billing to service %s", comp.GetName()))
+	}
+
+	org, err := getOrg(comp.GetName(), svc)
+	if err != nil {
+		log.Error(err, "billing not working, cannot get organization", "service", comp.GetName())
+		return runtime.NewWarningResult(fmt.Sprintf("cannot add billing to service %s", comp.GetName()))
+	}
+
+	controlNS, ok := svc.Config.Data["controlNamespace"]
+	if !ok {
+		log.Error(err, "billing not working, control namespace missing", "service", comp.GetName())
+		return runtime.NewWarningResult(fmt.Sprintf("cannot add billing to service %s", comp.GetName()))
+	}
+
+	disabled, err := isBillingDisabled(controlNS, comp.GetInstanceNamespace(), comp.GetName(), svc)
+	if err != nil {
+		log.Error(err, "billing not working, cannot determine if primary service", "service", comp.GetName())
+		return runtime.NewWarningResult(fmt.Sprintf("cannot add billing to service %s", comp.GetName()))
+	}
+
+	if disabled {
+		log.Info("secondary service, skipping billing", "service", comp.GetName())
+		return runtime.NewNormalResult(fmt.Sprintf("billing disabled for instance %s", comp.GetName()))
+	}
 
-	_ = svc.GetObservedKubeObject(s, kubeName)
-	mp := v12.ManagementPolicies{v12.ManagementActionObserve}
-	labels := s.GetPodTemplateLabels()
-	_, exists := labels[billingLabel]
-	if !s.GetCreationTimestamp().Time.IsZero() {
-		if !exists {
-			labels[billingLabel] = "true"
-			s.SetPodTemplateLabels(labels)
-			mp = append(mp, v12.ManagementActionCreate, v12.ManagementActionUpdate)
-		}
+	p := &v1.PrometheusRule{
+		Spec: v1.PrometheusRuleSpec{
+			Groups: []v1.RuleGroup{
+				{
+					Name: "appcat-metering-rules",
+					Rules: []v1.Rule{
+						{
+							Record: "appcat:metering",
+							Expr:   intstr.FromString(expr),
+							Labels: getLabels(org, comp, svc),
+						},
+					},
+				},
+			},
+		},
 	}
+	p.SetName(comp.GetName() + "-billing")
+	p.SetNamespace(comp.GetInstanceNamespace())
+	kubeName := comp.GetName() + "-billing"
 
-	err := svc.SetDesiredKubeObject(s.GetObject(), kubeName, func(obj *xkube.Object) {
-		obj.Spec.ManagementPolicies = mp
-	})
+	err = svc.SetDesiredKubeObject(p, kubeName)
 
-	if err != nil && !exists {
-		runtime.NewWarningResult(fmt.Sprintf("cannot add billing to service object %s", s.GetName()))
+	if err != nil {
+		log.Error(err, "cannot add billing to service, cannot set desired object", "service", comp.GetName())
+		return runtime.NewWarningResult(fmt.Sprintf("cannot add billing to service %s", p.GetName()))
 	}
 
-	return runtime.NewNormalResult("billing enabled")
+	return runtime.NewNormalResult(fmt.Sprintf("billing enabled for instance %s", comp.GetName()))
 }
 
-func getType(myvar interface{}) (res string) {
-	return strings.ToLower(reflect.TypeOf(myvar).Elem().Field(0).Name)
+func getLabels(org string, comp InfoGetter, svc *runtime.ServiceRuntime) map[string]string {
+	labels := map[string]string{
+		"label_appcat_vshn_io_claim_name":      comp.GetClaimName(),
+		"label_appcat_vshn_io_claim_namespace": comp.GetClaimNamespace(),
+		"label_appcat_vshn_io_sla":             comp.GetSLA(),
+		"label_appuio_io_billing_name":         comp.GetBillingName(),
+		"label_appuio_io_organization":         org,
+	}
+
+	so := svc.Config.Data["salesOrder"]
+	// if appuio managed then add the sales order
+	if so != "" {
+		labels["sales_order"] = so
+	}
+	return labels
+}
+
+func getExprFromTemplate(i int) (string, error) {
+	var buf bytes.Buffer
+	tmpl, err := template.New("billing").Parse(rawExpr)
+	if err != nil {
+		return "", err
+	}
+
+	err = tmpl.Execute(&buf, i)
+	if err != nil {
+		return "", err
+	}
+
+	return buf.String(), err
 }

pkg/comp-functions/functions/common/instance_namespace.go

@@ -29,6 +29,7 @@ func BootstrapInstanceNs(ctx context.Context, comp Composite, serviceName, names
 	compositionName := comp.GetName()
 	instanceNs := comp.GetInstanceNamespace()
 	claimName, ok := comp.GetLabels()[claimNameLabel]
+	billingName := comp.GetBillingName()
 	if !ok {
 		return errors.New("no claim name available in composite labels")
 	}
@@ -40,7 +41,7 @@ func BootstrapInstanceNs(ctx context.Context, comp Composite, serviceName, names
 	}
 
 	l.Info("Creating namespace for " + serviceName + " instance")
-	err = createInstanceNamespace(serviceName, compositionName, claimNs, instanceNs, namespaceResName, claimName, svc)
+	err = createInstanceNamespace(serviceName, compositionName, claimNs, instanceNs, namespaceResName, claimName, billingName, svc)
 	if err != nil {
 		return fmt.Errorf("cannot create %s namespace: %w", serviceName, err)
 	}
@@ -93,7 +94,7 @@ func createNamespaceObserver(claimNs string, instance string, svc *runtime.Servi
 }
 
 // Create the namespace for the service instance
-func createInstanceNamespace(serviceName, compName, claimNamespace, instanceNamespace, namespaceResName, claimName string, svc *runtime.ServiceRuntime) error {
+func createInstanceNamespace(serviceName, compName, claimNamespace, instanceNamespace, namespaceResName, claimName, billingName string, svc *runtime.ServiceRuntime) error {
 
 	org, err := getOrg(compName, svc)
 	if err != nil {
@@ -111,12 +112,13 @@ func createInstanceNamespace(serviceName, compName, claimNamespace, instanceName
 		ObjectMeta: metav1.ObjectMeta{
 			Name: instanceNamespace,
 			Labels: map[string]string{
-				"appcat.vshn.io/servicename":     serviceName + "-" + mode,
-				"appcat.vshn.io/claim-namespace": claimNamespace,
-				"appcat.vshn.io/claim-name":      claimName,
-				"appuio.io/no-rbac-creation":     "true",
-				"appuio.io/billing-name":         "appcat-" + serviceName,
-				"appuio.io/organization":         org,
+				"appcat.vshn.io/servicename":      serviceName + "-" + mode,
+				"appcat.vshn.io/claim-namespace":  claimNamespace,
+				"appcat.vshn.io/claim-name":       claimName,
+				"appuio.io/no-rbac-creation":      "true",
+				"appuio.io/billing-name":          billingName,
+				"appuio.io/organization":          org,
+				"openshift.io/cluster-monitoring": "true",
 			},
 		},
 	}

pkg/comp-functions/functions/common/interfaces.go

@@ -20,6 +20,9 @@ type InfoGetter interface {
 	GetPDBLabels() map[string]string
 	GetWorkloadPodTemplateLabelsManager() vshnv1.PodTemplateLabelsManager
 	GetWorkloadName() string
+	GetClaimName() string
+	GetSLA() string
+	GetBillingName() string
 }
 
 // InstanceNamespaceInfo provides all the necessary information to create

pkg/comp-functions/functions/vshnkeycloak/billing.go

@@ -16,5 +16,5 @@ func AddServiceBillingLabel(ctx context.Context, comp *v1.VSHNKeycloak, svc *run
 		return runtime.NewFatalResult(fmt.Errorf("can't get composite: %w", err))
 	}
 
-	return common.InjectBillingLabelToService(ctx, svc, comp)
+	return common.CreateBillingRecord(ctx, svc, comp)
 }

pkg/comp-functions/functions/vshnmariadb/billing.go

@@ -16,5 +16,5 @@ func AddServiceBillingLabel(ctx context.Context, comp *v1.VSHNMariaDB, svc *runt
 		return runtime.NewFatalResult(fmt.Errorf("can't get composite: %w", err))
 	}
 
-	return common.InjectBillingLabelToService(ctx, svc, comp)
+	return common.CreateBillingRecord(ctx, svc, comp)
 }

pkg/comp-functions/functions/vshnmariadb/mariadb_deploy.go

@@ -48,7 +48,7 @@ func DeployMariadb(ctx context.Context, comp *vshnv1.VSHNMariaDB, svc *runtime.S
 	}
 
 	l.Info("Bootstrapping instance namespace and rbac rules")
-	err = common.BootstrapInstanceNs(ctx, comp, "mariadb", comp.GetName()+"-instanceNs", svc)
+	err = common.BootstrapInstanceNs(ctx, comp, comp.GetServiceName(), comp.GetName()+"-instanceNs", svc)
 	if err != nil {
 		return runtime.NewWarningResult(fmt.Errorf("cannot bootstrap instance namespace: %w", err).Error())
 	}

pkg/comp-functions/functions/vshnminio/billing.go

@@ -16,5 +16,5 @@ func AddServiceBillingLabel(ctx context.Context, comp *v1.VSHNMinio, svc *runtim
 		return runtime.NewFatalResult(fmt.Errorf("can't get composite: %w", err))
 	}
 
-	return common.InjectBillingLabelToService(ctx, svc, comp)
+	return common.CreateBillingRecord(ctx, svc, comp)
 }

pkg/comp-functions/functions/vshnnextcloud/billing.go

@@ -11,10 +11,11 @@ import (
 
 // AddServiceBillingLabel adds billingLabel to all the objects of a services that must be billed
 func AddServiceBillingLabel(ctx context.Context, comp *v1.VSHNNextcloud, svc *runtime.ServiceRuntime) *xfnproto.Result {
+
 	err := svc.GetObservedComposite(comp)
 	if err != nil {
 		return runtime.NewFatalResult(fmt.Errorf("can't get composite: %w", err))
 	}
 
-	return common.InjectBillingLabelToService(ctx, svc, comp)
+	return common.CreateBillingRecord(ctx, svc, comp)
 }

pkg/comp-functions/functions/vshnpostgres/billing.go

@@ -16,5 +16,5 @@ func AddServiceBillingLabel(ctx context.Context, comp *v1.VSHNPostgreSQL, svc *r
 		return runtime.NewFatalResult(fmt.Errorf("can't get composite: %w", err))
 	}
 
-	return common.InjectBillingLabelToService(ctx, svc, comp)
+	return common.CreateBillingRecord(ctx, svc, comp)
 }

pkg/comp-functions/functions/vshnredis/billing.go

@@ -16,5 +16,5 @@ func AddServiceBillingLabel(ctx context.Context, comp *v1.VSHNRedis, svc *runtim
 		return runtime.NewFatalResult(fmt.Errorf("can't get composite: %w", err))
 	}
 
-	return common.InjectBillingLabelToService(ctx, svc, comp)
+	return common.CreateBillingRecord(ctx, svc, comp)
 }