💥 Use multiple notification webhooks (#213)

config.v2alpha2.notificationWebhook is depricated
Now use config.v2alpha2.notificationWebhooks. The field
notificationWebhooks.SystemDatasourceChanged is equivalent to the old notificationWebhook.Address, but now
also supports notificationWebhooks.LibraryDatasourceChanged.

Makefile

@@ -76,15 +76,15 @@ lint: golangci-lint ## Run linters
 
 .PHONY: test
 test: ginkgo manifests generate lint envtest generate-mocks ## Run all tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GINKGO) -r -p --output-interceptor-mode=none
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GINKGO) -r --output-interceptor-mode=none
 
 .PHONY: test-unit
 test-unit: ginkgo manifests generate lint generate-mocks ## Run unit tests.
 	$(GINKGO) -r --label-filter "!integration" --output-interceptor-mode=none
 
 .PHONY: test-integration ## Run integration tests.
 test-integration: ginkgo manifests generate lint envtest generate-mocks ## Run integration tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GINKGO) -r -p --label-filter "integration" --output-interceptor-mode=none
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GINKGO) -r --label-filter "integration" --output-interceptor-mode=none
 
 .PHONY: kind-create
 kind-create: kind ## Create kind cluster

api/config/v2alpha1/projectconfig_types.go

@@ -210,8 +210,9 @@ func (c *ProjectConfig) ToV2Alpha2() *v2alpha2.ProjectConfig {
 	}
 
 	if c.NotificationWebhook != nil {
-		v2cfg.NotificationWebhook = &v2alpha2.NotificationWebhookConfig{
-			Address: c.NotificationWebhook.Address,
+		v2cfg.NotificationWebhooks = &v2alpha2.NotificationWebhooksConfig{
+			SystemDatasourceChanged:  c.NotificationWebhook.Address,
+			LibraryDatasourceChanged: "",
 		}
 	}
 

api/config/v2alpha2/projectconfig_types.go

@@ -60,7 +60,7 @@ type ProjectConfig struct {
 
 	LeaderElection *LeaderElectionConfig `json:"leaderElection"`
 
-	NotificationWebhook *NotificationWebhookConfig `json:"notificationWebhook"`
+	NotificationWebhooks *NotificationWebhooksConfig `json:"notificationWebhooks,omitempty"`
 
 	Sentry *SentryConfig `json:"sentry"`
 
@@ -116,13 +116,14 @@ type SentryConfig struct {
 	HTTPSProxy string `json:"httpsProxy"`
 }
 
-// NotificationWebhookConfig contains configuration for how to call the notification
+// NotificationWebhooksConfig contains configuration for how to call the notification
 // webhook.
-type NotificationWebhookConfig struct {
+type NotificationWebhooksConfig struct {
 	// Address is the URL to be called when the controller should do a webhook
 	// notification. Currently the only supported notification is that a
 	// datasource configuration has changed.
-	Address string `json:"address"`
+	SystemDatasourceChanged  string `json:"systemDatasourceChanged,omitempty"`
+	LibraryDatasourceChanged string `json:"libraryDatasourceChanged,omitempty"`
 }
 
 // SSOConfig contains configuration for how to use SSO tokens for determining

api/styra/v1alpha1/library_types.go

@@ -33,7 +33,7 @@ type LibrarySpec struct {
 	Subjects []LibrarySubject `json:"subjects,omitempty"`
 
 	// SourceControl is the sourcecontrol configuration for the Library
-	SourceControl *SourceControl `json:"sourceControl"`
+	SourceControl *SourceControl `json:"sourceControl,omitempty"`
 
 	// Datasources is the list of datasources in the Library
 	Datasources []LibraryDatasource `json:"datasources,omitempty"`

api/styra/v1alpha1/library_webhook.go

@@ -44,6 +44,10 @@ var _ webhook.Defaulter = &Library{}
 func (r *Library) Default() {
 	librarylog.Info("default", "name", r.Name)
 
+	if r.Spec.SourceControl == nil || r.Spec.SourceControl.LibraryOrigin == nil {
+		return
+	}
+
 	if r.Spec.SourceControl.LibraryOrigin.Commit != "" && r.Spec.SourceControl.LibraryOrigin.Reference != "" {
 		r.Spec.SourceControl.LibraryOrigin.Reference = ""
 	}

cmd/main.go

@@ -175,8 +175,8 @@ func main() {
 		Config:   ctrlConfig,
 	}
 
-	if ctrlConfig.NotificationWebhook != nil {
-		r1.WebhookClient = webhook.New(ctrlConfig.NotificationWebhook.Address)
+	if ctrlConfig.NotificationWebhooks != nil && ctrlConfig.NotificationWebhooks.SystemDatasourceChanged != "" {
+		r1.WebhookClient = webhook.New(ctrlConfig.NotificationWebhooks.SystemDatasourceChanged, "")
 	}
 
 	if err = r1.SetupWithManager(mgr); err != nil {
@@ -214,8 +214,9 @@ func main() {
 		Config: ctrlConfig,
 		Styra:  styraClient,
 	}
-	if ctrlConfig.NotificationWebhook != nil {
-		libraryReconciler.WebhookClient = webhook.New(ctrlConfig.NotificationWebhook.Address)
+
+	if ctrlConfig.NotificationWebhooks != nil && ctrlConfig.NotificationWebhooks.LibraryDatasourceChanged != "" {
+		libraryReconciler.WebhookClient = webhook.New("", ctrlConfig.NotificationWebhooks.LibraryDatasourceChanged)
 	}
 
 	if err = libraryReconciler.SetupWithManager(mgr); err != nil {

config/crd/bases/styra.bankdata.dk_libraries.yaml

@@ -112,7 +112,6 @@ spec:
             required:
             - description
             - name
-            - sourceControl
             type: object
           status:
             description: LibraryStatus defines the observed state of Library

config/default/config.yaml

@@ -10,7 +10,9 @@ leaderElection:
   leaseDuration: "60s"
   renewDeadline: "30s"
   retryPeriod: "5s"
-#notificationWebhook:
+notificationWebhooks:
+  systemDatasourceChanged: google.com
+  libraryDatasourceChanged: test.dk
 #sentry:
 #sso:
 styra:

internal/controller/styra/library_controller.go

@@ -76,20 +76,22 @@ func (r *LibraryReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 	}
 
 	log.Info("Reconciling git credentials from default credentials")
-	gitCredential := r.Config.GetGitCredentialForRepo(k8sLib.Spec.SourceControl.LibraryOrigin.URL)
-	if gitCredential == nil {
-		log.Info("Could not find matching credentials", "url", k8sLib.Spec.SourceControl.LibraryOrigin.URL)
-	} else {
-		_, err := r.Styra.CreateUpdateSecret(
-			ctx,
-			path.Join("libraries", k8sLib.Spec.Name, "git"),
-			&styra.CreateUpdateSecretsRequest{
-				Name:   gitCredential.User,
-				Secret: gitCredential.Password,
-			},
-		)
-		if err != nil {
-			return ctrl.Result{}, ctrlerr.Wrap(err, "Could not update Styra secret")
+	if k8sLib.Spec.SourceControl != nil {
+		gitCredential := r.Config.GetGitCredentialForRepo(k8sLib.Spec.SourceControl.LibraryOrigin.URL)
+		if gitCredential == nil {
+			log.Info("Could not find matching credentials", "url", k8sLib.Spec.SourceControl.LibraryOrigin.URL)
+		} else {
+			_, err := r.Styra.CreateUpdateSecret(
+				ctx,
+				path.Join("libraries", k8sLib.Spec.Name, "git"),
+				&styra.CreateUpdateSecretsRequest{
+					Name:   gitCredential.User,
+					Secret: gitCredential.Password,
+				},
+			)
+			if err != nil {
+				return ctrl.Result{}, ctrlerr.Wrap(err, "Could not update Styra secret")
+			}
 		}
 	}
 
@@ -138,20 +140,22 @@ func (r *LibraryReconciler) specToUpdate(k8sLib *styrav1alpha1.Library) *styra.U
 		return nil
 	}
 	specs := k8sLib.Spec
-	k8sSourceControl := specs.SourceControl.LibraryOrigin
-
-	sourceControl := styra.LibraryGitRepoConfig{
-		Commit:      k8sSourceControl.Commit,
-		Credentials: path.Join("libraries", specs.Name, "git"),
-		Path:        k8sSourceControl.Path,
-		Reference:   k8sSourceControl.Reference,
-		URL:         k8sSourceControl.URL,
+	req := styra.UpsertLibraryRequest{
+		Description: specs.Description,
+		ReadOnly:    true,
 	}
 
-	req := styra.UpsertLibraryRequest{
-		Description:   specs.Description,
-		ReadOnly:      true,
-		SourceControl: &styra.LibrarySourceControlConfig{LibraryOrigin: &sourceControl},
+	if specs.SourceControl != nil {
+		k8sSourceControl := specs.SourceControl.LibraryOrigin
+
+		sourceControl := styra.LibraryGitRepoConfig{
+			Commit:      k8sSourceControl.Commit,
+			Credentials: path.Join("libraries", specs.Name, "git"),
+			Path:        k8sSourceControl.Path,
+			Reference:   k8sSourceControl.Reference,
+			URL:         k8sSourceControl.URL,
+		}
+		req.SourceControl = &styra.LibrarySourceControlConfig{LibraryOrigin: &sourceControl}
 	}
 
 	return &req
@@ -171,8 +175,10 @@ func (r *LibraryReconciler) needsUpdate(k8sLib *styrav1alpha1.Library, styraLib
 		return true
 	}
 
-	if styraLib.SourceControl.LibraryOrigin.Credentials != path.Join("libraries", k8sLib.Spec.Name, "git") {
-		if r.Config.GetGitCredentialForRepo(specs.SourceControl.LibraryOrigin.URL) != nil {
+	if styraLib.SourceControl != nil &&
+		styraLib.SourceControl.LibraryOrigin.Credentials != path.Join("libraries", k8sLib.Spec.Name, "git") {
+		if specs.SourceControl != nil &&
+			r.Config.GetGitCredentialForRepo(specs.SourceControl.LibraryOrigin.URL) != nil {
 			return true
 		}
 	}
@@ -181,6 +187,10 @@ func (r *LibraryReconciler) needsUpdate(k8sLib *styrav1alpha1.Library, styraLib
 }
 
 func sameSourceControl(k8sLib *styrav1alpha1.SourceControl, styraLib *styra.LibrarySourceControlConfig) bool {
+	if k8sLib == nil || styraLib == nil {
+		return k8sLib == nil && styraLib == nil
+	}
+
 	return k8sLib.LibraryOrigin.Path == styraLib.LibraryOrigin.Path &&
 		k8sLib.LibraryOrigin.Reference == styraLib.LibraryOrigin.Reference &&
 		k8sLib.LibraryOrigin.Commit == styraLib.LibraryOrigin.Commit &&
@@ -225,7 +235,7 @@ func (r *LibraryReconciler) reconcileDatasources(ctx context.Context, log logr.L
 
 			if r.WebhookClient != nil {
 				log.Info("Calling library datasource changed webhook")
-				if err := r.WebhookClient.DatasourceChanged(ctx, log, "jwt-library", ""); err != nil {
+				if err := r.WebhookClient.LibraryDatasourceChanged(ctx, log, id); err != nil {
 					err = ctrlerr.Wrap(err, "could not call 'library datasource changed' webhook")
 					log.Error(err, err.Error())
 				}

internal/controller/styra/system_controller.go

@@ -741,7 +741,7 @@ func (r *SystemReconciler) reconcileDatasources(
 
 			if r.WebhookClient != nil {
 				log.Info("Calling datasource changed webhook")
-				if err := r.WebhookClient.DatasourceChanged(ctx, log, system.Status.ID, id); err != nil {
+				if err := r.WebhookClient.SystemDatasourceChanged(ctx, log, system.Status.ID, id); err != nil {
 					err = ctrlerr.Wrap(err, "Could not call datasource changed webhook").
 						WithEvent("ErrorCallWebhook").
 						WithSystemCondition(v1beta1.ConditionTypeDatasourcesUpdated)

internal/webhook/webhook.go

@@ -32,24 +32,75 @@ import (
 
 // Client defines the interface for the notification webhook client.
 type Client interface {
-	DatasourceChanged(context.Context, logr.Logger, string, string) error
+	SystemDatasourceChanged(context.Context, logr.Logger, string, string) error
+	LibraryDatasourceChanged(context.Context, logr.Logger, string) error
 }
 
 type client struct {
-	hc  http.Client
-	url string
+	hc                       http.Client
+	libraryDatasourceChanged string
+	systemDatasourceChanged  string
 }
 
 // New creates a new webhook notification Client.
-func New(url string) Client {
+func New(systemDatasourceChanged string, libraryDatasourceChanged string) Client {
 	return &client{
-		hc:  http.Client{},
-		url: url,
+		hc:                       http.Client{},
+		systemDatasourceChanged:  systemDatasourceChanged,
+		libraryDatasourceChanged: libraryDatasourceChanged,
 	}
 }
 
+func (client *client) LibraryDatasourceChanged(ctx context.Context, log logr.Logger, datasourceID string) error {
+	if client.libraryDatasourceChanged == "" {
+		return errors.New("libraryDatasourceChanged webhook not configured")
+	}
+
+	body := map[string]string{"datasourceID": datasourceID}
+	jsonData, err := json.Marshal(body)
+
+	if err != nil {
+		log.Error(err, "Failed to marshal request body")
+		return errors.Wrap(err, "Failed to marshal request body")
+	}
+
+	r, err := http.NewRequestWithContext(ctx, http.MethodPost, client.libraryDatasourceChanged, bytes.NewBuffer(jsonData))
+	if err != nil {
+		log.Error(err, "Failed to create request to webhook")
+		return errors.Wrap(err, "Failed to create request to webhook")
+	}
+	r.Header.Set("Content-Type", "application/json")
+
+	resp, err := client.hc.Do(r)
+	if err != nil {
+		log.Error(err, "Failed in call to webhook")
+		return errors.Wrap(err, "Failed in call to webhook")
+	}
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		log.Info("Response status code is not 2XX")
+		bodyBytes, err := io.ReadAll(resp.Body)
+		if err != nil {
+			log.Error(err, "Could not read response body")
+			return errors.Errorf("Could not read response body")
+		}
+		bodyString := string(bodyBytes)
+		return errors.Errorf("response status code is %d, response body is %s", resp.StatusCode, bodyString)
+	}
+
+	log.Info("Called library webhook successfully")
+	return nil
+}
+
 // DatasourceChanged notifies the webhook that a datasource has changed.
-func (client *client) DatasourceChanged(ctx context.Context, log logr.Logger, systemID string, dsID string) error {
+func (client *client) SystemDatasourceChanged(
+	ctx context.Context,
+	log logr.Logger,
+	systemID string,
+	dsID string) error {
+	if client.systemDatasourceChanged == "" {
+		return errors.New("systemDatasourceChanged webhook not configured")
+	}
 
 	body := map[string]string{"systemId": systemID, "datasourceId": dsID}
 	jsonData, err := json.Marshal(body)
@@ -59,7 +110,7 @@ func (client *client) DatasourceChanged(ctx context.Context, log logr.Logger, sy
 		return errors.Wrap(err, "Failed to marshal request body")
 	}
 
-	r, err := http.NewRequestWithContext(ctx, http.MethodPost, client.url, bytes.NewBuffer(jsonData))
+	r, err := http.NewRequestWithContext(ctx, http.MethodPost, client.systemDatasourceChanged, bytes.NewBuffer(jsonData))
 	if err != nil {
 		log.Error(err, "Failed to create request to webhook")
 		return errors.Wrap(err, "Failed to create request to webhook")
@@ -82,9 +133,9 @@ func (client *client) DatasourceChanged(ctx context.Context, log logr.Logger, sy
 			return errors.Errorf("Could not read response body")
 		}
 		bodyString := string(bodyBytes)
-		return errors.Errorf("response status code is %d, request body is %s", resp.StatusCode, bodyString)
+		return errors.Errorf("response status code is %d, response body is %s", resp.StatusCode, bodyString)
 	}
 
-	log.Info("Called webhook successfully")
+	log.Info("Called system webhook successfully")
 	return nil
 }