diff --git a/controllers/shipwrightbuild_controller.go b/controllers/shipwrightbuild_controller.go
index 560cddc3..8f77724a 100644
--- a/controllers/shipwrightbuild_controller.go
+++ b/controllers/shipwrightbuild_controller.go
@@ -13,6 +13,7 @@ import (
 	"github.com/manifestival/manifestival"
 	tektonoperatorv1alpha1client "github.com/tektoncd/operator/pkg/client/clientset/versioned/typed/operator/v1alpha1"
 	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	crdclientv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
@@ -85,6 +86,24 @@ func (r *ShipwrightBuildReconciler) unsetFinalizer(ctx context.Context, b *v1alp
 	return r.Update(ctx, b, &client.UpdateOptions{})
 }
 
+// deleteObjectsIfPresent deletes all the given objects if they are present in the cluster.
+func deleteObjectsIfPresent(ctx context.Context, k8sClient client.Client, objs []client.Object) error {
+	for _, obj := range objs {
+		err := k8sClient.Get(ctx, types.NamespacedName{Name: obj.GetName(), Namespace: obj.GetNamespace()}, obj)
+		if err != nil {
+			if errors.IsNotFound(err) {
+				return nil
+			}
+			return fmt.Errorf("getting object %s: %v", obj.GetName(), err)
+		}
+		err = k8sClient.Delete(ctx, obj)
+		if err != nil {
+			return fmt.Errorf("deleting object %s: %v", obj.GetName(), err)
+		}
+	}
+	return nil
+}
+
 // Reconcile performs the resource reconciliation steps to deploy or remove Shipwright Build
 // instances. When deletion-timestamp is found, the removal of the previously deploy resources is
 // executed, otherwise the regular deploy workflow takes place.
@@ -233,6 +252,12 @@ func (r *ShipwrightBuildReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		err = r.Client.Status().Update(ctx, b)
 		return RequeueWithError(err)
 	}
+
+	deleteObjectsIfPresent(ctx, r.Client, []client.Object{
+		&rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "shipwright-build-webhook"}},
+		&rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "shipwright-build-webhook"}},
+	})
+
 	if err := r.setFinalizer(ctx, b); err != nil {
 		logger.Info(fmt.Sprintf("%#v", b))
 		logger.Error(err, "setting the finalizer")