diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go
index a15e3e7e..8f036d33 100644
--- a/internal/controller/object/object.go
+++ b/internal/controller/object/object.go
@@ -466,7 +466,7 @@ type objFinalizer struct {
 
 type refFinalizerFn func(context.Context, *unstructured.Unstructured, string) error
 
-func (f *objFinalizer) handleRefFinalizer(ctx context.Context, obj *v1alpha1.Object, finalizerFn refFinalizerFn) error {
+func (f *objFinalizer) handleRefFinalizer(ctx context.Context, obj *v1alpha1.Object, finalizerFn refFinalizerFn, ignoreNotFound bool) error {
 	// Loop through references to resolve each referenced resource
 	for _, ref := range obj.Spec.References {
 		if ref.DependsOn == nil && ref.PatchesFrom == nil {
@@ -484,6 +484,10 @@ func (f *objFinalizer) handleRefFinalizer(ctx context.Context, obj *v1alpha1.Obj
 		}, res)
 
 		if err != nil {
+			if ignoreNotFound && kerrors.IsNotFound(err) {
+				continue
+			}
+
 			return errors.Wrap(err, errGetReferencedResource)
 		}
 
@@ -523,7 +527,7 @@ func (f *objFinalizer) AddFinalizer(ctx context.Context, res resource.Object) er
 			}
 		}
 		return nil
-	})
+	}, false)
 	return errors.Wrap(err, errAddFinalizer)
 }
 
@@ -543,7 +547,7 @@ func (f *objFinalizer) RemoveFinalizer(ctx context.Context, res resource.Object)
 			}
 		}
 		return nil
-	})
+	}, true)
 	if err != nil {
 		return errors.Wrap(err, errRemoveFinalizer)
 	}
diff --git a/internal/controller/object/object_test.go b/internal/controller/object/object_test.go
index b74950d1..3d33a9b0 100644
--- a/internal/controller/object/object_test.go
+++ b/internal/controller/object/object_test.go
@@ -1337,6 +1337,25 @@ func Test_objFinalizer_RemoveFinalizer(t *testing.T) {
 				finalizers: []string{},
 			},
 		},
+		"ReferenceNotFound": {
+			args: args{
+				mg: kubernetesObject(func(obj *v1alpha1.Object) {
+					obj.ObjectMeta.Finalizers = append(obj.ObjectMeta.Finalizers, objFinalizerName)
+					obj.Spec.References = objectReferences()
+					obj.ObjectMeta.UID = "some-uid"
+				}),
+				client: resource.ClientApplicator{
+					Client: &test.MockClient{
+						MockGet:    test.NewMockGetFn(kerrors.NewNotFound(schema.GroupResource{}, testReferenceObjectName)),
+						MockUpdate: test.NewMockUpdateFn(nil),
+					},
+				},
+			},
+			want: want{
+				err:        nil,
+				finalizers: []string{},
+			},
+		},
 		"FailedToRemoveReferenceFinalizer": {
 			args: args{
 				mg: kubernetesObject(func(obj *v1alpha1.Object) {