Support flux v2.2.0

This adds the latest source-controller v1.2.2 as a marker for Flux
v2.1.2.

Dependencies are updated to allow us to pull in the latest Flux code.

And we now pull the internal Helm release storage version from the
history, rather than the latest version field.

If there's no history, we retain the old version.

Makefile

@@ -236,7 +236,7 @@ download-test-crds:
 		curl -sL "https://raw.githubusercontent.com/fluxcd/source-controller/v1.0.0/config/crd/bases/$${group}.toolkit.fluxcd.io_$${resource}.yaml" -o "tools/testcrds/$${group}.toolkit.fluxcd.io_$${resource}.yaml"; \
 	done
 	curl -sL "https://raw.githubusercontent.com/fluxcd/kustomize-controller/v1.0.0/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml" -o "tools/testcrds/kustomize.toolkit.fluxcd.io_kustomizations.yaml"
-	curl -sL "https://raw.githubusercontent.com/fluxcd/helm-controller/v0.35.0/config/crd/bases/helm.toolkit.fluxcd.io_helmreleases.yaml" -o "tools/testcrds/helm.toolkit.fluxcd.io_helmreleases.yaml"
+	curl -sL "https://raw.githubusercontent.com/fluxcd/helm-controller/v0.37.0/config/crd/bases/helm.toolkit.fluxcd.io_helmreleases.yaml" -o "tools/testcrds/helm.toolkit.fluxcd.io_helmreleases.yaml"
 
 .PHONY: help
 # Thanks to https://www.thapaliya.com/en/writings/well-documented-makefiles/

cmd/gitops/create/terraform/cmd.go

@@ -57,14 +57,9 @@ gitops create terraform -n default my-resource --source GitRepository/my-project
 			kubeConfigArgs.Namespace = &namespace
 			kubeConfigArgs.Context = &context
 
-			cfg, err := kubeConfigArgs.ToRESTConfig()
-			if err != nil {
-				return err
-			}
-
 			v := viper.New()
 			v.Set("namespace", namespace)
-			if err := app.Init(cfg, v); err != nil {
+			if err := app.Init(kubeConfigArgs, v); err != nil {
 				return err
 			}
 

cmd/gitops/delete/terraform/cmd.go

@@ -1,13 +1,14 @@
 package terraform
 
 import (
+	"os"
+
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/weaveworks/tf-controller/tfctl"
 	"github.com/weaveworks/weave-gitops/cmd/gitops/config"
 	"github.com/weaveworks/weave-gitops/pkg/run"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
-	"os"
 )
 
 var kubeConfigArgs *genericclioptions.ConfigFlags
@@ -38,14 +39,9 @@ gitops delete terraform -n default my-resource
 			kubeConfigArgs.Namespace = &namespace
 			kubeConfigArgs.Context = &context
 
-			cfg, err := kubeConfigArgs.ToRESTConfig()
-			if err != nil {
-				return err
-			}
-
 			v := viper.New()
 			v.Set("namespace", namespace)
-			if err := app.Init(cfg, v); err != nil {
+			if err := app.Init(kubeConfigArgs, v); err != nil {
 				return err
 			}
 

cmd/gitops/replan/terraform/cmd.go

@@ -1,13 +1,14 @@
 package terraform
 
 import (
+	"os"
+
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/weaveworks/tf-controller/tfctl"
 	"github.com/weaveworks/weave-gitops/cmd/gitops/config"
 	"github.com/weaveworks/weave-gitops/pkg/run"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
-	"os"
 )
 
 var kubeConfigArgs *genericclioptions.ConfigFlags
@@ -38,14 +39,9 @@ gitops replan terraform --namespace flux-system my-resource
 			kubeConfigArgs.Namespace = &namespace
 			kubeConfigArgs.Context = &context
 
-			cfg, err := kubeConfigArgs.ToRESTConfig()
-			if err != nil {
-				return err
-			}
-
 			v := viper.New()
 			v.Set("namespace", namespace)
-			if err := app.Init(cfg, v); err != nil {
+			if err := app.Init(kubeConfigArgs, v); err != nil {
 				return err
 			}
 

cmd/gitops/resume/terraform/cmd.go

@@ -1,13 +1,14 @@
 package terraform
 
 import (
+	"os"
+
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/weaveworks/tf-controller/tfctl"
 	"github.com/weaveworks/weave-gitops/cmd/gitops/config"
 	"github.com/weaveworks/weave-gitops/pkg/run"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
-	"os"
 )
 
 var kubeConfigArgs *genericclioptions.ConfigFlags
@@ -38,14 +39,9 @@ gitops resume terraform --namespace flux-system my-resource
 			kubeConfigArgs.Namespace = &namespace
 			kubeConfigArgs.Context = &context
 
-			cfg, err := kubeConfigArgs.ToRESTConfig()
-			if err != nil {
-				return err
-			}
-
 			v := viper.New()
 			v.Set("namespace", namespace)
-			if err := app.Init(cfg, v); err != nil {
+			if err := app.Init(kubeConfigArgs, v); err != nil {
 				return err
 			}
 

cmd/gitops/suspend/terraform/cmd.go

@@ -1,13 +1,14 @@
 package terraform
 
 import (
+	"os"
+
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/weaveworks/tf-controller/tfctl"
 	"github.com/weaveworks/weave-gitops/cmd/gitops/config"
 	"github.com/weaveworks/weave-gitops/pkg/run"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
-	"os"
 )
 
 var kubeConfigArgs *genericclioptions.ConfigFlags
@@ -38,14 +39,9 @@ gitops suspend terraform --namespace flux-system my-resource
 			kubeConfigArgs.Namespace = &namespace
 			kubeConfigArgs.Context = &context
 
-			cfg, err := kubeConfigArgs.ToRESTConfig()
-			if err != nil {
-				return err
-			}
-
 			v := viper.New()
 			v.Set("namespace", namespace)
-			if err := app.Init(cfg, v); err != nil {
+			if err := app.Init(kubeConfigArgs, v); err != nil {
 				return err
 			}
 

core/fluxsync/adapters.go

@@ -1,7 +1,7 @@
 package fluxsync
 
 import (
-	helmv2 "github.com/fluxcd/helm-controller/api/v2beta1"
+	helmv2 "github.com/fluxcd/helm-controller/api/v2beta2"
 	imgautomationv1 "github.com/fluxcd/image-automation-controller/api/v1beta1"
 	reflectorv1 "github.com/fluxcd/image-reflector-controller/api/v1beta2"
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1"

core/server/events_test.go

@@ -6,7 +6,7 @@ import (
 
 	"testing"
 
-	helmv2 "github.com/fluxcd/helm-controller/api/v2beta1"
+	helmv2 "github.com/fluxcd/helm-controller/api/v2beta2"
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1"
 	. "github.com/onsi/gomega"
 	pb "github.com/weaveworks/weave-gitops/pkg/api/core"

core/server/fluxruntime.go

@@ -19,7 +19,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	helmv2 "github.com/fluxcd/helm-controller/api/v2beta1"
+	helmv2 "github.com/fluxcd/helm-controller/api/v2beta2"
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1"
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"

core/server/helm_release.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"strings"
 
-	helmv2 "github.com/fluxcd/helm-controller/api/v2beta1"
+	helmv2 "github.com/fluxcd/helm-controller/api/v2beta2"
 	"github.com/weaveworks/weave-gitops/core/clustersmngr"
 	pb "github.com/weaveworks/weave-gitops/pkg/api/core"
 )

core/server/inventory.go

@@ -11,7 +11,7 @@ import (
 	"strings"
 	"sync"
 
-	helmv2 "github.com/fluxcd/helm-controller/api/v2beta1"
+	helmv2 "github.com/fluxcd/helm-controller/api/v2beta2"
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1"
 	"github.com/fluxcd/pkg/ssa"
 	"github.com/go-logr/logr"
@@ -26,6 +26,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"sigs.k8s.io/cli-utils/pkg/object"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 )
 
 // ObjectWithChildren is a recursive data structure containing a tree of Unstructured
@@ -70,10 +71,7 @@ func (cs *coreServer) GetInventory(ctx context.Context, msg *pb.GetInventoryRequ
 		}
 	}
 
-	objsWithChildren, err := GetObjectsWithChildren(ctx, inventoryRefs, client, msg.WithChildren, cs.logger)
-	if err != nil {
-		return nil, fmt.Errorf("failed getting objects with children: %w", err)
-	}
+	objsWithChildren := getObjectsWithChildren(ctx, msg.Namespace, inventoryRefs, client, msg.WithChildren, cs.logger)
 
 	entries := []*pb.InventoryEntry{}
 	clusterUserNamespaces := cs.clustersManager.GetUserNamespaces(auth.Principal(ctx))
@@ -145,29 +143,18 @@ func (cs *coreServer) getHelmReleaseInventory(ctx context.Context, k8sClient cli
 
 // Returns the list of resources applied in the helm chart.
 func getHelmReleaseObjects(ctx context.Context, k8sClient client.Client, helmRelease *helmv2.HelmRelease) ([]*unstructured.Unstructured, error) {
-	storageNamespace := helmRelease.GetStorageNamespace()
-
-	storageName := helmRelease.GetReleaseName()
-
-	storageVersion := helmRelease.Status.LastReleaseRevision
-	if storageVersion < 1 {
+	secretName := secretNameFromHelmRelease(helmRelease)
+	if secretName == nil {
 		// skip release if it failed to install
 		return nil, nil
 	}
-
 	storageSecret := &v1.Secret{}
-	secretName := fmt.Sprintf("sh.helm.release.v1.%s.v%v", storageName, storageVersion)
-	key := client.ObjectKey{
-		Name:      secretName,
-		Namespace: storageNamespace,
-	}
-
 	if helmRelease.Spec.KubeConfig != nil {
 		// helmrelease secret is on another cluster so we cannot inspect it to figure out the inventory and version and other things
 		return nil, nil
 	}
 
-	if err := k8sClient.Get(ctx, key, storageSecret); err != nil {
+	if err := k8sClient.Get(ctx, *secretName, storageSecret); err != nil {
 		return nil, err
 	}
 
@@ -258,18 +245,49 @@ func unstructuredToInventoryEntry(clusterName string, objWithChildren ObjectWith
 // GetObjectsWithChildren returns objects with their children populated if withChildren is true.
 // Objects are retrieved in parallel.
 // Children are retrieved recusively, e.g. Deployment -> ReplicaSet -> Pod
-func GetObjectsWithChildren(ctx context.Context, objects []*unstructured.Unstructured, k8sClient client.Client, withChildren bool, logger logr.Logger) ([]*ObjectWithChildren, error) {
+func getObjectsWithChildren(ctx context.Context, releaseNS string, objects []*unstructured.Unstructured, k8sClient client.Client, withChildren bool, logger logr.Logger) []*ObjectWithChildren {
 	result := []*ObjectWithChildren{}
-	resultMu := sync.Mutex{}
 
-	wg := sync.WaitGroup{}
+	var (
+		isNamespacedGVK = map[string]bool{}
+		resultMu        sync.Mutex
+		gvkMu           sync.RWMutex
+		wg              sync.WaitGroup
+	)
 
 	for _, o := range objects {
 		wg.Add(1)
 
 		go func(obj unstructured.Unstructured) {
 			defer wg.Done()
 
+			// Set the namespace of the object if it is not set.
+			if obj.GetNamespace() == "" {
+				// Manifest does not contain the namespace of the release.
+				// Figure out if the object is namespaced if the namespace is not
+				// explicitly set, and configure the namespace accordingly.
+				objGVK := obj.GetObjectKind().GroupVersionKind().String()
+				gvkMu.RLock()
+				namespaced, ok := isNamespacedGVK[objGVK]
+				gvkMu.RUnlock()
+
+				if !ok {
+					namespaced, err := apiutil.IsObjectNamespaced(&obj, k8sClient.Scheme(), k8sClient.RESTMapper())
+					if err != nil {
+						logger.Error(err, "failed to determine if %s is namespace scoped", obj.GetObjectKind().GroupVersionKind().Kind)
+						return
+					}
+					// Cache the result, so we don't have to do this for every object
+					gvkMu.Lock()
+					isNamespacedGVK[objGVK] = namespaced
+					gvkMu.Unlock()
+				}
+
+				if namespaced {
+					obj.SetNamespace(releaseNS)
+				}
+			}
+
 			if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(&obj), &obj); err != nil {
 				logger.Error(err, "failed to get object", "entry", obj)
 				return
@@ -291,14 +309,14 @@ func GetObjectsWithChildren(ctx context.Context, objects []*unstructured.Unstruc
 			}
 
 			resultMu.Lock()
+			defer resultMu.Unlock()
 			result = append(result, entry)
-			resultMu.Unlock()
 		}(*o)
 	}
 
 	wg.Wait()
 
-	return result, nil
+	return result
 }
 
 func getChildren(ctx context.Context, k8sClient client.Client, parentObj unstructured.Unstructured) ([]*ObjectWithChildren, error) {
@@ -446,3 +464,23 @@ func parseInventoryFromUnstructured(obj *unstructured.Unstructured) ([]*unstruct
 
 	return objects, nil
 }
+
+const helmSecretNameFmt = "sh.helm.release.v1.%s.v%v"
+
+func secretNameFromHelmRelease(helmRelease *helmv2.HelmRelease) *client.ObjectKey {
+	if latest := helmRelease.Status.History.Latest(); latest != nil {
+		return &client.ObjectKey{
+			Name:      fmt.Sprintf(helmSecretNameFmt, latest.Name, latest.Version),
+			Namespace: helmRelease.Status.StorageNamespace,
+		}
+	}
+
+	if latestRevision := helmRelease.Status.LastReleaseRevision; latestRevision > 0 {
+		return &client.ObjectKey{
+			Name:      fmt.Sprintf(helmSecretNameFmt, helmRelease.GetReleaseName(), latestRevision),
+			Namespace: helmRelease.GetStorageNamespace(),
+		}
+	}
+
+	return nil
+}

core/server/inventory_test.go

@@ -7,7 +7,7 @@ import (
 	"fmt"
 	"testing"
 
-	helmv2 "github.com/fluxcd/helm-controller/api/v2beta1"
+	helmv2 "github.com/fluxcd/helm-controller/api/v2beta2"
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1"
 	"github.com/fluxcd/pkg/apis/meta"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1"

core/server/objects.go

@@ -5,7 +5,7 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/fluxcd/helm-controller/api/v2beta1"
+	helmv2 "github.com/fluxcd/helm-controller/api/v2beta2"
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1"
 	"github.com/hashicorp/go-multierror"
 	"github.com/weaveworks/weave-gitops/core/clustersmngr"
@@ -25,7 +25,7 @@ const (
 )
 
 func getUnstructuredHelmReleaseInventory(ctx context.Context, obj unstructured.Unstructured, c clustersmngr.Client, cluster string) ([]*pb.GroupVersionKind, error) {
-	var release v2beta1.HelmRelease
+	var release helmv2.HelmRelease
 
 	err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &release)
 	if err != nil {
@@ -116,7 +116,7 @@ func (cs *coreServer) ListObjects(ctx context.Context, msg *pb.ListObjectsReques
 						respErrors = append(respErrors, &pb.ListError{ClusterName: clusterName, Message: fmt.Sprintf("error sanitizing secrets: %v", err)})
 						continue
 					}
-				case v2beta1.HelmReleaseKind:
+				case helmv2.HelmReleaseKind:
 					inventory, err = getUnstructuredHelmReleaseInventory(ctx, unstructuredObj, clustersClient, clusterName)
 					if err != nil {
 						respErrors = append(respErrors, &pb.ListError{ClusterName: clusterName, Message: err.Error()})
@@ -175,7 +175,7 @@ func parseSessionInfo(unstructuredObj unstructured.Unstructured) (string, string
 	if annotations["run.weave.works/automation-kind"] == "ks" {
 		kind = kustomizev1.KustomizationKind
 	} else {
-		kind = v2beta1.HelmReleaseKind
+		kind = helmv2.HelmReleaseKind
 	}
 
 	ns := annotations["run.weave.works/namespace"]
@@ -253,7 +253,7 @@ func (cs *coreServer) GetObject(ctx context.Context, msg *pb.GetObjectRequest) (
 		if err != nil {
 			return nil, fmt.Errorf("error sanitizing secrets: %w", err)
 		}
-	case v2beta1.HelmReleaseKind:
+	case helmv2.HelmReleaseKind:
 		inventory, err = getUnstructuredHelmReleaseInventory(ctx, unstructuredObj, clustersClient, msg.ClusterName)
 		if err != nil {
 			inventory = nil // We can still display most things without inventory