Add support for bootstrapping additional root apps

See [SDD-0030 - ArgoCD multitenancy] for the overall design for
additional root apps.

This commit introduces a new command line flag to specify a config map
from which Steward will read field `teams` which is expected to contain
a JSON-encoded list of team names for which to setup additional ArgoCD
AppProject and root Application (app of apps) resources.

Steward uses the team names as is for the additional AppProject
resources, and creates the additional root apps as `root-<team name>`.
As defined in the design document, the path for the root apps is
configured as `manifests/apps-<team name>/`.

[SDD-0030 - ArgoCD multitenancy]: https://syn.tools/syn/SDDs/0030-argocd-multitenancy.html

main.go

@@ -54,6 +54,12 @@ func main() {
 			"Additional facts added to the dynamic facts in the cluster object. Keys in the configmap's data field can't override existing keys.").
 		Default("additional-facts").
 		StringVar(&agent.AdditionalFactsConfigMap)
+	app.
+		Flag(
+			"additional-root-apps-config-map",
+			"Config map holding metadata for additional ArgoCD root apps and app projects.").
+		Default("additional-root-apps").
+		StringVar(&agent.AdditionalRootAppsConfigMap)
 	app.
 		Flag(
 			"ocp-oauth-route-namespace",

pkg/agent/agent.go

@@ -36,6 +36,9 @@ type Agent struct {
 	// The configmap containing additional facts to be added to the dynamic facts
 	AdditionalFactsConfigMap string
 
+	// The configmap containing metadata for additional root apps to deploy
+	AdditionalRootAppsConfigMap string
+
 	// Reference to the OpenShift OAuth route to be added to the dynamic facts
 	OCPOAuthRouteNamespace string
 	OCPOAuthRouteName      string
@@ -140,7 +143,7 @@ func (a *Agent) registerCluster(ctx context.Context, config *rest.Config, apiCli
 		return
 	}
 
-	if err := argocd.Apply(ctx, config, a.Namespace, a.OperatorNamespace, a.ArgoCDImage, a.RedisImage, apiClient, cluster); err != nil {
+	if err := argocd.Apply(ctx, config, a.Namespace, a.OperatorNamespace, a.ArgoCDImage, a.RedisImage, a.AdditionalRootAppsConfigMap, apiClient, cluster); err != nil {
 		klog.Error(err)
 	}
 }

pkg/argocd/argo-app.go

@@ -2,13 +2,16 @@ package argocd
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
 
 	"github.com/projectsyn/lieutenant-api/pkg/api"
 	k8err "k8s.io/apimachinery/pkg/api/errors"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/klog"
 )
@@ -23,9 +26,27 @@ var (
 	argoProjectGVR = argoGroupVersion.WithResource("appprojects")
 
 	localKubernetesAPI = "https://kubernetes.default.svc"
+
+	additionalRootAppsConfigKey = "teams"
 )
 
-func createArgoProject(ctx context.Context, cluster *api.Cluster, config *rest.Config, namespace string) error {
+func readAdditionalRootAppsConfigMap(ctx context.Context, clientset *kubernetes.Clientset, namespace, additionalRootAppsConfigMapName string) ([]string, error) {
+	cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(ctx, additionalRootAppsConfigMapName, v1.GetOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("unable to fetch the additional root apps config map: %w", err)
+	}
+	teamsJson, ok := cm.Data[additionalRootAppsConfigKey]
+	if !ok {
+		return nil, fmt.Errorf("additional root apps ConfigMap doesn't have key %s", additionalRootAppsConfigKey)
+	}
+	var teams []string
+	if err := json.Unmarshal([]byte(teamsJson), &teams); err != nil {
+		return nil, fmt.Errorf("unmarshalling additional root apps ConfigMap contents: %v", err)
+	}
+	return teams, nil
+}
+
+func createArgoProject(ctx context.Context, cluster *api.Cluster, config *rest.Config, namespace, name string) error {
 	dynamicClient, err := dynamic.NewForConfig(config)
 	if err != nil {
 		return err
@@ -36,7 +57,7 @@ func createArgoProject(ctx context.Context, cluster *api.Cluster, config *rest.C
 			"apiVersion": argoProjectGVR.Group + "/" + argoProjectGVR.Version,
 			"kind":       "AppProject",
 			"metadata": map[string]interface{}{
-				"name": argoProjectName,
+				"name": name,
 			},
 			"spec": map[string]interface{}{
 				"clusterResourceWhitelist": []map[string]interface{}{{
@@ -66,7 +87,7 @@ func createArgoProject(ctx context.Context, cluster *api.Cluster, config *rest.C
 	return nil
 }
 
-func createArgoApp(ctx context.Context, cluster *api.Cluster, config *rest.Config, namespace string) error {
+func createArgoApp(ctx context.Context, cluster *api.Cluster, config *rest.Config, namespace, projectName, name, appsPath string) error {
 	dynamicClient, err := dynamic.NewForConfig(config)
 	if err != nil {
 		return err
@@ -77,13 +98,13 @@ func createArgoApp(ctx context.Context, cluster *api.Cluster, config *rest.Confi
 			"apiVersion": argoAppGVR.Group + "/" + argoAppGVR.Version,
 			"kind":       "Application",
 			"metadata": map[string]interface{}{
-				"name": argoRootAppName,
+				"name": name,
 			},
 			"spec": map[string]interface{}{
-				"project": argoProjectName,
+				"project": projectName,
 				"source": map[string]interface{}{
 					"repoURL":        *cluster.GitRepo.Url,
-					"path":           argoAppsPath,
+					"path":           appsPath + "/",
 					"targetRevision": "HEAD",
 				},
 				"syncPolicy": map[string]interface{}{

pkg/argocd/argocd.go

@@ -26,23 +26,23 @@ var (
 	argoAnnotations = map[string]string{
 		"argocd.argoproj.io/sync-options": "Prune=false",
 	}
-	argoSSHSecretName     = "argo-ssh-key"
-	argoSSHPublicKey      = "sshPublicKey"
-	argoSSHPrivateKey     = "sshPrivateKey"
-	argoSSHConfigMapName  = "argocd-ssh-known-hosts-cm"
-	argoTLSConfigMapName  = "argocd-tls-certs-cm"
-	argoRbacConfigMapName = "argocd-rbac-cm"
-	argoConfigMapName     = "argocd-cm"
-	argoSecretName        = "argocd-secret"
-	argoClusterSecretName = "syn-argocd-cluster"
-	argoRbacName          = "argocd-application-controller"
-	argoRootAppName       = "root"
-	argoProjectName       = "syn"
-	argoAppsPath          = "manifests/apps/"
+	argoSSHSecretName      = "argo-ssh-key"
+	argoSSHPublicKey       = "sshPublicKey"
+	argoSSHPrivateKey      = "sshPrivateKey"
+	argoSSHConfigMapName   = "argocd-ssh-known-hosts-cm"
+	argoTLSConfigMapName   = "argocd-tls-certs-cm"
+	argoRbacConfigMapName  = "argocd-rbac-cm"
+	argoConfigMapName      = "argocd-cm"
+	argoSecretName         = "argocd-secret"
+	argoClusterSecretName  = "syn-argocd-cluster"
+	argoRbacName           = "argocd-application-controller"
+	defaultArgoRootAppName = "root"
+	defaultArgoProjectName = "syn"
+	argoAppsPathPrefix     = "manifests/apps"
 )
 
 // Apply reconciles the Argo CD deployments
-func Apply(ctx context.Context, config *rest.Config, namespace, operatorNamespace, argoImage, redisArgoImage string, apiClient *api.Client, cluster *api.Cluster) error {
+func Apply(ctx context.Context, config *rest.Config, namespace, operatorNamespace, argoImage, redisArgoImage, additionalRootAppsConfigMapName string, apiClient *api.Client, cluster *api.Cluster) error {
 	clientset, err := kubernetes.NewForConfig(config)
 	if err != nil {
 		return err
@@ -98,10 +98,15 @@ func Apply(ctx context.Context, config *rest.Config, namespace, operatorNamespac
 	}
 
 	klog.Infof("Found %d of expected %d deployments, found %d of expected %d statefulsets, bootstrapping now", foundDeploymentCount, expectedDeploymentCount, foundStatefulSetCount, expectedStatefulSetCount)
-	return bootstrapArgo(ctx, clientset, config, namespace, argoImage, redisArgoImage, apiClient, cluster)
+	return bootstrapArgo(ctx, clientset, config, namespace, argoImage, redisArgoImage, additionalRootAppsConfigMapName, apiClient, cluster)
 }
 
-func bootstrapArgo(ctx context.Context, clientset *kubernetes.Clientset, config *rest.Config, namespace, argoImage, redisArgoImage string, apiClient *api.Client, cluster *api.Cluster) error {
+func bootstrapArgo(ctx context.Context, clientset *kubernetes.Clientset, config *rest.Config, namespace, argoImage, redisArgoImage, additionalRootAppsConfigMapName string, apiClient *api.Client, cluster *api.Cluster) error {
+	teamNames, err := readAdditionalRootAppsConfigMap(ctx, clientset, namespace, additionalRootAppsConfigMapName)
+	if err != nil {
+		return err
+	}
+
 	if err := createArgoCDConfigMaps(ctx, cluster, clientset, namespace); err != nil {
 		return err
 	}
@@ -122,14 +127,25 @@ func bootstrapArgo(ctx context.Context, clientset *kubernetes.Clientset, config
 		return err
 	}
 
-	if err := createArgoProject(ctx, cluster, config, namespace); err != nil {
+	if err := createArgoProject(ctx, cluster, config, namespace, defaultArgoProjectName); err != nil {
 		return err
 	}
 
-	if err := createArgoApp(ctx, cluster, config, namespace); err != nil {
+	if err := createArgoApp(ctx, cluster, config, namespace, defaultArgoProjectName, defaultArgoRootAppName, argoAppsPathPrefix); err != nil {
 		return err
 	}
 
+	for _, name := range teamNames {
+		if err := createArgoProject(ctx, cluster, config, namespace, name); err != nil {
+			return err
+		}
+
+		// apps path for additional root apps is `manifests/apps-<team name>/`.
+		if err := createArgoApp(ctx, cluster, config, namespace, name, "root-"+name, argoAppsPathPrefix+"-"+name); err != nil {
+			return err
+		}
+	}
+
 	if err := createApplicationControllerStatefulSet(ctx, clientset, namespace, argoImage); err != nil {
 		return err
 	}