Merge pull request #72 from caos/next

fix: forceApply possibility, argocd scopes and handling of folders fix

api/v1beta1/argocd.go

@@ -10,8 +10,9 @@ type Argocd struct {
 }
 
 type Rbac struct {
-	Csv     string `json:"policy.csv,omitempty" yaml:"policy.csv,omitempty"`
-	Default string `json:"policy.default,omitempty" yaml:"policy.default,omitempty"`
+	Csv     string   `json:"policy.csv,omitempty" yaml:"policy.csv,omitempty"`
+	Default string   `json:"policy.default,omitempty" yaml:"policy.default,omitempty"`
+	Scopes  []string `json:"scopes,omitempty" yaml:"scopes,omitempty"`
 }
 
 type ArgocdRepository struct {

api/v1beta1/toolset_types.go

@@ -24,6 +24,7 @@ import (
 
 // ToolsetSpec defines the desired state of Toolset
 type ToolsetSpec struct {
+	ForceApply             bool                    `json:"forceApply,omitempty" yaml:"forceApply,omitempty"`
 	CurrentStateFolder     string                  `json:"currentStatePath,omitempty" yaml:"currentStatePath,omitempty"`
 	PreApply               *PreApply               `json:"preApply,omitempty" yaml:"preApply,omitempty"`
 	PostApply              *PostApply              `json:"postApply,omitempty" yaml:"postApply,omitempty"`

applySecretAndStartSkaffold.sh

@@ -3,5 +3,6 @@
 gopass sync -s caos-secrets
 ./scripts/0_imagepull-secrets.sh | kubectl apply -f -
 ./scripts/1_argocd-secrets.sh | kubectl apply -f -
+./scripts/2_ops-repo-read-secret.sh | kubectl apply -f -
 
 skaffold run -f ../build/skaffold/skaffold.yaml

cmd/boom/main.go

@@ -182,7 +182,7 @@ func main() {
 	}
 
 	if !localMode {
-		cmd, err := kustomize.New("/crd", true)
+		cmd, err := kustomize.New("/crd", true, false)
 		if err != nil {
 			setupLog.Error(err, "unable to locate crd")
 			os.Exit(1)
@@ -223,7 +223,7 @@ func main() {
 
 		setupLog.Info("starting manager")
 	} else {
-		cmd, err := kustomize.New("../../config/crd", true)
+		cmd, err := kustomize.New("../../config/crd", true, false)
 		if err != nil {
 			setupLog.Error(err, "unable to locate crd")
 			os.Exit(1)

config/crd/bases/toolsets.boom.caos.ch_toolsets.yaml

@@ -277,14 +277,16 @@ spec:
                           name:
                             type: string
                         required:
-                        - key
-                        - name
+                          - key
+                          - name
                         type: object
                     type: object
                   type: array
               type: object
             currentStatePath:
               type: string
+            forceApply:
+              type: boolean
             grafana:
               properties:
                 admin:
@@ -561,7 +563,6 @@ spec:
           description: ToolsetStatus defines the observed state of Toolset
           type: object
       type: object
-  version: v1beta1
   versions:
   - name: v1beta1
     served: true

go.mod

@@ -3,7 +3,7 @@ module github.com/caos/boom
 go 1.14
 
 require (
-	github.com/caos/orbiter v0.16.0
+	github.com/caos/orbiter v0.16.1
 	github.com/go-logr/logr v0.1.0
 	github.com/golang/mock v1.3.1
 	github.com/pkg/errors v0.8.1

go.sum

@@ -39,8 +39,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/caos/orbiter v0.16.0 h1:LLzOfXQAOA0l4I7TLVpc8mqQMYWAIJfmPyVNz1jcyAI=
-github.com/caos/orbiter v0.16.0/go.mod h1:5Iu0mEmWybNVOncYonervGAce3JhZ+ltBLGhd579leE=
+github.com/caos/orbiter v0.16.1 h1:kll/1MyJ2dQ1fY2Nr4s09DWWeEHTPLi3OO5uMMxcnoQ=
+github.com/caos/orbiter v0.16.1/go.mod h1:5Iu0mEmWybNVOncYonervGAce3JhZ+ltBLGhd579leE=
 github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=

internal/bundle/application/applications/argocd/helm.go

@@ -94,9 +94,22 @@ func (a *Argocd) SpecToHelmValues(monitor mntr.Monitor, toolsetCRDSpec *toolsets
 	}
 
 	if spec.Rbac != nil {
+		scopes := ""
+		for _, scope := range spec.Rbac.Scopes {
+			if scopes == "" {
+				scopes = scope
+			} else {
+				scopes = strings.Join([]string{scopes, scope}, ", ")
+			}
+		}
+		if scopes != "" {
+			scopes = strings.Join([]string{"[", scopes, "]"}, "")
+		}
+
 		values.Server.RbacConfig = &helm.RbacConfig{
 			Csv:     spec.Rbac.Csv,
 			Default: spec.Rbac.Default,
+			Scopes:  scopes,
 		}
 	}
 

internal/bundle/application/applications/argocd/helm/values.go

@@ -213,6 +213,7 @@ type Server struct {
 type RbacConfig struct {
 	Csv     string `yaml:"policy.csv,omitempty"`
 	Default string `yaml:"policy.default,omitempty"`
+	Scopes  string `yaml:"scopes,omitempty"`
 }
 
 type RepoServer struct {

internal/bundle/application/applications/grafana/helm.go

@@ -173,7 +173,7 @@ func getKustomizeOutput(folders []string) ([]string, error) {
 	ret := make([]string, len(folders))
 	for n, folder := range folders {
 
-		cmd, err := kustomize.New(folder, false)
+		cmd, err := kustomize.New(folder, false, false)
 		if err != nil {
 			return nil, err
 		}

internal/bundle/application/applications/grafanastandalone/app.go

@@ -202,7 +202,7 @@ func getKustomizeOutput(folders []string) ([]string, error) {
 	ret := make([]string, len(folders))
 	for n, folder := range folders {
 
-		cmd, err := kustomize.New(folder, false)
+		cmd, err := kustomize.New(folder, false, false)
 		if err != nil {
 			return nil, err
 		}

internal/bundle/apply.go

@@ -9,15 +9,15 @@ import (
 	"github.com/caos/orbiter/mntr"
 )
 
-func applyWithCurrentState(monitor mntr.Monitor, currentResourceList []*clientgo.Resource, app application.Application) func(resultFilePath, namespace string) error {
+func applyWithCurrentState(monitor mntr.Monitor, currentResourceList []*clientgo.Resource, app application.Application, force bool) func(resultFilePath, namespace string) error {
 
 	logFields := map[string]interface{}{
 		"command": "apply",
 	}
 	applyMonitor := monitor.WithFields(logFields)
 
 	resultFunc := func(resultFilePath, namespace string) error {
-		applyFunc := apply(monitor, app)
+		applyFunc := apply(monitor, app, force)
 
 		desiredResources, err := desired.Get(monitor, resultFilePath, namespace, app.GetName())
 		if err != nil {
@@ -74,15 +74,15 @@ func applyWithCurrentState(monitor mntr.Monitor, currentResourceList []*clientgo
 	return resultFunc
 }
 
-func apply(monitor mntr.Monitor, app application.Application) func(resultFilePath, namespace string) error {
+func apply(monitor mntr.Monitor, app application.Application, force bool) func(resultFilePath, namespace string) error {
 
 	logFields := map[string]interface{}{
 		"command": "apply",
 	}
 	applyMonitor := monitor.WithFields(logFields)
 
 	resultFunc := func(resultFilePath, namespace string) error {
-		return desired.Apply(applyMonitor, resultFilePath, namespace, app.GetName())
+		return desired.Apply(applyMonitor, resultFilePath, namespace, app.GetName(), force)
 	}
 
 	return resultFunc

internal/bundle/bundle.go

@@ -150,7 +150,7 @@ func (b *Bundle) ReconcileApplication(currentResourceList []*clientgo.Resource,
 		}
 	} else {
 		if deploy {
-			resultFunc = applyWithCurrentState(monitor, currentApplicationResourceList, app)
+			resultFunc = applyWithCurrentState(monitor, currentApplicationResourceList, app, spec.ForceApply)
 		} else {
 			resultFunc = deleteWithCurrentState(monitor, currentApplicationResourceList, app)
 		}

internal/desired/desired.go

@@ -14,15 +14,15 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-func Apply(monitor mntr.Monitor, resultFilePath, namespace string, appName name.Application) error {
+func Apply(monitor mntr.Monitor, resultFilePath, namespace string, appName name.Application, force bool) error {
 	resultFileDirPath := filepath.Dir(resultFilePath)
 
 	if err := prepareAdditionalFiles(resultFilePath, namespace, appName); err != nil {
 		return err
 	}
 
 	// apply resources
-	cmd, err := kustomize.New(resultFileDirPath, true)
+	cmd, err := kustomize.New(resultFileDirPath, true, force)
 	if err != nil {
 		return err
 	}
@@ -38,7 +38,7 @@ func Get(monitor mntr.Monitor, resultFilePath, namespace string, appName name.Ap
 	}
 
 	// apply resources
-	cmd, err := kustomize.New(resultFileDirPath, false)
+	cmd, err := kustomize.New(resultFileDirPath, false, false)
 	if err != nil {
 		return nil, err
 	}
@@ -74,11 +74,15 @@ func prepareAdditionalFiles(resultFilePath, namespace string, appName name.Appli
 	resultFileTransformerPath := filepath.Join(resultFileDirPath, "transformer.yaml")
 
 	if helper.FileExists(resultFileKustomizePath) {
-		os.Remove(resultFileKustomizePath)
+		if err := os.Remove(resultFileKustomizePath); err != nil {
+			return err
+		}
 	}
 
 	if helper.FileExists(resultFileTransformerPath) {
-		os.Remove(resultFileTransformerPath)
+		if err := os.Remove(resultFileTransformerPath); err != nil {
+			return err
+		}
 	}
 
 	transformer := &kustomize.LabelTransformer{

internal/git/git.go

@@ -242,6 +242,10 @@ func (g *Client) stage(files ...File) (bool, error) {
 
 		monitor.Debug("Overwriting local index")
 
+		if err := g.fs.MkdirAll(filepath.Dir(f.Path), os.ModePerm); err != nil {
+			return false, err
+		}
+
 		file, err := g.fs.Create(f.Path)
 		if err != nil {
 			return true, errors.Wrapf(err, "creating file %s in worktree failed", f.Path)

internal/gitcrd/v1beta1/gitcrd.go

@@ -124,6 +124,16 @@ func (c *GitCrd) Reconcile(currentResourceList []*clientgo.Resource) {
 	// pre-steps
 	if toolsetCRD.Spec.PreApply != nil {
 		pre := toolsetCRD.Spec.PreApply
+		if pre.Folder != "" {
+			c.status = errors.New("PreApply defined but no folder provided")
+		}
+		if !helper.FolderExists(pre.Folder) {
+			c.status = errors.New("PreApply provided folder is nonexistent")
+		}
+		if empty, err := helper.FolderEmpty(pre.Folder); empty == true || err != nil {
+			c.status = errors.New("PreApply provided folder is empty")
+		}
+
 		c.gitMutex.Lock()
 		err := helper.CopyFolderToLocal(c.git, c.crdDirectoryPath, pre.Folder)
 		c.gitMutex.Unlock()
@@ -148,6 +158,16 @@ func (c *GitCrd) Reconcile(currentResourceList []*clientgo.Resource) {
 	// post-steps
 	if toolsetCRD.Spec.PostApply != nil {
 		post := toolsetCRD.Spec.PostApply
+		if post.Folder != "" {
+			c.status = errors.New("PostApply defined but no folder provided")
+		}
+		if !helper.FolderExists(post.Folder) {
+			c.status = errors.New("PostApply provided folder is nonexistent")
+		}
+		if empty, err := helper.FolderEmpty(post.Folder); empty == true || err != nil {
+			c.status = errors.New("PostApply provided folder is empty")
+		}
+
 		c.gitMutex.Lock()
 		err := helper.CopyFolderToLocal(c.git, c.crdDirectoryPath, post.Folder)
 		c.gitMutex.Unlock()
@@ -199,7 +219,7 @@ func (c *GitCrd) WriteBackCurrentState(currentResourceList []*clientgo.Resource)
 
 	currentFolder := toolsetCRD.Spec.CurrentStateFolder
 	if currentFolder == "" {
-		currentFolder = filepath.Join("internal", "boom")
+		currentFolder = filepath.Join("caos-internal", "boom")
 	}
 
 	file := git.File{

internal/helper/files.go

@@ -1,6 +1,7 @@
 package helper
 
 import (
+	"io"
 	"os"
 )
 
@@ -11,3 +12,28 @@ func FileExists(filename string) bool {
 	}
 	return !info.IsDir()
 }
+
+func FolderExists(folder string) bool {
+	info, err := os.Stat(folder)
+	if os.IsNotExist(err) {
+		return false
+	}
+	return info.IsDir()
+}
+
+func FolderEmpty(folder string) (bool, error) {
+	f, err := os.Open(folder)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	// read in ONLY one file
+	_, err = f.Readdir(1)
+
+	// and if the file is EOF... well, the dir is empty.
+	if err == io.EOF {
+		return true, nil
+	}
+	return false, err
+}

internal/kustomize/kustomize.go

@@ -30,9 +30,10 @@ type FieldSpec struct {
 type Kustomize struct {
 	path  string
 	apply bool
+	force bool
 }
 
-func New(path string, apply bool) (*Kustomize, error) {
+func New(path string, apply bool, force bool) (*Kustomize, error) {
 	abspath, err := filepath.Abs(path)
 	if err != nil {
 		return nil, err
@@ -41,14 +42,19 @@ func New(path string, apply bool) (*Kustomize, error) {
 	return &Kustomize{
 		path:  abspath,
 		apply: apply,
+		force: force,
 	}, nil
 }
 
 func (k *Kustomize) Build() exec.Cmd {
 	all := strings.Join([]string{"kustomize", "build", k.path}, " ")
 	if k.apply {
 		all = strings.Join([]string{all, "| kubectl apply -f -"}, " ")
+		if k.force {
+			all = strings.Join([]string{all, "--force"}, " ")
+		}
 	}
+
 	cmd := exec.Command("/bin/sh", "-c", all)
 	return *cmd
 }