fixes, new tests, remote kubernetes manifest

.vscode/settings.json

@@ -120,6 +120,7 @@
     "subcommand",
     "submodules",
     "sysctl",
+    "templating",
     "testrepo",
     "tfmodule",
     "tfplan",

internal/project/scanners.go

@@ -86,6 +86,7 @@ func StateOutputsReplacer(data reflect.Value, unit Unit) (reflect.Value, error)
 	resString := subVal.String()
 	markersList := unit.Project().UnitLinks.ByLinkTypes(OutputLinkType).Map()
 	for key, marker := range markersList {
+		// log.Warnf("StateOutputsReplacer: %v", key)
 		if strings.Contains(resString, key) {
 			resString = strings.ReplaceAll(resString, key, fmt.Sprintf("<output %v.%v.%v>", marker.TargetStackName, marker.TargetUnitName, marker.OutputName))
 		}

internal/project/state.go

@@ -236,10 +236,6 @@ func (p *Project) LoadState() (*StateProject, error) {
 	if err != nil {
 		return nil, err
 	}
-	for key := range statePrj.Units {
-		log.Warnf("LoadState %v", key)
-	}
-
 	return statePrj, nil
 }
 

internal/project/templating.go

@@ -175,9 +175,9 @@ func (d *GlobalTemplateDriver) AddTemplateFunctions(mp template.FuncMap, p *Proj
 
 	addOutputMarkerFunk := func(path string) (string, error) {
 		splittedPath := strings.Split(path, ".")
-		// if len(splittedPath) != 3 {
-		// 	return "", fmt.Errorf("bad dependency path 2")
-		// }
+		if len(splittedPath) == 2 {
+			splittedPath = append([]string{"this"}, splittedPath...)
+		}
 
 		dep := ULinkT{
 			Unit:            nil,

internal/units/shell/common/files.go

@@ -12,7 +12,7 @@ import (
 	"github.com/apex/log"
 )
 
-var fileSizeLimit int64 = 102400
+var fileSizeLimit int64 = 1024000
 var ignorePatterns []string = []string{
 	".*\\.terraform/.*",
 	".*\\.git/.*",

internal/units/shell/common/state.go

@@ -56,10 +56,9 @@ func (u *Unit) GetUnitDiff() UnitDiffSpec {
 	for _, file := range *u.CreateFiles {
 		fileLines := strings.Split(file.Content, "\n")
 		if len(fileLines) < 2 {
-			filesListDiff[file.FileName][0] = file.Content
+			filesListDiff[file.FileName] = append(filesListDiff[file.FileName], file.Content)
 		} else {
 			for _, line := range fileLines {
-				//log.Warnf("filesListDiff %v", line)
 				if line == "" {
 					continue // Ignore empty lines
 				}

internal/units/shell/common/unit.go

@@ -28,11 +28,7 @@ type OperationConfig struct {
 	Commands []interface{} `yaml:"commands" json:"commands"`
 }
 
-type OutputsT struct {
-	List map[string]*project.ULinkT `json:"outputs_list,omitempty"`
-}
-
-// OutputsConfigSpec describe how to retrive parse unit outputs.
+// OutputsConfigSpec describe how to retrieve parse unit outputs.
 type OutputsConfigSpec struct {
 	Command   string `yaml:"command,omitempty" json:"command,omitempty"`
 	Type      string `yaml:"type" json:"type"`
@@ -105,7 +101,7 @@ func (u *Unit) ForceApply() bool {
 	return u.FApply
 }
 
-// WasApplied return true if unit's method Apply was runned.
+// WasApplied return true if unit's method Apply was ran.
 func (u *Unit) WasApplied() bool {
 	return u.AlreadyApplied
 }
@@ -314,8 +310,6 @@ func (u *Unit) Apply() error {
 		err = parser(string(u.OutputRaw), u.ProjectPtr.UnitLinks.ByTargetUnit(u).ByLinkTypes(project.OutputLinkType))
 		if err != nil {
 
-			//str := fmt.Sprintf("Outputs data: %s", string(u.OutputRaw))
-			// log.Warnf("Len: %v", len(str))
 			u.SetTainted(true, err)
 			return fmt.Errorf("parse outputs '%v': %w", u.GetOutputsConf.Type, err)
 		}
@@ -327,14 +321,6 @@ func (u *Unit) Apply() error {
 	return err
 }
 
-// func (u *Unit) MarkTainted(err error) {
-// 	u.ExecErr = err
-// 	if u.SavedState != nil {
-// 		u.SavedState.SetTainted(true)
-// 		log.Warnf("MarkTainted %v", u.SavedState.IsTainted())
-// 	}
-// }
-
 func (u *Unit) runCommands(commandsCnf OperationConfig, name string) ([]byte, error) {
 	if len(commandsCnf.Commands) == 0 {
 		log.Debugf("configuration for '%v' is empty for unit '%v'. Skip.", name, u.Key())
@@ -439,7 +425,6 @@ func (u *Unit) UpdateProjectRuntimeData(p *project.Project) error {
 }
 
 func (u *Unit) ScanData(scanner project.MarkerScanner) error {
-	// log.Warnf("Replacing markers...")
 	if u.Env != nil {
 		err := project.ScanMarkers(u.Env, scanner, u)
 		if err != nil {
@@ -547,7 +532,7 @@ func readHook(hookData interface{}, hookType string) (*HookSpec, error) {
 	cmd, cmdExists := hook["command"].(string)
 
 	if !cmdExists {
-		return nil, fmt.Errorf("error in %s config, use 'script' option", hookType)
+		return nil, fmt.Errorf("error in %s config, 'command' option is required", hookType)
 	}
 
 	ScriptData := HookSpec{
@@ -586,14 +571,19 @@ func (u *Unit) EnvSlice() []string {
 	return res
 }
 
+// SetExecStatus set unit execution status (backlog, in progress, etc).
 func (u *Unit) SetExecStatus(status project.ExecutionStatus) {
 	if u.GetExecStatus() != status {
 		u.ExecStatus = status
 	}
 }
+
+// GetExecStatus get current execution status.
 func (u *Unit) GetExecStatus() project.ExecutionStatus {
 	return u.ExecStatus
 }
+
+// ExecError return last execution error
 func (u *Unit) ExecError() error {
 	return u.ExecErr
 }

internal/units/shell/k8s_manifest/unit.go

@@ -123,7 +123,7 @@ func (u *Unit) ReadManifestsPath(src string) error {
 		}
 		if isDir {
 			u.ManifestsFiles.ReadDir(manifestsPath, baseDir, `.ya{0,1}ml$`)
-			// log.Debugf("List %v", u.ManifestsFiles.SPrintLs())
+			log.Debugf("List %v", u.ManifestsFiles.SPrintLs())
 			if u.ManifestsFiles.IsEmpty() {
 				return fmt.Errorf("read unit '%v': no manifests found in path %v", u.Name(), u.Path)
 			}
@@ -139,7 +139,7 @@ func (u *Unit) ReadManifestsPath(src string) error {
 	} else {
 		manifest, err := utils.GetFileByUrl(src)
 		if err != nil {
-			return fmt.Errorf("get remote file: %w", err)
+			return fmt.Errorf("get remote file by url (%v): %w", src, err)
 		}
 		err = u.ManifestsFiles.AddOverride("./main.yaml", manifest, fs.ModePerm)
 		if err != nil {
@@ -215,6 +215,9 @@ func (u *Unit) GetManifestsMap() (res map[string]interface{}, namespaces []strin
 	res = make(map[string]interface{})
 	namespaces = []string{}
 	nsUniq := map[string]bool{}
+	if u == nil {
+		return
+	}
 	for _, file := range *u.ManifestsFiles {
 		mns, err := utils.ReadYAMLObjects([]byte(file.Content))
 		if err != nil {

internal/units/shell/terraform/helm/state.go

@@ -43,16 +43,15 @@ func (u *Unit) GetUnitDiff() UnitDiffSpec {
 		Source:       u.Source,
 		HelmOpts:     u.HelmOpts,
 		Sets:         u.Sets,
-		//		Values:       u.ValuesFilesList,
 	}
 	filesListDiff := make([]interface{}, len(u.ValuesFilesList))
+	st.Outputs = nil
 	for i, str := range u.ValuesFilesList {
 		fileLines := strings.Split(str, "\n")
 		if len(fileLines) < 2 {
 			filesListDiff[i] = str
 		} else {
 			for _, line := range fileLines {
-				//log.Warnf("filesListDiff %v", line)
 				if line == "" {
 					continue // Ignore empty lines
 				}
@@ -69,6 +68,7 @@ func (u *Unit) GetDiffData() interface{} {
 	res := map[string]interface{}{}
 	utils.JSONCopy(st, &res)
 	project.ScanMarkers(res, base.StringRemStScanner, u)
+	project.ScanMarkers(res, project.OutputsReplacer, u)
 	return res
 }
 

internal/units/shell/terraform/kubernetes/main.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/apex/log"
 	"github.com/hashicorp/hcl/v2/hclwrite"
+	"github.com/shalb/cluster.dev/internal/config"
 	"github.com/shalb/cluster.dev/internal/units/shell/terraform/base"
 	"github.com/shalb/cluster.dev/pkg/hcltools"
 
@@ -71,30 +72,40 @@ func (u *Unit) ReadConfig(spec map[string]interface{}, stack *project.Stack) err
 	if !ok {
 		return fmt.Errorf("reading kubernetes unit '%v': malformed unit source", u.Key())
 	}
-	tmplDir := u.Stack().TemplateDir
 	var absSource string
-	if source[1:2] == "/" {
-		absSource = filepath.Join(tmplDir, source)
-	} else {
-		absSource = source
-	}
-	fileInfo, err := os.Stat(absSource)
-	if err != nil {
-		return fmt.Errorf("reading kubernetes unit '%v': reading kubernetes manifests form source '%v': %v", u.Key(), source, err.Error())
-	}
 	var filesList []string
-	if fileInfo.IsDir() {
-		filesList, err = filepath.Glob(absSource + "/*.yaml")
+	if utils.IsLocalPath(source) {
+		if utils.IsAbsolutePath(source) {
+			absSource = source
+		} else {
+			absSource = filepath.Join(config.Global.ProjectConfigsPath, u.Stack().TemplateDir, source)
+		}
+
+		fileInfo, err := os.Stat(absSource)
 		if err != nil {
-			return fmt.Errorf("reading kubernetes unit '%v': reading kubernetes manifests form source '%v': %v", u.Key(), source, err.Error())
+			return fmt.Errorf("reading kubernetes unit '%v': check file: '%v': %v", u.Key(), source, err.Error())
+		}
+		if fileInfo.IsDir() {
+			filesList, err = utils.ListFilesByRegex(absSource, `\.ya{0,1}ml$`) //filepath.Glob(absSource + "/*.yaml")
+			if err != nil {
+				return fmt.Errorf("reading kubernetes unit '%v': list manifests in dir '%v': %v", u.Key(), source, err.Error())
+			}
+		} else {
+			filesList = append(filesList, absSource)
 		}
 	} else {
-		filesList = append(filesList, absSource)
+		filesList = append(filesList, source)
 	}
 	for _, fileName := range filesList {
-		file, err := os.ReadFile(fileName)
+		var file []byte
+		var err error
+		if utils.IsLocalPath(fileName) {
+			file, err = os.ReadFile(fileName)
+		} else {
+			file, err = utils.GetFileByUrlByte(fileName)
+		}
 		if err != nil {
-			return fmt.Errorf("reading kubernetes unit '%v': reading kubernetes manifests form source '%v': %v", u.Key(), source, err.Error())
+			return fmt.Errorf("reading kubernetes unit '%v': read manifest from '%v': %v", u.Key(), source, err.Error())
 		}
 		manifest, errIsWarn, err := u.Stack().TemplateTry(file, fileName)
 		if err != nil {
@@ -119,7 +130,7 @@ func (u *Unit) ReadConfig(spec map[string]interface{}, stack *project.Stack) err
 		return fmt.Errorf("the kubernetes unit must contain at least one manifest")
 	}
 
-	err = utils.YAMLInterfaceToType(spec, u)
+	err := utils.YAMLInterfaceToType(spec, u)
 	if err != nil {
 		return err
 	}

internal/units/shell/terraform/kubernetes/state.go

@@ -18,13 +18,21 @@ func (u *Unit) GetStateUnit() *Unit {
 		log.Fatalf("read unit '%v': create state: %w", u.Name(), err)
 	}
 	unitState.Unit = *u.Unit.GetStateUnit()
+	// unitState.GetOutputsConf = nil
 	return &unitState
 }
 
+func printStateDebug(state interface{}, comment string) {
+	js, _ := utils.JSONEncodeString(state)
+	log.Warnf("printStateDebug [%v]: %v", comment, js)
+}
+
 func (u *Unit) GetState() project.Unit {
 	if u.StateData != nil {
+		// printStateDebug(u.StateData, "GetState")
 		return u.StateData
 	}
+	// printStateDebug(u.GetStateUnit(), "GetState")
 	return u.GetStateUnit()
 }
 
@@ -47,8 +55,10 @@ func (u *Unit) GetUnitDiff() UnitDiffSpec {
 func (u *Unit) GetDiffData() interface{} {
 	st := u.GetUnitDiff()
 	res := map[string]interface{}{}
+	st.Outputs = nil
 	utils.JSONCopy(st, &res)
 	project.ScanMarkers(res, base.StringRemStScanner, u)
+	// printStateDebug(res, "GetDiffData")
 	return res
 }
 
@@ -61,5 +71,8 @@ func (u *Unit) LoadState(stateData interface{}, modKey string, p *project.StateP
 	if err != nil {
 		return fmt.Errorf("load state: %v", err.Error())
 	}
+	// u.GetState()
+	// u.GetDiffData()
+	u.GetOutputsConf = nil
 	return nil
 }

pkg/utils/files.go

@@ -240,3 +240,32 @@ func RemoveDirContent(dir string) error {
 	}
 	return nil
 }
+
+func ListFilesByRegex(dirPath, pattern string) ([]string, error) {
+	re, err := regexp.Compile(pattern)
+	if err != nil {
+		return nil, fmt.Errorf("invalid regex pattern: %s", err)
+	}
+
+	var matches []string
+	err = filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			return nil
+		}
+
+		if re.MatchString(info.Name()) {
+			matches = append(matches, path)
+		}
+		return nil
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("error walking directory: %s", err)
+	}
+
+	return matches, nil
+}

pkg/utils/net.go

@@ -1,6 +1,8 @@
 package utils
 
 import (
+	"bytes"
+	"errors"
 	"fmt"
 	"io"
 	"math/big"
@@ -60,3 +62,20 @@ func GetFileByUrl(srcUrl string) (string, error) {
 
 	return buf.String(), nil
 }
+
+func GetFileByUrlByte(URL string) ([]byte, error) {
+	response, err := http.Get(URL)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return nil, errors.New(response.Status)
+	}
+	var data bytes.Buffer
+	_, err = io.Copy(&data, response.Body)
+	if err != nil {
+		return nil, err
+	}
+	return data.Bytes(), nil
+}

pkg/utils/yaml.go

@@ -20,7 +20,9 @@ func ReadYAMLObjects(objData []byte) ([]map[string]interface{}, error) {
 			// log.Debugf("can't decode config to yaml: %s", err.Error())
 			return nil, fmt.Errorf("can't decode config to yaml: %s", ResolveYamlError(objData, err))
 		}
-		objects = append(objects, parsedConf)
+		if len(parsedConf) > 0 { // Ignore empty yaml parts, like two '---' with only comment between.
+			objects = append(objects, parsedConf)
+		}
 	}
 	return objects, nil
 }