Fix load context bug

cmd/arcaflow/main.go

@@ -5,16 +5,13 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"os"
-	"os/signal"
-	"path/filepath"
-	"strings"
-
 	"go.arcalot.io/log/v2"
-
 	"go.flow.arcalot.io/engine"
 	"go.flow.arcalot.io/engine/config"
+	"go.flow.arcalot.io/engine/loadfile"
 	"gopkg.in/yaml.v3"
+	"os"
+	"os/signal"
 )
 
 // These variables are filled using ldflags during the build process with Goreleaser.
@@ -38,6 +35,15 @@ const ExitCodeWorkflowErrorOutput = 2
 // ExitCodeWorkflowFailed indicates that the workflow execution failed.
 const ExitCodeWorkflowFailed = 3
 
+// RequiredFileKeyWorkflow is the key for the workflow file in hash map of files required for execution.
+const RequiredFileKeyWorkflow = "workflow"
+
+// RequiredFileKeyConfig is the key for the config file in hash map of files required for execution.
+const RequiredFileKeyConfig = "config"
+
+// RequiredFileKeyInput is the key for the input file in hash map of files required for execution.
+const RequiredFileKeyInput = "input"
+
 func main() {
 	tempLogger := log.New(log.Config{
 		Level:       log.LevelInfo,
@@ -117,9 +123,23 @@ Options:
 	}
 
 	var err error
+
+	requiredFiles := map[string]string{
+		RequiredFileKeyConfig:   configFile,
+		RequiredFileKeyInput:    input,
+		RequiredFileKeyWorkflow: workflowFile,
+	}
+
+	requiredFilesAbsPaths, err := loadfile.AbsPathsWithContext(dir, requiredFiles)
+	if err != nil {
+		flag.Usage()
+		tempLogger.Errorf("context path resolution failed %s (%v)", dir, err)
+		os.Exit(ExitCodeInvalidData)
+	}
+
 	var configData any = map[any]any{}
 	if configFile != "" {
-		configData, err = loadYamlFile(configFile)
+		configData, err = loadYamlFile(requiredFilesAbsPaths[RequiredFileKeyConfig])
 		if err != nil {
 			tempLogger.Errorf("Failed to load configuration file %s (%v)", configFile, err)
 			flag.Usage()
@@ -132,13 +152,20 @@ Options:
 		flag.Usage()
 		os.Exit(ExitCodeInvalidData)
 	}
-	cfg.Log.Stdout = os.Stderr
 
+	// now we are ready to instantiate our main logger
+	cfg.Log.Stdout = os.Stderr
 	logger := log.New(cfg.Log).WithLabel("source", "main")
 
-	dirContext, err := loadContext(dir)
+	var requiredFilesAbsSlice = make([]string, len(requiredFilesAbsPaths))
+	var j int
+	for _, f := range requiredFilesAbsPaths {
+		requiredFilesAbsSlice[j] = f
+		j++
+	}
+	dirContext, err := loadfile.LoadContext(requiredFilesAbsSlice)
 	if err != nil {
-		logger.Errorf("Failed to load configuration file %s (%v)", configFile, err)
+		logger.Errorf("Failed to load required files into context (%v)", err)
 		flag.Usage()
 		os.Exit(ExitCodeInvalidData)
 	}
@@ -160,7 +187,7 @@ Options:
 		}
 	}
 
-	os.Exit(runWorkflow(flow, dirContext, workflowFile, logger, inputData))
+	os.Exit(runWorkflow(flow, dirContext, requiredFilesAbsPaths[RequiredFileKeyWorkflow], logger, inputData))
 }
 
 func runWorkflow(flow engine.WorkflowEngine, dirContext map[string][]byte, workflowFile string, logger log.Logger, inputData []byte) int {
@@ -224,32 +251,6 @@ func handleOSInterrupt(ctrlC chan os.Signal, cancel context.CancelFunc, logger l
 	os.Exit(1)
 }
 
-func loadContext(dir string) (map[string][]byte, error) {
-	absDir, err := filepath.Abs(dir)
-	if err != nil {
-		return nil, fmt.Errorf("failed to obtain absolute path of context directory %s (%w)", dir, err)
-	}
-	result := map[string][]byte{}
-	err = filepath.Walk(absDir,
-		func(path string, i os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-
-			if !i.IsDir() {
-				fileData, err := os.ReadFile(path) //nolint:gosec
-				if err != nil {
-					return fmt.Errorf("failed to read file from context directory: %s (%w)", path, err)
-				}
-				path = strings.TrimPrefix(path, absDir)
-				path = strings.TrimPrefix(path, string([]byte{os.PathSeparator}))
-				result[path] = fileData
-			}
-			return nil
-		})
-	return result, err
-}
-
 func loadYamlFile(configFile string) (any, error) {
 	fileContents, err := os.ReadFile(configFile) //nolint:gosec
 	if err != nil {

go.mod

@@ -3,7 +3,7 @@ module go.flow.arcalot.io/engine
 go 1.18
 
 require (
-	go.arcalot.io/assert v1.6.0
+	go.arcalot.io/assert v1.6.1
 	go.arcalot.io/dgraph v1.1.0
 	go.arcalot.io/lang v1.0.0
 	go.arcalot.io/log/v2 v2.0.0

go.sum

@@ -92,7 +92,6 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
-github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
@@ -152,8 +151,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.arcalot.io/assert v1.6.0 h1:iKA8SZZ1MRblMX5QAwwY5RbpR+VNyp//4IU7vo08Xu0=
-go.arcalot.io/assert v1.6.0/go.mod h1:Xy3ScX0p9IMY89gdsgexOKxnmDr0nGHG9dV7p8Uxg7w=
+go.arcalot.io/assert v1.6.1 h1:pr7wvTE/vZJ4pY0f88R8tXicTOGogkGS1mtY0Qb7GRE=
+go.arcalot.io/assert v1.6.1/go.mod h1:Xy3ScX0p9IMY89gdsgexOKxnmDr0nGHG9dV7p8Uxg7w=
 go.arcalot.io/dgraph v1.1.0 h1:c0LR7+xdUy7Ki6e4nR9rBvK0Upr4Nu49fu+poP/9WMg=
 go.arcalot.io/dgraph v1.1.0/go.mod h1:FuNv92OgHsJYepD6Unwn+S/4DioBnv06JxQ2BtQct7E=
 go.arcalot.io/lang v1.0.0 h1:mgDaieT4wWdZTnR4V7+/pgYRmzfU7VZZgIzHccuxAbY=
@@ -170,8 +169,6 @@ go.flow.arcalot.io/kubernetesdeployer v0.8.0 h1:UjH/aspPif/k+X65sLWlNDZAW5JlzUfg
 go.flow.arcalot.io/kubernetesdeployer v0.8.0/go.mod h1:BhERhKpvQMJkrcW9lbBF4kJEe+OGhz2NpSftZIgtVNQ=
 go.flow.arcalot.io/pluginsdk v0.5.1 h1:ebb2ThAqmjmwGpDyKpd1wEDUisPqPabgARjFohy47Io=
 go.flow.arcalot.io/pluginsdk v0.5.1/go.mod h1:2s2f//7uOkBjr1QaiWJD/bqDIeLlINJtD1BhiY4aGPM=
-go.flow.arcalot.io/podmandeployer v0.6.2 h1:iAAZGgwhxInEVAleakavGruHnW4qsD/v39JpfnTeXiE=
-go.flow.arcalot.io/podmandeployer v0.6.2/go.mod h1:BmKbyG2qZG9PMPLkIeXUvKVJfU+AZx+POLvydZN26IY=
 go.flow.arcalot.io/podmandeployer v0.7.0 h1:bXzWi4IjjLTIftUbH2NPgPiyTb82lzERVgfHP4zpmXI=
 go.flow.arcalot.io/podmandeployer v0.7.0/go.mod h1:tiWVDNpeNpPrY2GloihwjtnCEzJf8zNxBbiwVGRX7rs=
 go.flow.arcalot.io/pythondeployer v0.4.0 h1:l8nw6awYMVzgND+ZXdbnNJPYu3V0sgSUFsIzn+SRgh0=

loadfile/loadfile.go

@@ -0,0 +1,47 @@
+// Package loadfile provides functions to load files from a directory.
+package loadfile
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// LoadContext reads the content of each file into a map where the key
+// is the absolute filepath and the file content is the value.
+func LoadContext(neededFilepaths []string) (map[string][]byte, error) {
+	result := map[string][]byte{}
+	var err error
+	for _, filePath := range neededFilepaths {
+		absPath, err := filepath.Abs(filePath)
+		if err != nil {
+			return nil, fmt.Errorf("error obtaining absolute path of file %s (%w)",
+				filePath, err)
+		}
+		fileData, err := os.ReadFile(filepath.Clean(absPath))
+		if err != nil {
+			return nil, fmt.Errorf("error reading file %s (%w)", absPath, err)
+		}
+		result[absPath] = fileData
+	}
+	return result, err
+}
+
+// AbsPathsWithContext creates a map of absolute filepaths. If a required
+// file is not provided with an absolute path, then it is joined with the
+// root directory.
+func AbsPathsWithContext(rootDir string, requiredFiles map[string]string) (map[string]string, error) {
+	absDir, err := filepath.Abs(rootDir)
+	if err != nil {
+		return nil, fmt.Errorf("error determining context directory absolute path %s (%w)", rootDir, err)
+	}
+	requiredFilesAbs := map[string]string{}
+	for key, f := range requiredFiles {
+		abspath := f
+		if !filepath.IsAbs(f) {
+			abspath = filepath.Join(absDir, f)
+		}
+		requiredFilesAbs[key] = abspath
+	}
+	return requiredFilesAbs, nil
+}

loadfile/loadfile_test.go

@@ -0,0 +1,113 @@
+package loadfile_test
+
+import (
+	"go.arcalot.io/assert"
+	"go.flow.arcalot.io/engine/loadfile"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// This tests the functional behavior of LoadContext when it is given
+// a directory of files that cover more than the directory file node
+// type 'd' (as seen in *nix systems). Specifically, this test adds
+// symbolic link files for a regular file and a directory file. The
+// engine should only attempt to read files of a type that the os can
+// read (i.e. not throw an error on a call to os.ReadFile()), and
+// disregard (not throw an error) files with a type it cannot read.
+func TestLoadContext(t *testing.T) {
+	testdir := "/tmp/loadfile-test"
+	// cleanup directory even if it's there
+	_ = os.RemoveAll(testdir)
+
+	assert.NoError(t, os.MkdirAll(testdir, os.ModePerm))
+
+	// create a directory
+	dirname := "mydir"
+	dirpath := filepath.Join(testdir, dirname)
+	assert.NoError(t, os.MkdirAll(dirpath, os.ModePerm))
+
+	// create a file
+	filename := "myfile"
+	filePath := filepath.Join(testdir, filename)
+	f, err := os.Create(filepath.Clean(filePath))
+	assert.NoError(t, err)
+	assert.NoError(t, f.Close())
+
+	// create symlink to the directory
+	symlinkDirname := dirname + "_sym"
+	symlinkDirpath := filepath.Join(testdir, symlinkDirname)
+	assert.NoError(t, os.Symlink(dirpath, symlinkDirpath))
+
+	// create symlink to the file
+	symlinkFilepath := filePath + "_sym"
+	assert.NoError(t, os.Symlink(filePath, symlinkFilepath))
+
+	neededFiles := []string{
+		filePath,
+		symlinkFilepath,
+	}
+	filemap, err := loadfile.LoadContext(neededFiles)
+	// assert no error on attempting to read files
+	// that cannot be read
+	assert.NoError(t, err)
+
+	// assert only the regular and symlinked file are loaded
+	filemapExp := map[string][]byte{
+		filePath:        {},
+		symlinkFilepath: {},
+	}
+	assert.Equals(t, filemap, filemapExp)
+
+	// error on loading a directory
+	neededFiles = []string{
+		dirpath,
+	}
+
+	errFileRead := "reading file"
+	ctxFiles, err := loadfile.LoadContext(neededFiles)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), errFileRead)
+	assert.Nil(t, ctxFiles)
+
+	// error on loading a symlink directory
+	neededFiles = []string{
+		symlinkDirpath,
+	}
+	ctxFiles, err = loadfile.LoadContext(neededFiles)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), errFileRead)
+	assert.Nil(t, ctxFiles)
+
+	t.Cleanup(func() {
+		assert.NoError(t, os.RemoveAll(testdir))
+	})
+}
+
+// This tests AbsPathsWithContext which joins relative paths
+// with the context (root) directory, and passes through
+// absolute paths unmodified.
+func TestContextAbsFilepaths(t *testing.T) {
+	testdir, err := os.MkdirTemp(os.TempDir(), "")
+	assert.NoError(t, err)
+
+	testFilepaths := map[string]string{
+		"a": "a.yaml",
+		"b": "/b.toml",
+		"c": "rel/../subdir/c.txt",
+	}
+
+	absPathsExp := map[string]string{
+		"a": filepath.Join(testdir, testFilepaths["a"]),
+		"b": testFilepaths["b"],
+		"c": filepath.Join(testdir, testFilepaths["c"]),
+	}
+
+	absPathsGot, err := loadfile.AbsPathsWithContext(testdir, testFilepaths)
+	assert.NoError(t, err)
+	assert.Equals(t, absPathsExp, absPathsGot)
+
+	t.Cleanup(func() {
+		assert.NoError(t, os.RemoveAll(testdir))
+	})
+}