diff --git a/cmd/arcaflow/main.go b/cmd/arcaflow/main.go index b87ca800..9be3b58e 100644 --- a/cmd/arcaflow/main.go +++ b/cmd/arcaflow/main.go @@ -5,16 +5,13 @@ import ( "context" "flag" "fmt" - "os" - "os/signal" - "path/filepath" - "strings" - "go.arcalot.io/log/v2" - "go.flow.arcalot.io/engine" "go.flow.arcalot.io/engine/config" + "go.flow.arcalot.io/engine/loadfile" "gopkg.in/yaml.v3" + "os" + "os/signal" ) // These variables are filled using ldflags during the build process with Goreleaser. @@ -38,6 +35,15 @@ const ExitCodeWorkflowErrorOutput = 2 // ExitCodeWorkflowFailed indicates that the workflow execution failed. const ExitCodeWorkflowFailed = 3 +// RequiredFileKeyWorkflow is the key for the workflow file in hash map of files required for execution. +const RequiredFileKeyWorkflow = "workflow" + +// RequiredFileKeyConfig is the key for the config file in hash map of files required for execution. +const RequiredFileKeyConfig = "config" + +// RequiredFileKeyInput is the key for the input file in hash map of files required for execution. +const RequiredFileKeyInput = "input" + func main() { tempLogger := log.New(log.Config{ Level: log.LevelInfo, @@ -117,9 +123,23 @@ Options: } var err error + + requiredFiles := map[string]string{ + RequiredFileKeyConfig: configFile, + RequiredFileKeyInput: input, + RequiredFileKeyWorkflow: workflowFile, + } + + requiredFilesAbsPaths, err := loadfile.AbsPathsWithContext(dir, requiredFiles) + if err != nil { + flag.Usage() + tempLogger.Errorf("context path resolution failed %s (%v)", dir, err) + os.Exit(ExitCodeInvalidData) + } + var configData any = map[any]any{} if configFile != "" { - configData, err = loadYamlFile(configFile) + configData, err = loadYamlFile(requiredFilesAbsPaths[RequiredFileKeyConfig]) if err != nil { tempLogger.Errorf("Failed to load configuration file %s (%v)", configFile, err) flag.Usage() @@ -132,13 +152,20 @@ Options: flag.Usage() os.Exit(ExitCodeInvalidData) } - cfg.Log.Stdout = os.Stderr + // now we are ready to instantiate our main logger + cfg.Log.Stdout = os.Stderr logger := log.New(cfg.Log).WithLabel("source", "main") - dirContext, err := loadContext(dir) + var requiredFilesAbsSlice = make([]string, len(requiredFilesAbsPaths)) + var j int + for _, f := range requiredFilesAbsPaths { + requiredFilesAbsSlice[j] = f + j++ + } + dirContext, err := loadfile.LoadContext(requiredFilesAbsSlice) if err != nil { - logger.Errorf("Failed to load configuration file %s (%v)", configFile, err) + logger.Errorf("Failed to load required files into context (%v)", err) flag.Usage() os.Exit(ExitCodeInvalidData) } @@ -160,7 +187,7 @@ Options: } } - os.Exit(runWorkflow(flow, dirContext, workflowFile, logger, inputData)) + os.Exit(runWorkflow(flow, dirContext, requiredFilesAbsPaths[RequiredFileKeyWorkflow], logger, inputData)) } func runWorkflow(flow engine.WorkflowEngine, dirContext map[string][]byte, workflowFile string, logger log.Logger, inputData []byte) int { @@ -224,32 +251,6 @@ func handleOSInterrupt(ctrlC chan os.Signal, cancel context.CancelFunc, logger l os.Exit(1) } -func loadContext(dir string) (map[string][]byte, error) { - absDir, err := filepath.Abs(dir) - if err != nil { - return nil, fmt.Errorf("failed to obtain absolute path of context directory %s (%w)", dir, err) - } - result := map[string][]byte{} - err = filepath.Walk(absDir, - func(path string, i os.FileInfo, err error) error { - if err != nil { - return err - } - - if !i.IsDir() { - fileData, err := os.ReadFile(path) //nolint:gosec - if err != nil { - return fmt.Errorf("failed to read file from context directory: %s (%w)", path, err) - } - path = strings.TrimPrefix(path, absDir) - path = strings.TrimPrefix(path, string([]byte{os.PathSeparator})) - result[path] = fileData - } - return nil - }) - return result, err -} - func loadYamlFile(configFile string) (any, error) { fileContents, err := os.ReadFile(configFile) //nolint:gosec if err != nil { diff --git a/go.mod b/go.mod index fe72af32..0effad3e 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module go.flow.arcalot.io/engine go 1.18 require ( - go.arcalot.io/assert v1.6.0 + go.arcalot.io/assert v1.6.1 go.arcalot.io/dgraph v1.1.0 go.arcalot.io/lang v1.0.0 go.arcalot.io/log/v2 v2.0.0 diff --git a/go.sum b/go.sum index 2147b255..8742ddf4 100644 --- a/go.sum +++ b/go.sum @@ -92,7 +92,6 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -152,8 +151,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1: github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.arcalot.io/assert v1.6.0 h1:iKA8SZZ1MRblMX5QAwwY5RbpR+VNyp//4IU7vo08Xu0= -go.arcalot.io/assert v1.6.0/go.mod h1:Xy3ScX0p9IMY89gdsgexOKxnmDr0nGHG9dV7p8Uxg7w= +go.arcalot.io/assert v1.6.1 h1:pr7wvTE/vZJ4pY0f88R8tXicTOGogkGS1mtY0Qb7GRE= +go.arcalot.io/assert v1.6.1/go.mod h1:Xy3ScX0p9IMY89gdsgexOKxnmDr0nGHG9dV7p8Uxg7w= go.arcalot.io/dgraph v1.1.0 h1:c0LR7+xdUy7Ki6e4nR9rBvK0Upr4Nu49fu+poP/9WMg= go.arcalot.io/dgraph v1.1.0/go.mod h1:FuNv92OgHsJYepD6Unwn+S/4DioBnv06JxQ2BtQct7E= go.arcalot.io/lang v1.0.0 h1:mgDaieT4wWdZTnR4V7+/pgYRmzfU7VZZgIzHccuxAbY= @@ -170,8 +169,6 @@ go.flow.arcalot.io/kubernetesdeployer v0.8.0 h1:UjH/aspPif/k+X65sLWlNDZAW5JlzUfg go.flow.arcalot.io/kubernetesdeployer v0.8.0/go.mod h1:BhERhKpvQMJkrcW9lbBF4kJEe+OGhz2NpSftZIgtVNQ= go.flow.arcalot.io/pluginsdk v0.5.1 h1:ebb2ThAqmjmwGpDyKpd1wEDUisPqPabgARjFohy47Io= go.flow.arcalot.io/pluginsdk v0.5.1/go.mod h1:2s2f//7uOkBjr1QaiWJD/bqDIeLlINJtD1BhiY4aGPM= -go.flow.arcalot.io/podmandeployer v0.6.2 h1:iAAZGgwhxInEVAleakavGruHnW4qsD/v39JpfnTeXiE= -go.flow.arcalot.io/podmandeployer v0.6.2/go.mod h1:BmKbyG2qZG9PMPLkIeXUvKVJfU+AZx+POLvydZN26IY= go.flow.arcalot.io/podmandeployer v0.7.0 h1:bXzWi4IjjLTIftUbH2NPgPiyTb82lzERVgfHP4zpmXI= go.flow.arcalot.io/podmandeployer v0.7.0/go.mod h1:tiWVDNpeNpPrY2GloihwjtnCEzJf8zNxBbiwVGRX7rs= go.flow.arcalot.io/pythondeployer v0.4.0 h1:l8nw6awYMVzgND+ZXdbnNJPYu3V0sgSUFsIzn+SRgh0= diff --git a/loadfile/loadfile.go b/loadfile/loadfile.go new file mode 100644 index 00000000..242b5188 --- /dev/null +++ b/loadfile/loadfile.go @@ -0,0 +1,47 @@ +// Package loadfile provides functions to load files from a directory. +package loadfile + +import ( + "fmt" + "os" + "path/filepath" +) + +// LoadContext reads the content of each file into a map where the key +// is the absolute filepath and the file content is the value. +func LoadContext(neededFilepaths []string) (map[string][]byte, error) { + result := map[string][]byte{} + var err error + for _, filePath := range neededFilepaths { + absPath, err := filepath.Abs(filePath) + if err != nil { + return nil, fmt.Errorf("error obtaining absolute path of file %s (%w)", + filePath, err) + } + fileData, err := os.ReadFile(filepath.Clean(absPath)) + if err != nil { + return nil, fmt.Errorf("error reading file %s (%w)", absPath, err) + } + result[absPath] = fileData + } + return result, err +} + +// AbsPathsWithContext creates a map of absolute filepaths. If a required +// file is not provided with an absolute path, then it is joined with the +// root directory. +func AbsPathsWithContext(rootDir string, requiredFiles map[string]string) (map[string]string, error) { + absDir, err := filepath.Abs(rootDir) + if err != nil { + return nil, fmt.Errorf("error determining context directory absolute path %s (%w)", rootDir, err) + } + requiredFilesAbs := map[string]string{} + for key, f := range requiredFiles { + abspath := f + if !filepath.IsAbs(f) { + abspath = filepath.Join(absDir, f) + } + requiredFilesAbs[key] = abspath + } + return requiredFilesAbs, nil +} diff --git a/loadfile/loadfile_test.go b/loadfile/loadfile_test.go new file mode 100644 index 00000000..e5c7dd32 --- /dev/null +++ b/loadfile/loadfile_test.go @@ -0,0 +1,113 @@ +package loadfile_test + +import ( + "go.arcalot.io/assert" + "go.flow.arcalot.io/engine/loadfile" + "os" + "path/filepath" + "testing" +) + +// This tests the functional behavior of LoadContext when it is given +// a directory of files that cover more than the directory file node +// type 'd' (as seen in *nix systems). Specifically, this test adds +// symbolic link files for a regular file and a directory file. The +// engine should only attempt to read files of a type that the os can +// read (i.e. not throw an error on a call to os.ReadFile()), and +// disregard (not throw an error) files with a type it cannot read. +func TestLoadContext(t *testing.T) { + testdir := "/tmp/loadfile-test" + // cleanup directory even if it's there + _ = os.RemoveAll(testdir) + + assert.NoError(t, os.MkdirAll(testdir, os.ModePerm)) + + // create a directory + dirname := "mydir" + dirpath := filepath.Join(testdir, dirname) + assert.NoError(t, os.MkdirAll(dirpath, os.ModePerm)) + + // create a file + filename := "myfile" + filePath := filepath.Join(testdir, filename) + f, err := os.Create(filepath.Clean(filePath)) + assert.NoError(t, err) + assert.NoError(t, f.Close()) + + // create symlink to the directory + symlinkDirname := dirname + "_sym" + symlinkDirpath := filepath.Join(testdir, symlinkDirname) + assert.NoError(t, os.Symlink(dirpath, symlinkDirpath)) + + // create symlink to the file + symlinkFilepath := filePath + "_sym" + assert.NoError(t, os.Symlink(filePath, symlinkFilepath)) + + neededFiles := []string{ + filePath, + symlinkFilepath, + } + filemap, err := loadfile.LoadContext(neededFiles) + // assert no error on attempting to read files + // that cannot be read + assert.NoError(t, err) + + // assert only the regular and symlinked file are loaded + filemapExp := map[string][]byte{ + filePath: {}, + symlinkFilepath: {}, + } + assert.Equals(t, filemap, filemapExp) + + // error on loading a directory + neededFiles = []string{ + dirpath, + } + + errFileRead := "reading file" + ctxFiles, err := loadfile.LoadContext(neededFiles) + assert.Error(t, err) + assert.Contains(t, err.Error(), errFileRead) + assert.Nil(t, ctxFiles) + + // error on loading a symlink directory + neededFiles = []string{ + symlinkDirpath, + } + ctxFiles, err = loadfile.LoadContext(neededFiles) + assert.Error(t, err) + assert.Contains(t, err.Error(), errFileRead) + assert.Nil(t, ctxFiles) + + t.Cleanup(func() { + assert.NoError(t, os.RemoveAll(testdir)) + }) +} + +// This tests AbsPathsWithContext which joins relative paths +// with the context (root) directory, and passes through +// absolute paths unmodified. +func TestContextAbsFilepaths(t *testing.T) { + testdir, err := os.MkdirTemp(os.TempDir(), "") + assert.NoError(t, err) + + testFilepaths := map[string]string{ + "a": "a.yaml", + "b": "/b.toml", + "c": "rel/../subdir/c.txt", + } + + absPathsExp := map[string]string{ + "a": filepath.Join(testdir, testFilepaths["a"]), + "b": testFilepaths["b"], + "c": filepath.Join(testdir, testFilepaths["c"]), + } + + absPathsGot, err := loadfile.AbsPathsWithContext(testdir, testFilepaths) + assert.NoError(t, err) + assert.Equals(t, absPathsExp, absPathsGot) + + t.Cleanup(func() { + assert.NoError(t, os.RemoveAll(testdir)) + }) +}