Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix load context bug #137

Merged
merged 10 commits into from
Jan 5, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 38 additions & 37 deletions cmd/arcaflow/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,13 @@ import (
"context"
"flag"
"fmt"
"os"
"os/signal"
"path/filepath"
"strings"

"go.arcalot.io/log/v2"

"go.flow.arcalot.io/engine"
"go.flow.arcalot.io/engine/config"
"go.flow.arcalot.io/engine/loadfile"
"gopkg.in/yaml.v3"
"os"
"os/signal"
)

// These variables are filled using ldflags during the build process with Goreleaser.
Expand All @@ -38,6 +35,15 @@ const ExitCodeWorkflowErrorOutput = 2
// ExitCodeWorkflowFailed indicates that the workflow execution failed.
const ExitCodeWorkflowFailed = 3

// RequiredFileKeyWorkflow is the key for the workflow file in hash map of files required for execution.
const RequiredFileKeyWorkflow = "workflow"

// RequiredFileKeyConfig is the key for the config file in hash map of files required for execution.
const RequiredFileKeyConfig = "config"

// RequiredFileKeyInput is the key for the input file in hash map of files required for execution.
const RequiredFileKeyInput = "input"
mfleader marked this conversation as resolved.
Show resolved Hide resolved

func main() {
tempLogger := log.New(log.Config{
Level: log.LevelInfo,
Expand Down Expand Up @@ -117,9 +123,23 @@ Options:
}

var err error

requiredFiles := map[string]string{
RequiredFileKeyConfig: configFile,
RequiredFileKeyInput: input,
RequiredFileKeyWorkflow: workflowFile,
}

requiredFilesAbsPaths, err := loadfile.AbsPathsWithContext(dir, requiredFiles)
if err != nil {
flag.Usage()
tempLogger.Errorf("context path resolution failed %s (%v)", dir, err)
os.Exit(ExitCodeInvalidData)
}

var configData any = map[any]any{}
if configFile != "" {
configData, err = loadYamlFile(configFile)
configData, err = loadYamlFile(requiredFilesAbsPaths[RequiredFileKeyConfig])
if err != nil {
tempLogger.Errorf("Failed to load configuration file %s (%v)", configFile, err)
flag.Usage()
Expand All @@ -132,13 +152,20 @@ Options:
flag.Usage()
os.Exit(ExitCodeInvalidData)
}
cfg.Log.Stdout = os.Stderr

// now we are ready to instantiate our main logger
cfg.Log.Stdout = os.Stderr
logger := log.New(cfg.Log).WithLabel("source", "main")

dirContext, err := loadContext(dir)
var requiredFilesAbsSlice = make([]string, len(requiredFilesAbsPaths))
var j int
for _, f := range requiredFilesAbsPaths {
requiredFilesAbsSlice[j] = f
j++
}
dirContext, err := loadfile.LoadContext(requiredFilesAbsSlice)
if err != nil {
logger.Errorf("Failed to load configuration file %s (%v)", configFile, err)
logger.Errorf("Failed to load required files into context (%v)", err)
flag.Usage()
os.Exit(ExitCodeInvalidData)
}
Expand All @@ -160,7 +187,7 @@ Options:
}
}

os.Exit(runWorkflow(flow, dirContext, workflowFile, logger, inputData))
os.Exit(runWorkflow(flow, dirContext, requiredFilesAbsPaths[RequiredFileKeyWorkflow], logger, inputData))
}

func runWorkflow(flow engine.WorkflowEngine, dirContext map[string][]byte, workflowFile string, logger log.Logger, inputData []byte) int {
Expand Down Expand Up @@ -224,32 +251,6 @@ func handleOSInterrupt(ctrlC chan os.Signal, cancel context.CancelFunc, logger l
os.Exit(1)
}

func loadContext(dir string) (map[string][]byte, error) {
absDir, err := filepath.Abs(dir)
if err != nil {
return nil, fmt.Errorf("failed to obtain absolute path of context directory %s (%w)", dir, err)
}
result := map[string][]byte{}
err = filepath.Walk(absDir,
func(path string, i os.FileInfo, err error) error {
if err != nil {
return err
}

if !i.IsDir() {
fileData, err := os.ReadFile(path) //nolint:gosec
if err != nil {
return fmt.Errorf("failed to read file from context directory: %s (%w)", path, err)
}
path = strings.TrimPrefix(path, absDir)
path = strings.TrimPrefix(path, string([]byte{os.PathSeparator}))
result[path] = fileData
}
return nil
})
return result, err
}

func loadYamlFile(configFile string) (any, error) {
fileContents, err := os.ReadFile(configFile) //nolint:gosec
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module go.flow.arcalot.io/engine
go 1.18

require (
go.arcalot.io/assert v1.6.0
go.arcalot.io/assert v1.6.1
go.arcalot.io/dgraph v1.1.0
go.arcalot.io/lang v1.0.0
go.arcalot.io/log/v2 v2.0.0
Expand Down
7 changes: 2 additions & 5 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,6 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
Expand Down Expand Up @@ -152,8 +151,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.arcalot.io/assert v1.6.0 h1:iKA8SZZ1MRblMX5QAwwY5RbpR+VNyp//4IU7vo08Xu0=
go.arcalot.io/assert v1.6.0/go.mod h1:Xy3ScX0p9IMY89gdsgexOKxnmDr0nGHG9dV7p8Uxg7w=
go.arcalot.io/assert v1.6.1 h1:pr7wvTE/vZJ4pY0f88R8tXicTOGogkGS1mtY0Qb7GRE=
go.arcalot.io/assert v1.6.1/go.mod h1:Xy3ScX0p9IMY89gdsgexOKxnmDr0nGHG9dV7p8Uxg7w=
go.arcalot.io/dgraph v1.1.0 h1:c0LR7+xdUy7Ki6e4nR9rBvK0Upr4Nu49fu+poP/9WMg=
go.arcalot.io/dgraph v1.1.0/go.mod h1:FuNv92OgHsJYepD6Unwn+S/4DioBnv06JxQ2BtQct7E=
go.arcalot.io/lang v1.0.0 h1:mgDaieT4wWdZTnR4V7+/pgYRmzfU7VZZgIzHccuxAbY=
Expand All @@ -170,8 +169,6 @@ go.flow.arcalot.io/kubernetesdeployer v0.8.0 h1:UjH/aspPif/k+X65sLWlNDZAW5JlzUfg
go.flow.arcalot.io/kubernetesdeployer v0.8.0/go.mod h1:BhERhKpvQMJkrcW9lbBF4kJEe+OGhz2NpSftZIgtVNQ=
go.flow.arcalot.io/pluginsdk v0.5.1 h1:ebb2ThAqmjmwGpDyKpd1wEDUisPqPabgARjFohy47Io=
go.flow.arcalot.io/pluginsdk v0.5.1/go.mod h1:2s2f//7uOkBjr1QaiWJD/bqDIeLlINJtD1BhiY4aGPM=
go.flow.arcalot.io/podmandeployer v0.6.2 h1:iAAZGgwhxInEVAleakavGruHnW4qsD/v39JpfnTeXiE=
go.flow.arcalot.io/podmandeployer v0.6.2/go.mod h1:BmKbyG2qZG9PMPLkIeXUvKVJfU+AZx+POLvydZN26IY=
go.flow.arcalot.io/podmandeployer v0.7.0 h1:bXzWi4IjjLTIftUbH2NPgPiyTb82lzERVgfHP4zpmXI=
go.flow.arcalot.io/podmandeployer v0.7.0/go.mod h1:tiWVDNpeNpPrY2GloihwjtnCEzJf8zNxBbiwVGRX7rs=
go.flow.arcalot.io/pythondeployer v0.4.0 h1:l8nw6awYMVzgND+ZXdbnNJPYu3V0sgSUFsIzn+SRgh0=
Expand Down
47 changes: 47 additions & 0 deletions loadfile/loadfile.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// Package loadfile provides functions to load files from a directory.
package loadfile

import (
"fmt"
"os"
"path/filepath"
)

// LoadContext reads the content of each file into a map where the key
// is the absolute filepath and the file content is the value.
func LoadContext(neededFilepaths []string) (map[string][]byte, error) {
result := map[string][]byte{}
var err error
for _, filePath := range neededFilepaths {
absPath, err := filepath.Abs(filePath)
if err != nil {
return nil, fmt.Errorf("error obtaining absolute path of file %s (%w)",
filePath, err)
}
fileData, err := os.ReadFile(filepath.Clean(absPath))
dbutenhof marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return nil, fmt.Errorf("error reading file %s (%w)", absPath, err)
}
result[absPath] = fileData
}
return result, err
}

// AbsPathsWithContext creates a map of absolute filepaths. If a required
// file is not provided with an absolute path, then it is joined with the
// root directory.
func AbsPathsWithContext(rootDir string, requiredFiles map[string]string) (map[string]string, error) {
absDir, err := filepath.Abs(rootDir)
if err != nil {
return nil, fmt.Errorf("error determining context directory absolute path %s (%w)", rootDir, err)
}
requiredFilesAbs := map[string]string{}
for key, f := range requiredFiles {
abspath := f
if !filepath.IsAbs(f) {
abspath = filepath.Join(absDir, f)
}
requiredFilesAbs[key] = abspath
}
return requiredFilesAbs, nil
}
113 changes: 113 additions & 0 deletions loadfile/loadfile_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
package loadfile_test

import (
"go.arcalot.io/assert"
"go.flow.arcalot.io/engine/loadfile"
"os"
"path/filepath"
"testing"
)

// This tests the functional behavior of LoadContext when it is given
// a directory of files that cover more than the directory file node
// type 'd' (as seen in *nix systems). Specifically, this test adds
// symbolic link files for a regular file and a directory file. The
// engine should only attempt to read files of a type that the os can
// read (i.e. not throw an error on a call to os.ReadFile()), and
// disregard (not throw an error) files with a type it cannot read.
func TestLoadContext(t *testing.T) {
testdir := "/tmp/loadfile-test"
// cleanup directory even if it's there
_ = os.RemoveAll(testdir)
mfleader marked this conversation as resolved.
Show resolved Hide resolved

assert.NoError(t, os.MkdirAll(testdir, os.ModePerm))

// create a directory
dirname := "mydir"
dirpath := filepath.Join(testdir, dirname)
assert.NoError(t, os.MkdirAll(dirpath, os.ModePerm))

// create a file
filename := "myfile"
filePath := filepath.Join(testdir, filename)
f, err := os.Create(filepath.Clean(filePath))
assert.NoError(t, err)
assert.NoError(t, f.Close())

// create symlink to the directory
symlinkDirname := dirname + "_sym"
symlinkDirpath := filepath.Join(testdir, symlinkDirname)
assert.NoError(t, os.Symlink(dirpath, symlinkDirpath))

// create symlink to the file
symlinkFilepath := filePath + "_sym"
assert.NoError(t, os.Symlink(filePath, symlinkFilepath))

neededFiles := []string{
filePath,
symlinkFilepath,
}
filemap, err := loadfile.LoadContext(neededFiles)
// assert no error on attempting to read files
// that cannot be read
assert.NoError(t, err)

// assert only the regular and symlinked file are loaded
filemapExp := map[string][]byte{
filePath: {},
symlinkFilepath: {},
}
assert.Equals(t, filemap, filemapExp)

// error on loading a directory
neededFiles = []string{
dirpath,
}

errFileRead := "reading file"
ctxFiles, err := loadfile.LoadContext(neededFiles)
assert.Error(t, err)
assert.Contains(t, err.Error(), errFileRead)
assert.Nil(t, ctxFiles)

// error on loading a symlink directory
neededFiles = []string{
symlinkDirpath,
}
ctxFiles, err = loadfile.LoadContext(neededFiles)
assert.Error(t, err)
assert.Contains(t, err.Error(), errFileRead)
assert.Nil(t, ctxFiles)

t.Cleanup(func() {
assert.NoError(t, os.RemoveAll(testdir))
})
}

// This tests AbsPathsWithContext which joins relative paths
// with the context (root) directory, and passes through
// absolute paths unmodified.
func TestContextAbsFilepaths(t *testing.T) {
testdir, err := os.MkdirTemp(os.TempDir(), "")
assert.NoError(t, err)

testFilepaths := map[string]string{
"a": "a.yaml",
"b": "/b.toml",
"c": "rel/../subdir/c.txt",
}

absPathsExp := map[string]string{
"a": filepath.Join(testdir, testFilepaths["a"]),
"b": testFilepaths["b"],
"c": filepath.Join(testdir, testFilepaths["c"]),
}

absPathsGot, err := loadfile.AbsPathsWithContext(testdir, testFilepaths)
assert.NoError(t, err)
assert.Equals(t, absPathsExp, absPathsGot)

t.Cleanup(func() {
assert.NoError(t, os.RemoveAll(testdir))
})
}