Merge branch 'master' into develop

ohsu-comp-bio · Oct 7, 2024 · a3f0d2c · a3f0d2c
2 parents 1718725 + d6d0517
commit a3f0d2c
Show file tree

Hide file tree

Showing 36 changed files with 2,576 additions and 3,345 deletions.
diff --git a/cmd/run/parse_vars.go b/cmd/run/parse_vars.go
@@ -3,7 +3,7 @@ package run
 import (
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -197,7 +197,7 @@ func valsToTask(vals flagVals) (task *tes.Task, err error) {
 }
 
 func getContent(p string) string {
-	b, err := ioutil.ReadFile(p)
+	b, err := os.ReadFile(p)
 	if err != nil {
 		panic(err)
 	}

diff --git a/cmd/util/config.go b/cmd/util/config.go
@@ -1,7 +1,6 @@
 package util
 
 import (
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -42,7 +41,7 @@ func MergeConfigFileWithFlags(file string, flagConf config.Config) (config.Confi
 // - "path" is the path of the file.
 // - "cleanup" can be called to remove the temporary file.
 func TempConfigFile(c config.Config, name string) (path string, cleanup func()) {
-	tmpdir, err := ioutil.TempDir("", "")
+	tmpdir, err := os.MkdirTemp("", "")
 	if err != nil {
 		panic(err)
 	}

diff --git a/cmd/util/reader.go b/cmd/util/reader.go
@@ -3,13 +3,12 @@ package util
 import (
 	"bytes"
 	"io"
-	"io/ioutil"
 	"os"
 )
 
 // EmptyReader returns an io.Reader which is empty and immediately closed.
 func EmptyReader() io.Reader {
-	return ioutil.NopCloser(bytes.NewReader(nil))
+	return io.NopCloser(bytes.NewReader(nil))
 }
 
 // StdinPipe will return stdin if it's available, otherwise it will return

diff --git a/compute/gridengine/backend.go b/compute/gridengine/backend.go
@@ -3,7 +3,7 @@ package gridengine
 
 import (
 	"fmt"
-	"io/ioutil"
+	"os"
 	"regexp"
 
 	"github.com/ohsu-comp-bio/funnel/compute"
@@ -16,7 +16,7 @@ import (
 // NewBackend returns a new Grid Engine HPCBackend instance.
 func NewBackend(conf config.Config, reader tes.ReadOnlyServer, writer events.Writer, log *logger.Logger) (*compute.HPCBackend, error) {
 	if conf.GridEngine.TemplateFile != "" {
-		content, err := ioutil.ReadFile(conf.GridEngine.TemplateFile)
+		content, err := os.ReadFile(conf.GridEngine.TemplateFile)
 		if err != nil {
 			return nil, fmt.Errorf("reading template: %v", err)
 		}

diff --git a/compute/hpc_backend_test.go b/compute/hpc_backend_test.go
@@ -3,15 +3,15 @@ package compute
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"testing"
 
 	"github.com/ohsu-comp-bio/funnel/config"
 	"github.com/ohsu-comp-bio/funnel/tes"
 )
 
 func TestSetupTemplatedHPCSubmit(t *testing.T) {
-	tmp, err := ioutil.TempDir("", "funnel-test-scheduler")
+	tmp, err := os.MkdirTemp("", "funnel-test-scheduler")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -64,7 +64,7 @@ funnel worker run --taskID {{.TaskId}}
 		t.Fatal(err)
 	}
 
-	actual, rerr := ioutil.ReadFile(sf)
+	actual, rerr := os.ReadFile(sf)
 	if rerr != nil {
 		t.Fatal(rerr)
 	}

diff --git a/compute/htcondor/backend.go b/compute/htcondor/backend.go
@@ -5,7 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"os/exec"
 	"regexp"
 	"strings"
@@ -21,7 +21,7 @@ import (
 // NewBackend returns a new HTCondor backend instance.
 func NewBackend(ctx context.Context, conf config.Config, reader tes.ReadOnlyServer, writer events.Writer, log *logger.Logger) (*compute.HPCBackend, error) {
 	if conf.HTCondor.TemplateFile != "" {
-		content, err := ioutil.ReadFile(conf.HTCondor.TemplateFile)
+		content, err := os.ReadFile(conf.HTCondor.TemplateFile)
 		if err != nil {
 			return nil, fmt.Errorf("reading template: %v", err)
 		}

diff --git a/compute/kubernetes/backend.go b/compute/kubernetes/backend.go
@@ -7,7 +7,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"text/template"
 	"time"
 
@@ -28,7 +28,7 @@ import (
 // NewBackend returns a new local Backend instance.
 func NewBackend(ctx context.Context, conf config.Kubernetes, reader tes.ReadOnlyServer, writer events.Writer, log *logger.Logger) (*Backend, error) {
 	if conf.TemplateFile != "" {
-		content, err := ioutil.ReadFile(conf.TemplateFile)
+		content, err := os.ReadFile(conf.TemplateFile)
 		if err != nil {
 			return nil, fmt.Errorf("reading template: %v", err)
 		}

diff --git a/compute/pbs/backend.go b/compute/pbs/backend.go
@@ -5,7 +5,7 @@ import (
 	"context"
 	"encoding/xml"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"os/exec"
 	"time"
 
@@ -19,7 +19,7 @@ import (
 // NewBackend returns a new PBS (Portable Batch System) HPCBackend instance.
 func NewBackend(ctx context.Context, conf config.Config, reader tes.ReadOnlyServer, writer events.Writer, log *logger.Logger) (*compute.HPCBackend, error) {
 	if conf.PBS.TemplateFile != "" {
-		content, err := ioutil.ReadFile(conf.PBS.TemplateFile)
+		content, err := os.ReadFile(conf.PBS.TemplateFile)
 		if err != nil {
 			return nil, fmt.Errorf("reading template: %v", err)
 		}

diff --git a/compute/scheduler/testutils_test.go b/compute/scheduler/testutils_test.go
@@ -3,7 +3,7 @@ package scheduler
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"testing"
 	"time"
 
@@ -21,7 +21,7 @@ type testNode struct {
 }
 
 func newTestNode(conf config.Config, t *testing.T) testNode {
-	workDir, _ := ioutil.TempDir("", "funnel-test-storage-")
+	workDir, _ := os.MkdirTemp("", "funnel-test-storage-")
 	conf.Worker.WorkDir = workDir
 	log := logger.NewLogger("test-node", logger.DebugConfig())
 

diff --git a/compute/slurm/backend.go b/compute/slurm/backend.go
@@ -6,7 +6,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"os/exec"
 	"regexp"
 	"strings"
@@ -22,7 +22,7 @@ import (
 // NewBackend returns a new Slurm HPCBackend instance.
 func NewBackend(ctx context.Context, conf config.Config, reader tes.ReadOnlyServer, writer events.Writer, log *logger.Logger) (*compute.HPCBackend, error) {
 	if conf.Slurm.TemplateFile != "" {
-		content, err := ioutil.ReadFile(conf.Slurm.TemplateFile)
+		content, err := os.ReadFile(conf.Slurm.TemplateFile)
 		if err != nil {
 			return nil, fmt.Errorf("reading template: %v", err)
 		}

diff --git a/config/gce/meta.go b/config/gce/meta.go
@@ -3,7 +3,7 @@ package gce
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"strings"
 
@@ -101,7 +101,7 @@ func LoadMetadataFromURL(url string) (*Metadata, error) {
 		return nil, fmt.Errorf("Non-200 response status from GCE Metadata: %d", resp.StatusCode)
 	}
 	defer resp.Body.Close()
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, err
 	}

diff --git a/config/gce/meta_test.go b/config/gce/meta_test.go
@@ -1,17 +1,17 @@
 package gce
 
 import (
-	"io/ioutil"
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"testing"
 
 	"github.com/ohsu-comp-bio/funnel/config"
 )
 
 func loadTestData(name string) []byte {
-	b, err := ioutil.ReadFile(name + ".json")
+	b, err := os.ReadFile(name + ".json")
 	if err != nil {
 		panic(err)
 	}

diff --git a/config/internal/bundle.go b/config/internal/bundle.go
diff --git a/config/utils.go b/config/utils.go
@@ -3,7 +3,7 @@ package config
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"path/filepath"
 	"reflect"
 	"strings"
@@ -22,7 +22,7 @@ func ToYamlFile(c Config, path string) error {
 	if err != nil {
 		return err
 	}
-	return ioutil.WriteFile(path, b, 0600)
+	return os.WriteFile(path, b, 0600)
 }
 
 // Parse parses a YAML doc into the given Config instance.
@@ -56,7 +56,7 @@ func ParseFile(relpath string, conf *Config) error {
 	}
 
 	// Read file
-	source, err := ioutil.ReadFile(path)
+	source, err := os.ReadFile(path)
 	if err != nil {
 		return fmt.Errorf("failed to read config at path %s: \n%v", path, err)
 	}

diff --git a/examples/internal/bundle.go b/examples/internal/bundle.go
@@ -16,7 +16,6 @@ import (
 	"compress/gzip"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -398,7 +397,7 @@ func RestoreAsset(dir, name string) error {
 	if err != nil {
 		return err
 	}
-	err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode())
+	err = os.WriteFile(_filePath(dir, name), data, info.Mode())
 	if err != nil {
 		return err
 	}

diff --git a/logger/log.go b/logger/log.go
@@ -4,7 +4,6 @@ package logger
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"runtime"
 	"strings"
@@ -74,7 +73,7 @@ func (l *Logger) SetOutput(o io.Writer) {
 
 // Discard configures the logger to discard all logs.
 func (l *Logger) Discard() {
-	l.SetOutput(ioutil.Discard)
+	l.SetOutput(io.Discard)
 }
 
 // Debug logs a debug message.

diff --git a/storage/amazon_s3.go b/storage/amazon_s3.go
@@ -5,7 +5,6 @@ import (
 	"crypto/md5"
 	"encoding/base64"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -59,7 +58,7 @@ func NewAmazonS3(conf config.AmazonS3Storage) (*AmazonS3, error) {
 	}
 
 	if conf.SSE.CustomerKeyFile != "" {
-		key, err := ioutil.ReadFile(conf.SSE.CustomerKeyFile)
+		key, err := os.ReadFile(conf.SSE.CustomerKeyFile)
 		if err != nil {
 			return nil, fmt.Errorf("error reading sse-c file: %v", err)
 		}

diff --git a/storage/google_cloud.go b/storage/google_cloud.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"strings"
@@ -34,7 +33,7 @@ func NewGoogleCloud(conf config.GoogleCloudStorage) (*GoogleCloud, error) {
 	if conf.CredentialsFile != "" {
 		// Pull the client configuration (e.g. auth) from a given account file.
 		// This is likely downloaded from Google Cloud manually via IAM & Admin > Service accounts.
-		bytes, rerr := ioutil.ReadFile(conf.CredentialsFile)
+		bytes, rerr := os.ReadFile(conf.CredentialsFile)
 		if rerr != nil {
 			return nil, rerr
 		}

diff --git a/storage/local.go b/storage/local.go
@@ -168,19 +168,16 @@ func copyFile(ctx context.Context, source string, dest string) (err error) {
 
 // Hard links file source to destination dest.
 func linkFile(ctx context.Context, source string, dest string) error {
-
 	// If source has a glob or wildcard, get the filepath using the filepath.Glob function
 	if strings.Contains(source, "*") {
 		globs, err := filepath.Glob(source)
 		if err != nil {
 			return fmt.Errorf("failed to get filepath using Glob: %v", err)
 		}
 		for _, glob := range globs {
-			// Since tesOutput.path contains a wildcard, then the dest path must be a directory
-			// Ref: https://github.com/ga4gh/task-execution-schemas/blob/v1.1/openapi/task_execution_service.openapi.yaml#L528
-			// TODO: Verify that path_prefix is being removed from the dest filepath
-			dest = filepath.Join(dest, filepath.Base(glob))
-			err := processItem(ctx, glob, dest)
+			// Correctly calculate the destination for each file
+			destFile := filepath.Join(dest, filepath.Base(glob))
+			err := processItem(ctx, glob, destFile)
 			if err != nil {
 				return err
 			}
@@ -192,7 +189,7 @@ func linkFile(ctx context.Context, source string, dest string) error {
 }
 
 // Process a single item (file or directory)
-func processItem(ctx context.Context, source string, dest string) error {
+func processItem(ctx context.Context, source, dest string) error {
 	fileInfo, err := os.Stat(source)
 	if err != nil {
 		return err
@@ -206,7 +203,7 @@ func processItem(ctx context.Context, source string, dest string) error {
 }
 
 // Process a directory
-func processDirectory(ctx context.Context, source string, dest string) error {
+func processDirectory(ctx context.Context, source, dest string) error {
 	// Create destination directory
 	err := os.MkdirAll(dest, 0755) // Adjust permissions as needed
 	if err != nil {