Merge pull request #138 from ibuildthecloud/distribution

feat: Add golang tool support

pkg/engine/cmd.go

@@ -57,7 +57,6 @@ func (e *Engine) runCommand(ctx context.Context, tool types.Tool, input string)
 
 	output := &bytes.Buffer{}
 	all := &bytes.Buffer{}
-	cmd.Stdin = strings.NewReader(input)
 	cmd.Stderr = io.MultiWriter(all, os.Stderr)
 	cmd.Stdout = io.MultiWriter(all, output)
 
@@ -142,6 +141,7 @@ func appendInputAsEnv(env []string, input string) []string {
 		}
 	}
 
+	env = appendEnv(env, "GPTSCRIPT_INPUT", input)
 	return env
 }
 

pkg/repos/download/extract.go

@@ -30,6 +30,8 @@ func Extract(ctx context.Context, downloadURL, digest, targetDir string) error {
 	}
 	defer resp.Body.Close()
 
+	// NOTE: Because I'm validating the hash at the same time as extracting this isn't actually secure.
+	// Security is still assumed the source is trusted.  Which is bad and should be changed.
 	digester := sha256.New()
 	input := io.TeeReader(resp.Body, digester)
 

pkg/repos/runtimes/default.go

@@ -3,6 +3,7 @@ package runtimes
 import (
 	"github.com/gptscript-ai/gptscript/pkg/engine"
 	"github.com/gptscript-ai/gptscript/pkg/repos"
+	"github.com/gptscript-ai/gptscript/pkg/repos/runtimes/golang"
 	"github.com/gptscript-ai/gptscript/pkg/repos/runtimes/node"
 	"github.com/gptscript-ai/gptscript/pkg/repos/runtimes/python"
 )
@@ -22,6 +23,9 @@ var Runtimes = []repos.Runtime{
 		Version: "21",
 		Default: true,
 	},
+	&golang.Runtime{
+		Version: "1.22.1",
+	},
 }
 
 func Default(cacheDir string) engine.RuntimeManager {

pkg/repos/runtimes/golang/digests.txt

@@ -0,0 +1,10 @@
+3bc971772f4712fec0364f4bc3de06af22a00a12daab10b6f717fdcd13156cc0  go1.22.1.darwin-amd64.tar.gz
+943e4f9f038239f9911c44366f52ab9202f6ee13610322a668fe42406fb3deef  go1.22.1.darwin-amd64.pkg
+f6a9cec6b8a002fcc9c0ee24ec04d67f430a52abc3cfd613836986bcc00d8383  go1.22.1.darwin-arm64.tar.gz
+5f10b95e2678618f85ba9d87fbed506b3b87efc9d5a8cafda939055cb97949ba  go1.22.1.darwin-arm64.pkg
+8484df36d3d40139eaf0fe5e647b006435d826cc12f9ae72973bf7ec265e0ae4  go1.22.1.linux-386.tar.gz
+aab8e15785c997ae20f9c88422ee35d962c4562212bb0f879d052a35c8307c7f  go1.22.1.linux-amd64.tar.gz
+e56685a245b6a0c592fc4a55f0b7803af5b3f827aaa29feab1f40e491acf35b8  go1.22.1.linux-arm64.tar.gz
+8cb7a90e48c20daed39a6ac8b8a40760030ba5e93c12274c42191d868687c281  go1.22.1.linux-armv6l.tar.gz
+0c5ebb7eb39b7884ec99f92b425d4c03a96a72443562aafbf6e7d15c42a3108a  go1.22.1.windows-386.zip
+cf9c66a208a106402a527f5b956269ca506cfe535fc388e828d249ea88ed28ba  go1.22.1.windows-amd64.zip

pkg/repos/runtimes/golang/golang.go

@@ -0,0 +1,119 @@
+package golang
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	_ "embed"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/gptscript-ai/gptscript/pkg/debugcmd"
+	"github.com/gptscript-ai/gptscript/pkg/hash"
+	"github.com/gptscript-ai/gptscript/pkg/repos/download"
+	runtimeEnv "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/env"
+)
+
+//go:embed digests.txt
+var releasesData []byte
+
+const downloadURL = "https://go.dev/dl/"
+
+type Runtime struct {
+	// version something like "1.22.1"
+	Version string
+}
+
+func (r *Runtime) ID() string {
+	return "go" + r.Version
+}
+
+func (r *Runtime) Supports(cmd []string) bool {
+	return len(cmd) > 0 && cmd[0] == "${GPTSCRIPT_TOOL_DIR}/bin/gptscript-go-tool"
+}
+
+func (r *Runtime) Setup(ctx context.Context, dataRoot, toolSource string, env []string) ([]string, error) {
+	binPath, err := r.getRuntime(ctx, dataRoot)
+	if err != nil {
+		return nil, err
+	}
+
+	newEnv := runtimeEnv.AppendPath(env, binPath)
+	if err := r.runBuild(ctx, toolSource, binPath, append(env, newEnv...)); err != nil {
+		return nil, err
+	}
+
+	return newEnv, nil
+}
+
+func (r *Runtime) getReleaseAndDigest() (string, string, error) {
+	scanner := bufio.NewScanner(bytes.NewReader(releasesData))
+	key := r.ID() + "." + runtime.GOOS + "-" + runtime.GOARCH
+	for scanner.Scan() {
+		line := strings.Split(scanner.Text(), "  ")
+		file, digest := strings.TrimSpace(line[1]), strings.TrimSpace(line[0])
+		if strings.HasPrefix(file, key) {
+			return downloadURL + file, digest, nil
+		}
+	}
+
+	return "", "", fmt.Errorf("failed to find %s release for os=%s arch=%s", r.ID(), runtime.GOOS, runtime.GOARCH)
+}
+
+func stripGo(env []string) (result []string) {
+	for _, env := range env {
+		if strings.HasPrefix(env, "GO") {
+			continue
+		}
+		result = append(result, env)
+	}
+	return
+}
+
+func (r *Runtime) runBuild(ctx context.Context, toolSource, binDir string, env []string) error {
+	cmd := debugcmd.New(ctx, filepath.Join(binDir, "go"), "build", "-o", "bin/gptscript-go-tool")
+	cmd.Env = stripGo(env)
+	cmd.Dir = toolSource
+	return cmd.Run()
+}
+
+func (r *Runtime) binDir(rel string) string {
+	return filepath.Join(rel, "go", "bin")
+}
+
+func (r *Runtime) getRuntime(ctx context.Context, cwd string) (string, error) {
+	url, sha, err := r.getReleaseAndDigest()
+	if err != nil {
+		return "", err
+	}
+
+	target := filepath.Join(cwd, "golang", hash.ID(url, sha))
+	if _, err := os.Stat(target); err == nil {
+		return r.binDir(target), nil
+	} else if !errors.Is(err, fs.ErrNotExist) {
+		return "", err
+	}
+
+	log.Infof("Downloading Go %s", r.Version)
+	tmp := target + ".download"
+	defer os.RemoveAll(tmp)
+
+	if err := os.MkdirAll(tmp, 0755); err != nil {
+		return "", err
+	}
+
+	if err := download.Extract(ctx, url, sha, tmp); err != nil {
+		return "", err
+	}
+
+	if err := os.Rename(tmp, target); err != nil {
+		return "", err
+	}
+
+	return r.binDir(target), nil
+}

pkg/repos/runtimes/golang/golang_test.go

@@ -0,0 +1,35 @@
+package golang
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/adrg/xdg"
+	"github.com/samber/lo"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	testCacheHome = lo.Must(xdg.CacheFile("gptscript-test-cache/runtime"))
+)
+
+func TestRuntime(t *testing.T) {
+	t.Cleanup(func() {
+		os.RemoveAll("testdata/bin")
+	})
+	r := Runtime{
+		Version: "1.22.1",
+	}
+
+	s, err := r.Setup(context.Background(), testCacheHome, "testdata", os.Environ())
+	require.NoError(t, err)
+	p, v, _ := strings.Cut(s[0], "=")
+	v, _, _ = strings.Cut(v, string(filepath.ListSeparator))
+	assert.Equal(t, "PATH", p)
+	_, err = os.Stat(filepath.Join(v, "gofmt"))
+	assert.NoError(t, err)
+}

pkg/repos/runtimes/golang/log.go

@@ -0,0 +1,5 @@
+package golang
+
+import "github.com/gptscript-ai/gptscript/pkg/mvl"
+
+var log = mvl.Package()

pkg/repos/runtimes/golang/testdata/go.mod

@@ -0,0 +1,3 @@
+module example.com
+
+go 1.22.1

pkg/repos/runtimes/golang/testdata/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "fmt"
+
+func main() {
+	fmt.Println("Hello AI World!")
+}