From cb46358ec8b92cab677856f4592bbe2010563ab8 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Fri, 9 Aug 2024 11:44:39 -0700 Subject: [PATCH] feat: manage node/python runtime for local files in dev --- pkg/repos/get.go | 18 ++- pkg/repos/runtimes/busybox/busybox.go | 4 + pkg/repos/runtimes/golang/golang.go | 4 + pkg/repos/runtimes/node/node.go | 26 +++- pkg/repos/runtimes/python/python.go | 38 +++-- pkg/tests/runner_test.go | 23 +++ .../TestRuntimesLocalDev/call1-resp.golden | 16 +++ .../TestRuntimesLocalDev/call1.golden | 37 +++++ .../TestRuntimesLocalDev/call2-resp.golden | 16 +++ .../TestRuntimesLocalDev/call2.golden | 70 +++++++++ .../TestRuntimesLocalDev/call3-resp.golden | 16 +++ .../TestRuntimesLocalDev/call3.golden | 103 +++++++++++++ .../TestRuntimesLocalDev/call4-resp.golden | 9 ++ .../TestRuntimesLocalDev/call4.golden | 136 ++++++++++++++++++ .../TestRuntimesLocalDev/package.json | 15 ++ .../TestRuntimesLocalDev/requirements.txt | 1 + .../testdata/TestRuntimesLocalDev/test.gpt | 34 +++++ 17 files changed, 552 insertions(+), 14 deletions(-) create mode 100644 pkg/tests/testdata/TestRuntimesLocalDev/call1-resp.golden create mode 100644 pkg/tests/testdata/TestRuntimesLocalDev/call1.golden create mode 100644 pkg/tests/testdata/TestRuntimesLocalDev/call2-resp.golden create mode 100644 pkg/tests/testdata/TestRuntimesLocalDev/call2.golden create mode 100644 pkg/tests/testdata/TestRuntimesLocalDev/call3-resp.golden create mode 100644 pkg/tests/testdata/TestRuntimesLocalDev/call3.golden create mode 100644 pkg/tests/testdata/TestRuntimesLocalDev/call4-resp.golden create mode 100644 pkg/tests/testdata/TestRuntimesLocalDev/call4.golden create mode 100644 pkg/tests/testdata/TestRuntimesLocalDev/package.json create mode 100644 pkg/tests/testdata/TestRuntimesLocalDev/requirements.txt create mode 100644 pkg/tests/testdata/TestRuntimesLocalDev/test.gpt diff --git a/pkg/repos/get.go b/pkg/repos/get.go index fc675c58..b43bc63b 100644 --- a/pkg/repos/get.go +++ b/pkg/repos/get.go @@ -28,6 +28,7 @@ type Runtime interface { ID() string Supports(tool types.Tool, cmd []string) bool Setup(ctx context.Context, tool types.Tool, dataRoot, toolSource string, env []string) ([]string, error) + GetHash(tool types.Tool) (string, error) } type noopRuntime struct { @@ -37,6 +38,10 @@ func (n noopRuntime) ID() string { return "none" } +func (n noopRuntime) GetHash(_ types.Tool) (string, error) { + return "", nil +} + func (n noopRuntime) Supports(_ types.Tool, _ []string) bool { return false } @@ -183,8 +188,13 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e locker.Lock(tool.ID) defer locker.Unlock(tool.ID) + runtimeHash, err := runtime.GetHash(tool) + if err != nil { + return "", nil, err + } + target := filepath.Join(m.storageDir, tool.Source.Repo.Revision, tool.Source.Repo.Path, tool.Source.Repo.Name, runtime.ID()) - targetFinal := filepath.Join(target, tool.Source.Repo.Path) + targetFinal := filepath.Join(target, tool.Source.Repo.Path+runtimeHash) doneFile := targetFinal + ".done" envData, err := os.ReadFile(doneFile) if err == nil { @@ -251,7 +261,11 @@ func (m *Manager) GetContext(ctx context.Context, tool types.Tool, cmd, env []st for _, runtime := range m.runtimes { if runtime.Supports(tool, cmd) { log.Debugf("Runtime %s supports %v", runtime.ID(), cmd) - return m.setup(ctx, runtime, tool, env) + wd, env, err := m.setup(ctx, runtime, tool, env) + if isLocal { + wd = tool.WorkingDir + } + return wd, env, err } } diff --git a/pkg/repos/runtimes/busybox/busybox.go b/pkg/repos/runtimes/busybox/busybox.go index 542ba94a..481ed1fe 100644 --- a/pkg/repos/runtimes/busybox/busybox.go +++ b/pkg/repos/runtimes/busybox/busybox.go @@ -33,6 +33,10 @@ func (r *Runtime) ID() string { return "busybox" } +func (r *Runtime) GetHash(_ types.Tool) (string, error) { + return "", nil +} + func (r *Runtime) Supports(_ types.Tool, cmd []string) bool { if runtime.GOOS != "windows" { return false diff --git a/pkg/repos/runtimes/golang/golang.go b/pkg/repos/runtimes/golang/golang.go index b19cfe90..882e8a0b 100644 --- a/pkg/repos/runtimes/golang/golang.go +++ b/pkg/repos/runtimes/golang/golang.go @@ -35,6 +35,10 @@ func (r *Runtime) ID() string { return "go" + r.Version } +func (r *Runtime) GetHash(_ types.Tool) (string, error) { + return "", nil +} + func (r *Runtime) Supports(tool types.Tool, cmd []string) bool { return tool.Source.IsGit() && len(cmd) > 0 && cmd[0] == "${GPTSCRIPT_TOOL_DIR}/bin/gptscript-go-tool" diff --git a/pkg/repos/runtimes/node/node.go b/pkg/repos/runtimes/node/node.go index fde5103d..d0a9d8cb 100644 --- a/pkg/repos/runtimes/node/node.go +++ b/pkg/repos/runtimes/node/node.go @@ -39,10 +39,7 @@ func (r *Runtime) ID() string { return "node" + r.Version } -func (r *Runtime) Supports(tool types.Tool, cmd []string) bool { - if _, hasPackageJSON := tool.MetaData[packageJSON]; !hasPackageJSON && !tool.Source.IsGit() { - return false - } +func (r *Runtime) Supports(_ types.Tool, cmd []string) bool { for _, testCmd := range []string{"node", "npx", "npm"} { if r.supports(testCmd, cmd) { return true @@ -61,6 +58,15 @@ func (r *Runtime) supports(testCmd string, cmd []string) bool { return runtimeEnv.Matches(cmd, testCmd) } +func (r *Runtime) GetHash(tool types.Tool) (string, error) { + if !tool.Source.IsGit() && tool.WorkingDir != "" { + if s, err := os.Stat(filepath.Join(tool.WorkingDir, packageJSON)); err == nil { + return hash.Digest(tool.WorkingDir + s.ModTime().String())[:7], nil + } + } + return "", nil +} + func (r *Runtime) Setup(ctx context.Context, tool types.Tool, dataRoot, toolSource string, env []string) ([]string, error) { binPath, err := r.getRuntime(ctx, dataRoot) if err != nil { @@ -74,6 +80,8 @@ func (r *Runtime) Setup(ctx context.Context, tool types.Tool, dataRoot, toolSour if _, ok := tool.MetaData[packageJSON]; ok { newEnv = append(newEnv, "GPTSCRIPT_TMPDIR="+toolSource) + } else if !tool.Source.IsGit() && tool.WorkingDir != "" { + newEnv = append(newEnv, "GPTSCRIPT_TMPDIR="+tool.WorkingDir, "GPTSCRIPT_RUNTIME_DEV=true") } return newEnv, nil @@ -120,6 +128,16 @@ func (r *Runtime) runNPM(ctx context.Context, tool types.Tool, toolSource, binDi if err := os.WriteFile(filepath.Join(toolSource, packageJSON), []byte(contents+"\n"), 0644); err != nil { return err } + } else if !tool.Source.IsGit() { + if tool.WorkingDir == "" { + return nil + } + if _, err := os.Stat(filepath.Join(tool.WorkingDir, packageJSON)); errors.Is(fs.ErrNotExist, err) { + return nil + } else if err != nil { + return err + } + cmd.Dir = tool.WorkingDir } return cmd.Run() } diff --git a/pkg/repos/runtimes/python/python.go b/pkg/repos/runtimes/python/python.go index ae24f92a..87b072e5 100644 --- a/pkg/repos/runtimes/python/python.go +++ b/pkg/repos/runtimes/python/python.go @@ -24,8 +24,9 @@ import ( var releasesData []byte const ( - uvVersion = "uv==0.2.33" - requirementsTxt = "requirements.txt" + uvVersion = "uv==0.2.33" + requirementsTxt = "requirements.txt" + gptscriptRequirementsTxt = "requirements-gptscript.txt" ) type Release struct { @@ -47,10 +48,7 @@ func (r *Runtime) ID() string { return "python" + r.Version } -func (r *Runtime) Supports(tool types.Tool, cmd []string) bool { - if _, hasRequirements := tool.MetaData[requirementsTxt]; !hasRequirements && !tool.Source.IsGit() { - return false - } +func (r *Runtime) Supports(_ types.Tool, cmd []string) bool { if runtimeEnv.Matches(cmd, r.ID()) { return true } @@ -177,6 +175,22 @@ func (r *Runtime) getReleaseAndDigest() (string, string, error) { return "", "", fmt.Errorf("failed to find an python runtime for %s", r.Version) } +func (r *Runtime) GetHash(tool types.Tool) (string, error) { + if !tool.Source.IsGit() && tool.WorkingDir != "" { + if _, ok := tool.MetaData[requirementsTxt]; ok { + return "", nil + } + for _, req := range []string{gptscriptRequirementsTxt, requirementsTxt} { + reqFile := filepath.Join(tool.WorkingDir, req) + if s, err := os.Stat(reqFile); err == nil && !s.IsDir() { + return hash.Digest(tool.WorkingDir + s.ModTime().String())[:7], nil + } + } + } + + return "", nil +} + func (r *Runtime) runPip(ctx context.Context, tool types.Tool, toolSource, binDir string, env []string) error { log.InfofCtx(ctx, "Running pip in %s", toolSource) if content, ok := tool.MetaData[requirementsTxt]; ok { @@ -189,8 +203,16 @@ func (r *Runtime) runPip(ctx context.Context, tool types.Tool, toolSource, binDi return cmd.Run() } - for _, req := range []string{"requirements-gptscript.txt", requirementsTxt} { - reqFile := filepath.Join(toolSource, req) + reqPath := toolSource + if !tool.Source.IsGit() { + if tool.WorkingDir == "" { + return nil + } + reqPath = tool.WorkingDir + } + + for _, req := range []string{gptscriptRequirementsTxt, requirementsTxt} { + reqFile := filepath.Join(reqPath, req) if s, err := os.Stat(reqFile); err == nil && !s.IsDir() { cmd := debugcmd.New(ctx, uvBin(binDir), "pip", "install", "-r", reqFile) cmd.Env = env diff --git a/pkg/tests/runner_test.go b/pkg/tests/runner_test.go index 424c84c1..141e6aff 100644 --- a/pkg/tests/runner_test.go +++ b/pkg/tests/runner_test.go @@ -1018,3 +1018,26 @@ func TestRuntimes(t *testing.T) { }) r.RunDefault() } + +func TestRuntimesLocalDev(t *testing.T) { + r := tester.NewRunner(t) + r.RespondWith(tester.Result{ + Func: types.CompletionFunctionCall{ + Name: "py", + Arguments: "{}", + }, + }, tester.Result{ + Func: types.CompletionFunctionCall{ + Name: "node", + Arguments: "{}", + }, + }, tester.Result{ + Func: types.CompletionFunctionCall{ + Name: "bash", + Arguments: "{}", + }, + }) + r.RunDefault() + _ = os.RemoveAll("testdata/TestRuntimesLocalDev/node_modules") + _ = os.RemoveAll("testdata/TestRuntimesLocalDev/package-lock.json") +} diff --git a/pkg/tests/testdata/TestRuntimesLocalDev/call1-resp.golden b/pkg/tests/testdata/TestRuntimesLocalDev/call1-resp.golden new file mode 100644 index 00000000..1d53670a --- /dev/null +++ b/pkg/tests/testdata/TestRuntimesLocalDev/call1-resp.golden @@ -0,0 +1,16 @@ +`{ + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 0, + "id": "call_1", + "function": { + "name": "py", + "arguments": "{}" + } + } + } + ], + "usage": {} +}` diff --git a/pkg/tests/testdata/TestRuntimesLocalDev/call1.golden b/pkg/tests/testdata/TestRuntimesLocalDev/call1.golden new file mode 100644 index 00000000..7e775029 --- /dev/null +++ b/pkg/tests/testdata/TestRuntimesLocalDev/call1.golden @@ -0,0 +1,37 @@ +`{ + "model": "gpt-4o", + "tools": [ + { + "function": { + "toolID": "testdata/TestRuntimesLocalDev/test.gpt:py", + "name": "py", + "parameters": null + } + }, + { + "function": { + "toolID": "testdata/TestRuntimesLocalDev/test.gpt:node", + "name": "node", + "parameters": null + } + }, + { + "function": { + "toolID": "testdata/TestRuntimesLocalDev/test.gpt:bash", + "name": "bash", + "parameters": null + } + } + ], + "messages": [ + { + "role": "system", + "content": [ + { + "text": "Dummy" + } + ], + "usage": {} + } + ] +}` diff --git a/pkg/tests/testdata/TestRuntimesLocalDev/call2-resp.golden b/pkg/tests/testdata/TestRuntimesLocalDev/call2-resp.golden new file mode 100644 index 00000000..4806793c --- /dev/null +++ b/pkg/tests/testdata/TestRuntimesLocalDev/call2-resp.golden @@ -0,0 +1,16 @@ +`{ + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 1, + "id": "call_2", + "function": { + "name": "node", + "arguments": "{}" + } + } + } + ], + "usage": {} +}` diff --git a/pkg/tests/testdata/TestRuntimesLocalDev/call2.golden b/pkg/tests/testdata/TestRuntimesLocalDev/call2.golden new file mode 100644 index 00000000..cc1fd1b7 --- /dev/null +++ b/pkg/tests/testdata/TestRuntimesLocalDev/call2.golden @@ -0,0 +1,70 @@ +`{ + "model": "gpt-4o", + "tools": [ + { + "function": { + "toolID": "testdata/TestRuntimesLocalDev/test.gpt:py", + "name": "py", + "parameters": null + } + }, + { + "function": { + "toolID": "testdata/TestRuntimesLocalDev/test.gpt:node", + "name": "node", + "parameters": null + } + }, + { + "function": { + "toolID": "testdata/TestRuntimesLocalDev/test.gpt:bash", + "name": "bash", + "parameters": null + } + } + ], + "messages": [ + { + "role": "system", + "content": [ + { + "text": "Dummy" + } + ], + "usage": {} + }, + { + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 0, + "id": "call_1", + "function": { + "name": "py", + "arguments": "{}" + } + } + } + ], + "usage": {} + }, + { + "role": "tool", + "content": [ + { + "text": "py worked\r\n" + } + ], + "toolCall": { + "index": 0, + "id": "call_1", + "function": { + "name": "py", + "arguments": "{}" + } + }, + "usage": {} + } + ] +}` diff --git a/pkg/tests/testdata/TestRuntimesLocalDev/call3-resp.golden b/pkg/tests/testdata/TestRuntimesLocalDev/call3-resp.golden new file mode 100644 index 00000000..1103f824 --- /dev/null +++ b/pkg/tests/testdata/TestRuntimesLocalDev/call3-resp.golden @@ -0,0 +1,16 @@ +`{ + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 2, + "id": "call_3", + "function": { + "name": "bash", + "arguments": "{}" + } + } + } + ], + "usage": {} +}` diff --git a/pkg/tests/testdata/TestRuntimesLocalDev/call3.golden b/pkg/tests/testdata/TestRuntimesLocalDev/call3.golden new file mode 100644 index 00000000..7c928c07 --- /dev/null +++ b/pkg/tests/testdata/TestRuntimesLocalDev/call3.golden @@ -0,0 +1,103 @@ +`{ + "model": "gpt-4o", + "tools": [ + { + "function": { + "toolID": "testdata/TestRuntimesLocalDev/test.gpt:py", + "name": "py", + "parameters": null + } + }, + { + "function": { + "toolID": "testdata/TestRuntimesLocalDev/test.gpt:node", + "name": "node", + "parameters": null + } + }, + { + "function": { + "toolID": "testdata/TestRuntimesLocalDev/test.gpt:bash", + "name": "bash", + "parameters": null + } + } + ], + "messages": [ + { + "role": "system", + "content": [ + { + "text": "Dummy" + } + ], + "usage": {} + }, + { + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 0, + "id": "call_1", + "function": { + "name": "py", + "arguments": "{}" + } + } + } + ], + "usage": {} + }, + { + "role": "tool", + "content": [ + { + "text": "py worked\r\n" + } + ], + "toolCall": { + "index": 0, + "id": "call_1", + "function": { + "name": "py", + "arguments": "{}" + } + }, + "usage": {} + }, + { + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 1, + "id": "call_2", + "function": { + "name": "node", + "arguments": "{}" + } + } + } + ], + "usage": {} + }, + { + "role": "tool", + "content": [ + { + "text": "node worked\n" + } + ], + "toolCall": { + "index": 1, + "id": "call_2", + "function": { + "name": "node", + "arguments": "{}" + } + }, + "usage": {} + } + ] +}` diff --git a/pkg/tests/testdata/TestRuntimesLocalDev/call4-resp.golden b/pkg/tests/testdata/TestRuntimesLocalDev/call4-resp.golden new file mode 100644 index 00000000..8135a8c9 --- /dev/null +++ b/pkg/tests/testdata/TestRuntimesLocalDev/call4-resp.golden @@ -0,0 +1,9 @@ +`{ + "role": "assistant", + "content": [ + { + "text": "TEST RESULT CALL: 4" + } + ], + "usage": {} +}` diff --git a/pkg/tests/testdata/TestRuntimesLocalDev/call4.golden b/pkg/tests/testdata/TestRuntimesLocalDev/call4.golden new file mode 100644 index 00000000..b95b880d --- /dev/null +++ b/pkg/tests/testdata/TestRuntimesLocalDev/call4.golden @@ -0,0 +1,136 @@ +`{ + "model": "gpt-4o", + "tools": [ + { + "function": { + "toolID": "testdata/TestRuntimesLocalDev/test.gpt:py", + "name": "py", + "parameters": null + } + }, + { + "function": { + "toolID": "testdata/TestRuntimesLocalDev/test.gpt:node", + "name": "node", + "parameters": null + } + }, + { + "function": { + "toolID": "testdata/TestRuntimesLocalDev/test.gpt:bash", + "name": "bash", + "parameters": null + } + } + ], + "messages": [ + { + "role": "system", + "content": [ + { + "text": "Dummy" + } + ], + "usage": {} + }, + { + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 0, + "id": "call_1", + "function": { + "name": "py", + "arguments": "{}" + } + } + } + ], + "usage": {} + }, + { + "role": "tool", + "content": [ + { + "text": "py worked\r\n" + } + ], + "toolCall": { + "index": 0, + "id": "call_1", + "function": { + "name": "py", + "arguments": "{}" + } + }, + "usage": {} + }, + { + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 1, + "id": "call_2", + "function": { + "name": "node", + "arguments": "{}" + } + } + } + ], + "usage": {} + }, + { + "role": "tool", + "content": [ + { + "text": "node worked\n" + } + ], + "toolCall": { + "index": 1, + "id": "call_2", + "function": { + "name": "node", + "arguments": "{}" + } + }, + "usage": {} + }, + { + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 2, + "id": "call_3", + "function": { + "name": "bash", + "arguments": "{}" + } + } + } + ], + "usage": {} + }, + { + "role": "tool", + "content": [ + { + "text": "bash works\n" + } + ], + "toolCall": { + "index": 2, + "id": "call_3", + "function": { + "name": "bash", + "arguments": "{}" + } + }, + "usage": {} + } + ] +}` diff --git a/pkg/tests/testdata/TestRuntimesLocalDev/package.json b/pkg/tests/testdata/TestRuntimesLocalDev/package.json new file mode 100644 index 00000000..d5f400a1 --- /dev/null +++ b/pkg/tests/testdata/TestRuntimesLocalDev/package.json @@ -0,0 +1,15 @@ +{ + "name": "chalk-example", + "version": "1.0.0", + "type": "module", + "description": "A simple example project to demonstrate the use of chalk", + "main": "example.js", + "scripts": { + "start": "node example.js" + }, + "author": "Your Name", + "license": "MIT", + "dependencies": { + "chalk": "^5.0.0" + } +} diff --git a/pkg/tests/testdata/TestRuntimesLocalDev/requirements.txt b/pkg/tests/testdata/TestRuntimesLocalDev/requirements.txt new file mode 100644 index 00000000..f2293605 --- /dev/null +++ b/pkg/tests/testdata/TestRuntimesLocalDev/requirements.txt @@ -0,0 +1 @@ +requests diff --git a/pkg/tests/testdata/TestRuntimesLocalDev/test.gpt b/pkg/tests/testdata/TestRuntimesLocalDev/test.gpt new file mode 100644 index 00000000..454ffce0 --- /dev/null +++ b/pkg/tests/testdata/TestRuntimesLocalDev/test.gpt @@ -0,0 +1,34 @@ +name: first +tools: py, node, bash + +Dummy + +--- +name: py + +#!/usr/bin/env python3 + +import requests +import platform + +# this is dumb hack to get the line endings to always be \r\n so the golden files match +# on both linux and windows +if platform.system() == 'Windows': + print('py worked') +else: + print('py worked\r') + +--- +name: node + +#!/usr/bin/env node + +import chalk from 'chalk'; +console.log("node worked") + +--- +name: bash + +#!/bin/bash + +echo bash works \ No newline at end of file