feat: manage node/python runtime for local files in dev

gptscript-ai · Aug 10, 2024 · cb46358 · cb46358
1 parent a383d4f
commit cb46358
Show file tree

Hide file tree

Showing 17 changed files with 552 additions and 14 deletions.
diff --git a/pkg/repos/get.go b/pkg/repos/get.go
@@ -28,6 +28,7 @@ type Runtime interface {
 	ID() string
 	Supports(tool types.Tool, cmd []string) bool
 	Setup(ctx context.Context, tool types.Tool, dataRoot, toolSource string, env []string) ([]string, error)
+	GetHash(tool types.Tool) (string, error)
 }
 
 type noopRuntime struct {
@@ -37,6 +38,10 @@ func (n noopRuntime) ID() string {
 	return "none"
 }
 
+func (n noopRuntime) GetHash(_ types.Tool) (string, error) {
+	return "", nil
+}
+
 func (n noopRuntime) Supports(_ types.Tool, _ []string) bool {
 	return false
 }
@@ -183,8 +188,13 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e
 	locker.Lock(tool.ID)
 	defer locker.Unlock(tool.ID)
 
+	runtimeHash, err := runtime.GetHash(tool)
+	if err != nil {
+		return "", nil, err
+	}
+
 	target := filepath.Join(m.storageDir, tool.Source.Repo.Revision, tool.Source.Repo.Path, tool.Source.Repo.Name, runtime.ID())
-	targetFinal := filepath.Join(target, tool.Source.Repo.Path)
+	targetFinal := filepath.Join(target, tool.Source.Repo.Path+runtimeHash)
 	doneFile := targetFinal + ".done"
 	envData, err := os.ReadFile(doneFile)
 	if err == nil {
@@ -251,7 +261,11 @@ func (m *Manager) GetContext(ctx context.Context, tool types.Tool, cmd, env []st
 	for _, runtime := range m.runtimes {
 		if runtime.Supports(tool, cmd) {
 			log.Debugf("Runtime %s supports %v", runtime.ID(), cmd)
-			return m.setup(ctx, runtime, tool, env)
+			wd, env, err := m.setup(ctx, runtime, tool, env)
+			if isLocal {
+				wd = tool.WorkingDir
+			}
+			return wd, env, err
 		}
 	}
 

diff --git a/pkg/repos/runtimes/busybox/busybox.go b/pkg/repos/runtimes/busybox/busybox.go
@@ -33,6 +33,10 @@ func (r *Runtime) ID() string {
 	return "busybox"
 }
 
+func (r *Runtime) GetHash(_ types.Tool) (string, error) {
+	return "", nil
+}
+
 func (r *Runtime) Supports(_ types.Tool, cmd []string) bool {
 	if runtime.GOOS != "windows" {
 		return false

diff --git a/pkg/repos/runtimes/golang/golang.go b/pkg/repos/runtimes/golang/golang.go
@@ -35,6 +35,10 @@ func (r *Runtime) ID() string {
 	return "go" + r.Version
 }
 
+func (r *Runtime) GetHash(_ types.Tool) (string, error) {
+	return "", nil
+}
+
 func (r *Runtime) Supports(tool types.Tool, cmd []string) bool {
 	return tool.Source.IsGit() &&
 		len(cmd) > 0 && cmd[0] == "${GPTSCRIPT_TOOL_DIR}/bin/gptscript-go-tool"

diff --git a/pkg/repos/runtimes/node/node.go b/pkg/repos/runtimes/node/node.go
@@ -39,10 +39,7 @@ func (r *Runtime) ID() string {
 	return "node" + r.Version
 }
 
-func (r *Runtime) Supports(tool types.Tool, cmd []string) bool {
-	if _, hasPackageJSON := tool.MetaData[packageJSON]; !hasPackageJSON && !tool.Source.IsGit() {
-		return false
-	}
+func (r *Runtime) Supports(_ types.Tool, cmd []string) bool {
 	for _, testCmd := range []string{"node", "npx", "npm"} {
 		if r.supports(testCmd, cmd) {
 			return true
@@ -61,6 +58,15 @@ func (r *Runtime) supports(testCmd string, cmd []string) bool {
 	return runtimeEnv.Matches(cmd, testCmd)
 }
 
+func (r *Runtime) GetHash(tool types.Tool) (string, error) {
+	if !tool.Source.IsGit() && tool.WorkingDir != "" {
+		if s, err := os.Stat(filepath.Join(tool.WorkingDir, packageJSON)); err == nil {
+			return hash.Digest(tool.WorkingDir + s.ModTime().String())[:7], nil
+		}
+	}
+	return "", nil
+}
+
 func (r *Runtime) Setup(ctx context.Context, tool types.Tool, dataRoot, toolSource string, env []string) ([]string, error) {
 	binPath, err := r.getRuntime(ctx, dataRoot)
 	if err != nil {
@@ -74,6 +80,8 @@ func (r *Runtime) Setup(ctx context.Context, tool types.Tool, dataRoot, toolSour
 
 	if _, ok := tool.MetaData[packageJSON]; ok {
 		newEnv = append(newEnv, "GPTSCRIPT_TMPDIR="+toolSource)
+	} else if !tool.Source.IsGit() && tool.WorkingDir != "" {
+		newEnv = append(newEnv, "GPTSCRIPT_TMPDIR="+tool.WorkingDir, "GPTSCRIPT_RUNTIME_DEV=true")
 	}
 
 	return newEnv, nil
@@ -120,6 +128,16 @@ func (r *Runtime) runNPM(ctx context.Context, tool types.Tool, toolSource, binDi
 		if err := os.WriteFile(filepath.Join(toolSource, packageJSON), []byte(contents+"\n"), 0644); err != nil {
 			return err
 		}
+	} else if !tool.Source.IsGit() {
+		if tool.WorkingDir == "" {
+			return nil
+		}
+		if _, err := os.Stat(filepath.Join(tool.WorkingDir, packageJSON)); errors.Is(fs.ErrNotExist, err) {
+			return nil
+		} else if err != nil {
+			return err
+		}
+		cmd.Dir = tool.WorkingDir
 	}
 	return cmd.Run()
 }

diff --git a/pkg/repos/runtimes/python/python.go b/pkg/repos/runtimes/python/python.go
@@ -24,8 +24,9 @@ import (
 var releasesData []byte
 
 const (
-	uvVersion       = "uv==0.2.33"
-	requirementsTxt = "requirements.txt"
+	uvVersion                = "uv==0.2.33"
+	requirementsTxt          = "requirements.txt"
+	gptscriptRequirementsTxt = "requirements-gptscript.txt"
 )
 
 type Release struct {
@@ -47,10 +48,7 @@ func (r *Runtime) ID() string {
 	return "python" + r.Version
 }
 
-func (r *Runtime) Supports(tool types.Tool, cmd []string) bool {
-	if _, hasRequirements := tool.MetaData[requirementsTxt]; !hasRequirements && !tool.Source.IsGit() {
-		return false
-	}
+func (r *Runtime) Supports(_ types.Tool, cmd []string) bool {
 	if runtimeEnv.Matches(cmd, r.ID()) {
 		return true
 	}
@@ -177,6 +175,22 @@ func (r *Runtime) getReleaseAndDigest() (string, string, error) {
 	return "", "", fmt.Errorf("failed to find an python runtime for %s", r.Version)
 }
 
+func (r *Runtime) GetHash(tool types.Tool) (string, error) {
+	if !tool.Source.IsGit() && tool.WorkingDir != "" {
+		if _, ok := tool.MetaData[requirementsTxt]; ok {
+			return "", nil
+		}
+		for _, req := range []string{gptscriptRequirementsTxt, requirementsTxt} {
+			reqFile := filepath.Join(tool.WorkingDir, req)
+			if s, err := os.Stat(reqFile); err == nil && !s.IsDir() {
+				return hash.Digest(tool.WorkingDir + s.ModTime().String())[:7], nil
+			}
+		}
+	}
+
+	return "", nil
+}
+
 func (r *Runtime) runPip(ctx context.Context, tool types.Tool, toolSource, binDir string, env []string) error {
 	log.InfofCtx(ctx, "Running pip in %s", toolSource)
 	if content, ok := tool.MetaData[requirementsTxt]; ok {
@@ -189,8 +203,16 @@ func (r *Runtime) runPip(ctx context.Context, tool types.Tool, toolSource, binDi
 		return cmd.Run()
 	}
 
-	for _, req := range []string{"requirements-gptscript.txt", requirementsTxt} {
-		reqFile := filepath.Join(toolSource, req)
+	reqPath := toolSource
+	if !tool.Source.IsGit() {
+		if tool.WorkingDir == "" {
+			return nil
+		}
+		reqPath = tool.WorkingDir
+	}
+
+	for _, req := range []string{gptscriptRequirementsTxt, requirementsTxt} {
+		reqFile := filepath.Join(reqPath, req)
 		if s, err := os.Stat(reqFile); err == nil && !s.IsDir() {
 			cmd := debugcmd.New(ctx, uvBin(binDir), "pip", "install", "-r", reqFile)
 			cmd.Env = env

diff --git a/pkg/tests/runner_test.go b/pkg/tests/runner_test.go
@@ -1018,3 +1018,26 @@ func TestRuntimes(t *testing.T) {
 	})
 	r.RunDefault()
 }
+
+func TestRuntimesLocalDev(t *testing.T) {
+	r := tester.NewRunner(t)
+	r.RespondWith(tester.Result{
+		Func: types.CompletionFunctionCall{
+			Name:      "py",
+			Arguments: "{}",
+		},
+	}, tester.Result{
+		Func: types.CompletionFunctionCall{
+			Name:      "node",
+			Arguments: "{}",
+		},
+	}, tester.Result{
+		Func: types.CompletionFunctionCall{
+			Name:      "bash",
+			Arguments: "{}",
+		},
+	})
+	r.RunDefault()
+	_ = os.RemoveAll("testdata/TestRuntimesLocalDev/node_modules")
+	_ = os.RemoveAll("testdata/TestRuntimesLocalDev/package-lock.json")
+}
diff --git a/pkg/tests/testdata/TestRuntimesLocalDev/call1-resp.golden b/pkg/tests/testdata/TestRuntimesLocalDev/call1-resp.golden
@@ -0,0 +1,16 @@
+`{
+  "role": "assistant",
+  "content": [
+    {
+      "toolCall": {
+        "index": 0,
+        "id": "call_1",
+        "function": {
+          "name": "py",
+          "arguments": "{}"
+        }
+      }
+    }
+  ],
+  "usage": {}
+}`
diff --git a/pkg/tests/testdata/TestRuntimesLocalDev/call1.golden b/pkg/tests/testdata/TestRuntimesLocalDev/call1.golden
@@ -0,0 +1,37 @@
+`{
+  "model": "gpt-4o",
+  "tools": [
+    {
+      "function": {
+        "toolID": "testdata/TestRuntimesLocalDev/test.gpt:py",
+        "name": "py",
+        "parameters": null
+      }
+    },
+    {
+      "function": {
+        "toolID": "testdata/TestRuntimesLocalDev/test.gpt:node",
+        "name": "node",
+        "parameters": null
+      }
+    },
+    {
+      "function": {
+        "toolID": "testdata/TestRuntimesLocalDev/test.gpt:bash",
+        "name": "bash",
+        "parameters": null
+      }
+    }
+  ],
+  "messages": [
+    {
+      "role": "system",
+      "content": [
+        {
+          "text": "Dummy"
+        }
+      ],
+      "usage": {}
+    }
+  ]
+}`
diff --git a/pkg/tests/testdata/TestRuntimesLocalDev/call2-resp.golden b/pkg/tests/testdata/TestRuntimesLocalDev/call2-resp.golden
@@ -0,0 +1,16 @@
+`{
+  "role": "assistant",
+  "content": [
+    {
+      "toolCall": {
+        "index": 1,
+        "id": "call_2",
+        "function": {
+          "name": "node",
+          "arguments": "{}"
+        }
+      }
+    }
+  ],
+  "usage": {}
+}`
diff --git a/pkg/tests/testdata/TestRuntimesLocalDev/call2.golden b/pkg/tests/testdata/TestRuntimesLocalDev/call2.golden
@@ -0,0 +1,70 @@
+`{
+  "model": "gpt-4o",
+  "tools": [
+    {
+      "function": {
+        "toolID": "testdata/TestRuntimesLocalDev/test.gpt:py",
+        "name": "py",
+        "parameters": null
+      }
+    },
+    {
+      "function": {
+        "toolID": "testdata/TestRuntimesLocalDev/test.gpt:node",
+        "name": "node",
+        "parameters": null
+      }
+    },
+    {
+      "function": {
+        "toolID": "testdata/TestRuntimesLocalDev/test.gpt:bash",
+        "name": "bash",
+        "parameters": null
+      }
+    }
+  ],
+  "messages": [
+    {
+      "role": "system",
+      "content": [
+        {
+          "text": "Dummy"
+        }
+      ],
+      "usage": {}
+    },
+    {
+      "role": "assistant",
+      "content": [
+        {
+          "toolCall": {
+            "index": 0,
+            "id": "call_1",
+            "function": {
+              "name": "py",
+              "arguments": "{}"
+            }
+          }
+        }
+      ],
+      "usage": {}
+    },
+    {
+      "role": "tool",
+      "content": [
+        {
+          "text": "py worked\r\n"
+        }
+      ],
+      "toolCall": {
+        "index": 0,
+        "id": "call_1",
+        "function": {
+          "name": "py",
+          "arguments": "{}"
+        }
+      },
+      "usage": {}
+    }
+  ]
+}`
diff --git a/pkg/tests/testdata/TestRuntimesLocalDev/call3-resp.golden b/pkg/tests/testdata/TestRuntimesLocalDev/call3-resp.golden
@@ -0,0 +1,16 @@
+`{
+  "role": "assistant",
+  "content": [
+    {
+      "toolCall": {
+        "index": 2,
+        "id": "call_3",
+        "function": {
+          "name": "bash",
+          "arguments": "{}"
+        }
+      }
+    }
+  ],
+  "usage": {}
+}`