Skip to content

Commit

Permalink
feat: support different versions of the same box
Browse files Browse the repository at this point in the history
  • Loading branch information
@nalgeon
nalgeon committed Dec 20, 2023
1 parent 162ca55 commit ade821f
Show file tree
Hide file tree
Showing 7 changed files with 212 additions and 28 deletions.
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
module github.com/nalgeon/codapi

go 1.20
go 1.21
4 changes: 3 additions & 1 deletion internal/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,8 @@ type Box struct {
Runtime string `json:"runtime"`
Host

Files []string `json:"files"`
Versions []string `json:"versions"`
Files []string `json:"files"`
}

// A Host describes container Host attributes.
Expand Down Expand Up @@ -96,6 +97,7 @@ type Command struct {
// A Step describes a single step of a command.
type Step struct {
Box string `json:"box"`
Version string `json:"version"`
User string `json:"user"`
Action string `json:"action"`
Stdin bool `json:"stdin"`
Expand Down
74 changes: 51 additions & 23 deletions internal/engine/docker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"os"
"os/exec"
"path/filepath"
"slices"
"strconv"
"strings"
"time"
Expand Down Expand Up @@ -62,22 +63,22 @@ func (e *Docker) Exec(req Request) Execution {

// initialization step
if e.cmd.Before != nil {
out := e.execStep(e.cmd.Before, req.ID, dir, nil)
out := e.execStep(e.cmd.Before, req, dir, nil)
if !out.OK {
return out
}
}

// the first step is required
first, rest := e.cmd.Steps[0], e.cmd.Steps[1:]
out := e.execStep(first, req.ID, dir, req.Files)
out := e.execStep(first, req, dir, req.Files)

// the rest are optional
if out.OK && len(rest) > 0 {
// each step operates on the results of the previous one,
// without using the source files - hence `nil` instead of `files`
for _, step := range rest {
out = e.execStep(step, req.ID, dir, nil)
out = e.execStep(step, req, dir, nil)
if !out.OK {
break
}
Expand All @@ -86,7 +87,7 @@ func (e *Docker) Exec(req Request) Execution {

// cleanup step
if e.cmd.After != nil {
afterOut := e.execStep(e.cmd.After, req.ID, dir, nil)
afterOut := e.execStep(e.cmd.After, req, dir, nil)
if out.OK && !afterOut.OK {
return afterOut
}
Expand All @@ -96,27 +97,44 @@ func (e *Docker) Exec(req Request) Execution {
}

// execStep executes a step using the docker container.
func (e *Docker) execStep(step *config.Step, reqID, dir string, files Files) Execution {
func (e *Docker) execStep(step *config.Step, req Request, dir string, files Files) Execution {
box := e.cfg.Boxes[step.Box]
err := e.copyFiles(box, dir)
err := e.validateVersion(box, step, req)
if err != nil {
return Fail(req.ID, err)
}

err = e.copyFiles(box, dir)
if err != nil {
err = NewExecutionError("copy files to temp dir", err)
return Fail(reqID, err)
return Fail(req.ID, err)
}

stdout, stderr, err := e.exec(box, step, reqID, dir, files)
stdout, stderr, err := e.exec(box, step, req, dir, files)
if err != nil {
return Fail(reqID, err)
return Fail(req.ID, err)
}

return Execution{
ID: reqID,
ID: req.ID,
OK: true,
Stdout: stdout,
Stderr: stderr,
}
}

func (e *Docker) validateVersion(box *config.Box, step *config.Step, req Request) error {
// If the version is set in the step config, use it.
// If the version isn't set in the request, use the latest one.
// Otherwise, check that the version in the request is supported
// according to the box config.
if step.Version == "" && req.Version != "" && !slices.Contains(box.Versions, req.Version) {
err := fmt.Errorf("box %s does not support version %s", step.Box, req.Version)
return err
}
return nil
}

// copyFiles copies box files to the temporary directory.
func (e *Docker) copyFiles(box *config.Box, dir string) error {
if box == nil || len(box.Files) == 0 {
Expand Down Expand Up @@ -147,18 +165,18 @@ func (e *Docker) writeFiles(dir string, files Files) error {

// exec executes the step in the docker container
// using the files from in the temporary directory.
func (e *Docker) exec(box *config.Box, step *config.Step, reqID, dir string, files Files) (stdout string, stderr string, err error) {
func (e *Docker) exec(box *config.Box, step *config.Step, req Request, dir string, files Files) (stdout string, stderr string, err error) {
// limit the stdout/stderr size
prog := NewProgram(step.Timeout, int64(step.NOutput))
args := e.buildArgs(box, step, reqID, dir)
args := e.buildArgs(box, step, req, dir)

if step.Stdin {
// pass files to container from stdin
stdin := filesReader(files)
stdout, stderr, err = prog.RunStdin(stdin, reqID, "docker", args...)
stdout, stderr, err = prog.RunStdin(stdin, req.ID, "docker", args...)
} else {
// pass files to container from temp directory
stdout, stderr, err = prog.Run(reqID, "docker", args...)
stdout, stderr, err = prog.Run(req.ID, "docker", args...)
}

if err == nil {
Expand All @@ -172,11 +190,11 @@ func (e *Docker) exec(box *config.Box, step *config.Step, reqID, dir string, fil
// inside the container is not related to the "docker run" process,
// and will hang forever after the "docker run" process is killed
go func() {
err = dockerKill(reqID)
err = dockerKill(req.ID)
if err == nil {
logx.Debug("%s: docker kill ok", reqID)
logx.Debug("%s: docker kill ok", req.ID)
} else {
logx.Log("%s: docker kill failed: %v", reqID, err)
logx.Log("%s: docker kill failed: %v", req.ID, err)
}
}()
}
Expand All @@ -202,28 +220,28 @@ func (e *Docker) exec(box *config.Box, step *config.Step, reqID, dir string, fil
}

// buildArgs prepares the arguments for the `docker` command.
func (e *Docker) buildArgs(box *config.Box, step *config.Step, name, dir string) []string {
func (e *Docker) buildArgs(box *config.Box, step *config.Step, req Request, dir string) []string {
var args []string
if step.Action == actionRun {
args = dockerRunArgs(box, step, name, dir)
args = dockerRunArgs(box, step, req, dir)
} else if step.Action == actionExec {
args = dockerExecArgs(step)
} else {
// should never happen if the config is valid
args = []string{"version"}
}

command := expandVars(step.Command, name)
command := expandVars(step.Command, req.ID)
args = append(args, command...)
logx.Debug("%v", args)
return args
}

// buildArgs prepares the arguments for the `docker run` command.
func dockerRunArgs(box *config.Box, step *config.Step, name, dir string) []string {
func dockerRunArgs(box *config.Box, step *config.Step, req Request, dir string) []string {
args := []string{
actionRun, "--rm",
"--name", name,
"--name", req.ID,
"--runtime", box.Runtime,
"--cpus", strconv.Itoa(box.CPU),
"--memory", fmt.Sprintf("%dm", box.Memory),
Expand Down Expand Up @@ -255,7 +273,17 @@ func dockerRunArgs(box *config.Box, step *config.Step, name, dir string) []strin
for _, lim := range box.Ulimit {
args = append(args, "--ulimit", lim)
}
args = append(args, box.Image)

if step.Version != "" {
// if the version is set in the step config, use it
args = append(args, box.Image+":"+step.Version)
} else if req.Version != "" {
// if the version is set in the request, use it
args = append(args, box.Image+":"+req.Version)
} else {
// otherwise, use the latest version
args = append(args, box.Image)
}
return args
}

Expand Down
121 changes: 120 additions & 1 deletion internal/engine/docker_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,25 @@ import (

var dockerCfg = &config.Config{
Boxes: map[string]*config.Box{
"alpine": {
Image: "codapi/alpine",
Runtime: "runc",
Host: config.Host{
CPU: 1, Memory: 64, Network: "none",
Volume: "%s:/sandbox:ro",
NProc: 64,
},
},
"go": {
Image: "codapi/go",
Runtime: "runc",
Host: config.Host{
CPU: 1, Memory: 64, Network: "none",
Volume: "%s:/sandbox:ro",
NProc: 64,
},
Versions: []string{"dev"},
},
"postgresql": {
Image: "codapi/postgresql",
Runtime: "runc",
Expand All @@ -28,9 +47,28 @@ var dockerCfg = &config.Config{
Volume: "%s:/sandbox:ro",
NProc: 64,
},
Versions: []string{"dev"},
},
},
Commands: map[string]config.SandboxCommands{
"go": map[string]*config.Command{
"run": {
Engine: "docker",
Steps: []*config.Step{
{
Box: "go", User: "sandbox", Action: "run",
Command: []string{"go", "build"},
NOutput: 4096,
},
{
Box: "alpine", Version: "latest",
User: "sandbox", Action: "run",
Command: []string{"./main"},
NOutput: 4096,
},
},
},
},
"postgresql": map[string]*config.Command{
"run": {
Engine: "docker",
Expand Down Expand Up @@ -75,9 +113,10 @@ func TestDockerRun(t *testing.T) {
"docker run": {Stdout: "hello world", Stderr: "", Err: nil},
}
mem := execy.Mock(commands)
engine := NewDocker(dockerCfg, "python", "run")

t.Run("success", func(t *testing.T) {
mem.Clear()
engine := NewDocker(dockerCfg, "python", "run")
req := Request{
ID: "http_42",
Sandbox: "python",
Expand All @@ -103,8 +142,88 @@ func TestDockerRun(t *testing.T) {
if out.Err != nil {
t.Errorf("Err: expected nil, got %v", out.Err)
}
mem.MustHave(t, "codapi/python")
mem.MustHave(t, "python main.py")
})

t.Run("latest version", func(t *testing.T) {
mem.Clear()
engine := NewDocker(dockerCfg, "python", "run")
req := Request{
ID: "http_42",
Sandbox: "python",
Command: "run",
Files: map[string]string{
"": "print('hello world')",
},
}
out := engine.Exec(req)
if !out.OK {
t.Error("OK: expected true")
}
mem.MustHave(t, "codapi/python")
})

t.Run("custom version", func(t *testing.T) {
mem.Clear()
engine := NewDocker(dockerCfg, "python", "run")
req := Request{
ID: "http_42",
Sandbox: "python",
Version: "dev",
Command: "run",
Files: map[string]string{
"": "print('hello world')",
},
}
out := engine.Exec(req)
if !out.OK {
t.Error("OK: expected true")
}
mem.MustHave(t, "codapi/python:dev")
})

t.Run("step version", func(t *testing.T) {
mem.Clear()
engine := NewDocker(dockerCfg, "go", "run")
req := Request{
ID: "http_42",
Sandbox: "go",
Version: "dev",
Command: "run",
Files: map[string]string{
"": "var n = 42",
},
}
out := engine.Exec(req)
if !out.OK {
t.Error("OK: expected true")
}
mem.MustHave(t, "codapi/go:dev")
mem.MustHave(t, "codapi/alpine:latest")
})

t.Run("unsupported version", func(t *testing.T) {
mem.Clear()
engine := NewDocker(dockerCfg, "python", "run")
req := Request{
ID: "http_42",
Sandbox: "python",
Version: "42",
Command: "run",
Files: map[string]string{
"": "print('hello world')",
},
}
out := engine.Exec(req)
if out.OK {
t.Error("OK: expected false")
}
want := "box python does not support version 42"
if out.Stderr != want {
t.Errorf("Stderr: unexpected value: %s", out.Stderr)
}
})
}

func TestDockerExec(t *testing.T) {
Expand Down
7 changes: 6 additions & 1 deletion internal/engine/engine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,18 @@ import (
type Request struct {
ID string `json:"id"`
Sandbox string `json:"sandbox"`
Version string `json:"version,omitempty"`
Command string `json:"command"`
Files Files `json:"files"`
}

// GenerateID() sets a unique ID for the request.
func (r *Request) GenerateID() {
r.ID = fmt.Sprintf("%s_%s_%s", r.Sandbox, r.Command, stringx.RandString(8))
if r.Version != "" {
r.ID = fmt.Sprintf("%s.%s_%s_%s", r.Sandbox, r.Version, r.Command, stringx.RandString(8))
} else {
r.ID = fmt.Sprintf("%s_%s_%s", r.Sandbox, r.Command, stringx.RandString(8))
}
}

// An Execution is an output from the code execution engine.
Expand Down
Loading

0 comments on commit ade821f

Please sign in to comment.