tests: add Windows E2E testing

.github/workflows/e2e-tests.yaml

@@ -20,6 +20,7 @@ env:
   GO111MODULE: on
   GOFLAGS: -mod=vendor
 
+# windows blocklist: "up-podman", "integration", "machineprovider", "up"
 jobs:
   test-e2e:
     runs-on: ubuntu-latest

e2e/framework/exec.go

@@ -7,14 +7,15 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 )
 
 // ExecCommand executes the command string with the devpod test binary
 func (f *Framework) ExecCommandOutput(ctx context.Context, args []string) (string, error) {
 	var execOut bytes.Buffer
 
-	cmd := exec.CommandContext(ctx, f.DevpodBinDir+"/"+f.DevpodBinName, args...)
+	cmd := exec.CommandContext(ctx, filepath.Join(f.DevpodBinDir, f.DevpodBinName), args...)
 	cmd.Stdout = io.MultiWriter(os.Stdout, &execOut)
 	cmd.Stderr = os.Stderr
 
@@ -27,7 +28,7 @@ func (f *Framework) ExecCommandOutput(ctx context.Context, args []string) (strin
 
 // ExecCommandStdout executes the command string with the devpod test binary
 func (f *Framework) ExecCommandStdout(ctx context.Context, args []string) error {
-	cmd := exec.CommandContext(ctx, f.DevpodBinDir+"/"+f.DevpodBinName, args...)
+	cmd := exec.CommandContext(ctx, filepath.Join(f.DevpodBinDir, f.DevpodBinName), args...)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	if err := cmd.Run(); err != nil {
@@ -41,7 +42,7 @@ func (f *Framework) ExecCommandStdout(ctx context.Context, args []string) error
 func (f *Framework) ExecCommand(ctx context.Context, captureStdOut, searchForString bool, searchString string, args []string) error {
 	var execOut bytes.Buffer
 
-	cmd := exec.CommandContext(ctx, f.DevpodBinDir+"/"+f.DevpodBinName, args...)
+	cmd := exec.CommandContext(ctx, filepath.Join(f.DevpodBinDir, f.DevpodBinName), args...)
 	cmd.Stdout = io.MultiWriter(os.Stdout, &execOut)
 	cmd.Stderr = os.Stderr
 
@@ -65,7 +66,7 @@ func (f *Framework) ExecCommandCapture(ctx context.Context, args []string) (stri
 	var execOut bytes.Buffer
 	var execErr bytes.Buffer
 
-	cmd := exec.CommandContext(ctx, f.DevpodBinDir+"/"+f.DevpodBinName, args...)
+	cmd := exec.CommandContext(ctx, filepath.Join(f.DevpodBinDir, f.DevpodBinName), args...)
 	cmd.Stdout = io.MultiWriter(os.Stdout, &execOut)
 	cmd.Stderr = io.MultiWriter(os.Stderr, &execErr)
 

e2e/framework/framework.go

@@ -10,12 +10,14 @@ type Framework struct {
 }
 
 func NewDefaultFramework(path string) *Framework {
-	var binName = "devpod-"
+	binName := "devpod-"
 	switch runtime.GOOS {
 	case "darwin":
 		binName = binName + "darwin-"
 	case "linux":
 		binName = binName + "linux-"
+	case "windows":
+		binName = binName + "windows-"
 	}
 
 	switch runtime.GOARCH {
@@ -24,5 +26,10 @@ func NewDefaultFramework(path string) *Framework {
 	case "arm64":
 		binName = binName + "arm64"
 	}
+
+	if runtime.GOOS == "windows" {
+		binName = binName + ".exe"
+	}
+
 	return &Framework{DevpodBinDir: path, DevpodBinName: binName}
 }

e2e/framework/util.go

@@ -1,12 +1,24 @@
 package framework
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
 
 	"github.com/otiai10/copy"
 )
 
+func GetTimeout() time.Duration {
+	if runtime.GOOS == "windows" {
+		return 300 * time.Second
+	}
+
+	return 60 * time.Second
+}
+
 func CreateTempDir() (string, error) {
 	// Create temporary directory
 	dir, err := os.MkdirTemp("", "temp-*")
@@ -78,8 +90,15 @@ func CopyToTempDir(relativePath string) (string, error) {
 
 func CleanupTempDir(initialDir, tempDir string) {
 	err := os.RemoveAll(tempDir)
-	ExpectNoError(err)
+	if err != nil {
+		fmt.Println("WARN:", err)
+	}
 
 	err = os.Chdir(initialDir)
 	ExpectNoError(err)
 }
+
+func CleanString(input string) string {
+	input = strings.ReplaceAll(input, "\\", "")
+	return strings.ReplaceAll(input, "/", "")
+}

e2e/tests/build/build.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"os"
 	"path/filepath"
+	"runtime"
 
 	"github.com/loft-sh/devpod/e2e/framework"
 	"github.com/loft-sh/devpod/pkg/devcontainer/config"
@@ -157,6 +158,11 @@ var _ = DevPodDescribe("devpod build test suite", func() {
 		})
 
 		ginkgo.It("build kubernetes dockerless", func() {
+			// skip windows for now
+			if runtime.GOOS == "windows" {
+				return
+			}
+
 			ctx := context.Background()
 
 			f := framework.NewDefaultFramework(initialDir + "/bin")

e2e/tests/machineprovider/machineprovider.go

@@ -81,7 +81,7 @@ var _ = DevPodDescribe("devpod machine provider test suite", func() {
 			// delete workspace
 			err = f.DevPodWorkspaceDelete(ctx, tempDir)
 			framework.ExpectNoError(err)
-		}, ginkgo.SpecTimeout(60*time.Second))
+		}, ginkgo.SpecTimeout(framework.GetTimeout()))
 
 		ginkgo.It("test devpod inactivity timeout", func(ctx context.Context) {
 			f := framework.NewDefaultFramework(initialDir + "/bin")
@@ -155,6 +155,6 @@ var _ = DevPodDescribe("devpod machine provider test suite", func() {
 
 				time.Sleep(time.Second * 2)
 			}
-		}, ginkgo.SpecTimeout(300*time.Second))
+		}, ginkgo.SpecTimeout(framework.GetTimeout()*5))
 	})
 })

e2e/tests/proxyprovider/proxyprovider.go

@@ -6,7 +6,6 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/loft-sh/devpod/e2e/framework"
 	"github.com/loft-sh/devpod/pkg/client"
@@ -81,7 +80,7 @@ var _ = DevPodDescribe("devpod proxy provider test suite", func() {
 			// delete workspace
 			err = f.DevPodWorkspaceDelete(ctx, tempDir)
 			framework.ExpectNoError(err)
-		}, ginkgo.SpecTimeout(120*time.Second))
+		}, ginkgo.SpecTimeout(framework.GetTimeout()*2))
 
 		ginkgo.It("create & stop workspace via proxy provider", func(ctx context.Context) {
 			f := framework.NewDefaultFramework(initialDir + "/bin")
@@ -128,7 +127,7 @@ var _ = DevPodDescribe("devpod proxy provider test suite", func() {
 			// delete workspace
 			err = f.DevPodWorkspaceDelete(ctx, tempDir)
 			framework.ExpectNoError(err)
-		}, ginkgo.SpecTimeout(120*time.Second))
+		}, ginkgo.SpecTimeout(framework.GetTimeout()*2))
 
 		ginkgo.It("recreate workspace", func(ctx context.Context) {
 			f := framework.NewDefaultFramework(initialDir + "/bin")
@@ -176,7 +175,7 @@ var _ = DevPodDescribe("devpod proxy provider test suite", func() {
 			// delete workspace
 			err = f.DevPodWorkspaceDelete(ctx, tempDir)
 			framework.ExpectNoError(err)
-		}, ginkgo.SpecTimeout(120*time.Second))
+		}, ginkgo.SpecTimeout(framework.GetTimeout()*2))
 
 		ginkgo.It("devcontainer path workspace", func(ctx context.Context) {
 			f := framework.NewDefaultFramework(initialDir + "/bin")
@@ -210,6 +209,6 @@ var _ = DevPodDescribe("devpod proxy provider test suite", func() {
 			// delete workspace
 			err = f.DevPodWorkspaceDelete(ctx, tempDir)
 			framework.ExpectNoError(err)
-		}, ginkgo.SpecTimeout(120*time.Second))
+		}, ginkgo.SpecTimeout(framework.GetTimeout()*2))
 	})
 })

e2e/tests/ssh/ssh.go

@@ -8,6 +8,7 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"runtime"
 	"strconv"
 	"time"
 
@@ -82,6 +83,11 @@ var _ = DevPodDescribe("devpod ssh test suite", func() {
 		})
 
 		ginkgo.It("should start a new workspace with a docker provider (default) and forward a port into it", func() {
+			// skip windows for now
+			if runtime.GOOS == "windows" {
+				return
+			}
+
 			tempDir, err := framework.CopyToTempDir("tests/ssh/testdata/forward-test")
 			framework.ExpectNoError(err)
 			defer framework.CleanupTempDir(initialDir, tempDir)

e2e/tests/up/docker.go

@@ -8,7 +8,6 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/loft-sh/devpod/e2e/framework"
 	"github.com/loft-sh/devpod/pkg/devcontainer/config"
@@ -51,7 +50,7 @@ var _ = DevPodDescribe("devpod up test suite", func() {
 				// Wait for devpod workspace to come online (deadline: 30s)
 				err = f.DevPodUp(ctx, tempDir)
 				framework.ExpectNoError(err)
-			}, ginkgo.SpecTimeout(60*time.Second))
+			}, ginkgo.SpecTimeout(framework.GetTimeout()))
 			ginkgo.It("should start a new workspace and substitute devcontainer.json variables", func(ctx context.Context) {
 				tempDir, err := framework.CopyToTempDir("tests/up/testdata/docker-variables")
 				framework.ExpectNoError(err)
@@ -91,20 +90,22 @@ var _ = DevPodDescribe("devpod up test suite", func() {
 
 				localWorkspaceFolder, _, err := f.ExecCommandCapture(ctx, []string{"ssh", "--command", "cat $HOME/local-workspace-folder.out", projectName})
 				framework.ExpectNoError(err)
-				gomega.Expect(localWorkspaceFolder).To(gomega.Equal(tempDir))
+				gomega.Expect(framework.CleanString(localWorkspaceFolder)).To(gomega.Equal(framework.CleanString(tempDir)))
 
 				localWorkspaceFolderBasename, _, err := f.ExecCommandCapture(ctx, []string{"ssh", "--command", "cat $HOME/local-workspace-folder-basename.out", projectName})
 				framework.ExpectNoError(err)
 				gomega.Expect(localWorkspaceFolderBasename).To(gomega.Equal(filepath.Base(tempDir)))
 
 				containerWorkspaceFolder, _, err := f.ExecCommandCapture(ctx, []string{"ssh", "--command", "cat $HOME/container-workspace-folder.out", projectName})
 				framework.ExpectNoError(err)
-				gomega.Expect(containerWorkspaceFolder).To(gomega.Equal(filepath.Join("/workspaces", filepath.Base(tempDir))))
+				gomega.Expect(framework.CleanString(containerWorkspaceFolder)).To(gomega.Equal(
+					framework.CleanString("workspaces" + filepath.Base(tempDir)),
+				))
 
 				containerWorkspaceFolderBasename, _, err := f.ExecCommandCapture(ctx, []string{"ssh", "--command", "cat $HOME/container-workspace-folder-basename.out", projectName})
 				framework.ExpectNoError(err)
 				gomega.Expect(containerWorkspaceFolderBasename).To(gomega.Equal(filepath.Base(tempDir)))
-			}, ginkgo.SpecTimeout(60*time.Second))
+			}, ginkgo.SpecTimeout(framework.GetTimeout()))
 
 			ginkgo.It("should start a new workspace with mounts", func(ctx context.Context) {
 				tempDir, err := framework.CopyToTempDir("tests/up/testdata/docker-mounts")
@@ -138,7 +139,7 @@ var _ = DevPodDescribe("devpod up test suite", func() {
 				bar, _, err := f.ExecCommandCapture(ctx, []string{"ssh", "--command", "cat $HOME/mnt2/bar.txt", projectName})
 				framework.ExpectNoError(err)
 				gomega.Expect(bar).To(gomega.Equal("FOO"))
-			}, ginkgo.SpecTimeout(60*time.Second))
+			}, ginkgo.SpecTimeout(framework.GetTimeout()))
 
 			ginkgo.It("should start a new workspace with multistage build", func(ctx context.Context) {
 				tempDir, err := framework.CopyToTempDir("tests/up/testdata/docker-with-multi-stage-build")
@@ -155,7 +156,7 @@ var _ = DevPodDescribe("devpod up test suite", func() {
 				// Wait for devpod workspace to come online (deadline: 30s)
 				err = f.DevPodUp(ctx, tempDir, "--debug")
 				framework.ExpectNoError(err)
-			}, ginkgo.SpecTimeout(180*time.Second))
+			}, ginkgo.SpecTimeout(framework.GetTimeout()*3))
 
 			ginkgo.Context("should start a new workspace with features", func() {
 				ginkgo.It("ensure dependencies installed via features are accessible in lifecycle hooks", func(ctx context.Context) {
@@ -173,7 +174,7 @@ var _ = DevPodDescribe("devpod up test suite", func() {
 					// Wait for devpod workspace to come online (deadline: 30s)
 					err = f.DevPodUp(ctx, tempDir, "--debug")
 					framework.ExpectNoError(err)
-				}, ginkgo.SpecTimeout(60*time.Second))
+				}, ginkgo.SpecTimeout(framework.GetTimeout()))
 			})
 			ginkgo.It("should start a new workspace with dotfiles - no install script", func(ctx context.Context) {
 				tempDir, err := framework.CopyToTempDir("tests/up/testdata/docker")
@@ -202,7 +203,7 @@ var _ = DevPodDescribe("devpod up test suite", func() {
 /home/vscode/.file3
 `
 				framework.ExpectEqual(out, expectedOutput, "should match")
-			}, ginkgo.SpecTimeout(60*time.Second))
+			}, ginkgo.SpecTimeout(framework.GetTimeout()))
 			ginkgo.It("should start a new workspace with dotfiles - install script", func(ctx context.Context) {
 				tempDir, err := framework.CopyToTempDir("tests/up/testdata/docker")
 				framework.ExpectNoError(err)
@@ -228,7 +229,7 @@ var _ = DevPodDescribe("devpod up test suite", func() {
 				expectedOutput := "/tmp/worked\n"
 
 				framework.ExpectEqual(out, expectedOutput, "should match")
-			}, ginkgo.SpecTimeout(60*time.Second))
+			}, ginkgo.SpecTimeout(framework.GetTimeout()))
 
 			ginkgo.It("should start a new workspace with custom image", func(ctx context.Context) {
 				tempDir, err := framework.CopyToTempDir("tests/up/testdata/docker")
@@ -257,7 +258,7 @@ var _ = DevPodDescribe("devpod up test suite", func() {
 
 				framework.ExpectEqual(out, expectedOutput, "should match")
 				framework.ExpectNotEqual(out, unexpectedOutput, "should NOT match")
-			}, ginkgo.SpecTimeout(60*time.Second))
+			}, ginkgo.SpecTimeout(framework.GetTimeout()))
 			ginkgo.It("should start a new workspace with custom image and skip building", func(ctx context.Context) {
 				tempDir, err := framework.CopyToTempDir("tests/up/testdata/docker-with-multi-stage-build")
 				framework.ExpectNoError(err)
@@ -285,7 +286,7 @@ var _ = DevPodDescribe("devpod up test suite", func() {
 
 				framework.ExpectEqual(out, expectedOutput, "should match")
 				framework.ExpectNotEqual(out, unexpectedOutput, "should NOT match")
-			}, ginkgo.SpecTimeout(60*time.Second))
+			}, ginkgo.SpecTimeout(framework.GetTimeout()))
 
 			ginkgo.Context("should start a workspace from a Dockerfile build", func() {
 				ginkgo.It("should rebuild image in case of changes in files in build context", func(ctx context.Context) {
@@ -339,7 +340,7 @@ var _ = DevPodDescribe("devpod up test suite", func() {
 					image2 := container.Config.LegacyImage
 
 					gomega.Expect(image2).ShouldNot(gomega.Equal(image1), "images should be different")
-				}, ginkgo.SpecTimeout(60*time.Second))
+				}, ginkgo.SpecTimeout(framework.GetTimeout()))
 				ginkgo.It("should not rebuild image for changes in files mentioned in .dockerignore", func(ctx context.Context) {
 					tempDir, err := framework.CopyToTempDir("tests/up/testdata/docker-dockerfile-buildcontext")
 					framework.ExpectNoError(err)
@@ -391,7 +392,7 @@ var _ = DevPodDescribe("devpod up test suite", func() {
 					image2 := container.Config.LegacyImage
 
 					gomega.Expect(image2).Should(gomega.Equal(image1), "image should be same")
-				}, ginkgo.SpecTimeout(60*time.Second))
+				}, ginkgo.SpecTimeout(framework.GetTimeout()))
 			})
 			ginkgo.It("should use http headers to download feature", func(ctx context.Context) {
 				server := ghttp.NewServer()
@@ -441,7 +442,7 @@ var _ = DevPodDescribe("devpod up test suite", func() {
 				err = f.DevPodUp(ctx, tempDir)
 				framework.ExpectNoError(err)
 				server.Close()
-			}, ginkgo.SpecTimeout(60*time.Second))
+			}, ginkgo.SpecTimeout(framework.GetTimeout()))
 		})
 	})
 })