Persistent worker: clean-up working directory used by the agent (#356)

* Persistent worker: clean-up working directory used by the agent

* Worker: clean-up old static and dynamic working directories

* Use "cirrus-build-" when creating directory with ioutil.TempDir()

* Worker: a proper place to defer an instance Close()'ure

* Don't cleanup old working directories if running in Cirrus CI

internal/executor/instance/abstract/abstract.go

@@ -8,4 +8,5 @@ import (
 type Instance interface {
 	Run(context.Context, *runconfig.RunConfig) error
 	WorkingDirectory(projectDir string, dirtyMode bool) string
+	Close() error
 }

internal/executor/instance/container.go

@@ -63,3 +63,7 @@ func (inst *ContainerInstance) WorkingDirectory(projectDir string, dirtyMode boo
 
 	return inst.Platform.GenericWorkingDir()
 }
+
+func (inst *ContainerInstance) Close() error {
+	return nil
+}

internal/executor/instance/persistentworker/isolation/none/none.go

@@ -25,7 +25,7 @@ type PersistentWorkerInstance struct {
 
 func New() (*PersistentWorkerInstance, error) {
 	// Create a working directory that will be used if no dirty mode is requested in Run()
-	tempDir, err := ioutil.TempDir("", "")
+	tempDir, err := ioutil.TempDir("", "cirrus-build-")
 	if err != nil {
 		return nil, err
 	}
@@ -54,7 +54,8 @@ func (pwi *PersistentWorkerInstance) Run(ctx context.Context, config *runconfig.
 		config.ClientSecret,
 		"-task-id",
 		strconv.FormatInt(config.TaskID, 10),
-		"-clean-working-dir", // to make sure no garbage is left
+		"-pre-created-working-dir",
+		pwi.tempDir,
 	)
 
 	// Determine the working directory for the agent
@@ -83,3 +84,7 @@ func (pwi *PersistentWorkerInstance) WorkingDirectory(projectDir string, dirtyMo
 
 	return pwi.tempDir
 }
+
+func (pwi *PersistentWorkerInstance) Close() error {
+	return pwi.cleanup()
+}

internal/executor/instance/persistentworker/isolation/parallels/parallels.go

@@ -190,6 +190,10 @@ func (parallels *Parallels) WorkingDirectory(projectDir string, dirtyMode bool)
 	return platform.NewUnix().GenericWorkingDir()
 }
 
+func (parallels *Parallels) Close() error {
+	return nil
+}
+
 func TimeSyncCommand(t time.Time) string {
 	return fmt.Sprintf("sudo date -u %s\n", t.Format("010215042006"))
 }

internal/executor/instance/pipe.go

@@ -108,3 +108,7 @@ func (pi *PipeInstance) WorkingDirectory(projectDir string, dirtyMode bool) stri
 
 	return platform.NewUnix().GenericWorkingDir()
 }
+
+func (pi *PipeInstance) Close() error {
+	return nil
+}

internal/executor/instance/prebuilt.go

@@ -153,3 +153,7 @@ Outer:
 func (prebuilt *PrebuiltInstance) WorkingDirectory(projectDir string, dirtyMode bool) string {
 	return ""
 }
+
+func (prebuilt *PrebuiltInstance) Close() error {
+	return nil
+}

internal/executor/platform/platform.go

@@ -11,7 +11,7 @@ const (
 	agentImageBase = "gcr.io/cirrus-ci-community/cirrus-ci-agent:v"
 
 	// DefaultAgentVersion represents the default version of the https://github.com/cirruslabs/cirrus-ci-agent to use.
-	DefaultAgentVersion = "1.35.0"
+	DefaultAgentVersion = "1.38.0"
 )
 
 type CopyCommand struct {

internal/worker/task.go

@@ -25,6 +25,11 @@ func (worker *Worker) runTask(ctx context.Context, agentAwareTask *api.PollRespo
 
 	go func() {
 		defer func() {
+			if err := inst.Close(); err != nil {
+				worker.logger.Errorf("failed to close persistent worker instance for task %d: %v",
+					agentAwareTask.TaskId, err)
+			}
+
 			worker.taskCompletions <- agentAwareTask.TaskId
 		}()
 

internal/worker/worker.go

@@ -12,8 +12,11 @@ import (
 	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"runtime"
+	"strings"
 	"time"
 )
 
@@ -109,7 +112,55 @@ func (worker *Worker) info() *api.WorkerInfo {
 	}
 }
 
+// https://github.com/cirruslabs/cirrus-cli/issues/357
+func (worker *Worker) oldWorkingDirectoryCleanup() {
+	// Fix tests failing due to /tmp/cirrus-ci-build removal
+	if _, runningInCi := os.LookupEnv("CIRRUS_CI"); runningInCi {
+		return
+	}
+
+	tmpDir := os.TempDir()
+
+	// Clean-up static directory[1]
+	//
+	// nolint:lll
+	// [1]: https://github.com/cirruslabs/cirrus-ci-agent/blob/f88afe342106a6691d9e5b2d2e9187080c69fd2d/internal/executor/executor.go#L190
+	staticWorkingDir := filepath.Join(tmpDir, "cirrus-ci-build")
+	if err := os.RemoveAll(staticWorkingDir); err != nil {
+		worker.logger.Infof("failed to clean up old cirrus-ci-build static working directory %s: %v",
+			staticWorkingDir, err)
+	}
+
+	// Clean-up dynamic directories[1]
+	//
+	// nolint:lll
+	// [1]: https://github.com/cirruslabs/cirrus-ci-agent/blob/f88afe342106a6691d9e5b2d2e9187080c69fd2d/internal/executor/executor.go#L197
+	entries, err := ioutil.ReadDir(tmpDir)
+	if err != nil {
+		worker.logger.Infof("failed to clean up old cirrus-task-* dynamic working directories: %v", err)
+		return
+	}
+
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			continue
+		}
+
+		if strings.HasPrefix(entry.Name(), "cirrus-task-") {
+			dynamicWorkingDir := filepath.Join(tmpDir, entry.Name())
+
+			if err := os.RemoveAll(dynamicWorkingDir); err != nil {
+				worker.logger.Infof("failed to clean up old cirrus-task-* dynamic working directory %s: %v",
+					dynamicWorkingDir, err)
+			}
+		}
+	}
+}
+
 func (worker *Worker) Run(ctx context.Context) error {
+	// https://github.com/cirruslabs/cirrus-cli/issues/357
+	worker.oldWorkingDirectoryCleanup()
+
 	// A sub-context to cancel out all Run() side-effects when it finishes
 	subCtx, cancel := context.WithCancel(ctx)
 	defer cancel()