preprocessor: use a shared /caches mount in multi-step scripts

For Go API guides, use of cuelang.org/go will be common. In serve mode,
making changes and re-running a guide as it progresses is common. Where
a dependency on cuelang.org/go exists, this requires downloading that
module and its dependencies on every run because we currently start from
a cold set of caches.

For such guides it is therefore useful to be able to leverage at least
modules download caches. But build and test caches are sometimes useful
too.

This change introduces a pattern whereby images used by guides must
provide a /caches directory which is world-writable. This cache area is
then the basis for a convention used by scripts (in later CLs):

    /caches/gobuild - a shared GOCACHE cache
    /caches/gomodcache - a shared GOMODCACHE
    ...

This can only be a convention, because it requires some degree of
coordination between guides and images. The area in general needs to be
world-writable because there is no guarantee that each image uses the
same UID as the user for the guide. In general, at least for Go caches,
this is not a problem.

A further convention is therefore that each image must establish a umask
of 0000 so that writes to the cache area remain world-writable. Again,
there is no risk in general here because everything is running in a
container.

Using a docker volume for these shared caches means that all caches, and
any potential damage to those caches, is entirely isolated from the host
machine.

In a totally unrelated change (but in the interests of keeping this
stack small) we also bump to go1.22 in our go.mod.

Preprocessor-No-Write-Cache: true
Signed-off-by: Paul Jolly <[email protected]>
Change-Id: I67f17f53e68a8b719546ccf05e6767ec6f8a9e5d

_scripts/build.bash

@@ -32,7 +32,7 @@ then
 	minify=""
 	if [ "${CI:-}" == "true" ]
 	then
-		skipcache="--skipcache=true"
+		skipcache="--skipcache=true --nocachevolume"
 	fi
 fi
 

go.mod

@@ -1,6 +1,6 @@
 module github.com/cue-lang/cuelang.org
 
-go 1.20
+go 1.22
 
 require (
 	cuelang.org/go v0.7.0

go.sum

@@ -20,22 +20,30 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/emicklei/proto v1.10.0 h1:pDGyFRVV5RvV+nkBK9iy3q67FBy9Xa7vwrOTE+g5aGw=
 github.com/emicklei/proto v1.10.0/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A=
 github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA=
+github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=
+github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw=
+github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
 github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
 github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de h1:D5x39vF5KCwKQaw+OC9ZPiLVHXz3UFw2+psEX+gYcto=

internal/cmd/preprocessor/cmd/_docker/entrypoint.sh

@@ -22,14 +22,30 @@ then
 		exit 1
 	fi
 fi
+
 # In case we didn't actually create a user
 # add the $USER_UID user
 user=$(getent passwd $USER_UID | cut -d: -f1)
 usermod -a -G $group $user
 
+# We are still root at this point. Ensure /caches exists and that everything
+# inside /caches is rwX for all.
+mkdir -p /caches
+chmod a+rwX /caches
+if compgen -G "/caches/*" > /dev/null; then
+	chmod a+rwX /caches/*
+fi
+
 # Create the home dir if it does not exist
 mkdir -p /home/runner
 chown $user:$group /home/runner
 cd /home/runner
 export HOME=/home/runner
+
+# Set a umask to 0000 so that writes to /caches (if that is a mount) are done
+# so in a way which won't conflict with users of other UIDs in other
+# containers. Worst case, writes to /caches between containers will interfere
+# with each other but this will be caught in CI.
+umask 0000
+
 exec setpriv --reuid $USER_UID --regid $USER_GID --init-groups "$@"

internal/cmd/preprocessor/cmd/execute.go

@@ -16,14 +16,18 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"crypto/sha256"
 	"fmt"
 	"hash"
 	"io"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"regexp"
 	"strings"
+	"sync"
+	"time"
 
 	"cuelang.org/go/cmd/cue/cmd"
 	"cuelang.org/go/cue"
@@ -35,16 +39,18 @@ type lang string
 const (
 	langEn lang = "en"
 
-	flagDir           flagName = "dir"
-	flagDebug         flagName = "debug"
-	flagServe         flagName = "serve"
-	flagUpdate        flagName = "update"
-	flagReadonlyCache flagName = "readonlycache"
-	flagSkipCache     flagName = "skipcache"
-	flagHugoFlag      flagName = "hugo"
-	flagNoWriteCache  flagName = "nowritecache"
-	flagCheck         flagName = "check"
-	flagList          flagName = "ls"
+	flagDir             flagName = "dir"
+	flagDebug           flagName = "debug"
+	flagServe           flagName = "serve"
+	flagUpdate          flagName = "update"
+	flagReadonlyCache   flagName = "readonlycache"
+	flagSkipCache       flagName = "skipcache"
+	flagHugoFlag        flagName = "hugo"
+	flagNoWriteCache    flagName = "nowritecache"
+	flagCheck           flagName = "check"
+	flagList            flagName = "ls"
+	flagCacheVolumeName flagName = "cachevolumename"
+	flagNoCacheVolume   flagName = "nocachevolume"
 )
 
 const (
@@ -144,6 +150,29 @@ type executionContext struct {
 
 	// siteSchema is a CUE schema that validates a preprocessor site
 	siteSchema cue.Value
+
+	// noCacheVolume can be set to avoid using a shared docker volume for
+	// multi-step scripts.
+	noCacheVolume bool
+
+	// cacheVolumeCheck ensures we run our docker cache volume check only
+	// once per instance of the preprocessor (which is why this field is on
+	// executionContext and not executeContext). It returns the name of the
+	// volume to use for caches if the volume exists, and an error otherwise.
+	cacheVolumeCheck func() (string, error)
+}
+
+// dockerCacheVolumeCheck ensures that a docker volume volumeName exists, and
+// returns an error if not.
+func dockerCacheVolumeCheck(volumeName string) (string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	cmd := exec.CommandContext(ctx, "docker", "volume", "create", volumeName)
+	out, err := cmd.CombinedOutput()
+	cancel()
+	if err != nil {
+		err = fmt.Errorf("failed to ensure cache volume exists via [%v]: %v\n%s", cmd, err, out)
+	}
+	return volumeName, err
 }
 
 // tempDir creates a new temporary directory within the
@@ -215,6 +244,10 @@ func executeDef(c *Command, args []string) error {
 		skipCache:         flagSkipCache.Bool(c),
 		noWriteCache:      flagNoWriteCache.Bool(c),
 		siteSchema:        schema,
+		cacheVolumeCheck: sync.OnceValues(func() (string, error) {
+			return dockerCacheVolumeCheck(flagCacheVolumeName.String(c))
+		}),
+		noCacheVolume: flagNoCacheVolume.Bool(c),
 	}
 
 	// Calculate which levels of debug-level logging to enable, processing each

internal/cmd/preprocessor/cmd/execute_doc.go

@@ -158,5 +158,16 @@ func newExecuteCmd(c *Command) *cobra.Command {
 	cmd.Flags().Bool(string(flagCheck), false, "check CUE in page roots is properly namespaced")
 	cmd.Flags().Bool(string(flagList), false, "list all .cue files that form part of site configuration")
 	cmd.Flags().StringSliceVar(&hugoArgs, string(flagHugoFlag), nil, "list of flags to pass to hugo")
+	cmd.Flags().String(string(flagCacheVolumeName), envOrDefault("CUE_CACHE_VOLUME", "cuelang_org_caches"), "the name of the cache volume to use; this flag overrides CUE_CACHE_VOLUME")
+	cmd.Flags().Bool(string(flagNoCacheVolume), false, "do not use a shared docker volume cache for mult-step scripts")
 	return cmd
 }
+
+// envOrDefault returns the value of the environment variable named by v if
+// non-empty, else the value d.
+func envOrDefault(v, d string) string {
+	if res := os.Getenv(v); res != "" {
+		return res
+	}
+	return d
+}

internal/cmd/preprocessor/cmd/rootfile.go

@@ -598,6 +598,15 @@ func (m *multiStepScript) run() (runerr error) {
 		"-v", fmt.Sprintf("%s:/scripts", scriptsDir),
 	)
 
+	// Ensure we have a docker cache volume if one is required
+	if !m.noCacheVolume {
+		volumeName, err := m.cacheVolumeCheck()
+		if err != nil {
+			return m.errorf("%v: failed to ensure cache volume exists: %v", m, err)
+		}
+		args = append(args, "-v", fmt.Sprintf("%s:/caches", volumeName))
+	}
+
 	// We cannot perform the --network=host trick here, even if the user wants
 	// to be unsafe, because we might, for example, run cue mod registry which
 	// requires its own networking isolation for binding to the port it will

internal/cmd/preprocessor/cmd/testdata/execute_multistagescript.txtar

@@ -215,12 +215,12 @@ package site
 			page: {
 				cache: {
 					upload: {
-						"upload-some-cue":  "Qeh5dBYv44+X5MjEvYJg7vlFPAdlUWMv82XbC1Sz2PE="
-						"upload-some-json": "UcdyFGMTs9LUPE01F0z18B1V3EABbFvRtJ2WgBWhW7Y="
-						"in-subdir":        "zHoWoXaq9p6lYZoULKSi7hLssBPZmLU6v3l1DfYXv/o="
+						"upload-some-cue":  "7IDjGd2+UaDqL/CtCzWTF/n9JAghLOal/XYmGOXLTwU="
+						"upload-some-json": "r7ZxaLhkjrkTxjeWFPVjLXXQEAk5pMHlkexqx+RnHKU="
+						"in-subdir":        "J43PmA4U4WvVNMgjwuRuxRYV01FPIWNFmKMuvNPDah4="
 					}
 					multi_step: {
-						"3UEAH8JLUEFNADS5TNBJMM22ME58OUESIJUIE3BI40BJ4O3O3AO0====": [{
+						"8OE47KRJN899MV3TUJQ0JF5DR4IV4E6K09S3J26JI2QFPPH0A1EG====": [{
 							doc: """
 									# script doc comment
 									#scripttag