From dde5e7c710b103081ba98cbb747a8b4aedfd6bd3 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Thu, 15 Feb 2024 14:05:06 +0000 Subject: [PATCH] preprocessor: use a shared /caches mount in multi-step scripts For Go API guides, use of cuelang.org/go will be common. In serve mode, making changes and re-running a guide as it progresses is common. Where a dependency on cuelang.org/go exists, this requires downloading that module and its dependencies on every run because we currently start from a cold set of caches. For such guides it is therefore useful to be able to leverage at least modules download caches. But build and test caches are sometimes useful too. This change introduces a pattern whereby images used by guides must provide a /caches directory which is world-writable. This cache area is then the basis for a convention used by scripts (in later CLs): /caches/gobuild - a shared GOCACHE cache /caches/gomodcache - a shared GOMODCACHE ... This can only be a convention, because it requires some degree of coordination between guides and images. The area in general needs to be world-writable because there is no guarantee that each image uses the same UID as the user for the guide. In general, at least for Go caches, this is not a problem. A further convention is therefore that each image must establish a umask of 0000 so that writes to the cache area remain world-writable. Again, there is no risk in general here because everything is running in a container. Using a docker volume for these shared caches means that all caches, and any potential damage to those caches, is entirely isolated from the host machine. In a totally unrelated change (but in the interests of keeping this stack small) we also bump to go1.22 in our go.mod. Fixes cue-lang/cue#2831 Preprocessor-No-Write-Cache: true Signed-off-by: Paul Jolly Change-Id: I67f17f53e68a8b719546ccf05e6767ec6f8a9e5d Dispatch-Trailer: {"type":"trybot","CL":1177000,"patchset":5,"ref":"refs/changes/00/1177000/5","targetBranch":"alpha"} --- _scripts/build.bash | 2 +- go.mod | 2 +- go.sum | 8 +++ .../preprocessor/cmd/_docker/entrypoint.sh | 16 ++++++ internal/cmd/preprocessor/cmd/execute.go | 51 +++++++++++++++---- internal/cmd/preprocessor/cmd/execute_doc.go | 11 ++++ .../preprocessor/cmd/gen_dockerimagetag.go | 2 +- internal/cmd/preprocessor/cmd/rootfile.go | 5 ++ .../testdata/execute_multistagescript.txtar | 8 +-- 9 files changed, 88 insertions(+), 17 deletions(-) diff --git a/_scripts/build.bash b/_scripts/build.bash index ae2e959788..d258665be8 100755 --- a/_scripts/build.bash +++ b/_scripts/build.bash @@ -32,7 +32,7 @@ then minify="" if [ "${CI:-}" == "true" ] then - skipcache="--skipcache=true" + skipcache="--skipcache=true --nocachevolume" fi fi diff --git a/go.mod b/go.mod index 882406b7ee..29c7322cc1 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/cue-lang/cuelang.org -go 1.20 +go 1.22 require ( cuelang.org/go v0.7.0 diff --git a/go.sum b/go.sum index bedc4b21cc..1e2d2ab1d8 100644 --- a/go.sum +++ b/go.sum @@ -20,22 +20,30 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/emicklei/proto v1.10.0 h1:pDGyFRVV5RvV+nkBK9iy3q67FBy9Xa7vwrOTE+g5aGw= github.com/emicklei/proto v1.10.0/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= +github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de h1:D5x39vF5KCwKQaw+OC9ZPiLVHXz3UFw2+psEX+gYcto= diff --git a/internal/cmd/preprocessor/cmd/_docker/entrypoint.sh b/internal/cmd/preprocessor/cmd/_docker/entrypoint.sh index ebe3fdf04c..5a9e00469c 100644 --- a/internal/cmd/preprocessor/cmd/_docker/entrypoint.sh +++ b/internal/cmd/preprocessor/cmd/_docker/entrypoint.sh @@ -22,14 +22,30 @@ then exit 1 fi fi + # In case we didn't actually create a user # add the $USER_UID user user=$(getent passwd $USER_UID | cut -d: -f1) usermod -a -G $group $user +# We are still root at this point. Ensure /caches exists and that everything +# inside /caches is rwX for all. +mkdir -p /caches +chmod a+rwX /caches +if compgen -G "/caches/*" > /dev/null; then + chmod a+rwX /caches/* +fi + # Create the home dir if it does not exist mkdir -p /home/runner chown $user:$group /home/runner cd /home/runner export HOME=/home/runner + +# Set a umask to 0000 so that writes to /caches (if that is a mount) are done +# so in a way which won't conflict with users of other UIDs in other +# containers. Worst case, writes to /caches between containers will interfere +# with each other but this will be caught in CI. +umask 0000 + exec setpriv --reuid $USER_UID --regid $USER_GID --init-groups "$@" diff --git a/internal/cmd/preprocessor/cmd/execute.go b/internal/cmd/preprocessor/cmd/execute.go index e5061e2eef..9c018370dd 100644 --- a/internal/cmd/preprocessor/cmd/execute.go +++ b/internal/cmd/preprocessor/cmd/execute.go @@ -21,6 +21,7 @@ import ( "hash" "io" "os" + "os/exec" "path/filepath" "regexp" "strings" @@ -35,16 +36,18 @@ type lang string const ( langEn lang = "en" - flagDir flagName = "dir" - flagDebug flagName = "debug" - flagServe flagName = "serve" - flagUpdate flagName = "update" - flagReadonlyCache flagName = "readonlycache" - flagSkipCache flagName = "skipcache" - flagHugoFlag flagName = "hugo" - flagNoWriteCache flagName = "nowritecache" - flagCheck flagName = "check" - flagList flagName = "ls" + flagDir flagName = "dir" + flagDebug flagName = "debug" + flagServe flagName = "serve" + flagUpdate flagName = "update" + flagReadonlyCache flagName = "readonlycache" + flagSkipCache flagName = "skipcache" + flagHugoFlag flagName = "hugo" + flagNoWriteCache flagName = "nowritecache" + flagCheck flagName = "check" + flagList flagName = "ls" + flagCacheVolumeName flagName = "cachevolumename" + flagNoCacheVolume flagName = "nocachevolume" ) const ( @@ -144,6 +147,14 @@ type executionContext struct { // siteSchema is a CUE schema that validates a preprocessor site siteSchema cue.Value + + // noCacheVolume can be set to avoid using a shared docker volume for + // multi-step scripts. + noCacheVolume bool + + // cacheVolumeName is the name of the cache volume to use to optimise + // multi-step scripts. + cacheVolumeName string } // tempDir creates a new temporary directory within the @@ -215,6 +226,15 @@ func executeDef(c *Command, args []string) error { skipCache: flagSkipCache.Bool(c), noWriteCache: flagNoWriteCache.Bool(c), siteSchema: schema, + cacheVolumeName: flagCacheVolumeName.String(c), + noCacheVolume: flagNoCacheVolume.Bool(c), + } + + // Ensure we have a docker cache volume if one is required + if !ctx.noCacheVolume { + if err := dockerCacheVolumeCheck(flagCacheVolumeName.String(c)); err != nil { + return fmt.Errorf("failed to ensure cache volume exists: %v", err) + } } // Calculate which levels of debug-level logging to enable, processing each @@ -277,6 +297,17 @@ func executeDef(c *Command, args []string) error { return nil } +// dockerCacheVolumeCheck ensures that a docker volume volumeName exists, and +// returns an error if not. +func dockerCacheVolumeCheck(volumeName string) error { + cmd := exec.Command("docker", "volume", "create", volumeName) + out, err := cmd.CombinedOutput() + if err != nil { + err = fmt.Errorf("command [%v] failed: %v\n%s", cmd, err, out) + } + return err +} + func buildRootFileRegexp(langs []lang) *regexp.Regexp { var qlangs []string for _, lang := range langs { diff --git a/internal/cmd/preprocessor/cmd/execute_doc.go b/internal/cmd/preprocessor/cmd/execute_doc.go index 94764a7e7d..b7b24846d8 100644 --- a/internal/cmd/preprocessor/cmd/execute_doc.go +++ b/internal/cmd/preprocessor/cmd/execute_doc.go @@ -158,5 +158,16 @@ func newExecuteCmd(c *Command) *cobra.Command { cmd.Flags().Bool(string(flagCheck), false, "check CUE in page roots is properly namespaced") cmd.Flags().Bool(string(flagList), false, "list all .cue files that form part of site configuration") cmd.Flags().StringSliceVar(&hugoArgs, string(flagHugoFlag), nil, "list of flags to pass to hugo") + cmd.Flags().String(string(flagCacheVolumeName), envOrDefault("CUE_CACHE_VOLUME", "cuelang_org_caches"), "the name of the cache volume to use; this flag overrides CUE_CACHE_VOLUME") + cmd.Flags().Bool(string(flagNoCacheVolume), false, "do not use a shared docker volume cache for mult-step scripts") return cmd } + +// envOrDefault returns the value of the environment variable named by v if +// non-empty, else the value d. +func envOrDefault(v, d string) string { + if res := os.Getenv(v); res != "" { + return res + } + return d +} diff --git a/internal/cmd/preprocessor/cmd/gen_dockerimagetag.go b/internal/cmd/preprocessor/cmd/gen_dockerimagetag.go index e8797f051a..59a442b40e 100644 --- a/internal/cmd/preprocessor/cmd/gen_dockerimagetag.go +++ b/internal/cmd/preprocessor/cmd/gen_dockerimagetag.go @@ -2,4 +2,4 @@ package cmd -const dockerImageTag = "preprocessor:c57835e52cdeefd7917ee22e47b2d846d4c7b19107e243378aa3776e2fc3310e" +const dockerImageTag = "preprocessor:6e7cb00a0a5ab45565a39b9daa398f8fa8e2faa58a7b3baae3101c59a43764b2" diff --git a/internal/cmd/preprocessor/cmd/rootfile.go b/internal/cmd/preprocessor/cmd/rootfile.go index f55aa37e88..9b60868722 100644 --- a/internal/cmd/preprocessor/cmd/rootfile.go +++ b/internal/cmd/preprocessor/cmd/rootfile.go @@ -598,6 +598,11 @@ func (m *multiStepScript) run() (runerr error) { "-v", fmt.Sprintf("%s:/scripts", scriptsDir), ) + // Ensure we have a docker cache volume if one is required + if !m.noCacheVolume { + args = append(args, "-v", fmt.Sprintf("%s:/caches", m.cacheVolumeName)) + } + // We cannot perform the --network=host trick here, even if the user wants // to be unsafe, because we might, for example, run cue mod registry which // requires its own networking isolation for binding to the port it will diff --git a/internal/cmd/preprocessor/cmd/testdata/execute_multistagescript.txtar b/internal/cmd/preprocessor/cmd/testdata/execute_multistagescript.txtar index 756f17a97a..e4fb989869 100644 --- a/internal/cmd/preprocessor/cmd/testdata/execute_multistagescript.txtar +++ b/internal/cmd/preprocessor/cmd/testdata/execute_multistagescript.txtar @@ -215,12 +215,12 @@ package site page: { cache: { upload: { - "upload-some-cue": "Qeh5dBYv44+X5MjEvYJg7vlFPAdlUWMv82XbC1Sz2PE=" - "upload-some-json": "UcdyFGMTs9LUPE01F0z18B1V3EABbFvRtJ2WgBWhW7Y=" - "in-subdir": "zHoWoXaq9p6lYZoULKSi7hLssBPZmLU6v3l1DfYXv/o=" + "upload-some-cue": "7IDjGd2+UaDqL/CtCzWTF/n9JAghLOal/XYmGOXLTwU=" + "upload-some-json": "r7ZxaLhkjrkTxjeWFPVjLXXQEAk5pMHlkexqx+RnHKU=" + "in-subdir": "J43PmA4U4WvVNMgjwuRuxRYV01FPIWNFmKMuvNPDah4=" } multi_step: { - "3UEAH8JLUEFNADS5TNBJMM22ME58OUESIJUIE3BI40BJ4O3O3AO0====": [{ + "8OE47KRJN899MV3TUJQ0JF5DR4IV4E6K09S3J26JI2QFPPH0A1EG====": [{ doc: """ # script doc comment #scripttag