preprocessor: take statement doc comments into account in cache key

In https://cuelang.org/cl/1176702 we added support for the first
statement directives, in that case adding a directive to apply an
ellipsis sanitiser. This means that directives in doc comments need to
be considered as part of the cache key for a multi-step script
(directives are not yet supported in code blocks... we need to
consolidate that code).

Strictly speaking only directives need to participate as part of the
cache key. However for now, in the interests of expediency, we add the
entire statement doc comment to be part of the cache key.

Fixes cue-lang/cue#2814

Preprocessor-No-Write-Cache: true
Signed-off-by: Paul Jolly <[email protected]>
Change-Id: Id80a8e3916f4b5f0c89f960094ac7bdc949b2b69

internal/cmd/preprocessor/cmd/rootfile.go

@@ -406,6 +406,7 @@ func (rf *rootFile) buildMultistepScript() (*multiStepScript, error) {
 				// echo the command we will run
 				cmdEchoFence := rf.getFence()
 				pf("cat <<'%s'\n", cmdEchoFence)
+				pf("%s\n", stmt.Doc)
 				pf("$ %s\n", stmt.Cmd)
 				pf("%s\n", cmdEchoFence)
 				stmt.outputFence = rf.getFence()

internal/cmd/preprocessor/cmd/testdata/execute_doc_comment_change.txtar

@@ -0,0 +1,67 @@
+# Verify that changing a doc comment in any way causes a cache miss
+
+unquote content/dir/en.md content/dir/en.md.doc_comment
+
+# Run the preprocessor
+exec preprocessor execute
+
+# Verify that we generated the expected output
+cmp hugo/content/en/dir/index.md golden/hugo/content/en/dir/index.md
+
+# Re-run the preprocessor and ensure we have a cache hit
+exec preprocessor execute --debug=all
+stderr $WORK${/}'content'${/}'dir'${/}'en.md: cache hit for multi-step script; not running'
+
+# Verify that we still generated the expected output
+cmp hugo/content/en/dir/index.md golden/hugo/content/en/dir/index.md
+
+# Modify the doc comment alone and verify we get a cache miss
+cp content/dir/en.md.doc_comment content/dir/en.md
+exec preprocessor execute --debug=all
+stderr $WORK${/}'content'${/}'dir'${/}'en.md: cache miss for multi-step script'
+
+# Verify that we generated the expected output with a doc comment
+cmp hugo/content/en/dir/index.md golden/hugo/content/en/dir/index_doc_comment.md
+
+-- hugo/.keep --
+-- content/site.cue --
+package site
+-- content/dir/page.cue --
+package site
+
+content: dir: page: {
+	leftDelim:  "{{{"
+	rightDelim: "}}}"
+}
+-- content/dir/en.md --
+>---
+>title: A page
+>---
+>{{{with script "en" "run-the-cue"}}}
+>echo hello
+>{{{end}}}
+-- content/dir/en.md.doc_comment --
+>---
+>title: A page
+>---
+>{{{with script "en" "run-the-cue"}}}
+># This is a doc comment
+>echo hello
+>{{{end}}}
+-- golden/hugo/content/en/dir/index.md --
+---
+title: A page
+---
+```text { title="TERMINAL" codeToCopy="ZWNobyBoZWxsbwo=" }
+$ echo hello
+hello
+```
+-- golden/hugo/content/en/dir/index_doc_comment.md --
+---
+title: A page
+---
+```text { title="TERMINAL" codeToCopy="ZWNobyBoZWxsbwo=" }
+# This is a doc comment
+$ echo hello
+hello
+```

internal/cmd/preprocessor/cmd/testdata/execute_multistagescript.txtar

@@ -220,7 +220,7 @@ package site
 						"in-subdir":        "zHoWoXaq9p6lYZoULKSi7hLssBPZmLU6v3l1DfYXv/o="
 					}
 					multi_step: {
-						"A836Q874LN0IJ68A6NTQAC0QOKK8R6Q2CMU52VQ1FV0LPS6NL11G====": [{
+						"3UEAH8JLUEFNADS5TNBJMM22ME58OUESIJUIE3BI40BJ4O3O3AO0====": [{
 							doc: """
 									# script doc comment
 									#scripttag