Skip to content

Commit

Permalink
internal/ci: use writefs
Browse files Browse the repository at this point in the history
As a further experiment, switch to using writefs, which is copied by
hand into this repo as an internal command.

There is no speed advantage to using it, rather it is a usability and
readability win.

Kicking the tyres with it more in this repo will help to see whether the
pattern works or not.

Signed-off-by: Paul Jolly <[email protected]>
Change-Id: Ia9203e2138ad74947ba49614df3298d84b7aa860
Dispatch-Trailer: {"type":"trybot","CL":1170145,"patchset":6,"ref":"refs/changes/45/1170145/6","targetBranch":"alpha"}
  • Loading branch information
myitcv authored and cueckoo committed Sep 30, 2023
1 parent 387ca47 commit af2837f
Show file tree
Hide file tree
Showing 4 changed files with 155 additions and 64 deletions.
61 changes: 61 additions & 0 deletions internal/ci/ci.cue
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
// Copyright 2023 The CUE Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ci

import (
"encoding/yaml"
"strings"

"github.com/cue-lang/cuelang.org/internal/ci/base"
"github.com/cue-lang/cuelang.org/internal/ci/repo"
"github.com/cue-lang/cuelang.org/internal/ci/github"
_netlify "github.com/cue-lang/cuelang.org/internal/ci/netlify"
)

// #writefs mirrors the type of the arguments expected by
// internal/cmd/writefs
#writefs: {
Remove: [...string]
Create: [string]: {
Type: "symlink" | *"file"
Contents: *"" | string
}
}

fs: #writefs & {
Remove: [
"../../.github/workflows/*.yml",
]

Create: {
[_]: {
// TODO: do not hardcode this to ci_tool
let donotedit = base.doNotEditMessage & {#generatedBy: "internal/ci/ci_tool.cue", _}
_res: string
Contents: "# \(donotedit)\n\n\(strings.TrimSpace(_res))\n"
}
for _name, _workflow in github.workflows {
"../../.github/workflows/\(_name).yml": {
_res: yaml.Marshal(_workflow)
}
}
"../../codereview.cfg": {
_res: base.toCodeReviewCfg & {#input: repo.codeReview, _}
}
"../../netlify.toml": {
_res: _netlify.#toToml & {#input: _netlify.config, _}
}
}
}
68 changes: 5 additions & 63 deletions internal/ci/ci_tool.cue
Original file line number Diff line number Diff line change
Expand Up @@ -15,69 +15,11 @@
package ci

import (
"path"
"encoding/yaml"
"tool/file"

"github.com/cue-lang/cuelang.org/internal/ci/base"
"github.com/cue-lang/cuelang.org/internal/ci/repo"
"github.com/cue-lang/cuelang.org/internal/ci/github"
_netlify "github.com/cue-lang/cuelang.org/internal/ci/netlify"
"encoding/json"
"tool/exec"
)

// For the commands below, note we use simple yet hacky path resolution, rather
// than anything that might derive the module root using go list or similar, in
// order that we have zero dependencies. This is important because this CUE
// package is "vendored" to an external dependency so that that unity can
// re-run and verify these steps as part of a the test suite that runs against
// new CUE versions.

_goos: string @tag(os,var=os)

// gen.workflows regenerates the GitHub workflow Yaml definitions.
//
// See internal/ci/gen.go for details on how this step fits into the sequence
// of generating our CI workflow definitions, and updating various txtar tests
// with files from that process.
command: gen: {
_dir: path.FromSlash("../../.github/workflows", path.Unix)

workflows: {
remove: {
glob: file.Glob & {
glob: path.Join([_dir, "*.yml"], _goos)
files: [...string]
}
for _, _filename in glob.files {
"delete \(_filename)": file.RemoveAll & {
path: _filename
}
}
}
for _workflowName, _workflow in github.workflows {
let _filename = _workflowName + ".yml"
"generate \(_filename)": file.Create & {
$after: [ for v in remove {v}]
filename: path.Join([_dir, _filename], _goos)
let donotedit = base.doNotEditMessage & {#generatedBy: "internal/ci/ci_tool.cue", _}
contents: "# \(donotedit)\n\n\(yaml.Marshal(_workflow))"
}
}
}
}

command: gen: netlify: file.Create & {
_dir: path.FromSlash("../../", path.Unix)
filename: path.Join([_dir, "netlify.toml"], _goos)
let res = _netlify.#toToml & {#input: _netlify.config, _}
let donotedit = base.doNotEditMessage & {#generatedBy: "internal/ci/ci_tool.cue", _}
contents: "# \(donotedit)\n\n\(res)\n"
}

command: gen: codereviewcfg: file.Create & {
_dir: path.FromSlash("../../", path.Unix)
filename: path.Join([_dir, "codereview.cfg"], _goos)
let res = base.toCodeReviewCfg & {#input: repo.codeReview, _}
let donotedit = base.doNotEditMessage & {#generatedBy: "internal/ci/ci_tool.cue", _}
contents: "# \(donotedit)\n\n\(res)\n"
command: gen: exec.Run & {
cmd: ["go", "run", "github.com/cue-lang/cuelang.org/internal/cmd/writefs"]
stdin: json.Marshal(fs)
}
89 changes: 89 additions & 0 deletions internal/cmd/writefs/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package main

import (
"bytes"
"encoding/json"
"log"
"os"
"path/filepath"
)

type file struct {
Type string

// If typ == "symlink" this is the filepath of the target
// otherwise it is the Contents of the file
Contents string
}

type args struct {
// Remove is a list of globs of files to remove. File
// removal happens before file creation.
Remove []string

// Create is a map of file path (in Unix format) to contents.
Create map[string]file
}

func main() {
var todo args
dec := json.NewDecoder(os.Stdin)
if err := dec.Decode(&todo); err != nil {
log.Fatalf("failed to decode arguments from stdin: %v", err)
}
for _, glob := range todo.Remove {
files, err := filepath.Glob(glob)
if err != nil {
log.Fatalf("failed to glob %q: %v", glob, err)
}
if len(files) == 0 {
continue
}
for _, f := range files {
if err := os.Remove(f); err != nil {
log.Fatalf("failed to remove %s: %v", f, err)
}
}
}
for fp, f := range todo.Create {
fp = filepath.FromSlash(fp)
dir := filepath.Dir(fp)
if err := os.MkdirAll(dir, 0o777); err != nil {
log.Fatalf("failed to mkdir %s: %v", dir, err)
}
switch f.Type {
case "symlink":
target := filepath.FromSlash(f.Contents)
actualTarget, err := os.Readlink(fp)
if err == nil && actualTarget == target {
continue
}
if err := os.Symlink(target, fp); err != nil {
log.Fatalf("failed to symlink %s -> %s: %v", fp, target, err)
}
case "file":
// If we have a .json file, normalise the contents first
// as a special case.
//
// TODO: this actually belongs as an option in CUE.
contents := []byte(f.Contents)
if filepath.Ext(fp) == ".json" {
var i any
err := json.Unmarshal(contents, &i)
if err == nil {
var b bytes.Buffer
enc := json.NewEncoder(&b)
enc.SetEscapeHTML(false)
enc.SetIndent("", " ")
if err := enc.Encode(i); err == nil {
// Add a trailing newline
contents = b.Bytes()
}
}
}
if err := os.WriteFile(fp, []byte(contents), 0o666); err != nil {
log.Fatalf("failed to write file %s: %v", fp, err)
}
}
}
}
1 change: 0 additions & 1 deletion netlify.toml
Original file line number Diff line number Diff line change
Expand Up @@ -79,4 +79,3 @@
to = "https://calendar.google.com/calendar/u/0?cid=Y19lNzkxMWQ5OWQ4ZGIyMmU2ZTVjMzhkMTVkNjY2ZTVlNjdiNWE5ODNkZWU4N2JmNTU2NDY3NzI1OGIxYjJhMTFhQGdyb3VwLmNhbGVuZGFyLmdvb2dsZS5jb20"
status = 302
force = true

0 comments on commit af2837f

Please sign in to comment.