From bab26ffe871e9baad63446fff92b525df35438fb Mon Sep 17 00:00:00 2001 From: Zeke Lu Date: Tue, 25 May 2021 16:07:51 +0800 Subject: [PATCH 1/3] make the call to easyjson more reliable 1. remove "easyjson.go" from the white list so that deprecated files (if any) won't stay in the repository. 2. generate "easyjson.go" stubs first so that the source codes are valid from the perspetive of syntax, which in turn makes it possible to run bootstrap.Generator one by one. 3. run bootstrap.Generator one by one to avoid the occasional failure. Error messages when it failed (both the source file and import path changed from time to time when it failed): # github.com/chromedp/cdproto/cachestorage cachestorage/easyjson.go:8:3: can't find import: "encoding/json" # github.com/chromedp/cdproto/cast cast/easyjson.go:6:3: can't find import: "github.com/mailru/easyjson" --- main.go | 52 ++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/main.go b/main.go index 0251f27..759a2cc 100644 --- a/main.go +++ b/main.go @@ -59,7 +59,7 @@ var ( flagNoDump = flag.Bool("no-dump", false, "toggle not dumping generated protocol file to out directory") flagGoPkg = flag.String("go-pkg", "github.com/chromedp/cdproto", "go base package name") - flagGoWl = flag.String("go-wl", "LICENSE,README.md,*.pdl,go.mod,go.sum,"+easyjsonGo, "comma-separated list of files to whitelist (ignore)") + flagGoWl = flag.String("go-wl", "LICENSE,README.md,*.pdl,go.mod,go.sum", "comma-separated list of files to whitelist (ignore)") // flagWorkers = flag.Int("workers", runtime.NumCPU(), "number of workers") ) @@ -436,7 +436,46 @@ func goimports(fileBuffers map[string]*bytes.Buffer) error { // easyjson runs easy json on the list of packages. func easyjson(pkgs []string) error { + util.Logf("WRITING: easyjson stubs") + // All the easyjson.go files are removed in the CLEANING step, + // so that deprecated files (if any) won't stay in the repository. + // Now generate the stubs first so that the source codes are valid + // from the perspective of syntax. + if err := easyjsonStubs(pkgs); err != nil { + return err + } + util.Logf("RUNNING: easyjson") + // Got error messages like this when running g.Run() concurrently: + // # github.com/chromedp/cdproto/cachestorage + // cachestorage/easyjson.go:8:3: can't find import: "encoding/json" + // # github.com/chromedp/cdproto/cast + // cast/easyjson.go:6:3: can't find import: "github.com/mailru/easyjson" + // It seems that it fails more often on slow machines. The root cause is not clear yet, + // maybe it's relevant to the issue https://github.com/golang/go/issues/26794. + // The workaround for now is to run g.Run() one by one (take longer to finish). + for _, n := range pkgs { + n = filepath.Join(*flagOut, n) + p := parser.Parser{AllStructs: true} + if err := p.Parse(n, true); err != nil { + return err + } + g := bootstrap.Generator{ + OutName: filepath.Join(n, easyjsonGo), + PkgPath: p.PkgPath, + PkgName: p.PkgName, + Types: p.StructNames, + NoFormat: true, + } + if err := g.Run(); err != nil { + return err + } + } + return nil +} + +// easyjsonStubs runs easy json to generate stubs for the list of packages. +func easyjsonStubs(pkgs []string) error { eg, _ := errgroup.WithContext(context.Background()) for _, k := range pkgs { eg.Go(func(n string) func() error { @@ -447,11 +486,12 @@ func easyjson(pkgs []string) error { return err } g := bootstrap.Generator{ - OutName: filepath.Join(n, easyjsonGo), - PkgPath: p.PkgPath, - PkgName: p.PkgName, - Types: p.StructNames, - NoFormat: true, + OutName: filepath.Join(n, easyjsonGo), + PkgPath: p.PkgPath, + PkgName: p.PkgName, + Types: p.StructNames, + NoFormat: true, + StubsOnly: true, } return g.Run() } From a73cd0b55bd62a1e8e1a9114f3aa7ccbbec8dd51 Mon Sep 17 00:00:00 2001 From: Zeke Lu Date: Wed, 26 May 2021 12:57:15 +0800 Subject: [PATCH 2/3] configure github action to generate cdproto 1. It can be triggered by push and pull request. The purpose is to check the changes. 2. It can also be triggered manually. In this case, the versions of chromium and v8 can be specified. If not specified, it defaults to the latest version. The changes to the cdproto repository will be committed to the "build-[id]" branch. --- .github/workflows/gen.yml | 75 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 .github/workflows/gen.yml diff --git a/.github/workflows/gen.yml b/.github/workflows/gen.yml new file mode 100644 index 0000000..88f4691 --- /dev/null +++ b/.github/workflows/gen.yml @@ -0,0 +1,75 @@ +name: Gen + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + workflow_dispatch: + inputs: + chromium: + description: 'chromium protocol version' + required: false + default: '' + v8: + description: 'v8 protocol version' + required: false + default: '' + +jobs: + + build: + runs-on: ubuntu-latest + steps: + - name: Checkout pdlgen + uses: actions/checkout@v2 + with: + path: pdlgen + + - name: Checkout chromedp + uses: actions/checkout@v2 + with: + repository: chromedp/chromedp + path: chromedp + + # "Checkout cdproto" should be the last checkout action, + # otherwise, its token will be overwritten and the "Commit changes" + # step will fail. + - name: Checkout cdproto + uses: actions/checkout@v2 + with: + repository: chromedp/cdproto + token: ${{ secrets.CI_PDLGEN }} + fetch-depth: 0 + path: cdproto + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.16.x + + - name: Build cdproto-gen + working-directory: ./pdlgen + run: go build -v + + - name: Run cdproto-gen + working-directory: ./pdlgen + run: ./cdproto-gen -chromium=${{ github.event.inputs.chromium }} -v8=${{ github.event.inputs.v8 }} -out=../cdproto + + # test chromedp with the updated cdproto + - name: Test chromedp + working-directory: ./chromedp + run: | + go mod edit -replace github.com/chromedp/cdproto=../cdproto + TMPDIR=$RUNNER_TEMP go test -v ./... + ./contrib/docker-test.sh + + - name: Commit changes + working-directory: ./cdproto + run: | + git config user.name github-actions + git config user.email github-actions@github.com + git checkout -b build-$GITHUB_RUN_NUMBER + git commit -a -m "Updating to $(basename $HOME/.cache/cdproto-gen/pdl/combined/*.pdl .pdl) definitions" + git push origin build-$GITHUB_RUN_NUMBER + if: github.event == 'workflow_dispatch' From 62019ebf61a5e294b8de92166323f6a4726817b3 Mon Sep 17 00:00:00 2001 From: Zeke Lu Date: Wed, 26 May 2021 16:06:45 +0800 Subject: [PATCH 3/3] remove .travis.yml --- .travis.yml | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 353da18..0000000 --- a/.travis.yml +++ /dev/null @@ -1,34 +0,0 @@ -dist: bionic -language: go -go: - - 1.13.x -env: - - GO111MODULE=off -services: - - docker -addons: - apt: - chrome: stable -before_install: - - go get -d -t - - mkdir -p $GOPATH/src/github.com/mailru $GOPATH/src/github.com/chromedp - - rm -rf $GOPATH/src/github.com/mailru/easyjson - - git clone https://github.com/kenshaw/easyjson.git $GOPATH/src/github.com/mailru/easyjson - - cd $GOPATH/src/github.com/mailru/easyjson - - go get -d -t - - git clone https://github.com/chromedp/cdproto.git $GOPATH/src/github.com/chromedp/cdproto - - cd $GOPATH/src/github.com/chromedp/cdproto - - go get -d -t - - go install - - git clone https://github.com/chromedp/chromedp.git $GOPATH/src/github.com/chromedp/chromedp - - cd $GOPATH/src/github.com/chromedp/chromedp - - go get -d -t - - go install -script: - - cd $GOPATH/src/github.com/chromedp/cdproto-gen - - go build - - ./cdproto-gen - - cd $GOPATH/src/github.com/chromedp/chromedp - - go test -v - - go test -c - - docker run --rm --volume=$PWD:/chromedp --entrypoint=/chromedp/chromedp.test --workdir=/chromedp --env=PATH=/headless-shell chromedp/headless-shell:latest -test.v