diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index f56d98adf4c..8bb6522c958 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -4,7 +4,8 @@ on: push: branches: [ main ] pull_request: - branches: [ main ] + schedule: + - cron: "0 0 * * *" jobs: build: diff --git a/.github/workflows/protobufs.yml b/.github/workflows/protobufs.yml new file mode 100644 index 00000000000..d2163899348 --- /dev/null +++ b/.github/workflows/protobufs.yml @@ -0,0 +1,105 @@ +name: Protobufs + +on: + push: + branches: [ main ] + pull_request: + schedule: + - cron: "0 0 * * *" + +jobs: + validate_protobufs: + name: Validate Protobufs + runs-on: ubuntu-latest + steps: + - name: Install go + uses: actions/setup-go@v2 + with: + go-version: '1.19' + - name: Checkout code + uses: actions/checkout@v3 + - name: Cache + uses: actions/cache@v3 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ github.job }}-${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }} + restore-keys: ${{ github.job }}-${{ runner.os }}-go-build- + - name: Install protobuf + uses: arduino/setup-protoc@v1 + with: + version: '3.x' + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Lint protobufs + run: | + go install github.com/googleapis/api-linter/cmd/api-linter@latest + # Set directory to hold symlink + readonly PROTOBUF_IMPORT_DIR='protobuf-import' + mkdir -p "${PROTOBUF_IMPORT_DIR}" + # Remove any existing symlinks & empty directories + find "${PROTOBUF_IMPORT_DIR}" -type l -delete + find "${PROTOBUF_IMPORT_DIR}" -type d -empty -delete + # Download the required dependencies + go mod download + # Get ondatra modules we use and create required directory structure + go list -f "${PROTOBUF_IMPORT_DIR}/{{ .Path }}" -m github.com/openconfig/ondatra | xargs -L1 dirname | sort | uniq | xargs mkdir -p + # Create symlink + go list -f "{{ .Dir }} "${PROTOBUF_IMPORT_DIR}"/{{ .Path }}" -m github.com/openconfig/ondatra | xargs -L1 -- ln -s + find . -name \*.proto -exec api-linter -I./"${PROTOBUF_IMPORT_DIR}" --disable-rule all --enable-rule core {} \+ + - name: Compile topology binding textprotos + run: | + fail=0 + # Set directory to hold symlink + readonly PROTOBUF_IMPORT_DIR='protobuf-import' + mkdir -p "${PROTOBUF_IMPORT_DIR}" + # Remove any existing symlinks & empty directories + find "${PROTOBUF_IMPORT_DIR}" -type l -delete + find "${PROTOBUF_IMPORT_DIR}" -type d -empty -delete + # Download the required dependencies + go mod download + # Get ondatra modules we use and create required directory structure + go list -f "${PROTOBUF_IMPORT_DIR}/{{ .Path }}" -m github.com/openconfig/ondatra | xargs -L1 dirname | sort | uniq | xargs mkdir -p + # Create symlink + go list -f "{{ .Dir }} \"${PROTOBUF_IMPORT_DIR}\"/{{ .Path }}" -m github.com/openconfig/ondatra | xargs -L1 -- ln -s + for i in `find topologies/ -type f -name "*.binding"`; do + if ! output=$(protoc -I="${PROTOBUF_IMPORT_DIR}" --proto_path=topologies/proto --encode=openconfig.testing.Binding topologies/proto/binding.proto < $i 2>&1 >/dev/null); then + fail=1 + echo -e "Compile $i failed:\n$output\n" + fi + done + if [ "$fail" == "1" ]; then exit 1; fi + - name: Compile feature profile textprotos + run: | + fail=0 + for i in `find feature/ -type f -name "feature.textproto"`; do + if ! output=$(protoc --encode=openconfig.profiles.FeatureProfile proto/feature.proto < $i 2>&1 >/dev/null); then + fail=1 + echo -e "Compile $i failed:\n$output\n" + fi + done + if [ "$fail" == "1" ]; then exit 1; fi + + validate_oc_paths: + name: Validate OpenConfig Paths + runs-on: ubuntu-latest + steps: + - name: Install go + uses: actions/setup-go@v2 + with: + go-version: '1.19' + - name: Checkout code + uses: actions/checkout@v3 + - name: Cache + uses: actions/cache@v2 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ github.job }}-${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }} + restore-keys: ${{ github.job }}-${{ runner.os }}-go-build- + - name: Fetch Openconfig Models + run: make openconfig_public + - name: Validate Paths + run: make validate_paths + diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 2c9936bbcdf..fb1dbbf3755 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -1,101 +1,6 @@ name: Pull Request on: [pull_request] jobs: - validate_protobufs: - name: Validate Protobufs - runs-on: ubuntu-latest - steps: - - name: Install go - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Checkout code - uses: actions/checkout@v3 - - name: Cache - uses: actions/cache@v3 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: ${{ github.job }}-${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ github.job }}-${{ runner.os }}-go-build- - - name: Install protobuf - uses: arduino/setup-protoc@v1 - with: - version: '3.x' - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Lint protobufs - run: | - go install github.com/googleapis/api-linter/cmd/api-linter@latest - # Set directory to hold symlink - readonly PROTOBUF_IMPORT_DIR='protobuf-import' - mkdir -p "${PROTOBUF_IMPORT_DIR}" - # Remove any existing symlinks & empty directories - find "${PROTOBUF_IMPORT_DIR}" -type l -delete - find "${PROTOBUF_IMPORT_DIR}" -type d -empty -delete - # Download the required dependencies - go mod download - # Get ondatra modules we use and create required directory structure - go list -f "${PROTOBUF_IMPORT_DIR}/{{ .Path }}" -m github.com/openconfig/ondatra | xargs -L1 dirname | sort | uniq | xargs mkdir -p - # Create symlink - go list -f "{{ .Dir }} "${PROTOBUF_IMPORT_DIR}"/{{ .Path }}" -m github.com/openconfig/ondatra | xargs -L1 -- ln -s - find . -name \*.proto -exec api-linter -I./"${PROTOBUF_IMPORT_DIR}" --disable-rule all --enable-rule core {} \+ - - name: Compile topology binding textprotos - run: | - fail=0 - # Set directory to hold symlink - readonly PROTOBUF_IMPORT_DIR='protobuf-import' - mkdir -p "${PROTOBUF_IMPORT_DIR}" - # Remove any existing symlinks & empty directories - find "${PROTOBUF_IMPORT_DIR}" -type l -delete - find "${PROTOBUF_IMPORT_DIR}" -type d -empty -delete - # Download the required dependencies - go mod download - # Get ondatra modules we use and create required directory structure - go list -f "${PROTOBUF_IMPORT_DIR}/{{ .Path }}" -m github.com/openconfig/ondatra | xargs -L1 dirname | sort | uniq | xargs mkdir -p - # Create symlink - go list -f "{{ .Dir }} \"${PROTOBUF_IMPORT_DIR}\"/{{ .Path }}" -m github.com/openconfig/ondatra | xargs -L1 -- ln -s - for i in `find topologies/ -type f -name "*.binding"`; do - if ! output=$(protoc -I="${PROTOBUF_IMPORT_DIR}" --proto_path=topologies/proto --encode=openconfig.testing.Binding topologies/proto/binding.proto < $i 2>&1 >/dev/null); then - fail=1 - echo -e "Compile $i failed:\n$output\n" - fi - done - if [ "$fail" == "1" ]; then exit 1; fi - - name: Compile feature profile textprotos - run: | - fail=0 - for i in `find feature/ -type f -name "feature.textproto"`; do - if ! output=$(protoc --encode=openconfig.profiles.FeatureProfile proto/feature.proto < $i 2>&1 >/dev/null); then - fail=1 - echo -e "Compile $i failed:\n$output\n" - fi - done - if [ "$fail" == "1" ]; then exit 1; fi - - validate_oc_paths: - name: Validate OpenConfig Paths - runs-on: ubuntu-latest - steps: - - name: Install go - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Checkout code - uses: actions/checkout@v3 - - name: Cache - uses: actions/cache@v2 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: ${{ github.job }}-${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ github.job }}-${{ runner.os }}-go-build- - - name: Fetch Openconfig Models - run: make openconfig_public - - name: Validate Paths - run: make validate_paths - check_style: name: Check style against CONTRIBUTING.md runs-on: ubuntu-latest diff --git a/go.mod b/go.mod index 98e467a667b..aa9fa2ac45e 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,6 @@ require ( github.com/openconfig/ygot v0.28.3 github.com/p4lang/p4runtime v1.4.0-rc.5 github.com/protocolbuffers/txtpbfmt v0.0.0-20220608084003-fc78c767cd6a - github.com/yuin/goldmark v1.4.13 golang.org/x/crypto v0.10.0 golang.org/x/text v0.10.0 google.golang.org/api v0.122.0 diff --git a/go.sum b/go.sum index 58831fdf98a..236a2208fb5 100644 --- a/go.sum +++ b/go.sum @@ -1027,7 +1027,6 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= diff --git a/tools/ci-trigger/pr.go b/tools/ci-trigger/pr.go index ce1e1370e08..6b8e707a673 100644 --- a/tools/ci-trigger/pr.go +++ b/tools/ci-trigger/pr.go @@ -82,6 +82,9 @@ func (p *pullRequest) createBuild(ctx context.Context, buildClient *cloudbuild.S for _, d := range devices { for i, virtualDevice := range p.Virtual { if virtualDevice.Type == d { + if len(virtualDevice.Tests) == 0 { + continue + } skip := false for _, v := range virtualDevice.Tests { if v.Status != "pending authorization" { diff --git a/tools/ci-trigger/trigger.go b/tools/ci-trigger/trigger.go index 4912b3ed82a..6b9b7db4d15 100644 --- a/tools/ci-trigger/trigger.go +++ b/tools/ci-trigger/trigger.go @@ -136,6 +136,16 @@ func (t *trigger) processPullRequest(ctx context.Context, e *github.PullRequestE return fmt.Errorf("identify modified tests: %w", err) } + auth, err := t.authorizedUser(ctx, e.GetPullRequest().GetUser().GetLogin()) + if err != nil { + return fmt.Errorf("validating user auth: %w", err) + } + if auth { + if err := pr.createBuild(ctx, t.buildClient, t.storClient, virtualDeviceTypes); err != nil { + return fmt.Errorf("create build: %w", err) + } + } + if err := pr.updateBadges(ctx, t.storClient); err != nil { return fmt.Errorf("update GCS badges: %w", err) } diff --git a/tools/wikidoc/wikidoc.go b/tools/wikidoc/wikidoc.go index d8a98f9f26b..7a50c9f7859 100644 --- a/tools/wikidoc/wikidoc.go +++ b/tools/wikidoc/wikidoc.go @@ -17,25 +17,31 @@ package main import ( - "errors" "io/fs" "os" "path/filepath" + "regexp" + "sort" + "strconv" "strings" "text/template" "flag" log "github.com/golang/glog" - "github.com/yuin/goldmark" - "github.com/yuin/goldmark/text" + "google.golang.org/protobuf/encoding/prototext" + + mpb "github.com/openconfig/featureprofiles/proto/metadata_go_proto" ) // testDoc stores test plan metadata. type testDoc struct { - Name string + // Test directory name, e.g. "example_test" + Name string + // Full test title, e.g. "XX-01: Example Test" Title string - Path string + // Path is the file location of the test documentation, typically named README.md. + Path string } // path relative from outputRoot containing all test plan documents. @@ -60,6 +66,7 @@ func main() { if err != nil { log.Fatal(err) } + sortTests(docs) err = writeTestDocs(docs, *outputRoot) if err != nil { @@ -118,7 +125,7 @@ func writeTestDocs(docs []testDoc, rootPath string) error { // fetchTestDocs gathers all valid test plan documents in rootPath func fetchTestDocs(rootPath string) ([]testDoc, error) { - docs := []testDoc{} + docMap := make(map[string]testDoc) err := filepath.WalkDir(rootPath, func(path string, e fs.DirEntry, err error) error { @@ -126,56 +133,60 @@ func fetchTestDocs(rootPath string) ([]testDoc, error) { return err } - if !validDoc(path) { + if filepath.Base(path) != "metadata.textproto" { return nil } - title, err := docTitle(path) + bytes, err := os.ReadFile(path) if err != nil { return err } - - doc := testDoc{ + md := new(mpb.Metadata) + if err := prototext.Unmarshal(bytes, md); err != nil { + return err + } + docMap[md.GetUuid()] = testDoc{ Name: filepath.Base(filepath.Dir(path)), - Title: title, - Path: path, + Path: filepath.Dir(path) + "/README.md", + Title: md.GetPlanId() + ": " + md.GetDescription(), } - docs = append(docs, doc) return nil }) + docs := make([]testDoc, 0, len(docMap)) + for _, v := range docMap { + docs = append(docs, v) + } return docs, err } -// validDoc checks if a given file path is eligible to contain a testplan doc. -func validDoc(path string) bool { - if filepath.Base(path) != "README.md" { - return false - } +func sortTests(docs []testDoc) { + re := regexp.MustCompile("[0-9]+|[a-z]+") + sort.Slice(docs, func(i, j int) bool { + in := re.FindAllString(strings.ToLower(docs[i].Title), -1) + jn := re.FindAllString(strings.ToLower(docs[j].Title), -1) - validPaths := []string{"/ate_tests/", "/tests/"} - for _, validPath := range validPaths { - if strings.Contains(path, validPath) { - return true + minLen := len(in) + if len(in) > len(jn) { + minLen = len(jn) } - } - return false -} + for k := 0; k < minLen; k++ { + if in[k] == jn[k] { + continue + } -// docTitle fetches the first header string from a markdown file -func docTitle(path string) (string, error) { - b, err := os.ReadFile(path) - if err != nil { - return "", err - } + iv, errI := strconv.Atoi(in[k]) + jv, errJ := strconv.Atoi(jn[k]) - markdown := goldmark.New() - doc := markdown.Parser().Parse(text.NewReader(b)) - if doc.ChildCount() == 0 { - return "", errors.New("no children") - } + if errI == nil && errJ == nil { + return iv < jv + } + + return strings.Compare(in[k], jn[k]) < 0 + } - return string(doc.FirstChild().Text(b)), nil + return len(in) < len(jn) + }) }