DNM: Combine ceph-pull-requests and API jobs

ceph-pr-api/config/definitions/ceph-pr-api.yml

@@ -1,4 +1,5 @@
 - job:
+    disabled: true
     name: ceph-api
     project-type: freestyle
     defaults: global

ceph-pr-combined/build/api/start

@@ -0,0 +1,7 @@
+#!/bin/bash
+github_status_setup
+
+# the following two methods exist in scripts/build_utils.sh
+pkgs=( "github-status>0.0.3" )
+install_python_packages "pkgs[@]"
+GITHUB_STATUS_STATE="pending" $VENV/github-status create

ceph-pr-combined/build/api/success

@@ -0,0 +1,7 @@
+#!/bin/bash
+github_status_setup
+
+# the following two methods exist in scripts/build_utils.sh
+pkgs=( "github-status>0.0.3" )
+install_python_packages "pkgs[@]"
+GITHUB_STATUS_STATE="success" $VENV/github-status create

ceph-pr-combined/build/failure

@@ -0,0 +1,14 @@
+#!/bin/bash
+github_status_setup
+
+# the following two methods exist in scripts/build_utils.sh
+pkgs=( "github-status>0.0.3" )
+install_python_packages "pkgs[@]"
+
+# The end of the build/make-check/build writes this sentinel file if `make check` passes.
+if [[ -f "$WORKSPACE/.makecheckpassed" ]]; then
+  GITHUB_STATUS_CONTEXT="ceph PR - API tests"
+else
+  GITHUB_STATUS_CONTEXT="ceph PR - make check"
+fi
+GITHUB_STATUS_STATE="failure" $VENV/github-status create

ceph-pr-combined/build/make-check/build

@@ -0,0 +1,18 @@
+#!/bin/bash
+github_status_setup
+
+# the following two methods exist in scripts/build_utils.sh
+pkgs=( "github-status>0.0.3" )
+install_python_packages "pkgs[@]"
+GITHUB_STATUS_STATE="pending" $VENV/github-status create
+
+export NPROC=$(nproc)
+export WITH_SEASTAR=true
+timeout 3h ./run-make-check.sh
+sleep 5
+ps -ef | grep -v jnlp | grep ceph || true
+
+# This is so ugly but in order for github-status to know if `make check` or API tests failed in the postbuildscript, the ceph-pr-combined/build/failure script can check for this sentinel file.
+touch $WORKSPACE/.makecheckpassed
+
+GITHUB_STATUS_STATE="success" $VENV/github-status create

ceph-pr-combined/build/setup

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -ex
+
+# set up variables needed for
+# githubstatus to report back to the github PR
+# if this project was started manually
+github_status_setup
+
+# the following two methods exist in scripts/build_utils.sh
+pkgs=( "github-status>0.0.3" )
+install_python_packages "pkgs[@]"
+
+# The github-status project checks if a job was triggered by github-pull-request-builder and will not updated PR statuses if it was.
+# This is because github-pull-request-builder already updates PR statuses.  However, we can't set stages in the status message.
+# So I'm unsetting the ghprb vars to trick github-status into running.
+# https://github.com/alfredodeza/github-status/blob/master/github_status/util.py
+for var in ghprbActualCommit ghprbTriggerAuthor ghprbTargetBranch ghprbTriggerAuthorLogin ghprbCredentialsId ghprbGhRepository; do
+  unset $var
+done

ceph-pr-combined/config/definitions/ceph-pr-combined.yml

@@ -0,0 +1,135 @@
+- job:
+    name: ceph-pr-combined
+    description: 'make check and ceph API tests combined'
+    project-type: freestyle
+    defaults: global
+    concurrent: true
+    node: huge && bionic && x86_64 && !xenial && !trusty
+    display-name: 'ceph PRs: make check and API tests'
+    quiet-period: 5
+    block-downstream: false
+    block-upstream: false
+    retry-count: 3
+    properties:
+      - build-discarder:
+          days-to-keep: 15
+          num-to-keep: 300
+          artifact-days-to-keep: -1
+          artifact-num-to-keep: -1
+      - github:
+          url: https://github.com/ceph/ceph/
+      - rebuild:
+          auto-rebuild: true
+      - inject:
+          properties-content: |
+            TERM=xterm
+
+    parameters:
+      - string:
+          name: ghprbPullId
+          description: "the GitHub pull id, like '72' in 'ceph/pull/72'"
+
+    triggers:
+      - github-pull-request:
+          cancel-builds-on-update: true
+          allow-whitelist-orgs-as-admins: true
+          org-list:
+            - ceph
+          white-list-target-branches:
+            - master
+            - octopus
+            - nautilus
+          trigger-phrase: '^jenkins test make check|^jenkins test pr.*|^jenkins retest.*|^jenkins test api'
+          skip-build-phrase: '^jenkins do not test.*'
+          only-trigger-phrase: false
+          github-hooks: true
+          permit-all: true
+          auto-close-on-fail: false
+          # The option below requires jenkins-job-builder>=3.5.0
+          no-commit-status: true
+          # I had hoped to get these working by running 'make check' and the API tests in the same job
+          # and somehow changing {stage} throughout the job but that doesn't appear to be possible.
+          # So I'm using github.com/alfredodeza/github-status
+#          status-context: "ceph PR: {stage}"
+#          started-status: "ceph PR: {stage} running"
+#          success-status: "ceph PR: {stage} succeeded"
+#          failure-status: "ceph PR: {stage} failed"
+
+    scm:
+      - git:
+          url: https://github.com/ceph/ceph.git
+          branches:
+            - ${sha1}
+          refspec: +refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*
+          browser: auto
+          timeout: 20
+          skip-tag: true
+          shallow-clone: true
+          wipe-workspace: true
+
+    ## TODO: (maybe) I'm sure there's a cleaner/prettier way to do this.  Maybe doing an actual Jenkins pipeline.
+    builders:
+      - inject:
+          properties-content: |
+            GITHUB_REPOSITORY="ceph/ceph"
+            GITHUB_STATUS_CONTEXT="ceph PR - make check"
+            GITHUB_STATUS_STARTED="running"
+            GITHUB_STATUS_SUCCESS="OK"
+            GITHUB_STATUS_FAILURE="failed"
+            GITHUB_STATUS_ERROR="completed with errors"
+            GITHUB_SHA=${ghprbActualCommit}
+      - shell:
+          !include-raw:
+            - ../../../scripts/build_utils.sh
+            - ../../build/setup
+            - ../../build/make-check/build
+      - inject:
+          properties-content: |
+            GITHUB_REPOSITORY="ceph/ceph"
+            GITHUB_STATUS_CONTEXT="ceph PR - API tests"
+            GITHUB_STATUS_STARTED="running"
+            GITHUB_STATUS_SUCCESS="OK"
+            GITHUB_STATUS_FAILURE="failed"
+            GITHUB_STATUS_ERROR="completed with errors"
+            GITHUB_SHA=${ghprbActualCommit}
+      - shell:
+          !include-raw:
+            - ../../../scripts/build_utils.sh
+            - ../../build/setup
+            - ../../build/api/start
+            - ../../../scripts/dashboard/install-backend-api-test-deps.sh
+      - shell: "cd src/pybind/mgr/dashboard; timeout 7200 ./run-backend-api-tests.sh"
+      - shell:
+          !include-raw:
+            - ../../../scripts/build_utils.sh
+            - ../../build/setup
+            - ../../build/api/success
+
+    publishers:
+      - postbuildscript:
+          builders:
+            - role: SLAVE
+              build-on:
+                  - FAILURE
+                  - ABORTED
+              build-steps:
+                - inject:
+                    properties-content: |
+                      GITHUB_REPOSITORY="ceph/ceph"
+                      GITHUB_STATUS_STARTED="running"
+                      GITHUB_STATUS_SUCCESS="OK"
+                      GITHUB_STATUS_FAILURE="failed"
+                      GITHUB_STATUS_ERROR="completed with errors"
+                      GITHUB_SHA=${ghprbActualCommit}
+                - shell:
+                    !include-raw:
+                      - ../../../scripts/build_utils.sh
+                      - ../../build/setup
+                      - ../../build/failure
+
+    wrappers:
+      - ansicolor
+      - credentials-binding:
+          - text:
+              credential-id: 8dff73ff-506c-4397-969d-023f15aedea9
+              variable: GITHUB_OAUTH_TOKEN

ceph-pull-requests/config/definitions/ceph-pull-requests.yml

@@ -1,4 +1,5 @@
 - job:
+    disabled: true
     name: ceph-pull-requests
     project-type: freestyle
     defaults: global