From e2493b6553ca7821226b9134ba5b5d18dbc2f474 Mon Sep 17 00:00:00 2001 From: Michal Karzynski Date: Mon, 8 Apr 2024 11:28:47 +0200 Subject: [PATCH 1/2] Transition to GitHub actions Signed-off-by: Michal Karzynski --- .github/workflows/benchmark-backends.yml | 159 +++++++++++++++++++++ .github/workflows/update-website.yml | 40 ++++++ ADD-BACKEND.md | 45 +++--- azure-pipelines-website.yml | 45 ------ azure-pipelines.yml | 170 ----------------------- examples/job.yml | 39 +++--- requirements_web.txt | 4 +- setup/git-setup.sh | 9 +- 8 files changed, 248 insertions(+), 263 deletions(-) create mode 100644 .github/workflows/benchmark-backends.yml create mode 100644 .github/workflows/update-website.yml delete mode 100644 azure-pipelines-website.yml delete mode 100644 azure-pipelines.yml diff --git a/.github/workflows/benchmark-backends.yml b/.github/workflows/benchmark-backends.yml new file mode 100644 index 0000000000..16df11f61a --- /dev/null +++ b/.github/workflows/benchmark-backends.yml @@ -0,0 +1,159 @@ +name: Run benchmarks + +on: + schedule: + - cron: '0 0 * * *' + +jobs: + onnxruntime_stable: + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Build docker image + run: docker build -t scoreboard/onnx -f runtimes/onnx-runtime/stable/Dockerfile . + + - name: Set up SSH + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: | + ${{ secrets.DEPLOY_KEY }} + + - name: Git setup + run: . setup/git-setup.sh + + - name: Run docker container + run: docker run --name onnx-runtime --env-file setup/env.list -v ${{ github.workspace }}/results/onnx-runtime/stable:/root/results scoreboard/onnx || true + + - name: Deploy results + run: . setup/git-deploy-results.sh + + + onnxruntime_dev: + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Build docker image + run: docker build -t scoreboard/ort-dev -f runtimes/onnx-runtime/development/Dockerfile . + + - name: Set up SSH + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.DEPLOY_KEY }} + + - name: Git setup + run: . setup/git-setup.sh + + - name: Run docker container + run: docker run --name onnx-runtime --env-file setup/env.list -v ${{ github.workspace }}/results/onnx-runtime/development:/root/results scoreboard/ort-dev || true + + - name: Deploy results + run: . setup/git-deploy-results.sh + + + onnx_tf_stable: + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Build docker image + run: docker build -t scoreboard/onnxtf -f runtimes/onnx-tf/stable/Dockerfile . + + - name: Set up SSH + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.DEPLOY_KEY }} + + - name: Git setup + run: . setup/git-setup.sh + + - name: Run docker container + run: docker run --name onnxtf --env-file setup/env.list -v ${{ github.workspace }}/results/onnx-tf/stable:/root/results scoreboard/onnxtf || true + + - name: Deploy results + run: . setup/git-deploy-results.sh + + + onnx_tf_dev: + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Build docker image + run: docker build -t scoreboard/onnxtf-dev -f runtimes/onnx-tf/development/Dockerfile . + + - name: Set up SSH + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.DEPLOY_KEY }} + + - name: Git setup + run: . setup/git-setup.sh + + - name: Run docker container + run: docker run --name onnxtf --env-file setup/env.list -v ${{ github.workspace }}/results/onnx-tf/development:/root/results scoreboard/onnxtf-dev || true + + - name: Deploy results + run: . setup/git-deploy-results.sh + + + jaxonnxruntime_stable: + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Build docker image + run: docker build -t scoreboard/jaxonnxruntime -f runtimes/jaxonnxruntime/stable/Dockerfile . + + - name: Set up SSH + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.DEPLOY_KEY }} + + - name: Git setup + run: . setup/git-setup.sh + + - name: Run docker container + run: docker run --name jaxonnxruntime --env-file setup/env.list -v ${{ github.workspace }}/results/jaxonnxruntime/stable:/root/results scoreboard/jaxonnxruntime || true + + - name: Deploy results + run: . setup/git-deploy-results.sh + + jaxonnxruntime_dev: + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: true + fetch-depth: 0 + clean: true + + - name: Build docker image + run: docker build -t scoreboard/jaxonnxruntime-dev -f runtimes/jaxonnxruntime/development/Dockerfile . + + - name: Set up SSH + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.DEPLOY_KEY }} + + - name: Git setup + run: . setup/git-setup.sh + + - name: Run docker container + run: docker run --name jaxonnxruntime --env-file setup/env.list -v ${{ github.workspace }}/results/jaxonnxruntime/development:/root/results scoreboard/jaxonnxruntime-dev || true + + - name: Deploy results + run: . setup/git-deploy-results.sh diff --git a/.github/workflows/update-website.yml b/.github/workflows/update-website.yml new file mode 100644 index 0000000000..1831508a48 --- /dev/null +++ b/.github/workflows/update-website.yml @@ -0,0 +1,40 @@ +name: Website Update + +on: + schedule: + - cron: '0 6 * * *' + +jobs: + generate_website: + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + persist-credentials: false + clean: true + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install --upgrade -r requirements_web.txt + + - name: Set up SSH + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: | + ${{ secrets.DEPLOY_KEY }} + + - name: Git setup + run: . setup/git-setup.sh + + - name: Generate website + run: python3 website-generator/generator.py --config ./setup/config.json + + - name: Deploy website + run: . setup/git-deploy-website.sh diff --git a/ADD-BACKEND.md b/ADD-BACKEND.md index 74d09af7e2..2ff86542a0 100644 --- a/ADD-BACKEND.md +++ b/ADD-BACKEND.md @@ -49,37 +49,40 @@ For `development` version the `core_packages` list is optional: } ``` -### 3. Add new job to Azure pipelines +### 3. Add new job to GitHub Actions workflow -* Edit [azure-pipelines.yml](azure-pipelines.yml) file. +* Edit [benchmark-backends.yml](.github/workflows/benchmark-backends.yml) file. * Copy and paste this [job template](examples/job.yml) to the end of file. * Fill `new_backend` with the new backend unique name. +* Change the `docker build` and `docker run` commands to match your backend. ```yml -- job: new_backend - timeoutInMinutes: 90 - steps: - - checkout: self - persistCredentials: true - clean: true + jobs: + ... # other jobs - - task: InstallSSHKey@0 - inputs: - knownHostsEntry: ~/.ssh/known_hosts - sshPublicKey: $(public_deploy_key) - sshKeySecureFile: deploy_key + new_backend: + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - name: Checkout code + uses: actions/checkout@v2 - - script: docker build -t scoreboard/new_backend -f runtimes/new_backend/stable/Dockerfile . - displayName: 'Build docker image' + - name: Build docker image + run: docker build -t scoreboard/new_backend -f runtimes/new_backend/stable/Dockerfile . - - script: . setup/git-setup.sh - displayName: 'Git setup' + - name: Set up SSH + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: | + ${{ secrets.DEPLOY_KEY }} - - script: docker run --name new_backend --env-file setup/env.list -v `pwd`/results/new_backend/stable:/root/results scoreboard/new_backend || true - displayName: 'Run docker container' + - name: Git setup + run: . setup/git-setup.sh - - script: . setup/git-deploy-results.sh - displayName: 'Deploy results' + - name: Run docker container + run: docker run --name new_backend --env-file setup/env.list -v `pwd`/results/new_backend/stable:/root/results scoreboard/new_backend || true + - name: Deploy results + run: . setup/git-deploy-results.sh ``` diff --git a/azure-pipelines-website.yml b/azure-pipelines-website.yml deleted file mode 100644 index 44e8211b68..0000000000 --- a/azure-pipelines-website.yml +++ /dev/null @@ -1,45 +0,0 @@ -schedules: -- cron: "0 6 * * *" - displayName: Website update - branches: - include: - - master - -# Disable CI trigger (don't run on commits) -trigger: none -# and don't run on Pull Requests -pr: none - -pool: - vmImage: 'ubuntu-20.04' - -jobs: - - job: generate_website - steps: - - checkout: self - persistCredentials: true - clean: true - - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.6' - - - task: InstallSSHKey@0 - inputs: - knownHostsEntry: ~/.ssh/known_hosts - sshPublicKey: $(public_deploy_key) - sshKeySecureFile: deploy_key - - - script: | - pip install --upgrade pip - pip install --upgrade -r requirements_web.txt - displayName: 'Install requirements' - - - script: . setup/git-setup.sh - displayName: 'Git setup' - - - script: python3 website-generator/generator.py --config ./setup/config.json - displayName: 'Generate website' - - - script: . setup/git-deploy-website.sh - displayName: 'Deploy website' diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index 796a80c6c8..0000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,170 +0,0 @@ -schedules: -- cron: "0 0 * * *" - displayName: Midnight update - branches: - include: - - master - -# Disable CI trigger (don't run on commits) -trigger: none -# and don't run on Pull Requests -pr: none - -pool: - vmImage: 'ubuntu-latest' - -jobs: - - job: onnxruntime_stable - timeoutInMinutes: 90 - steps: - - checkout: self - persistCredentials: true - clean: true - - task: InstallSSHKey@0 - inputs: - knownHostsEntry: ~/.ssh/known_hosts - sshPublicKey: $(public_deploy_key) - sshKeySecureFile: deploy_key - - - script: docker build -t scoreboard/ort -f runtimes/onnx-runtime/stable/Dockerfile . - displayName: 'Build docker image' - - - script: . setup/git-setup.sh - displayName: 'Git setup' - - - script: docker run --name onnx-runtime --env-file setup/env.list -v `pwd`/results/onnx-runtime/stable:/root/results scoreboard/ort || true - displayName: 'Run docker container' - - - script: . setup/git-deploy-results.sh - displayName: 'Deploy results' - - - - job: onnx_tf_stable - timeoutInMinutes: 90 - steps: - - checkout: self - persistCredentials: true - clean: true - - - task: InstallSSHKey@0 - inputs: - knownHostsEntry: ~/.ssh/known_hosts - sshPublicKey: $(public_deploy_key) - sshKeySecureFile: deploy_key - - - - script: docker build -t scoreboard/onnxtf -f runtimes/onnx-tf/stable/Dockerfile . - displayName: 'Build docker image' - - - script: . setup/git-setup.sh - displayName: 'Git setup' - - - script: docker run --name onnxtf --env-file setup/env.list -v `pwd`/results/onnx-tf/stable:/root/results scoreboard/onnxtf || true - displayName: 'Run docker container' - - - script: . setup/git-deploy-results.sh - displayName: 'Deploy results' - - - job: jaxonnxruntime_stable - timeoutInMinutes: 90 - steps: - - checkout: self - persistCredentials: true - clean: true - - - task: InstallSSHKey@0 - inputs: - knownHostsEntry: ~/.ssh/known_hosts - sshPublicKey: $(public_deploy_key) - sshKeySecureFile: deploy_key - - - - script: docker build -t scoreboard/jaxonnxruntime -f runtimes/jaxonnxruntime/stable/Dockerfile . - displayName: 'Build docker image' - - - script: . setup/git-setup.sh - displayName: 'Git setup' - - - script: docker run --name jaxonnxruntime --env-file setup/env.list -v `pwd`/results/jaxonnxruntime/stable:/root/results scoreboard/jaxonnxruntime || true - displayName: 'Run docker container' - - - script: . setup/git-deploy-results.sh - displayName: 'Deploy results' - - - job: onnxruntime_dev - timeoutInMinutes: 90 - steps: - - checkout: self - persistCredentials: true - clean: true - - - task: InstallSSHKey@0 - inputs: - knownHostsEntry: ~/.ssh/known_hosts - sshPublicKey: $(public_deploy_key) - sshKeySecureFile: deploy_key - - - script: docker build -t scoreboard/ort-dev -f runtimes/onnx-runtime/development/Dockerfile . - displayName: 'Build docker image' - - - script: . setup/git-setup.sh - displayName: 'Git setup' - - - script: docker run --name onnx-runtime --env-file setup/env.list -v `pwd`/results/onnx-runtime/development:/root/results scoreboard/ort-dev || true - displayName: 'Run docker container' - - - script: . setup/git-deploy-results.sh - displayName: 'Deploy results' - - - - job: onnx_tf_dev - timeoutInMinutes: 90 - steps: - - checkout: self - persistCredentials: true - clean: true - - - task: InstallSSHKey@0 - inputs: - knownHostsEntry: ~/.ssh/known_hosts - sshPublicKey: $(public_deploy_key) - sshKeySecureFile: deploy_key - - - - script: docker build -t scoreboard/onnxtf-dev -f runtimes/onnx-tf/development/Dockerfile . - displayName: 'Build docker image' - - - script: . setup/git-setup.sh - displayName: 'Git setup' - - - script: docker run --name onnxtf --env-file setup/env.list -v `pwd`/results/onnx-tf/development:/root/results scoreboard/onnxtf-dev || true - displayName: 'Run docker container' - - - script: . setup/git-deploy-results.sh - displayName: 'Deploy results' - - - job: jaxonnxruntime_dev - timeoutInMinutes: 90 - steps: - - checkout: self - persistCredentials: true - clean: true - - - task: InstallSSHKey@0 - inputs: - knownHostsEntry: ~/.ssh/known_hosts - sshPublicKey: $(public_deploy_key) - sshKeySecureFile: deploy_key - - - - script: docker build -t scoreboard/jaxonnxruntime-dev -f runtimes/jaxonnxruntime/development/Dockerfile . - displayName: 'Build docker image' - - - script: . setup/git-setup.sh - displayName: 'Git setup' - - - script: docker run --name jaxonnxruntime --env-file setup/env.list -v `pwd`/results/jaxonnxruntime/development:/root/results scoreboard/jaxonnxruntime-dev || true - displayName: 'Run docker container' - - - script: . setup/git-deploy-results.sh - displayName: 'Deploy results' diff --git a/examples/job.yml b/examples/job.yml index ac9f651e46..da7aaddb16 100644 --- a/examples/job.yml +++ b/examples/job.yml @@ -1,26 +1,25 @@ -- job: new_backend - timeoutInMinutes: 90 +# add to `jobs` section: + new_backend: + runs-on: ubuntu-latest + timeout-minutes: 90 steps: - - checkout: self - persistCredentials: true - clean: true + - name: Checkout code + uses: actions/checkout@v2 - # This task enables ssh authentication. - # It keeps git permissions even after long build time. - - task: InstallSSHKey@0 - inputs: - knownHostsEntry: ~/.ssh/known_hosts - sshPublicKey: $(public_deploy_key) - sshKeySecureFile: deploy_key + - name: Build docker image + run: docker build -t scoreboard/new_backend -f runtimes/new_backend/stable/Dockerfile . - - script: docker build -t scoreboard/new_backend -f runtimes/new_backend/stable/Dockerfile . - displayName: 'Build docker image' + - name: Set up SSH + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: | + ${{ secrets.DEPLOY_KEY }} - - script: . setup/git-setup.sh - displayName: 'Git setup' + - name: Git setup + run: . setup/git-setup.sh - - script: docker run --name new_backend --env-file setup/env.list -v `pwd`/results/new_backend/stable:/root/results scoreboard/new_backend || true - displayName: 'Run docker container' + - name: Run docker container + run: docker run --name new_backend --env-file setup/env.list -v `pwd`/results/new_backend/stable:/root/results scoreboard/new_backend || true - - script: . setup/git-deploy-results.sh - displayName: 'Deploy results' + - name: Deploy results + run: . setup/git-deploy-results.sh diff --git a/requirements_web.txt b/requirements_web.txt index 8e7d47cff1..19837cb9a9 100644 --- a/requirements_web.txt +++ b/requirements_web.txt @@ -1,2 +1,2 @@ -Jinja2==2.11.3 -MarkupSafe==1.1.1 +Jinja2==3.1.3 +MarkupSafe==2.1.5 diff --git a/setup/git-setup.sh b/setup/git-setup.sh index a7b9ac0a90..6e9d2cf37f 100644 --- a/setup/git-setup.sh +++ b/setup/git-setup.sh @@ -6,11 +6,10 @@ set -e # Exit on error set -x # Command echo on -ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts -git remote set-url origin git@github.com:onnx/backend-scoreboard.git +git remote set-url origin git@github.com:postrational/backend-scoreboard.git -git config --global user.email "onnx_scoreboard_bot@azure" -git config --global user.name "ONNX Scoreboard Bot @ Azure Pipelines" +git config --global user.name "GitHub Action" +git config --global user.email "action@github.com" -git pull origin master +git fetch origin master git checkout master From be8ed01162bedb3f2dd7b6c402bd03ca819e1fdb Mon Sep 17 00:00:00 2001 From: Michal Karzynski Date: Mon, 8 Apr 2024 13:24:27 +0200 Subject: [PATCH 2/2] Bugfix Signed-off-by: Michal Karzynski --- setup/git-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/git-setup.sh b/setup/git-setup.sh index 6e9d2cf37f..bb0bd6ca43 100644 --- a/setup/git-setup.sh +++ b/setup/git-setup.sh @@ -6,7 +6,7 @@ set -e # Exit on error set -x # Command echo on -git remote set-url origin git@github.com:postrational/backend-scoreboard.git +git remote set-url origin git@github.com:onnx/backend-scoreboard.git git config --global user.name "GitHub Action" git config --global user.email "action@github.com"