[ci] Collect subsystem-benchmarks results and add graphs for them (pa…

…ritytech#3853)

PR adds CI jobs that collect subsystem-benchmarks results and publishes
them to gh-pages.

cc paritytech/ci_cd#934
cc @AndreiEres

.github/workflows/subsystem-benchmarks.yml

@@ -0,0 +1,42 @@
+# The actions takes json file as input and runs github-action-benchmark for it.
+
+on:
+  workflow_dispatch:
+    inputs:
+      benchmark-data-dir-path:
+        description: "Path to the benchmark data directory"
+        required: true
+        type: string
+      output-file-path:
+        description: "Path to the benchmark data file"
+        required: true
+        type: string
+
+jobs:
+  subsystem-benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Sources
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+          ref: "gh-pages"
+
+      - name: Copy bench results
+        id: step_one
+        run: |
+          cp bench/gitlab/${{ github.event.inputs.output-file-path }} ${{ github.event.inputs.output-file-path }}
+
+      - name: Switch branch
+        id: step_two
+        run: |
+          git checkout master
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: "customSmallerIsBetter"
+          output-file-path: ${{ github.event.inputs.output-file-path }}
+          benchmark-data-dir-path: "bench/${{ github.event.inputs.benchmark-data-dir-path }}"
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true

.gitlab-ci.yml

@@ -147,6 +147,13 @@ default:
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
     - if: $CI_COMMIT_REF_NAME =~ /^gh-readonly-queue.*$/ # merge queues
 
+.publish-gh-pages-refs:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME == "master"
+
 # handle the specific case where benches could store incorrect bench data because of the downstream staging runs
 # exclude cargo-check-benches from such runs
 .test-refs-check-benches:

.gitlab/pipeline/publish.yml

@@ -3,16 +3,13 @@
 
 publish-rustdoc:
   stage: publish
-  extends: .kubernetes-env
+  extends:
+    - .kubernetes-env
+    - .publish-gh-pages-refs
   variables:
     CI_IMAGE: node:18
     GIT_DEPTH: 100
     RUSTDOCS_DEPLOY_REFS: "master"
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "pipeline"
-      when: never
-    - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME == "master"
   needs:
     - job: build-rustdoc
       artifacts: true
@@ -60,9 +57,76 @@ publish-rustdoc:
     - git commit -m "___Updated docs for ${CI_COMMIT_REF_NAME}___" ||
       echo "___Nothing to commit___"
     - git push origin gh-pages --force
+    # artificial sleep to publish gh-pages
+    - sleep 300
   after_script:
     - rm -rf .git/ ./*
 
+publish-subsystem-benchmarks:
+  stage: publish
+  variables:
+    CI_IMAGE: "paritytech/tools:latest"
+  extends:
+    - .kubernetes-env
+    - .publish-gh-pages-refs
+  needs:
+    - job: subsystem-regression-tests
+      artifacts: true
+    - job: publish-rustdoc
+      artifacts: false
+  script:
+    # setup ssh
+    - eval $(ssh-agent)
+    - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY}
+    - mkdir ~/.ssh && touch ~/.ssh/known_hosts
+    - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
+    # Set git config
+    - rm -rf .git/config
+    - git config user.email "[email protected]"
+    - git config user.name "${GITHUB_USER}"
+    - git config remote.origin.url "[email protected]:/paritytech/${CI_PROJECT_NAME}.git"
+    - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
+    - git fetch origin gh-pages
+    # Push result to github
+    - git checkout gh-pages
+    - mkdir -p bench/gitlab/ || echo "Directory exists"
+    - rm -rf bench/gitlab/*.json || echo "No json files"
+    - cp -r charts/*.json bench/gitlab/
+    - git add bench/gitlab/
+    - git commit -m "Add json files with benchmark results for ${CI_COMMIT_REF_NAME}"
+    - git push origin gh-pages
+    # artificial sleep to publish gh-pages
+    - sleep 300
+  allow_failure: true
+  after_script:
+    - rm -rf .git/ ./*
+
+trigger_workflow:
+  stage: deploy
+  extends:
+    - .kubernetes-env
+    - .publish-gh-pages-refs
+  needs:
+    - job: publish-subsystem-benchmarks
+      artifacts: false
+    - job: subsystem-regression-tests
+      artifacts: true
+  script:
+    - echo "Triggering workflow"
+    - |
+      for benchmark in $(ls charts/*.json); do
+        export bencmark_name=$(basename $benchmark)
+        echo "Benchmark: $bencmark_name"
+        export benchmark_dir=$(echo $bencmark_name | sed 's/\.json//')
+        curl -q -X POST \
+              -H "Accept: application/vnd.github.v3+json" \
+              -H "Authorization: token $GITHUB_TOKEN" \
+               https://api.github.com/repos/paritytech-stg/${CI_PROJECT_NAME}/actions/workflows/subsystem-benchmarks.yml/dispatches \
+                -d '{"ref":"refs/heads/master","inputs":{"benchmark-data-dir-path":"'$benchmark_dir'","output-file-path":"'$bencmark_name'"}}'
+        sleep 300
+      done
+  allow_failure: true
+
 # note: images are used not only in zombienet but also in rococo, wococo and versi
 .build-push-image:
   image: $BUILDAH_IMAGE

.gitlab/pipeline/test.yml

@@ -497,6 +497,12 @@ test-syscalls:
 
 subsystem-regression-tests:
   stage: test
+  artifacts:
+    name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
+    when: on_success
+    expire_in: 1 days
+    paths:
+      - charts/
   extends:
     - .docker-env
     - .common-refs