OPCT-289: refact openshift-tests plugin to Golang

Refact the the `openshift-tests-plugin` (conformance step) from bash version to Golang.

This PR split core components for the main conformance plugin, implemented in bash, to Golang targeting to:
- decrease the maitenance burden in the project: the amount of tasks executed in this plugin have increased, and more complex while new requirements was added to the project, making the maintenance and stability very complex to a limited scripting language
- review the architecture to isolate components: the component/step responsible to schedule the conformance workflow have been isolated from artifacts collector, decreasing the requirements in the base image
- review the architecture to decrease the "interference" of OPCT environment to the conformance runtime: the `openshift-tests` utility implements the conformance tests shipped in OCP payload, the new tests are handled by the utility, following it's dependencies. It is impossible to track OPCT plugin workflow to resolve all the dependencies, so this change will introduce a workflow of making the plugin orchestrate the conformance suite execution in an isolated sidecard container (`tests`) shipped in OCP payload, monitoring the completion and reporting to aggregator server.
- introduce a new plugin in the pipeline to process failures from conformance suite and re-run it in serial mode, to decrease the possibility of flakes, and get more confidence in results when reviewing the artifacts.

Those changes resulted in an encapsulated conformance test execution by `openshift-tests` utility by keeping the flexibility of a modern language to orchestrate, monitor, report, and process results in the execution, decreasing the post-processing/manual steps

.github/workflows/ci.yaml

@@ -1,38 +1,39 @@
 ---
+# test-build-release is the main workflow which will trigger
+# linters, tests, and builds for the project for all events.
+# It will also trigger the release workflows when a tag is created
+# or when a push is made to the main branch.
 name: test-build-release
-on: 
+on:
   pull_request:
+    branches:
+      - main
+      - release-*
   push:
-    branches: [main]
-    tags: ['v*.*.*']
+    branches:
+      - main
+      - release-*
+    tags:
+      - 'v*.*.*'
 
 jobs:
+  # Pre jobs: Linter
   linters:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Shellcheck - hack
-        uses: azohra/shell-linter@latest
-        with:
-          path: "hack/*.sh"
-
-      - name: Shellcheck - Plugin - openshift-tests
-        uses: azohra/shell-linter@latest
-        with:
-          path: './openshift-tests-provider-cert/plugin/*.sh'
-
-      - name: Shellcheck - Plugin - openshift-tests - hack
-        uses: azohra/shell-linter@latest
-        with:
-          path: './openshift-tests-provider-cert/hack/*.sh'
+    uses: ./.github/workflows/pre_linters.yaml
 
+  # Pre jobs: unit tests and integration tests
+  test-openshift-tests-plugin:
+    uses: ./.github/workflows/pre_test-openshift-tests-plugin.yaml
 
+  # build-container-amd64: triggered when a PR is made
   build-container-amd64:
     runs-on: ubuntu-latest
-    needs: [linters]
+    needs:
+      - linters
+      - test-openshift-tests-plugin
     env:
       VERSION: "v0.0.0-devel-pr.${{ github.event.pull_request.number }}"
+      VERSION_TOOLS: v0.4.0
       PLATFORMS: linux/amd64
       EXPIRATION: 1d
       PUSH: false
@@ -53,7 +54,7 @@ jobs:
           username: ${{ secrets.QUAY_USER }}
           password: ${{ secrets.QUAY_PASS }}
 
-      - name: Build and push - plugin
+      - name: "Build and push: plugin-openshift-tests"
         uses: docker/build-push-action@v6
         with:
           platforms: ${{ env.PLATFORMS }}
@@ -62,12 +63,37 @@ jobs:
             quay.expires-after=${{ env.EXPIRATION }}
           build-args: |
             QUAY_EXPIRATION=${{ env.EXPIRATION }}
-          tags: quay.io/opct/plugin-openshift-tests:${{ env.VERSION }}
-          context: ./openshift-tests-provider-cert
-          file: ./openshift-tests-provider-cert/Containerfile
+            RELEASE_TAG=${{ env.VERSION }}
+          tags: "localhost/plugin-openshift-tests:latest"
+          outputs: |
+            type=image,name=localhost/plugin-openshift-tests:latest,push=false
+          load: true
+          context: ./openshift-tests-plugin
+          file: ./openshift-tests-plugin/Containerfile
 
-      - name: Build and push - must-gather-monitoring
+      - name: show images
+        run: docker images
+
+      # disable while not able to access the images previously built.
+      # Is it possible with cache-from/to?
+      # - name: "Build and push: plugin-artifacts-collector"
+      #   uses: docker/build-push-action@v6
+      #   with:
+      #     platforms: ${{ env.PLATFORMS }}
+      #     push: ${{ env.PUSH }}
+      #     labels: |
+      #       quay.expires-after=${{ env.EXPIRATION }}
+      #     build-args: |
+      #       QUAY_EXPIRATION=${{ env.EXPIRATION }}
+      #       TOOLS_VERSION=${{ env.VERSION_TOOLS }}
+      #     tags: quay.io/opct/plugin-artifacts-collector:${{ env.VERSION }}
+      #     context: ./artifacts-collector
+      #     file: ./artifacts-collector/Containerfile
+
+      - name: "Build and push: must-gather-monitoring"
         uses: docker/build-push-action@v6
+        env:
+          PLUGIN_IMAGE_OVERRIDE: "localhost/plugin-openshift-tests:latest"
         with:
           platforms: ${{ env.PLATFORMS }}
           push: ${{ env.PUSH }}
@@ -79,9 +105,9 @@ jobs:
           context: ./must-gather-monitoring
           file: ./must-gather-monitoring/Containerfile
 
-#
-# Releasing: triggered when push to main
-#
+  #
+  # Releasing latest: triggered when push to main
+  #
   release-latest:
     if: github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
@@ -108,7 +134,7 @@ jobs:
           username: ${{ secrets.QUAY_USER }}
           password: ${{ secrets.QUAY_PASS }}
 
-      - name: Build and push - plugin
+      - name: "Build and push: plugin-openshift-tests"
         uses: docker/build-push-action@v6
         with:
           platforms: ${{ env.PLATFORMS }}
@@ -117,11 +143,26 @@ jobs:
             quay.expires-after=${{ env.EXPIRATION }}
           build-args: |
             QUAY_EXPIRATION=${{ env.EXPIRATION }}
+            RELEASE_TAG=${{ env.VERSION }}
           tags: quay.io/opct/plugin-openshift-tests:${{ env.VERSION }}
-          context: ./openshift-tests-provider-cert
-          file: ./openshift-tests-provider-cert/Containerfile
+          context: ./openshift-tests-plugin
+          file: ./openshift-tests-plugin/Containerfile
+
+      - name: "Build and push: plugin-artifacts-collector"
+        uses: docker/build-push-action@v6
+        with:
+          platforms: ${{ env.PLATFORMS }}
+          push: ${{ env.PUSH }}
+          labels: |
+            quay.expires-after=${{ env.EXPIRATION }}
+          build-args: |
+            QUAY_EXPIRATION=${{ env.EXPIRATION }}
+            TOOLS_VERSION=${{ env.VERSION_TOOLS }}
+          tags: quay.io/opct/plugin-artifacts-collector:${{ env.VERSION }}
+          context: ./artifacts-collector
+          file: ./artifacts-collector/Containerfile
 
-      - name: Build and push - must-gather-monitoring
+      - name: "Build and push: must-gather-monitoring"
         uses: docker/build-push-action@v6
         with:
           platforms: ${{ env.PLATFORMS }}
@@ -134,10 +175,9 @@ jobs:
           context: ./must-gather-monitoring
           file: ./must-gather-monitoring/Containerfile
 
-
-#
-# Releasing: triggered when a tag is created
-#
+  #
+  # Releasing tag: triggered when a tag is created
+  #
   release-tag:
     if: startsWith(github.ref, 'refs/tags/')
     runs-on: ubuntu-latest
@@ -169,7 +209,7 @@ jobs:
           username: ${{ secrets.QUAY_USER }}
           password: ${{ secrets.QUAY_PASS }}
 
-      - name: Build and push - plugin
+      - name: "Build and push: plugin-openshift-tests"
         uses: docker/build-push-action@v6
         env:
           VERSION: ${{ steps.vars.outputs.tag }}
@@ -180,11 +220,28 @@ jobs:
             quay.expires-after=${{ env.EXPIRATION }}
           build-args: |
             QUAY_EXPIRATION=${{ env.EXPIRATION }}
+            RELEASE_TAG=${{ env.VERSION }}
           tags: quay.io/opct/plugin-openshift-tests:${{ env.VERSION }}
-          context: ./openshift-tests-provider-cert
-          file: ./openshift-tests-provider-cert/Containerfile
+          context: ./openshift-tests-plugin
+          file: ./openshift-tests-plugin/Containerfile
 
-      - name: Build and push - must-gather-monitoring
+      - name: "Build and push: plugin-artifacts-collector"
+        uses: docker/build-push-action@v6
+        env:
+          VERSION: ${{ steps.vars.outputs.tag }}
+        with:
+          platforms: ${{ env.PLATFORMS }}
+          push: ${{ env.PUSH }}
+          labels: |
+            quay.expires-after=${{ env.EXPIRATION }}
+          build-args: |
+            QUAY_EXPIRATION=${{ env.EXPIRATION }}
+            TOOLS_VERSION=${{ env.VERSION_TOOLS }}
+          tags: quay.io/opct/plugin-artifacts-collector:${{ env.VERSION }}
+          context: ./artifacts-collector
+          file: ./artifacts-collector/Containerfile
+
+      - name: "Build and push: must-gather-monitoring"
         uses: docker/build-push-action@v6
         env:
           VERSION: ${{ steps.vars.outputs.tag }}
@@ -210,9 +267,11 @@ jobs:
       - name: Create Release on Github
         uses: softprops/[email protected]
         env:
-          RELEASE_VERSION: ${{ steps.vars.outputs.tag }}
+          VERSION: ${{ steps.vars.outputs.tag }}
+          REPO: quay.io/opct/plugin-openshift-tests
+          URL: https://quay.io/repository/opct/plugin-openshift-tests?tab=tags
         with:
           body: |
             ## Changelog
-            Image published to [quay.io/ocp-cert/openshift-tests-provider-cert:$RELEASE_VERSION](https://quay.io/repository/ocp-cert/openshift-tests-provider-cert?tab=tags)
+            Image published to [${REPO}:$VERSION](${URL})
             ${{steps.github_release.outputs.changelog}}

.github/workflows/pre_linters.yaml

@@ -0,0 +1,85 @@
+---
+# pre-linter is a chain of jobs that must be called
+# by other workflows to ensure that the code is linted.
+name: linters
+
+on:
+  workflow_call: {}
+
+# golangci-lint-action requires those permissions to annotate issues in the PR.
+permissions:
+  # Required for analysis.
+  contents: read
+  # Used by `only-new-issues``.
+  pull-requests: read
+  # Allow access to checks to annotate code in the PR.
+  checks: write
+
+env:
+  GO_VERSION: 1.22
+  GOLANGCI_LINT_VERSION: v1.59
+
+jobs:
+  shell:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Shellcheck - must-gather-monitoring plugin - entrypoint
+        uses: azohra/shell-linter@latest
+        with:
+          path: './build.sh'
+
+      - name: Shellcheck - openshift-tests-plugin
+        uses: azohra/shell-linter@latest
+        with:
+          path: './openshift-tests-plugin/plugin/*.sh'
+
+      - name: Shellcheck - must-gather-monitoring plugin - entrypoint
+        uses: azohra/shell-linter@latest
+        with:
+          path: './must-gather-monitoring/runner_plugin'
+
+      - name: Shellcheck - must-gather-monitoring plugin - collection scripts
+        uses: azohra/shell-linter@latest
+        with:
+          path: './must-gather-monitoring/collection-scripts/*'
+
+      - name: Shellcheck - hack
+        uses: azohra/shell-linter@latest
+        with:
+          path: "hack/*.sh"
+
+  yaml:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install yamllint
+        run: pip install yamllint
+
+      - name: Lint YAML files
+        run: |
+          echo ">> Running YAML lint on data/templates/plugins/"
+          yamllint .github/workflows/*.yaml
+
+  go_plugin-tests:
+    name: "go / openshift-tests-plugin"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: openshift-tests-plugin/go.mod
+          cache-dependency-path: |
+             openshift-tests-plugin/go.sum
+
+      # https://github.com/golangci/golangci-lint-action
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v6
+        with:
+          version: ${{ env.GOLANGCI_LINT_VERSION }}
+          args: --timeout=10m
+          working-directory: openshift-tests-plugin
+          only-new-issues: true