feat: add conformance test

.chainsaw.yaml

@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/configuration-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Configuration
+metadata:
+  creationTimestamp: null
+  name: configuration
+spec:
+  parallel: 1
+  timeouts:
+    apply: 1m30s
+    assert: 1m30s
+    cleanup: 1m30s
+    delete: 1m30s
+    error: 1m30s
+    exec: 1m30s
+  fullName: true
+  forceTerminationGracePeriod: 5s
+  delayBeforeCleanup: 3s

.github/actions/failure-logs/action.yaml

@@ -0,0 +1,18 @@
+name: Logs
+
+description: Show pods logs
+
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      run: |
+        kubectl get apiservices v1alpha2.wgpolicyk8s.io v1.reports.kyverno.io
+        kubectl -n reports-server get pod
+        kubectl -n reports-server describe pod | grep -i events -A10
+    - shell: bash
+      run: |
+        kubectl -n reports-server logs deploy/reports-server --all-containers -p || true
+    - shell: bash
+      run: |
+        kubectl -n reports-server logs deploy/reports-server --all-containers

.github/kind.yml

@@ -0,0 +1,36 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+kubeadmConfigPatches:
+  - |-
+    kind: ClusterConfiguration
+    controllerManager:
+      extraArgs:
+        bind-address: 0.0.0.0
+    etcd:
+      local:
+        extraArgs:
+          listen-metrics-urls: http://0.0.0.0:2382
+    scheduler:
+      extraArgs:
+        bind-address: 0.0.0.0
+  - |-
+    kind: KubeProxyConfiguration
+    metricsBindAddress: 0.0.0.0
+nodes:
+  - role: control-plane
+    kubeadmConfigPatches:
+      - |-
+        kind: InitConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            node-labels: "ingress-ready=true"
+    extraPortMappings:
+      - containerPort: 80
+        hostPort: 80
+        protocol: TCP
+      - containerPort: 443
+        hostPort: 443
+        protocol: TCP
+  - role: worker
+  - role: worker
+  - role: worker

.github/workflows/conformance-tests.yaml

@@ -0,0 +1,102 @@
+name: E2E Tests
+
+permissions: {}
+
+on:
+  workflow_dispatch: {}
+  pull_request:
+    branches:
+      - "main"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  required:
+    strategy:
+      fail-fast: false
+      matrix:
+        k8s-version:
+          - name: v1.26
+            version: v1.26.6
+          - name: v1.27
+            version: v1.27.3
+          - name: v1.28
+            version: v1.28.0
+          - name: v1.29
+            version: v1.29.0
+        tests:
+          - ^reports$
+    runs-on: ubuntu-latest
+    name: ${{ matrix.k8s-version.name }} - ${{ matrix.tests }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Checkout kyverno/kyverno
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          repository: kyverno/kyverno
+          path: kyverno
+      - name: Setup Go
+        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        with:
+          go-version: ~1.21.1
+      - name: Install helm
+        id: helm
+        uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install Tools
+        run: |
+          set -e
+          curl -LO "https://dl.k8s.io/release/${{ matrix.k8s-version.version }}/bin/linux/amd64/kubectl"
+          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+      - name: Install kind
+        shell: bash
+        run: |
+          set -e
+          # For AMD64 / x86_64
+          [ $(uname -m) = x86_64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
+          # For ARM64
+          [ $(uname -m) = aarch64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-arm64
+          chmod +x ./kind
+          sudo mv ./kind /usr/local/bin/kind
+      - name: Create kind cluster
+        run: |
+          set -e
+          kind create cluster --image kindest/node:${{ matrix.k8s-version.version }} --config ./.github/kind.yml
+      - name: Install report server testing
+        run: |
+          set -e
+          export HELM=${{ steps.helm.outputs.helm-path }}
+          make kind-install
+      - name: Wait for report server ready
+        run: |
+          set -e
+          kubectl wait --namespace reports-server --for=condition=ready pod --selector '!job-name' --timeout=120s
+      - name: Install latest kyverno
+        run: |
+          set -e
+          kubectl create -f https://github.com/kyverno/kyverno/raw/main/config/install-latest-testing.yaml
+      - name: Wait for kyverno ready
+        run: |
+          set -e
+          kubectl wait --namespace kyverno --for=condition=ready pod --selector '!job-name' --timeout=60s
+      - name: API Service status
+        run: |
+          set -e
+          kubectl get apiservices v1alpha2.wgpolicyk8s.io v1.reports.kyverno.io
+      - name: Install Chainsaw
+        uses: kyverno/action-install-chainsaw@56be3cb4ec65a987b6ef4d7ab3a55ee17760a57c # v0.1.2
+        with:
+          release: v0.0.9
+      - name: Test with Chainsaw
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -e
+          cd ./kyverno/test/conformance/chainsaw && chainsaw test --config ./../../../../.chainsaw.yaml --include-test-regex '^chainsaw$/${{ matrix.tests }}' --no-color=false
+      - name: Debug failure
+        if: failure()
+        uses: ./.github/actions/failure-logs

Makefile

@@ -143,7 +143,7 @@ codegen-helm-docs: ## Generate helm docs
 	@echo Generate helm docs... >&2
 	@docker run -v ${PWD}/charts:/work -w /work jnorwood/helm-docs:v1.11.0 -s file
 
-.PHONY: codegen-manifest-install-latest
+.PHONY: codegen-install-manifest
 codegen-install-manifest: $(HELM) ## Create install manifest
 	@echo Generate latest install manifest... >&2
 	@$(HELM) template reports-server --namespace reports-server ./charts/reports-server/ \