Merge pull request #4 from p-se/master

Introduce E2E testing

.github/workflows/e2e.yml

@@ -0,0 +1,233 @@
+name: E2E
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Run everyday day at 9:00 AM
+    - cron: '0 9 * * *'
+  pull_request:
+    paths-ignore:
+      - 'scripts/**'
+      - '*.md'
+  push:
+    branches:
+      - 'release/*"'
+      - 'main'
+      - 'master'
+
+env:
+  GOARCH: amd64
+  CGO_ENABLED: 0
+  SETUP_GO_VERSION: '^1.19'
+  HOST_NAME: telemetry-test  # for `start` script
+  CLUSTER_NAME: telemetry-test
+  TOKEN: ${{ secrets.TOKEN }}
+  REPO: telemetry-stats
+  REPO_URL: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY
+  REPOSITORY: ${{ github.repository }}
+  BRANCH: ${{ github.ref_name }}
+  TELEMETRY_REPO: https://github.com/rancher/telemetry
+  TELEMETRY_BRANCH: ${{ vars.TELEMETRY_BRANCH || 'release' }}
+
+jobs:
+  single-cluster:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          path: telemetry-stats
+
+      - name: Get latest tag for telemetry
+        uses: actions/github-script@v6
+        id: latest_tag
+        if: ${{ env.TELEMETRY_BRANCH == 'release' }}
+        with:
+          script: |
+            const { data } = await github.rest.repos.getLatestRelease({
+              owner: 'rancher',
+              repo: 'telemetry'
+            })
+            return data.tag_name;
+
+      - name: Replace TELEMETRY_BRANCH with latest tag
+        if: ${{ env.TELEMETRY_BRANCH == 'release' }}
+        run: |
+          export ACTUAL_TELEMETRY_BRANCH=${{ steps.latest_tag.outputs.result }}
+          echo "ACTUAL_TELEMETRY_BRANCH=${ACTUAL_TELEMETRY_BRANCH}" >> $GITHUB_ENV
+
+      - name: Set ACTUAL_TELEMETRY_BRANCH to the value of TELEMETRY_BRANCH if still unset
+        if: ${{ env.TELEMETRY_BRANCH != 'release' }}
+        run: |
+          echo "ACTUAL_TELEMETRY_BRANCH=${TELEMETRY_BRANCH}" >> $GITHUB_ENV
+
+      - name: Print TELEMETRY_BRANCH and ACTUAL_TELEMETRY_BRANCH
+        run: |
+          echo "TELEMETRY_BRANCH=${TELEMETRY_BRANCH}"
+          echo "ACTUAL_TELEMETRY_BRANCH=${ACTUAL_TELEMETRY_BRANCH}"
+
+      - uses: actions/checkout@v3
+        with: 
+          repository: rancher/telemetry
+          path: telemetry
+          ref: ${{ env.ACTUAL_TELEMETRY_BRANCH }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.SETUP_GO_VERSION }}
+
+      - name: Compile & containerize Telemetry
+        if: ${{ env.TELEMETRY_BRANCH != 'release' }}
+        run: |
+          set -x
+
+          # Compile using dapper
+          cd telemetry
+          curl -sL https://releases.rancher.com/dapper/latest/dapper-$(uname -s)-$(uname -m) > /usr/local/bin/dapper
+          chmod +x /usr/local/bin/dapper
+          dapper
+
+          # Build container image
+          export image_name="rancher/telemetry:${ACTUAL_TELEMETRY_BRANCH}"
+
+          docker build \
+            --network=host \
+            -f package/Dockerfile \
+            -t ${image_name} \
+            .
+
+          # Make name of resulting image available to other steps
+          echo "TELEMETRY_IMAGE=${image_name}" >> $GITHUB_ENV
+
+      - name: Set remote container image for Telemetry release branch
+        if: ${{ env.TELEMETRY_BRANCH == 'release' }}
+        run: |
+          export image_name="rancher/telemetry:${ACTUAL_TELEMETRY_BRANCH}-amd64"
+          echo "TELEMETRY_IMAGE=${image_name}" >> $GITHUB_ENV
+
+      - name: Compile & containerize Telemetry Stats
+        run: |
+          cd telemetry-stats
+          set +x
+          docker build \
+            --network=host \
+            --build-arg VERSION=dev \
+            --build-arg upstream= \
+            -t rancherlabs/rancher-telemetry-stats:dev \
+            .
+
+      - name: Provision k3d Cluster
+        uses: AbsaOSS/k3d-action@v2
+        with:
+          cluster-name: ${{ env.CLUSTER_NAME }}
+          args: >-
+            --api-port 6443
+            --network "nw01"
+
+      - name: Upload Telemetry Docker image
+        if: ${{ env.TELEMETRY_BRANCH != 'release' }}
+        run: |
+          set -x
+          k3d image import -c ${{ env.CLUSTER_NAME }} rancher/telemetry:${ACTUAL_TELEMETRY_BRANCH}
+
+      - name: Upload Telemetry Stats Docker image
+        run: |
+          k3d image import -c ${{ env.CLUSTER_NAME }} rancherlabs/rancher-telemetry-stats:dev
+
+      - name: Install Rancher
+        run: |
+          set -ex
+
+          echo "working directory: $(pwd)"
+          echo "files in working directory: $(find)"
+
+          pip install httpie
+          cd telemetry-stats/e2e
+          source ./start
+          install-rancher
+
+          # Verify
+          for i in {0..20}; do
+            ip=$(
+              kubectl get -A ingress -o json |
+                jq -r '.items[0].status.loadBalancer.ingress[0].ip'
+            )
+
+            if [[ "${ip}" == "null" ]]; then
+              sleep 1
+              continue
+            fi
+
+            if http --check-status --verify=no GET "https://${ip}" Host:${HOST_NAME} ; then
+              >&2 echo "Rancher installation successfully verified"
+              exit 0
+            fi
+
+            sleep 1
+          done
+
+          >&2 echo "Rancher installation test failed"
+          exit 1
+
+      - name: Configure Rancher
+        run: |
+          cd telemetry-stats/e2e && \
+            source ./start && \
+            CLUSTER_IP=$(get-cluster-ip) && \
+            init-base "admin"
+
+      - name: Install PostgreSQL
+        run: |
+          cd telemetry-stats/e2e && source ./start && install-postgres-13.2
+
+      - name: Install Telemetry Server from $TELEMETRY_BRANCH branch
+        run: |
+          set -x
+          cd telemetry-stats/e2e && source ./start && install-telemetry-server
+
+      - name: Install Telemetry Client from $TELEMETRY_BRANCH branch
+        run: |
+          set -x
+          cd telemetry-stats/e2e && source ./start && install-telemetry-client
+
+      - name: Install InfluxDB
+        run: |
+          cd telemetry-stats/e2e && source ./start && install-influx
+
+      - name: Install Telemetry Stats
+        run: |
+          cd telemetry-stats/e2e && source ./start && \
+            install-telemetry-stats-dev
+
+      - name: Test installation
+        run: |
+          set -x
+          ns="${ns:-telemetry}"
+          cd telemetry-stats/e2e
+          kubectl -n "${ns}" apply --wait -k manifests/tests
+
+          kubectl -n telemetry wait \
+            --for=condition=complete \
+            --timeout=300s \
+            job/test-telemetry-server
+
+          kubectl -n telemetry wait \
+            --for=condition=complete \
+            --timeout=300s \
+            job/test-influxdb
+
+      - name: Collect logs
+        if: failure()
+        run: |
+          bash telemetry-stats/e2e/kubectl-dump telemetry
+
+      - name: Upload logs
+        uses: actions/upload-artifact@v3
+        if: ${{ failure() && github.action }}
+        with:
+          name: gha-telemetry-dev-e2e-logs-${{ github.sha }}-${{ github.run_id }}
+          path: |
+            kubectl-dump-*
+          retention-days: 2

.gitignore

@@ -0,0 +1,3 @@
+.env
+src/rancher-telemetry-stats
+act-*

Dockerfile

@@ -1,13 +1,23 @@
 ARG GO_IMAGE=registry.suse.com/bci/golang:1.17
 
 FROM ${GO_IMAGE} AS build
+SHELL ["/bin/bash", "-c"]
 ARG upstream=https://github.com/rancherlabs/rancher-telemetry-stats.git
 ARG version
 WORKDIR /src
-
 ENV NAME=rancher-telemetry-stats
 RUN zypper install -y tar wget gzip
-RUN git clone --depth=1 --branch=${version} ${upstream} .
+
+# Conditionally clone the upstream repo or copy the local source, depending on
+# whether the upstream is a git repo or a local directory.
+ADD . /src-local
+RUN if [[ "${upstream}" =~ "http" ]]; then \
+        git clone --depth=1 --branch=${version} ${upstream} . ; \
+    else \
+        cp -r /src-local/* . ; \
+    fi
+
+RUN echo DEBUG $(pwd)
 RUN cd src/ && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 GO111MODULE=on go build -v -a -tags netgo -o release/${NAME}
 RUN gzip -d src/GeoLite2-City.mmdb.gz
 

e2e/README.md

@@ -0,0 +1,64 @@
+# E2E Tests
+
+This directory contains end-to-end tests for the project. These tests are
+intended to be run in a Github Actions workflow if a Pull Request on Github is
+created. You can run them locally, but you will need to set up a few things
+first. And it's a bit more flaky.
+
+The tests will create a Kubernetes cluster using [k3d](https://k3d.io/). k3d
+uses Docker and [k3s](https://k3s.io/) to create a lightweight Kubernetes
+cluster for testing.
+
+The current amount of tests is very limited. For Telemetry, it merely tests if
+the Telemetry client has successfully sent data to the Telemetry Server, which
+stores the data in PostgreSQL. The test verifies that the data is there and can
+be retrieved using Telemetry Server. For Telemetry Stats, the test verifies that
+the data has been successfully stored in InfluxDB. The test retrieves the data
+directly from InfluxDB.
+
+## Configuration
+
+### Telemetry branch to be used for testing
+
+The tests are configured via environment variables. They can be configured on
+Github specifically for
+[Actions](https://docs.github.com/en/actions/learn-github-actions/variables).
+The following variables can be configured:
+
+- `TELEMETRY_BRANCH`: The branch to use for the test of Telemetry (not Telemetry
+  Stats). This can be any branch, tag, or commit hash. It defaults to `release`.
+  `release` is a special value that will result in the latest released container
+  image being downloaded and used for testing. Any other value will be taken as
+  a Git reference (branch, tag or reference) and the source code will be cloned,
+  built and a container image will be created and uploaded to the Kubernetes
+  cluster, so that the images will be available for use.
+
+# Running Tests Locally
+
+## Requirements
+
+- [act](https://github.com/nektos/act)
+- [Docker](https://www.docker.com/)
+
+## Configuration
+
+A Personal Access token (PAT) from Github is required to run the tests locally.
+
+## Usage
+
+```console
+act --env TELEMETRY_BRANCH=master --secret-file .env
+```
+
+Where `master` is the branch you want to test. You can also set it to "release"
+to test an already released version, for which only the container image will be
+downloaded but not built. In all other cases, the source code will be cloned,
+built and a corresponding container image is being created and uploaded to the
+Kubernetes cluster.
+
+The `--secret-file` option is required to pass the PAT to act. The file should
+contain the following:
+
+```console
+GITHUB_TOKEN=your-pat-here
+```