Skip to content

Commit

Permalink
[KOGITO-10013] Migrate from Minikube to Kind for E2E tests and export…
Browse files Browse the repository at this point in the history
… logs (#354)

* [KOGITO-10013] Migrate from Minikube to Kind for E2E tests

Signed-off-by: Jordi Gil <[email protected]>

* Add headers and set log retention to minimum (1 day)

Signed-off-by: Jordi Gil <[email protected]>

* Changed registry port reference to 5001 and reference to Minikube to Kind

Signed-off-by: Jordi Gil <[email protected]>

* Add makefile targets to install kind, create and delete cluster and amend github action to use the create cluster target

Signed-off-by: Jordi Gil <[email protected]>

* Load controller built image to node with kind load docker-image

Signed-off-by: Jordi Gil <[email protected]>

* Remove podman from build process in e2e lane

Signed-off-by: Jordi Gil <[email protected]>

* Install python's docker libraries: docker and python-docker

Signed-off-by: Jordi Gil <[email protected]>

* Add docker-squash as pip egg to build the image

Signed-off-by: Jordi Gil <[email protected]>

---------

Signed-off-by: Jordi Gil <[email protected]>
  • Loading branch information
jordigilh authored Jan 18, 2024
1 parent 64f688b commit a8dc5ed
Show file tree
Hide file tree
Showing 3 changed files with 129 additions and 47 deletions.
69 changes: 23 additions & 46 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,6 @@ on:

env:
GO_VERSION: 1.19
# Kubernetes version should aligned with OCP LTS, aka OCP 4.10 (k8s 1.23) for now
# https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.23.md
# WARNING: kindest/node is not always existing with given version ...
KUBERNETES_VERSION: v1.26.3
MINIKUBE_VERSION: v1.31.1
DEBUG: true

jobs:
Expand All @@ -27,7 +22,7 @@ jobs:
cancel-in-progress: true
timeout-minutes: 120
runs-on: ubuntu-latest
name: End-to-end tests (Minikube)
name: End-to-end tests (Kind)
steps:
- name: Install package
run: |
Expand All @@ -50,45 +45,13 @@ jobs:
go-version: ${{ env.GO_VERSION }}
cache: true

- name: Cache Minikube Download
id: cache-minikube-download
uses: actions/cache@v3
with:
key: ${{ runner.os }}-minikube-${{ env.MINIKUBE_VERSION }}
path: |
~/.minikube/download/
- name: Setup Minikube cluster
run: |
# Install minikube at $HOME/.local/bin
./hack/ci/install-minikube.sh $HOME/.local/bin
echo "$HOME/.local/bin" >> $GITHUB_PATH
minikube start --kubernetes-version ${{ env.KUBERNETES_VERSION }} --addons=registry --insecure-registry=localhost:5000,192.168.0.0/16 --cpus=max --memory=max
- name: Wait for Minikube up and running
- name: Setup Kind cluster
run: |
minikube ssh cat /lib/systemd/system/docker.service
kubectl get pods -A
set -x
MINIKUBE_COMPONENTS=(etcd kube-apiserver kube-controller-manager kube-scheduler)
for component in "${MINIKUBE_COMPONENTS[@]}"
do
echo "Check component '${component}' is in 'Running' state"
COMPONENT_NAME=${component} timeout 60s bash -c 'kubectl get pods -l tier=control-plane -l component=${COMPONENT_NAME} -n kube-system && while [[ "$(kubectl get pods -l tier=control-plane -l component=${COMPONENT_NAME} -n kube-system -o jsonpath={.items[0].status.phase})" != "Running" ]] ; do sleep 2 && kubectl get pods -l tier=control-plane -l component=${COMPONENT_NAME} -n kube-system -o jsonpath={.items[0].status.phase}; done'
done
make create-cluster
- name: Wait for Minikube registry
- name: Set OPERATOR_IMAGE_NAME to point to Kind's local registry
run: |
kubectl get pods -A
timeout 60s bash -c 'kubectl get pods -l kubernetes.io/minikube-addons=registry -l actual-registry=true -n kube-system && while [[ "$(kubectl get pods -l kubernetes.io/minikube-addons=registry -l actual-registry=true -n kube-system -o jsonpath={.items[0].status.phase})" != "Running" ]] ; do sleep 2 && kubectl get pods -l kubernetes.io/minikube-addons=registry -l actual-registry=true -n kube-system -o jsonpath={.items[0].status.phase}; done'
- name: Retrieve Minikube registry
run: |
minikube_registry="$(minikube ip):5000"
echo "MINIKUBE_REGISTRY=${minikube_registry}" >> $GITHUB_ENV
echo "OPERATOR_IMAGE_NAME=${minikube_registry}/kogito-serverless-operator:0.0.1" >> $GITHUB_ENV
echo "OPERATOR_IMAGE_NAME=127.0.0.1:5001/kogito-serverless-operator:0.0.1" >> $GITHUB_ENV
- name: Setup Python for cekit
uses: actions/setup-python@v4
Expand All @@ -99,12 +62,12 @@ jobs:
- name: Build operator image
run: |
pip install cekit==4.5.0
pip install odcs podman behave lxml krb5
make container-build BUILDER=podman IMG=${{ env.OPERATOR_IMAGE_NAME }}
pip install odcs docker python-docker behave lxml krb5 docker-squash
make container-build BUILDER=docker IMG=${{ env.OPERATOR_IMAGE_NAME }}
- name: Load image in Minikube
- name: Load image in Kind
run: |
podman push --tls-verify=false ${{ env.OPERATOR_IMAGE_NAME }}
kind load docker-image 127.0.0.1:5001/kogito-serverless-operator:0.0.1
- name: Check pods
run: |
Expand All @@ -115,3 +78,17 @@ jobs:
- name: Run tests
run: |
make test-e2e
- name: Export kind logs
if: always()
run: |
mkdir -p /tmp/kind/logs
kind export logs --loglevel=debug /tmp/kind/logs
- name: Upload kind logs
if: always()
uses: actions/upload-artifact@v3
with:
name: kind-logs-${{ env.JOB_NAME }}-${{ github.run_id }}
path: /tmp/kind/logs
retention-days: 1
16 changes: 15 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,7 @@ ENVTEST ?= $(LOCALBIN)/setup-envtest
## Tool Versions
KUSTOMIZE_VERSION ?= v4.5.2
CONTROLLER_TOOLS_VERSION ?= v0.9.2
KIND_VERSION ?= v0.20.0

KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"
.PHONY: kustomize
Expand Down Expand Up @@ -349,4 +350,17 @@ test-e2e: install-operator-sdk
go test ./test/e2e/* -v -ginkgo.v -timeout 30m

.PHONY: before-pr
before-pr: test generate-all
before-pr: test generate-all


.PHONY: install-kind
install-kind:
command -v kind >/dev/null || go install sigs.k8s.io/kind@$(KIND_VERSION)

.PHONY: create-cluster
create-cluster: install-kind
kind get clusters | grep kind >/dev/null || ./hack/ci/create-kind-cluster-with-registry.sh

.PHONY: delete-cluster
delete-cluster: install-kind
kind delete cluster && docker rm -f kind-registry
91 changes: 91 additions & 0 deletions hack/ci/create-kind-cluster-with-registry.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
# Copyright 2024 Apache Software Foundation (ASF)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#!/bin/sh

set -o errexit

reg_name='kind-registry'
reg_port='5001'


# 1. Create kind cluster with containerd registry config dir enabled
# TODO: kind will eventually enable this by default and this patch will
# be unnecessary.
#
# See:
# https://github.com/kubernetes-sigs/kind/issues/2875
# https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration
# See: https://github.com/containerd/containerd/blob/main/docs/hosts.md
cat <<EOF | kind create cluster -n kind --config=-
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
containerdConfigPatches:
- |-
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d"
EOF

# 2. Wait for kube system pods to reach running state
if ! kubectl wait -n kube-system --for=condition=ready pods --all --timeout=120s ; then
echo "some pods in the system are not running"
kubectl get pods -A -o wide || true
exit 1
fi

# 3 Create registry container
docker run \
-d --restart=always -p "127.0.0.1:${reg_port}:5000" --network bridge --name "${reg_name}" \
-v /tmp/certs:/certs \
registry:2

# 4. Connect the registry to the cluster network if not already connected
# This allows kind to bootstrap the network but ensures they're on the same network
if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
docker network connect "kind" "${reg_name}"
fi

# 5. Add the registry config to the nodes
#
# This is necessary because localhost resolves to loopback addresses that are
# network-namespace local.
# In other words: localhost in the container is not localhost on the host.
#
# We want a consistent name that works from both ends, so we tell containerd to
# alias localhost:${reg_port} to the registry container when pulling images
REGISTRY_DIR="/etc/containerd/certs.d/172.18.0.3:5000"
# retrieve IP address of the container connected to the cluster network
IP_ADDRESS=$(docker inspect --format='{{(index (index .NetworkSettings.Networks "kind") ).IPAddress}}' ${reg_name})
for node in $(kind get nodes); do
docker exec "${node}" mkdir -p "${REGISTRY_DIR}"
cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${REGISTRY_DIR}/hosts.toml"
[host."http://${IP_ADDRESS}:5000"]
capabilities = ["pull", "resolve", "push"]
skip_verify = true
EOF
done

# 6. Document the local registry
# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: ConfigMap
metadata:
name: local-registry-hosting
namespace: kube-public
data:
localRegistryHosting.v1: |
hostFromClusterNetwork: "${IP_ADDRESS}:5000"
help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
EOF

0 comments on commit a8dc5ed

Please sign in to comment.