Merge pull request #73 from cynepco3hahue/add_kubevirtci

Use KubeVirtCI providers for local development

.gitignore

@@ -27,3 +27,8 @@ tools/manifest-templator/manifest-templator
 
 # Ignore _out directory
 _out
+_ci-configs
+
+# Ignore IDE's directories
+.idea
+

.travis.yml

@@ -8,6 +8,7 @@ install:
 - git reset --hard
 script:
 - make generate
+- make kubevirtci-download
 - if [[ -n "$(git status --porcelain)" ]] ; then echo "It seems like you need to run
   `make generate`. Please run it and commit the changes"; git status --porcelain;
   false; fi

Makefile

@@ -18,6 +18,29 @@ bazel-tests:
 		--cache_test_results=no \
         --test_output=errors -- //pkg/... //tools/utils/..."
 
+.PHONY: cluster-build
+cluster-build:
+	KUBEVIRT_PROVIDER=okd-4.1 ./hack/cluster/build.sh
+
+.PHONY: cluster-clean
+cluster-clean:
+	./hack/cluster/clean.sh
+
+.PHONY: cluster-deploy
+cluster-deploy: cluster-clean
+	./hack/cluster/deploy.sh
+
+.PHONY: cluster-down
+cluster-down:
+	KUBEVIRT_PROVIDER=okd-4.1 ./cluster-up/down.sh
+
+.PHONY: cluster-sync
+cluster-sync: cluster-build cluster-deploy
+
+.PHONY: cluster-up
+cluster-up:
+	KUBEVIRT_PROVIDER=okd-4.1 ./cluster-up/up.sh
+
 .PHONY: deps-update
 deps-update:
 	SYNC_VENDOR=true ./hack/dockerized "./hack/deps-update.sh"
@@ -27,6 +50,14 @@ distclean:
 	hack/dockerized "rm -rf vendor/ && rm -f go.sum && GO111MODULE=on go clean -modcache"
 	rm -rf vendor/
 
+.PHONY: e2e-tests-build
+e2e-tests-build:
+	./hack/dockerized "./hack/testing/build.sh"
+
+.PHONY: e2e-tests-run
+e2e-tests-run: e2e-tests-build
+	./hack/testing/run.sh
+
 .PHONY: fmt
 fmt:
 	./hack/dockerized "./hack/bazel/fmt.sh"
@@ -63,10 +94,6 @@ generate-manifests: generate-templates
 generate-templates:
 	./hack/dockerized "./hack/generate/templates.sh"
 
-.PHONY: e2e-tests-build
-e2e-tests-build:
-	./hack/dockerized "./hack/testing/build.sh"
-
-.PHONY: e2e-tests-run
-e2e-tests-run: e2e-tests-build
-	./hack/testing/run.sh
+.PHONY: kubevirtci-download
+kubevirtci-download:
+	./hack/dockerized "./hack/kubevirtci/download.sh"

cluster-up/cli.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+#
+# This file is part of the KubeVirt project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Copyright 2019 Red Hat, Inc.
+#
+
+set -e
+
+if [ -z "$KUBEVIRTCI_PATH" ]; then
+    KUBEVIRTCI_PATH="$(
+        cd "$(dirname "$BASH_SOURCE[0]")/"
+        echo "$(pwd)/"
+    )"
+fi
+
+source ${KUBEVIRTCI_PATH}/hack/common.sh
+
+test -t 1 && USE_TTY="-it"
+source ${KUBEVIRTCI_PATH}/cluster/$KUBEVIRT_PROVIDER/provider.sh
+source ${KUBEVIRTCI_PATH}/hack/config.sh
+
+${_cli} --prefix $provider_prefix "$@"

cluster-up/cluster/ephemeral-provider-common.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+set -e
+
+_cli_container="kubevirtci/gocli@sha256:a7880757e2d2755c6a784c1b64c64b096769ed3ccfac9d8e535df481731c2144"
+_cli_with_tty="docker run --privileged --net=host --rm -t -v /var/run/docker.sock:/var/run/docker.sock ${_cli_container}"
+_cli="docker run --privileged --net=host --rm ${USE_TTY} -v /var/run/docker.sock:/var/run/docker.sock ${_cli_container}"
+
+function _main_ip() {
+    echo 127.0.0.1
+}
+
+function _port() {
+    ${_cli} ports --prefix $provider_prefix "$@"
+}
+
+function prepare_config() {
+    BASE_PATH=${KUBEVIRTCI_CONFIG_PATH:-$PWD}
+    cat >$BASE_PATH/$KUBEVIRT_PROVIDER/config-provider-$KUBEVIRT_PROVIDER.sh <<EOF
+master_ip=$(_main_ip)
+kubeconfig=${BASE_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig
+kubectl=${BASE_PATH}/$KUBEVIRT_PROVIDER/.kubectl
+gocli=${BASE_PATH}/../cluster-up/cli.sh
+docker_prefix=\${DOCKER_PREFIX:-localhost:$(_port registry)/kubevirt}
+manifest_docker_prefix=\${DOCKER_PREFIX:-registry:5000/kubevirt}
+EOF
+}
+
+function _registry_volume() {
+    echo ${job_prefix}_registry
+}
+
+function _add_common_params() {
+    local params="--nodes ${KUBEVIRT_NUM_NODES} --memory ${KUBEVIRT_MEMORY_SIZE} --cpu 5 --secondary-nics ${KUBEVIRT_NUM_SECONDARY_NICS} --random-ports --background --prefix $provider_prefix --registry-volume $(_registry_volume) kubevirtci/${image} ${KUBEVIRT_PROVIDER_EXTRA_ARGS}"
+    if [[ $TARGET =~ windows.* ]] && [ -n "$WINDOWS_NFS_DIR" ]; then
+        params=" --nfs-data $WINDOWS_NFS_DIR $params"
+    elif [[ $TARGET =~ os-.* ]] && [ -n "$RHEL_NFS_DIR" ]; then
+        params=" --nfs-data $RHEL_NFS_DIR $params"
+    fi
+    echo $params
+}
+
+function _kubectl() {
+    export KUBECONFIG=${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig
+    ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl "$@"
+}
+
+function down() {
+    ${_cli} rm --prefix $provider_prefix
+}

cluster-up/cluster/okd-4.1/README.md

@@ -0,0 +1,44 @@
+# OKD 4.1 in ephemeral containers
+
+Provides a pre-deployed OKD with version 4.1 purely in docker
+containers with libvirt. The provided VMs are completely ephemeral and are
+recreated on every cluster restart. The KubeVirt containers are built on the
+local machine and are then pushed to a registry which is exposed at
+`localhost:5000`.
+
+## Bringing the cluster up
+
+```bash
+export KUBEVIRT_PROVIDER=okd-4.1
+make cluster-up
+```
+
+The cluster can be accessed as usual:
+
+```bash
+$ cluster/kubectl.sh get nodes
+NAME                          STATUS   ROLES    AGE   VERSION
+test-1-82xp6-master-0         Ready    master   62m   v1.12.4+509916ce1
+test-1-82xp6-worker-0-wxf27   Ready    worker   57m   v1.12.4+509916ce1
+```
+
+## Bringing the cluster down
+
+```bash
+export KUBEVIRT_PROVIDER=okd-4.1
+make cluster-down
+```
+
+This destroys the whole cluster. Recreating the cluster is fast, since OKD is
+already pre-deployed. The only state which is kept is the state of the local
+docker registry.
+
+## Destroying the docker registry state
+
+The docker registry survives a `make cluster-down`. It's state is stored in a
+docker volume called `kubevirt_registry`. If the volume gets too big or the
+volume contains corrupt data, it can be deleted with
+
+```bash
+docker volume rm kubevirt_registry
+```

cluster-up/cluster/okd-4.1/provider.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+set -e
+
+image="okd-4.1@sha256:84ab0dcc301b1e5c801de4f30e9950641f65a39a8756663d02da6de70e22e19e"
+
+source ${KUBEVIRTCI_PATH}/cluster/ephemeral-provider-common.sh
+
+function _port() {
+    ${_cli} ports --prefix $provider_prefix --container-name cluster "$@"
+}
+
+function up() {
+    workers=$(($KUBEVIRT_NUM_NODES-1))
+    if [[ ( $workers < 1 ) ]]; then
+        workers=1
+    fi
+    echo "Number of workers: $workers"
+    params="--random-ports --background --prefix $provider_prefix --master-cpu 6 --workers-cpu 6 --registry-volume $(_registry_volume) --workers $workers kubevirtci/${image}"
+    if [[ ! -z "${RHEL_NFS_DIR}" ]]; then
+        params=" --nfs-data $RHEL_NFS_DIR ${params}"
+    fi
+
+    if [[ ! -z "${OKD_CONSOLE_PORT}" ]]; then
+        params=" --ocp-console-port $OKD_CONSOLE_PORT ${params}"
+    fi
+
+    ${_cli} run okd ${params}
+
+    # Copy k8s config and kubectl
+    cluster_container_id=$(docker ps -f "name=$provider_prefix-cluster" --format "{{.ID}}")
+    docker cp $cluster_container_id:/bin/oc ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl
+    chmod u+x ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl
+    docker cp $cluster_container_id:/root/install/auth/kubeconfig ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig
+
+    # Set server and disable tls check
+    export KUBECONFIG=${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig
+    ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl config set-cluster test-1 --server=https://$(_main_ip):$(_port k8s)
+    ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl config set-cluster test-1 --insecure-skip-tls-verify=true
+
+    # Make sure that local config is correct
+    prepare_config
+}

cluster-up/down.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+if [ -z "$KUBEVIRTCI_PATH" ]; then
+    KUBEVIRTCI_PATH="$(
+        cd "$(dirname "$BASH_SOURCE[0]")/"
+        echo "$(pwd)/"
+    )"
+fi
+
+source ${KUBEVIRTCI_PATH}hack/common.sh
+source ${KUBEVIRTCI_PATH}cluster/$KUBEVIRT_PROVIDER/provider.sh
+down

cluster-up/hack/common.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI
+w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP
+kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2
+hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO
+Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW
+yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd
+ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1
+Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf
+TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK
+iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A
+sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf
+4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP
+cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk
+EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN
+CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX
+3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG
+YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj
+3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+
+dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz
+6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC
+P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF
+llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ
+kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH
++vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ
+NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s=
+-----END RSA PRIVATE KEY-----

cluster-up/hack/common.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+if [ -z "$KUBEVIRTCI_PATH" ]; then
+    KUBEVIRTCI_PATH="$(
+        cd "$(dirname "$BASH_SOURCE[0]")/../"
+        echo "$(pwd)/"
+    )"
+fi
+
+
+if [ -z "$KUBEVIRTCI_CONFIG_PATH" ]; then
+    KUBEVIRTCI_CONFIG_PATH="$(
+        cd "$(dirname "$BASH_SOURCE[0]")/../../"
+        echo "$(pwd)/_ci-configs"
+    )"
+fi
+
+KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-k8s-1.13.3}
+KUBEVIRT_NUM_NODES=${KUBEVIRT_NUM_NODES:-1}
+KUBEVIRT_MEMORY_SIZE=${KUBEVIRT_MEMORY_SIZE:-5120M}
+KUBEVIRT_NUM_SECONDARY_NICS=${KUBEVIRT_NUM_SECONDARY_NICS:-0}
+
+# If on a developer setup, expose ocp on 8443, so that the openshift web console can be used (the port is important because of auth redirects)
+if [ -z "${JOB_NAME}" ]; then
+    KUBEVIRT_PROVIDER_EXTRA_ARGS="${KUBEVIRT_PROVIDER_EXTRA_ARGS} --ocp-port 8443"
+fi
+
+#If run on jenkins, let us create isolated environments based on the job and
+# the executor number
+provider_prefix=${JOB_NAME:-${KUBEVIRT_PROVIDER}}${EXECUTOR_NUMBER}
+job_prefix=${JOB_NAME:-kubevirt}${EXECUTOR_NUMBER}
+
+mkdir -p $KUBEVIRTCI_CONFIG_PATH/$KUBEVIRT_PROVIDER

cluster-up/hack/config-default.sh

@@ -0,0 +1,4 @@
+
+docker_prefix=${DOCKER_PREFIX:-kubevirt}
+master_ip=192.168.200.2
+network_provider=flannel

cluster-up/hack/config.sh

@@ -0,0 +1,10 @@
+unset docker_prefix master_ip network_provider kubeconfig manifest_docker_prefix
+
+KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-${PROVIDER}}
+
+source ${KUBEVIRTCI_PATH}hack/config-default.sh
+
+# Allow different providers to override default config values
+test -f "${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/config-provider-${KUBEVIRT_PROVIDER}.sh" && source ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/config-provider-${KUBEVIRT_PROVIDER}.sh
+
+export docker_prefix master_ip network_provider kubeconfig manifest_docker_prefix

cluster-up/kubeconfig.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+#
+# This file is part of the KubeVirt project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Copyright 2019 Red Hat, Inc.
+#
+
+set -e
+
+if [ -z "$KUBEVIRTCI_PATH" ]; then
+    KUBEVIRTCI_PATH="$(
+        cd "$(dirname "$BASH_SOURCE[0]")/"
+        echo "$(pwd)/"
+    )"
+fi
+
+source ${KUBEVIRTCI_PATH}/hack/common.sh
+
+echo "${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig"