From 5892418ce590c4754c25d84f9922e01028720f0b Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Mon, 22 Oct 2018 03:32:07 -0500 Subject: [PATCH 01/34] tests: Enable docker run integration test. Enable docker integration test that verifies than when a host networking is requested the runtime fails. Fixes #833 Signed-off-by: Gabriela Cervantes --- integration/docker/run_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/integration/docker/run_test.go b/integration/docker/run_test.go index 7e71f2179..01a9f75bb 100644 --- a/integration/docker/run_test.go +++ b/integration/docker/run_test.go @@ -253,7 +253,6 @@ var _ = Describe("run host networking", func() { Context("Run with host networking", func() { It("should error out", func() { - Skip("Issue: https://github.com/kata-containers/runtime/issues/652") args = []string{"--name", id, "-d", "--net=host", DebianImage, "sh"} _, stderr, exitCode = dockerRun(args...) Expect(exitCode).NotTo(Equal(0)) From 7315a16f8fb7c3eb2f7ce63d22ea30460aa87d80 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Wed, 5 Dec 2018 12:39:23 -0600 Subject: [PATCH 02/34] integration/docker: unskip docer cp test Unskip docker cp test to check mount points are not left after running docker cp. fixes #970 Signed-off-by: Julio Montes --- integration/docker/cp_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/integration/docker/cp_test.go b/integration/docker/cp_test.go index 818edb0fa..019b21047 100644 --- a/integration/docker/cp_test.go +++ b/integration/docker/cp_test.go @@ -123,7 +123,6 @@ var _ = Describe("docker cp with volume", func() { Context("check mount points", func() { It("should be removed", func() { - Skip("Issue: https://github.com/kata-containers/runtime/issues/794") file, err := ioutil.TempFile(os.TempDir(), "file") Expect(err).ToNot(HaveOccurred()) err = file.Close() From ba9c3ca0fa9071117310e68d4248d5f1b24b5a20 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Tue, 22 Jan 2019 02:07:01 -0600 Subject: [PATCH 03/34] test: Modify soak test to support Firecracker This will enable and modify the soak test to run with Firecracker. We need it to skip checking the processes of proxy and netmon as they are not present when running with Firecracker. Fixes #1053 Signed-off-by: Gabriela Cervantes --- integration/stability/soak_parallel_rm.sh | 47 +++++++++++------------ 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/integration/stability/soak_parallel_rm.sh b/integration/stability/soak_parallel_rm.sh index 244c5fa3c..12a8ae3cb 100755 --- a/integration/stability/soak_parallel_rm.sh +++ b/integration/stability/soak_parallel_rm.sh @@ -41,12 +41,7 @@ MAX_CONTAINERS="${MAX_CONTAINERS:-110}" KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" -if [ "$KATA_HYPERVISOR" == "firecracker" ]; then - echo "Skip soak test on $KATA_HYPERVISOR (see: https://github.com/kata-containers/tests/issues/1029)" - exit -fi - -if [ "$ID" == debian ]; then +if [ "$ID" == "debian" ]; then echo "Skip soak test on ${ID} (see: https://github.com/kata-containers/runtime/issues/1132)" exit fi @@ -100,19 +95,21 @@ check_all_running() { # Only check for Kata components if we are using a Kata runtime if (( $check_kata_components )); then - # Check we have one proxy per container - how_many_proxys=$(pgrep -a -f ${PROXY_PATH} | wc -l) - if check_vsock_active; then - if (( ${how_many_proxys} != 0 )); then - echo "Wrong number of proxys running (${how_many_running} containers, ${how_many_proxys} proxys)" - echo "When using vsocks, the number of proxies should be Zero - stopping" - ((goterror++)) - fi - - else - if (( ${how_many_running} != ${how_many_proxys} )); then - echo "Wrong number of proxys running (${how_many_running} containers, ${how_many_proxys} proxys) - stopping" - ((goterror++)) + if [ "$KATA_HYPERVISOR" == "qemu" ]; then + # Check we have one proxy per container + how_many_proxys=$(pgrep -a -f ${PROXY_PATH} | wc -l) + if check_vsock_active; then + if (( ${how_many_proxys} != 0 )); then + echo "Wrong number of proxys running (${how_many_running} containers, ${how_many_proxys} proxys)" + echo "When using vsocks, the number of proxies should be Zero - stopping" + ((goterror++)) + fi + + else + if (( ${how_many_running} != ${how_many_proxys} )); then + echo "Wrong number of proxys running (${how_many_running} containers, ${how_many_proxys} proxys) - stopping" + ((goterror++)) + fi fi fi @@ -131,11 +128,13 @@ check_all_running() { ((goterror++)) fi - # check we have the right number of netmon's - how_many_netmons=$(pgrep -a -f ${NETMON_PATH} | wc -l) - if (( ${how_many_running} != ${how_many_netmons} )); then - echo "Wrong number of netmons running (${how_many_running} != ${how_many_netmons}) - stopping" - ((goterror++)) + if [ "$KATA_HYPERVISOR" == "qemu" ]; then + # check we have the right number of netmon's + how_many_netmons=$(pgrep -a -f ${NETMON_PATH} | wc -l) + if (( ${how_many_running} != ${how_many_netmons} )); then + echo "Wrong number of netmons running (${how_many_running} != ${how_many_netmons}) - stopping" + ((goterror++)) + fi fi # check we have no runtimes running (they should be transient, we should not 'see them') From 7779e72959aee7b66b440e3efa6e5838f0635f6b Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Tue, 22 Jan 2019 04:58:35 -0600 Subject: [PATCH 04/34] ci: Modify Firecracker installation As we are installing from the tar, we are getting the static binaries for runtime, shim, proxy, etc. With this change, we will avoid the duplication of installing twice the kata elements. Fixes #1075 Signed-off-by: Gabriela Cervantes --- .ci/install_firecracker.sh | 5 +++++ .ci/install_kata.sh | 11 ++--------- .ci/install_runtime.sh | 14 ++++---------- .ci/setup.sh | 12 +++++++++++- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/.ci/install_firecracker.sh b/.ci/install_firecracker.sh index 2ea9053e8..34812f12a 100755 --- a/.ci/install_firecracker.sh +++ b/.ci/install_firecracker.sh @@ -85,3 +85,8 @@ check_vsock=$(sudo modprobe vhost_vsock) if [ $? != 0 ]; then die "vsock is not supported on your host system" fi + +# FIXME - we need to create a symbolic link for kata-runtime +# in order that kata-runtime kata-env works +# https://github.com/kata-containers/runtime/issues/1144 +ln -s /opt/kata/bin/kata-runtime /usr/local/bin/ diff --git a/.ci/install_kata.sh b/.ci/install_kata.sh index 3238f8fc3..d70b504df 100755 --- a/.ci/install_kata.sh +++ b/.ci/install_kata.sh @@ -13,21 +13,14 @@ cidir=$(dirname "$0") source /etc/os-release || source /usr/lib/os-release source "${cidir}/lib.sh" -KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" - echo "Install kata-containers image" "${cidir}/install_kata_image.sh" echo "Install Kata Containers Kernel" "${cidir}/install_kata_kernel.sh" -if [ "$KATA_HYPERVISOR" == firecracker ]; then - echo "Install Firecracker" - "${cidir}/install_firecracker.sh" -else - echo "Install Qemu" - "${cidir}/install_qemu.sh" -fi +echo "Install Qemu" +"${cidir}/install_qemu.sh" echo "Install shim" "${cidir}/install_shim.sh" diff --git a/.ci/install_runtime.sh b/.ci/install_runtime.sh index 007375c37..8b83097a4 100755 --- a/.ci/install_runtime.sh +++ b/.ci/install_runtime.sh @@ -12,8 +12,6 @@ cidir=$(dirname "$0") source "${cidir}/lib.sh" source /etc/os-release || source /usr/lib/os-release -KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" - # Modify the runtimes build-time defaults # enable verbose build @@ -79,11 +77,7 @@ if [ "$USE_VSOCK" == "yes" ]; then fi fi -if [ "$KATA_HYPERVISOR" == qemu ]; then - echo "Add runtime as a new/default Docker runtime. Docker version \"$(docker --version)\" could change according to updates." - docker_options="-D --add-runtime kata-runtime=/usr/local/bin/kata-runtime" - echo "Add kata-runtime as a new/default Docker runtime." - "${cidir}/../cmd/container-manager/manage_ctr_mgr.sh" docker configure -r kata-runtime -f -else - echo "Kata runtime will not set as a default in Docker" -fi +echo "Add runtime as a new/default Docker runtime. Docker version \"$(docker --version)\" could change according to updates." +docker_options="-D --add-runtime kata-runtime=/usr/local/bin/kata-runtime" +echo "Add kata-runtime as a new/default Docker runtime." +"${cidir}/../cmd/container-manager/manage_ctr_mgr.sh" docker configure -r kata-runtime -f diff --git a/.ci/setup.sh b/.ci/setup.sh index 154906f2c..5f7871b90 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -16,6 +16,7 @@ source "${cidir}/lib.sh" arch=$("${cidir}"/kata-arch.sh -d) INSTALL_KATA="${INSTALL_KATA:-yes}" CI=${CI:-false} +KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" # values indicating whether related intergration tests have been supported CRIO="${CRIO:-yes}" @@ -83,6 +84,11 @@ install_kata() { fi } +install_firecracker() { + echo "Install Firecracker" + bash -f ${cidir}/install_firecracker.sh +} + install_extra_tools() { echo "Install CNI plugins" bash -f "${cidir}/install_cni_plugins.sh" @@ -120,7 +126,11 @@ main() { setup_distro_env install_docker enable_nested_virtualization - install_kata + if [ "$KATA_HYPERVISOR" == "firecracker" ]; then + install_firecracker + else + install_kata + fi install_extra_tools echo "Disable systemd-journald rate limit" sudo crudini --set /etc/systemd/journald.conf Journal RateLimitInterval 0s From 7641321829b6fe7a5b2afe63462749f4aeaabdd8 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Wed, 23 Jan 2019 01:34:58 -0600 Subject: [PATCH 05/34] test: Change runtime name for Firecracker All our tests are looking for the kata-runtime name so we need to modify the name of the runtime for Firecracker in order that the tests run correctly. Fixes #1078 Signed-off-by: Gabriela Cervantes --- .ci/install_firecracker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/install_firecracker.sh b/.ci/install_firecracker.sh index 34812f12a..6de93431e 100755 --- a/.ci/install_firecracker.sh +++ b/.ci/install_firecracker.sh @@ -67,7 +67,7 @@ else cat <<-EOF | sudo tee "$docker_configuration_file" { "runtimes": { - "kata": { + "kata-runtime": { "path": "${path}" } }, From fee6b8b9d7f7ce2a7caf7f4800d4514ca4087758 Mon Sep 17 00:00:00 2001 From: Yang Bo Date: Thu, 24 Jan 2019 14:09:18 +0800 Subject: [PATCH 06/34] kata-manager: Remove directory if git clone fails. Remove directory if git clone fails, otherwise the following installation fails. Fixes: #1084 Signed-off-by: Yang Bo --- cmd/kata-manager/kata-manager.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/kata-manager/kata-manager.sh b/cmd/kata-manager/kata-manager.sh index 5b415f3ef..6d4263dd0 100755 --- a/cmd/kata-manager/kata-manager.sh +++ b/cmd/kata-manager/kata-manager.sh @@ -270,7 +270,7 @@ get_git_repo() fi info "getting repo $1 using git" - git clone "$repo_url" "$local_dest" + git clone "$repo_url" "$local_dest" || (rm -fr "$local_dest" && exit 1) } exec_document() From 5ed1ddbce9992b4d1a53b8209aa4548b519dc1ed Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Thu, 24 Jan 2019 02:50:54 -0600 Subject: [PATCH 07/34] test: Modify run.sh for Firecracker tests This will modify the run.sh in order to run the soak and the oci call test for Firecracker. Fixes #1087 Signed-off-by: Gabriela Cervantes --- .ci/run.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.ci/run.sh b/.ci/run.sh index 5b0c3419a..a822191b6 100755 --- a/.ci/run.sh +++ b/.ci/run.sh @@ -24,6 +24,12 @@ case "${CI_JOB}" in sudo -E PATH="$PATH" bash -c "make cri-containerd" sudo -E PATH="$PATH" CRI_RUNTIME="containerd" bash -c "make kubernetes" ;; + "FIRECRACKER") + echo "INFO: Running soak test" + sudo -E PATH="$PATH" bash -c "make docker-stability" + echo "INFO: Running oci call test" + sudo -E PATH="$PATH" bash -c "make oci" + ;; *) echo "INFO: Running checks" sudo -E PATH="$PATH" bash -c "make check" From 376b03827ad5cfa0ae73b44a67a50c7316021c4c Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Thu, 24 Jan 2019 03:30:08 -0600 Subject: [PATCH 08/34] test: Add missing sudo when doing the symbolic link We are missing a sudo when we are creating a symbolic link for kata-runtime in order that kata-runtime kata-env works Fixes #1090 Signed-off-by: Gabriela Cervantes --- .ci/install_firecracker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/install_firecracker.sh b/.ci/install_firecracker.sh index 6de93431e..b2f18c6f4 100755 --- a/.ci/install_firecracker.sh +++ b/.ci/install_firecracker.sh @@ -89,4 +89,4 @@ fi # FIXME - we need to create a symbolic link for kata-runtime # in order that kata-runtime kata-env works # https://github.com/kata-containers/runtime/issues/1144 -ln -s /opt/kata/bin/kata-runtime /usr/local/bin/ +sudo ln -s /opt/kata/bin/kata-runtime /usr/local/bin/ From 8360e488f6d2155846b9e38e45bf1c5b90b05046 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Thu, 24 Jan 2019 10:32:41 +0000 Subject: [PATCH 09/34] CI: Fix new URL check logic in static check script Fix a big in the static check script `check_docs()` function where regex anchoring was being used with `grep`. That command doesn't understand anchors so it needs to be `egrep` instead. Fixes #1085. Signed-off-by: James O. D. Hunt --- .ci/static-checks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/static-checks.sh b/.ci/static-checks.sh index a280a30ce..2dea07fb7 100755 --- a/.ci/static-checks.sh +++ b/.ci/static-checks.sh @@ -473,7 +473,7 @@ check_docs() if [ "$specific_branch" != "true" ] then # If the URL is new on this PR, it cannot be checked. - echo "$new_urls" | grep -q "\<${url}\>" && \ + echo "$new_urls" | egrep -q "\<${url}\>" && \ info "ignoring new (but correct) URL: $url" && continue fi From 035d545fd059dc012500c9a5602315d937795744 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Thu, 24 Jan 2019 08:47:33 -0600 Subject: [PATCH 10/34] ci: Modify Firecracker to install from repository Instead of installing firecracker, runtime, and all the other Kata components from the tar, this will change to install from the sources. Fixes #1091 Signed-off-by: Gabriela Cervantes --- .ci/install_firecracker.sh | 37 ++++++++++++++++++------------------- .ci/install_kata.sh | 10 ++++++++-- .ci/install_runtime.sh | 19 +++++++++++++++---- .ci/setup.sh | 12 +----------- 4 files changed, 42 insertions(+), 36 deletions(-) diff --git a/.ci/install_firecracker.sh b/.ci/install_firecracker.sh index b2f18c6f4..55714fbc0 100755 --- a/.ci/install_firecracker.sh +++ b/.ci/install_firecracker.sh @@ -11,6 +11,7 @@ set -o pipefail cidir=$(dirname "$0") arch=$("${cidir}"/kata-arch.sh -d) source "${cidir}/lib.sh" +KATA_DEV_MODE="${KATA_DEV_MODE:-false}" if [ "$arch" != "x86_64" ]; then die "Static binaries for Firecracker only available with x86_64." @@ -26,17 +27,23 @@ if [ "$docker_version" != "18.06" ]; then die "Firecracker hypervisor only works with docker 18.06" fi -# This is the initial release of Kata -# Containers that introduces support for -# the Firecracker hypervisor -release_version="1.5.0-rc2" -file_name="kata-fc-static-${release_version}-${arch}.tar.gz" -url="https://github.com/kata-containers/runtime/releases/download/${release_version}/${file_name}" -echo "Get static binaries from release version ${release_version}" -curl -OL ${url} +# Get url for firecracker from runtime/versions.yaml +firecracker_repo=$(get_version "assets.hypervisor.firecracker.url") +[ -n "$firecracker_repo" ] || die "failed to get firecracker repo" +firecracker_repo=${firecracker_repo/https:\/\//} -echo "Decompress binaries from release version ${release_version}" -sudo tar -xvf ${file_name} -C / +# Get version for firecracker from runtime/versions.yaml +firecracker_version=$(get_version "assets.hypervisor.firecracker.version") +[ -n "$firecracker_version" ] || die "failed to get firecracker version" + +# Get firecracker +go get -d ${firecracker_repo} || true +# Checkout to specific version +pushd "${GOPATH}/src/${firecracker_repo}" +git checkout tags/${firecracker_version} +./tools/devtool --unattended build --release -- --features vsock +sudo install ${GOPATH}/src/${firecracker_repo}/build/release/firecracker /usr/bin/ +popd echo "Install and configure docker" docker_configuration_path="/etc/docker" @@ -51,10 +58,7 @@ docker_configuration_file=$docker_configuration_path/daemon.json # is required driver="devicemapper" -# From decompressing the tarball, all the files are placed within -# /opt/kata. The runtime configuration is expected to land at -# /opt/kata/share/defaults/kata-containers/configuration.toml -path="/opt/kata/bin/kata-runtime" +path="/usr/local/bin/kata-runtime" if [ -f $docker_configuration_file ]; then # Check devicemapper flag @@ -85,8 +89,3 @@ check_vsock=$(sudo modprobe vhost_vsock) if [ $? != 0 ]; then die "vsock is not supported on your host system" fi - -# FIXME - we need to create a symbolic link for kata-runtime -# in order that kata-runtime kata-env works -# https://github.com/kata-containers/runtime/issues/1144 -sudo ln -s /opt/kata/bin/kata-runtime /usr/local/bin/ diff --git a/.ci/install_kata.sh b/.ci/install_kata.sh index d70b504df..f464c11d8 100755 --- a/.ci/install_kata.sh +++ b/.ci/install_kata.sh @@ -12,6 +12,7 @@ set -o pipefail cidir=$(dirname "$0") source /etc/os-release || source /usr/lib/os-release source "${cidir}/lib.sh" +KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" echo "Install kata-containers image" "${cidir}/install_kata_image.sh" @@ -19,8 +20,13 @@ echo "Install kata-containers image" echo "Install Kata Containers Kernel" "${cidir}/install_kata_kernel.sh" -echo "Install Qemu" -"${cidir}/install_qemu.sh" +if [ "$KATA_HYPERVISOR" == "firecracker" ]; then + echo "Install Firecracker" + "${cidir}/install_firecracker.sh" +else + echo "Install Qemu" + "${cidir}/install_qemu.sh" +fi echo "Install shim" "${cidir}/install_shim.sh" diff --git a/.ci/install_runtime.sh b/.ci/install_runtime.sh index 8b83097a4..110edce02 100755 --- a/.ci/install_runtime.sh +++ b/.ci/install_runtime.sh @@ -11,6 +11,7 @@ cidir=$(dirname "$0") source "${cidir}/lib.sh" source /etc/os-release || source /usr/lib/os-release +KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" # Modify the runtimes build-time defaults @@ -77,7 +78,17 @@ if [ "$USE_VSOCK" == "yes" ]; then fi fi -echo "Add runtime as a new/default Docker runtime. Docker version \"$(docker --version)\" could change according to updates." -docker_options="-D --add-runtime kata-runtime=/usr/local/bin/kata-runtime" -echo "Add kata-runtime as a new/default Docker runtime." -"${cidir}/../cmd/container-manager/manage_ctr_mgr.sh" docker configure -r kata-runtime -f +if [ "$KATA_HYPERVISOR" == "qemu" ]; then + echo "Add runtime as a new/default Docker runtime. Docker version \"$(docker --version)\" could change according to updates." + docker_options="-D --add-runtime kata-runtime=/usr/local/bin/kata-runtime" + echo "Add kata-runtime as a new/default Docker runtime." + "${cidir}/../cmd/container-manager/manage_ctr_mgr.sh" docker configure -r kata-runtime -f +else + echo "Kata runtime will not set as a default in Docker" +fi + +if [ "$KATA_HYPERVISOR" == "firecracker" ]; then + echo "Enable firecracker configuration.toml" + path="/usr/share/defaults/kata-containers" + sudo mv ${path}/configuration-fc.toml ${path}/configuration.toml +fi diff --git a/.ci/setup.sh b/.ci/setup.sh index 5f7871b90..154906f2c 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -16,7 +16,6 @@ source "${cidir}/lib.sh" arch=$("${cidir}"/kata-arch.sh -d) INSTALL_KATA="${INSTALL_KATA:-yes}" CI=${CI:-false} -KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" # values indicating whether related intergration tests have been supported CRIO="${CRIO:-yes}" @@ -84,11 +83,6 @@ install_kata() { fi } -install_firecracker() { - echo "Install Firecracker" - bash -f ${cidir}/install_firecracker.sh -} - install_extra_tools() { echo "Install CNI plugins" bash -f "${cidir}/install_cni_plugins.sh" @@ -126,11 +120,7 @@ main() { setup_distro_env install_docker enable_nested_virtualization - if [ "$KATA_HYPERVISOR" == "firecracker" ]; then - install_firecracker - else - install_kata - fi + install_kata install_extra_tools echo "Disable systemd-journald rate limit" sudo crudini --set /etc/systemd/journald.conf Journal RateLimitInterval 0s From c70237a3628d960754e5ef1cc39b7884b5a027ed Mon Sep 17 00:00:00 2001 From: Nitesh Konkar Date: Fri, 25 Jan 2019 11:56:30 +0530 Subject: [PATCH 11/34] CI: fix qemu install from build/package Fix qemu installation which is part of the CI irrespective of the installation being from source or package. Fixes: #1093 Signed-off-by: Nitesh Konkar niteshkonkar@in.ibm.com --- .ci/ppc64le/lib_install_qemu_ppc64le.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/ppc64le/lib_install_qemu_ppc64le.sh b/.ci/ppc64le/lib_install_qemu_ppc64le.sh index 4316e1e21..66c332dea 100755 --- a/.ci/ppc64le/lib_install_qemu_ppc64le.sh +++ b/.ci/ppc64le/lib_install_qemu_ppc64le.sh @@ -7,7 +7,8 @@ set -e CURRENT_QEMU_VERSION=$(get_version "assets.hypervisor.qemu.version") -PACKAGED_QEMU="qemu" +PACKAGED_QEMU="qemu-system-ppc" +BUILT_QEMU="qemu-system-ppc64" get_packaged_qemu_version() { if [ "$ID" == "ubuntu" ]; then @@ -66,7 +67,6 @@ build_and_install_qemu() { echo "Install Qemu" sudo -E make install - # Add link from /usr/local/bin to /usr/bin - sudo ln -sf $(command -v qemu-system-${QEMU_ARCH}) "/usr/bin/qemu-system-${QEMU_ARCH}" + sudo ln -sf $(command -v ${BUILT_QEMU}) "/usr/bin/qemu-system-${QEMU_ARCH}" popd } From b21327901f8e037d18701658036cd04cbf09400b Mon Sep 17 00:00:00 2001 From: running Date: Tue, 25 Dec 2018 15:21:16 +0800 Subject: [PATCH 12/34] integration/stability: test for bind-mounted volumes Test if a container stops properly: start a container with a bind mount and set bind propagation to a sub-directory of the mount point. In 1.5, add an umount after clean_env(). Fixes: #1016 Signed-off-by: --- Makefile | 1 + integration/stability/bind_mount_linux.sh | 86 +++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100755 integration/stability/bind_mount_linux.sh diff --git a/Makefile b/Makefile index 4cc7d7a3f..f8e7fc1ef 100644 --- a/Makefile +++ b/Makefile @@ -64,6 +64,7 @@ docker-stability: systemctl is-active --quiet docker || sudo systemctl start docker cd integration/stability && \ export ITERATIONS=2 && export MAX_CONTAINERS=20 && ./soak_parallel_rm.sh + cd integration/stability && ./bind_mount_linux.sh kubernetes: bash -f .ci/install_bats.sh diff --git a/integration/stability/bind_mount_linux.sh b/integration/stability/bind_mount_linux.sh new file mode 100755 index 000000000..006018875 --- /dev/null +++ b/integration/stability/bind_mount_linux.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# +# Copyright (c) 2019 Ning Lu +# +# SPDX-License-Identifier: Apache-2.0 +# +# This test will start a container with a bind mount +# and set bind propagation, the purpose of this +# test is to check if the container stops properly + +set -e + +cidir=$(dirname "$0") +testname="${0##*/}" +sysname=$(uname) + +if [ "${sysname}" != "Linux" ]; then + echo "Skip ${testname} on ${sysname}" + exit 0 +fi + +source "${cidir}/../../lib/common.bash" + +# Environment variables +IMAGE="${IMAGE:-busybox}" +CONTAINER_NAME="${CONTAINER_NAME:-test}" +PAYLOAD_ARGS="${PAYLOAD_ARGS:-tail -f /dev/null}" +TMP_DIR=$(mktemp -d --tmpdir=/tmp ${testname}.XXX) +MOUNT_DIR="${TMP_DIR}/mount" +BIND_DST="${MOUNT_DIR}/dst" +BIND_SRC="${TMP_DIR}/src" +DOCKER_ARGS="-v ${MOUNT_DIR}:${MOUNT_DIR}:rslave" +CONTAINER_ID= + +# Set the runtime if not set already +RUNTIME="${RUNTIME:-kata-runtime}" + +function setup { + clean_env + docker run --runtime=${RUNTIME} -d ${DOCKER_ARGS} --name ${CONTAINER_NAME} ${IMAGE} ${PAYLOAD_ARGS} + CONTAINER_ID=$(docker ps -q -f "name=${CONTAINER_NAME}") +} + +function cmd_bind_mount { + mkdir -p ${BIND_SRC} + mkdir -p ${BIND_DST} + mount --bind ${BIND_SRC} ${BIND_DST} + docker rm -f ${CONTAINER_NAME} + + KATA_PROC=$(ps aux | grep ${CONTAINER_ID} | grep -v grep | tee) +} + +function clean_kata_proc { + kata_pids=$(echo -n "${KATA_PROC}" | awk '{print $2}') + [ -n "${kata_pids}" ] && echo "${kata_pids}" | xargs kill + + kata_mount=$(mount | grep ${CONTAINER_ID} | awk '{print $3}'| sort -r) + [ -n "${kata_mount}" ] && echo "${kata_mount}" | xargs -n1 umount + + rm -rf ${TMP_DIR} +} + +function check { + if [ -n "${KATA_PROC}" ]; then + clean_kata_proc + die "Left kata processes, quitting: ${KATA_PROC}" + fi +} + +function teardown { + clean_env + if mountpoint -q ${BIND_DST}; then + umount ${BIND_DST} + fi + rm -rf ${TMP_DIR} +} + +echo "Starting stability test: ${testname}" +setup + +echo "Running stability test: ${testname}" +cmd_bind_mount +check + +echo "Ending stability test: ${testname}" +teardown From 7fd1fa3c3241be67e21fb776da39d9d5db3d73ed Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Mon, 28 Jan 2019 04:40:14 -0600 Subject: [PATCH 13/34] test: Modify process.go and config.go to handle Firecracker We need to add Firecracker as a hypervisor in order that the docker functional and integration tests work. Fixes #1096 Signed-off-by: Gabriela Cervantes --- config.go | 3 +++ process.go | 20 ++++++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/config.go b/config.go index d960707d3..26585eed8 100644 --- a/config.go +++ b/config.go @@ -70,6 +70,9 @@ const ( // DefaultHypervisor default hypervisor DefaultHypervisor = "qemu" + // FirecrackerHypervisor is firecracker + FirecrackerHypervisor = "firecracker" + // DefaultProxy default proxy DefaultProxy = "kata" diff --git a/process.go b/process.go index f9dc521c8..2d51829d2 100644 --- a/process.go +++ b/process.go @@ -53,17 +53,25 @@ func processRunning(regexps []string) bool { // HypervisorRunning returns true if the hypervisor is still running, otherwise false func HypervisorRunning(containerID string) bool { - hypervisorPath := KataConfig.Hypervisor[DefaultHypervisor].Path - if hypervisorPath == "" { - log.Fatal("Could not determine if hypervisor is running: hypervisor path is empty") - return false + var typeHypervisor = map[string]string{ + DefaultHypervisor: (".*-name.*" + containerID + ".*-qmp.*unix:.*/" + containerID + "/.*"), + FirecrackerHypervisor: (".*--api-sock.*" + containerID + ".*firecracker.sock.*"), + } + for h, r := range typeHypervisor { + config, ok := KataConfig.Hypervisor[h] + if ok { + return processRunning([]string{config.Path + r}) + } } - hypervisorRegexps := []string{hypervisorPath + ".*-name.*" + containerID + ".*-qmp.*unix:.*/" + containerID + "/.*"} - return processRunning(hypervisorRegexps) + log.Fatal("Could not determine if hypervisor is running") + return false } // ProxyRunning returns true if the proxy is still running, otherwise false func ProxyRunning(containerID string) bool { + if _, ok := KataConfig.Hypervisor[FirecrackerHypervisor]; ok { + return false + } proxyPath := KataConfig.Proxy[DefaultProxy].Path if proxyPath == "" { log.Fatal("Could not determine if proxy is running: proxy path is empty") From 06ec9d6dbedb1ee9c4a1eb041e1b6b91f8b1c72a Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Tue, 29 Jan 2019 14:33:24 -0600 Subject: [PATCH 14/34] ci: install parted in debian partprobe command is required to build the image fixes #1098 Signed-off-by: Julio Montes --- .ci/setup_env_debian.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/setup_env_debian.sh b/.ci/setup_env_debian.sh index 3cb44d655..d2915b3c0 100755 --- a/.ci/setup_env_debian.sh +++ b/.ci/setup_env_debian.sh @@ -22,7 +22,7 @@ echo "Install git" sudo -E apt install -y git echo "Install kata containers dependencies" -chronic sudo -E apt install -y libtool automake autotools-dev autoconf bc alien libpixman-1-dev coreutils +chronic sudo -E apt install -y libtool automake autotools-dev autoconf bc alien libpixman-1-dev coreutils parted echo "Install qemu dependencies" chronic sudo -E apt install -y libcap-dev libattr1-dev libcap-ng-dev librbd-dev From f989f942ef0eb2773e86fada6a407bf2152f6f61 Mon Sep 17 00:00:00 2001 From: Graham Whaley Date: Wed, 30 Jan 2019 11:04:45 +0000 Subject: [PATCH 15/34] ci: Add a CODEOWNERS file for github ack checks Add a CODEOWNERS file so we can get github to automatically request reviews. In this instance, specifically the docs team for markdown documents. Fixes: #1102 Signed-off-by: Graham Whaley --- CODEOWNERS | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 CODEOWNERS diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 000000000..e48c094af --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,13 @@ +# Copyright 2019 Intel Corporation. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Define any code owners for this repository. +# The code owners lists are used to help automatically enforce +# reviews and acks of the right groups on the right PRs. + +# Order in this file is important. Only the last match will be +# used. See https://help.github.com/articles/about-code-owners/ + +*.md @kata-containers/documentation + From 01120d39fbf91aace0cf4deb78871989409cc99e Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Wed, 30 Jan 2019 02:52:54 -0600 Subject: [PATCH 16/34] test: Filter docker integration tests for Firecracker Currently not all the docker integration tests are running with Firecracker so with this change we filter the tests. Fixes #1104 Signed-off-by: Gabriela Cervantes --- .../configuration_firecracker.yaml | 44 +++++++++++++++ .../firecracker/filter_docker_firecracker.sh | 53 +++++++++++++++++++ .ci/run.sh | 2 + Makefile | 2 +- 4 files changed, 100 insertions(+), 1 deletion(-) create mode 100755 .ci/hypervisors/firecracker/configuration_firecracker.yaml create mode 100755 .ci/hypervisors/firecracker/filter_docker_firecracker.sh diff --git a/.ci/hypervisors/firecracker/configuration_firecracker.yaml b/.ci/hypervisors/firecracker/configuration_firecracker.yaml new file mode 100755 index 000000000..696dc623b --- /dev/null +++ b/.ci/hypervisors/firecracker/configuration_firecracker.yaml @@ -0,0 +1,44 @@ +# +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +# We need to skip some docker integration tests as they are not +# running correctly using Firecracker. We will skip them using +# the ginkgo flag 'skip=REGEXP'. +test: + - docker +docker: + Describe: + - restart + - docker exec + - capabilities + - package manager update test + - build with docker + - inspect + - docker top + - users and groups + - terminal with docker + - docker commit + - ulimits + - docker cp with volume attached + - load with docker + - docker volume + - docker env + - CPUs and CPU set + - docker exit code + - run container with docker + - run hot plug block devices + - pause with docker + - Update number of CPUs + - docker cp + - docker privileges + - diff + - Hot plug CPUs + - Update CPU constraints + - memory constraints + - Hotplug memory when create containers + - run container and update its memory constraints + Context: + - remove bind-mount source before container exits + It: diff --git a/.ci/hypervisors/firecracker/filter_docker_firecracker.sh b/.ci/hypervisors/firecracker/filter_docker_firecracker.sh new file mode 100755 index 000000000..f76a7428b --- /dev/null +++ b/.ci/hypervisors/firecracker/filter_docker_firecracker.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -e + +cidir=$(dirname "$0") +source "${cidir}/../../lib.sh" + +test_config_file="${cidir}/configuration_firecracker.yaml" + +describe_skip_flag="docker.Describe" +context_skip_flag="docker.Context" +it_skip_flag="docker.It" + +# value for '-skip' in ginkgo +_skip_options=() + +filter_and_build() { + local dependency="$1" + local array_docker=$("${GOPATH}/bin/yq" read "${test_config_file}" "${dependency}") + [ "${array_docker}" = "null" ] && return + mapfile -t _array_docker <<< "${array_docker}" + for entry in "${_array_docker[@]}" + do + _skip_options+=("${entry#- }|") + done +} + +main() { + # Check GOPATH is set + check_gopath + + # Check if yq is installed + [ -z "$(command -v yq)" ] && install_yq + + # Build skip option based on Describe block + filter_and_build "${describe_skip_flag}" + + # Build skip option based on context block + filter_and_build "${context_skip_flag}" + + # Build skip option based on it block + filter_and_build "${it_skip_flag}" + + skip_options=$(IFS= ; echo "${_skip_options[*]}") + + echo "${skip_options%|}" +} + +main diff --git a/.ci/run.sh b/.ci/run.sh index a822191b6..aa642a368 100755 --- a/.ci/run.sh +++ b/.ci/run.sh @@ -25,6 +25,8 @@ case "${CI_JOB}" in sudo -E PATH="$PATH" CRI_RUNTIME="containerd" bash -c "make kubernetes" ;; "FIRECRACKER") + echo "INFO: Running docker integration tests" + sudo -E PATH="$PATH" bash -c "make docker" echo "INFO: Running soak test" sudo -E PATH="$PATH" bash -c "make docker-stability" echo "INFO: Running oci call test" diff --git a/Makefile b/Makefile index f8e7fc1ef..ff9e0dc1a 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ TIMEOUT := 60 UNION := functional docker crio docker-compose network netmon docker-stability oci openshift kubernetes swarm vm-factory entropy ramdisk shimv2 # skipped test suites for docker integration tests -SKIP := +SKIP := $(shell bash -f .ci/hypervisors/$(KATA_HYPERVISOR)/filter_docker_$(KATA_HYPERVISOR).sh) # get arch ARCH := $(shell bash -c '.ci/kata-arch.sh -d') From d779fa986d56868bc158fbd48ca66b78d8896903 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Wed, 30 Jan 2019 06:03:53 -0600 Subject: [PATCH 17/34] test: Add and skip networking tests for Firecracker Add networking tests (ipvlan and macvlan) and skip the disable network test for Firecracker. Fixes #1108 Signed-off-by: Gabriela Cervantes --- .ci/run.sh | 2 ++ integration/network/disable_net/net_none.bats | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/.ci/run.sh b/.ci/run.sh index aa642a368..bfb48a155 100755 --- a/.ci/run.sh +++ b/.ci/run.sh @@ -31,6 +31,8 @@ case "${CI_JOB}" in sudo -E PATH="$PATH" bash -c "make docker-stability" echo "INFO: Running oci call test" sudo -E PATH="$PATH" bash -c "make oci" + echo "INFO: Running networking tests" + sudo -E PATH="$PATH" bash -c "make network" ;; *) echo "INFO: Running checks" diff --git a/integration/network/disable_net/net_none.bats b/integration/network/disable_net/net_none.bats index 273f1feaa..0bfb7a5d2 100644 --- a/integration/network/disable_net/net_none.bats +++ b/integration/network/disable_net/net_none.bats @@ -10,8 +10,11 @@ load "${BATS_TEST_DIRNAME}/../../../lib/common.bash" IMAGE="busybox" PAYLOAD="tail -f /dev/null" NAME="test" +KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" +issue="https://github.com/kata-containers/runtime/issues/1197" setup () { + [ "${KATA_HYPERVISOR}" = "firecracker" ] && skip "test not working see: ${issue}" clean_env # Check that processes are not running @@ -21,6 +24,7 @@ setup () { } @test "Disable_new_netns equal to false" { + [ "${KATA_HYPERVISOR}" = "firecracker" ] && skip "test not working see: ${issue}" extract_kata_env sudo sed -i 's/#disable_new_netns = true/disable_new_netns = false/g' ${RUNTIME_CONFIG_PATH} @@ -47,6 +51,7 @@ setup () { } @test "Disable net" { + [ "${KATA_HYPERVISOR}" = "firecracker" ] && skip "test not working see: ${issue}" extract_kata_env # Get the name of the network name at the configuration.toml @@ -80,6 +85,7 @@ setup () { } teardown() { + [ "${KATA_HYPERVISOR}" = "firecracker" ] && skip "test not working see: ${issue}" clean_env # Check that processes are not running From df2d93119853717549991f1b609293dbf6670dab Mon Sep 17 00:00:00 2001 From: Nitesh Konkar Date: Fri, 1 Feb 2019 17:12:08 +0530 Subject: [PATCH 18/34] ci: Build memory-tested vish/stress image for non-amd64 arch Since vish/stress is a single-arch image for amd64, we need to build it for other architectures like arm64/ppc64le. Fixes: #1113 Signed-off-by: Nitesh Konkar niteshkonkar@in.ibm.com --- integration/docker/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/docker/main_test.go b/integration/docker/main_test.go index 408dd0cfe..6658aff83 100644 --- a/integration/docker/main_test.go +++ b/integration/docker/main_test.go @@ -38,7 +38,7 @@ func TestIntegration(t *testing.T) { for _, i := range images { // vish/stress is single-arch image only for amd64 - if i == StressImage && runtime.GOARCH == "arm64" { + if i == StressImage && runtime.GOARCH != "amd64" { //check if vish/stress has already been built argsImage := []string{"--format", "'{{.Repository}}:{{.Tag}}'", StressImage} imagesStdout, _, imagesExitcode := dockerImages(argsImage...) From 519ae86f7b29ff0c00b4ee2687dd5194a08ae615 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Fri, 1 Feb 2019 11:08:41 -0600 Subject: [PATCH 19/34] ci: qemu: clone capstone and keycodemapdb from github Instead of checking if capstone and keycodemapdb directories exist, installation script should check if those directories are empty then clone those projects from github. Signed-off-by: Julio Montes --- .ci/install_qemu.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/install_qemu.sh b/.ci/install_qemu.sh index 41e2d82e4..1a7a6ed32 100755 --- a/.ci/install_qemu.sh +++ b/.ci/install_qemu.sh @@ -63,8 +63,8 @@ build_and_install_qemu() { pushd "${GOPATH}/src/${QEMU_REPO}" git fetch git checkout "$CURRENT_QEMU_COMMIT" - [ -d "capstone" ] || git clone https://github.com/qemu/capstone.git capstone - [ -d "ui/keycodemapdb" ] || git clone https://github.com/qemu/keycodemapdb.git ui/keycodemapdb + [ -n "$(ls -A capstone)" ] || git clone https://github.com/qemu/capstone.git capstone + [ -n "$(ls -A ui/keycodemapdb)" ] || git clone https://github.com/qemu/keycodemapdb.git ui/keycodemapdb # Apply required patches QEMU_PATCHES_PATH="${GOPATH}/src/${PACKAGING_REPO}/obs-packaging/qemu-lite/patches" From 17b43bd6a7e949fcc3432c8b6fa93acb103fcab5 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Fri, 1 Feb 2019 11:08:19 -0600 Subject: [PATCH 20/34] ci: build and install qemu behind a proxy Use `git` instead of `go get` to clone qemu and its repositories since `go get` downloads the repo and its modules hence it's too slow, this behaviour can fail in systems that run behind a proxy. fixes #1119 Signed-off-by: Julio Montes --- .ci/install_qemu.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.ci/install_qemu.sh b/.ci/install_qemu.sh index 1a7a6ed32..b22fd0851 100755 --- a/.ci/install_qemu.sh +++ b/.ci/install_qemu.sh @@ -11,6 +11,7 @@ cidir=$(dirname "$0") source "${cidir}/lib.sh" source /etc/os-release || source /usr/lib/os-release +CURRENT_QEMU_BRANCH=$(get_version "assets.hypervisor.qemu-lite.branch") CURRENT_QEMU_COMMIT=$(get_version "assets.hypervisor.qemu-lite.commit") PACKAGED_QEMU="qemu-lite" QEMU_ARCH=$(${cidir}/kata-arch.sh -d) @@ -51,13 +52,14 @@ install_packaged_qemu() { } build_and_install_qemu() { - QEMU_REPO=$(get_version "assets.hypervisor.qemu-lite.url") + QEMU_REPO_URL=$(get_version "assets.hypervisor.qemu-lite.url") # Remove 'https://' from the repo url to be able to clone the repo using 'go get' - QEMU_REPO=${QEMU_REPO/https:\/\//} + QEMU_REPO=${QEMU_REPO_URL/https:\/\//} PACKAGING_REPO="github.com/kata-containers/packaging" QEMU_CONFIG_SCRIPT="${GOPATH}/src/${PACKAGING_REPO}/scripts/configure-hypervisor.sh" - go get -d "${QEMU_REPO}" || true + mkdir -p "${GOPATH}/src" + git clone --branch "$CURRENT_QEMU_BRANCH" --single-branch "${QEMU_REPO_URL}" "${GOPATH}/src/${QEMU_REPO}" go get -d "$PACKAGING_REPO" || true pushd "${GOPATH}/src/${QEMU_REPO}" From 3bd0227d416efa44031236840f259f1b18f85d43 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Fri, 1 Feb 2019 13:42:29 -0600 Subject: [PATCH 21/34] integration/docker: don't exceed the actual number of CPUs in cpuset tests Change cpuset tests dinamically according to the actual number of CPUs fixes #1123 Signed-off-by: Julio Montes --- integration/docker/cpu_test.go | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/integration/docker/cpu_test.go b/integration/docker/cpu_test.go index fe7ca8d6c..b4bb9823f 100644 --- a/integration/docker/cpu_test.go +++ b/integration/docker/cpu_test.go @@ -7,6 +7,7 @@ package docker import ( "fmt" "math" + "runtime" "strings" . "github.com/kata-containers/tests" @@ -312,7 +313,13 @@ func withCPUConstraintCheckPeriodAndQuota(cpus float64, fail bool) TableEntry { return Entry(fmt.Sprintf("quota/period should be equal to %.1f", cpus), cpus, fail) } -func withCPUSetConstraint(cpuset string, fail bool) TableEntry { +func withCPUSetConstraint(cpuset string, minCpusNeeded int, fail bool) TableEntry { + // test should fail when the actual number of cpus is less than the minimum number + // of cpus needed to run the test, for example cpuset=0-2 requires 3 cpus(0,1,2) + if runtime.NumCPU() < minCpusNeeded { + fail = true + } + return Entry(fmt.Sprintf("cpuset should be equal to %s", cpuset), cpuset, fail) } @@ -371,7 +378,9 @@ var _ = Describe("Update CPU constraints", func() { DescribeTable("Update CPU set", func(cpuset string, fail bool) { - runArgs = []string{"--rm", "--cpus=4", "--name", id, "-dt", DebianImage, "bash"} + // Use the actual number of CPUs + runArgs = []string{"--rm", fmt.Sprintf("--cpus=%d", runtime.NumCPU()), + "--name", id, "-dt", DebianImage, "bash"} _, _, exitCode = dockerRun(runArgs...) Expect(exitCode).To(BeZero()) @@ -388,15 +397,15 @@ var _ = Describe("Update CPU constraints", func() { Expect(exitCode).To(BeZero()) Expect(cpuset).To(Equal(strings.Trim(stdout, "\n\t "))) }, - withCPUSetConstraint("0", shouldNotFail), - withCPUSetConstraint("2", shouldNotFail), - withCPUSetConstraint("0-1", shouldNotFail), - withCPUSetConstraint("0-2", shouldNotFail), - withCPUSetConstraint("0-3", shouldNotFail), - withCPUSetConstraint("0,2", shouldNotFail), - withCPUSetConstraint("0,3", shouldNotFail), - withCPUSetConstraint("0,-2,3", shouldFail), - withCPUSetConstraint("-1-3", shouldFail), + withCPUSetConstraint("0", 1, shouldNotFail), + withCPUSetConstraint("2", 3, shouldNotFail), + withCPUSetConstraint("0-1", 2, shouldNotFail), + withCPUSetConstraint("0-2", 3, shouldNotFail), + withCPUSetConstraint("0-3", 4, shouldNotFail), + withCPUSetConstraint("0,2", 3, shouldNotFail), + withCPUSetConstraint("0,3", 4, shouldNotFail), + withCPUSetConstraint("0,-2,3", 0, shouldFail), + withCPUSetConstraint("-1-3", 0, shouldFail), ) }) From 43e297a9934f5bafa9606ddfb94b7ed268a2f216 Mon Sep 17 00:00:00 2001 From: Nitesh Konkar Date: Mon, 4 Feb 2019 22:52:31 +0530 Subject: [PATCH 22/34] CI: add filter scheme for CI tests on ppc64le Not all tests are supported on ppc64le. Use the filter scheme to filter out test cases that are not meant for ppc64le. Fixes: #1131 Signed-off-by: Nitesh Konkar niteshkonkar@in.ibm.com --- .ci/ppc64le/configuration_ppc64le.yaml | 24 ++++++++++++ .ci/ppc64le/filter_docker_ppc64le.sh | 54 ++++++++++++++++++++++++++ .ci/ppc64le/filter_test_ppc64le.sh | 36 +++++++++++++++++ arch/ppc64le-options.mk | 10 +++++ 4 files changed, 124 insertions(+) create mode 100644 .ci/ppc64le/configuration_ppc64le.yaml create mode 100755 .ci/ppc64le/filter_docker_ppc64le.sh create mode 100755 .ci/ppc64le/filter_test_ppc64le.sh create mode 100644 arch/ppc64le-options.mk diff --git a/.ci/ppc64le/configuration_ppc64le.yaml b/.ci/ppc64le/configuration_ppc64le.yaml new file mode 100644 index 000000000..a5a5f202b --- /dev/null +++ b/.ci/ppc64le/configuration_ppc64le.yaml @@ -0,0 +1,24 @@ +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 + +# for now, not all integration test suites are fully passed in ppc64le. +# some need to be tested, and some need to be refined. +# sequence of 'test' holds supported integration tests components. +test: + - functional + - docker + - docker-compose + +# for now, not all test suites under docker integration are fully passed in aarch64. +# some need to be tested, and some need to be refined. +# ginkgo offers '-skip=REGEXP' flag to skip specific ones. +# you can use infos from docker.Describe, docker.Context or docker.It to point to +# specific test specs or whole container of specs. +docker: + Describe: + - Update CPU set + Context: + It: + - should have the right number of vCPUs diff --git a/.ci/ppc64le/filter_docker_ppc64le.sh b/.ci/ppc64le/filter_docker_ppc64le.sh new file mode 100755 index 000000000..8531dd516 --- /dev/null +++ b/.ci/ppc64le/filter_docker_ppc64le.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 + +set -e + +GOPATH_LOCAL="${GOPATH%%:*}" +kata_dir="${GOPATH_LOCAL}/src/github.com/kata-containers" +test_dir="${kata_dir}/tests" +ci_dir="${test_dir}/.ci" +test_config_file="${ci_dir}/ppc64le/configuration_ppc64le.yaml" + +describe_skip_flag="docker.Describe" +context_skip_flag="docker.Context" +it_skip_flag="docker.It" + +# value for '-skip' in ginkgo +_skip_options=() + +source "${ci_dir}/lib.sh" + +filter_and_build() +{ + local dependency="$1" + local array_docker=$("${GOPATH_LOCAL}/bin/yq" read "${test_config_file}" "${dependency}") + [ "${array_docker}" = "null" ] && return + mapfile -t _array_docker <<< "${array_docker}" + for entry in "${_array_docker[@]}" + do + _skip_options+=("${entry#- }|") + done +} + +main() +{ + # install yq if not exist + [ -z "$(command -v yq)" ] && install_yq + # build skip option based on Describe block + filter_and_build "${describe_skip_flag}" + + # build skip option based on context block + filter_and_build "${context_skip_flag}" + + # build skip option based on it block + filter_and_build "${it_skip_flag}" + + skip_options=$(IFS= ; echo "${_skip_options[*]}") + + echo "${skip_options%|}" +} + +main diff --git a/.ci/ppc64le/filter_test_ppc64le.sh b/.ci/ppc64le/filter_test_ppc64le.sh new file mode 100755 index 000000000..b068b3c8f --- /dev/null +++ b/.ci/ppc64le/filter_test_ppc64le.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 + +set -e + +GOPATH_LOCAL="${GOPATH%%:*}" +kata_dir="${GOPATH_LOCAL}/src/github.com/kata-containers" +test_dir="${kata_dir}/tests" +ci_dir="${test_dir}/.ci" +test_config_file="${ci_dir}/ppc64le/configuration_ppc64le.yaml" + +test_filter_flag="test" + +_test_union=() + +source "${ci_dir}/lib.sh" + +main() +{ + # install yq if not exist + [ -z "$(command -v yq)" ] && install_yq + local array_test=$("${GOPATH_LOCAL}/bin/yq" read "${test_config_file}" "${test_filter_flag}") + [ "${array_test}" = "null" ] && return + mapfile -t _array_test <<< "${array_test}" + for entry in "${_array_test[@]}" + do + _test_union+=("${entry#- }") + done + test_union=$(IFS=" "; echo "${_test_union[*]}") + echo "${test_union}" +} + +main diff --git a/arch/ppc64le-options.mk b/arch/ppc64le-options.mk new file mode 100644 index 000000000..805cc7284 --- /dev/null +++ b/arch/ppc64le-options.mk @@ -0,0 +1,10 @@ +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 + +# union for 'make test' +UNION := $(shell bash -f .ci/ppc64le/filter_test_ppc64le.sh) + +# skiped test suites for docker integration tests +SKIP := $(shell bash -f .ci/ppc64le/filter_docker_ppc64le.sh) From 57990ffc20c08da4a09db92b3ec2a3441a4861ec Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Tue, 5 Feb 2019 11:31:34 +0000 Subject: [PATCH 23/34] CI: Set GOPATH once The `.ci/lib.sh` script was setting `GOPATH` *three times*. Set it once in the correct manner so that all scripts which source `.ci/lib.sh` will have `GOPATH` set. By setting `GOPATH` at the top of the script, this also resolves the current CI error we're seeing where `GOPATH` is being referenced before it's being set: ```sh /home/centos/tests/.ci/lib.sh: line 19: GOPATH: unbound variable ``` Fixes #1133. Signed-off-by: James O. D. Hunt --- .../firecracker/filter_docker_firecracker.sh | 3 --- .ci/lib.sh | 12 +++--------- .ci/run.sh | 2 -- .ci/setup.sh | 1 - 4 files changed, 3 insertions(+), 15 deletions(-) diff --git a/.ci/hypervisors/firecracker/filter_docker_firecracker.sh b/.ci/hypervisors/firecracker/filter_docker_firecracker.sh index f76a7428b..6fc4221b6 100755 --- a/.ci/hypervisors/firecracker/filter_docker_firecracker.sh +++ b/.ci/hypervisors/firecracker/filter_docker_firecracker.sh @@ -30,9 +30,6 @@ filter_and_build() { } main() { - # Check GOPATH is set - check_gopath - # Check if yq is installed [ -z "$(command -v yq)" ] && install_yq diff --git a/.ci/lib.sh b/.ci/lib.sh index 05533dcca..c1761dcfd 100755 --- a/.ci/lib.sh +++ b/.ci/lib.sh @@ -15,6 +15,9 @@ export KATA_RUNTIME=${KATA_RUNTIME:-kata-runtime} # more formats). export KATA_DOCKER_TIMEOUT=30 +# Ensure GOPATH set +export GOPATH=${GOPATH:-$(go env GOPATH)} + tests_repo="${tests_repo:-github.com/kata-containers/tests}" lib_script="${GOPATH}/src/${tests_repo}/lib/common.bash" source "${lib_script}" @@ -86,7 +89,6 @@ function build_and_install() { } function install_yq() { - GOPATH=${GOPATH:-${HOME}/go} local yq_path="${GOPATH}/bin/yq" local yq_pkg="github.com/mikefarah/yq" [ -x "${GOPATH}/bin/yq" ] && return @@ -153,7 +155,6 @@ function get_dep_from_yaml_db(){ function get_version(){ dependency="$1" - GOPATH=${GOPATH:-${HOME}/go} runtime_repo="github.com/kata-containers/runtime" runtime_repo_dir="$GOPATH/src/${runtime_repo}" versions_file="${runtime_repo_dir}/versions.yaml" @@ -177,13 +178,6 @@ function get_test_version(){ get_dep_from_yaml_db "${db}" "${dependency}" } -function check_gopath() { - # Verify GOPATH is set - if [ -z "$GOPATH" ]; then - export GOPATH=$(go env GOPATH) - fi -} - function waitForProcess(){ wait_time="$1" sleep_time="$2" diff --git a/.ci/run.sh b/.ci/run.sh index bfb48a155..f98aa46ff 100755 --- a/.ci/run.sh +++ b/.ci/run.sh @@ -12,8 +12,6 @@ set -e cidir=$(dirname "$0") source "${cidir}/lib.sh" -check_gopath - export RUNTIME="kata-runtime" export CI_JOB="${CI_JOB:-default}" diff --git a/.ci/setup.sh b/.ci/setup.sh index 154906f2c..3c769cec8 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -116,7 +116,6 @@ install_extra_tools() { } main() { - check_gopath setup_distro_env install_docker enable_nested_virtualization From b2a1243070b8ccc684ebfc7f2aded4aad4644196 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Tue, 5 Feb 2019 15:01:22 -0600 Subject: [PATCH 24/34] functional: add cgroup path add cgroupsPath to test cgroup path creation in functional tests fixes #1140 Signed-off-by: Julio Montes --- data/config.json | 1 + 1 file changed, 1 insertion(+) diff --git a/data/config.json b/data/config.json index 167d42ee5..c8e2ebd0c 100644 --- a/data/config.json +++ b/data/config.json @@ -122,6 +122,7 @@ } ] }, + "cgroupsPath": "kata", "namespaces": [ { "type": "pid" From 35a8ef88ce5c16b4eb930250e6c29732efefee47 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Wed, 6 Feb 2019 11:16:46 +0000 Subject: [PATCH 25/34] kata-manager: Update for Debian, SuSE and RHEL Support Debian, OpenSuSE, SLES and RHEL. Fixes #1142. Signed-off-by: James O. D. Hunt --- cmd/kata-manager/kata-manager.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cmd/kata-manager/kata-manager.sh b/cmd/kata-manager/kata-manager.sh index 6d4263dd0..b9515ffb0 100755 --- a/cmd/kata-manager/kata-manager.sh +++ b/cmd/kata-manager/kata-manager.sh @@ -349,25 +349,24 @@ cmd_remove_packages() info "removing packages" case "$distro" in - centos|fedora) + centos|fedora|opensuse|rhel|sles) packages=$(rpm -qa|egrep "${packages_regex}" || true) ;; - ubuntu) + debian|ubuntu) packages=$(dpkg-query -W -f='${Package}\n'|egrep "${packages_regex}" || true) ;; - *) - die "invalid distro: '$distro'" - ;; + *) die "invalid distro: '$distro'" ;; esac [ -z "$packages" ] && die "packages not installed" case "$distro" in - centos) sudo yum -y remove $packages ;; + centos|rhel) sudo yum -y remove $packages ;; + debian|ubuntu) sudo apt-get -y remove $packages ;; fedora) sudo dnf -y remove $packages ;; - ubuntu) sudo apt-get -y remove $packages ;; + opensuse|sles) sudo zypper remove -y $packages ;; esac } From b92ef7aa0bb455bc83bdc7356cab322da426512b Mon Sep 17 00:00:00 2001 From: jschintag Date: Tue, 29 Jan 2019 15:38:30 +0100 Subject: [PATCH 26/34] CI: Added s390x support for CI scripts Adds support for s390x to the CI scripts. Hot Plug CPUs is currently not working on s390x and was disabled in the Test-Configuration Fixes: #1129 Signed-off-by: Jan Schintag --- .ci/install_go.sh | 5 +- .ci/install_qemu.sh | 44 ++++++++++-------- .ci/s390x/clean_up_s390x.sh | 11 +++++ .ci/s390x/configuration_s390x.yaml | 22 +++++++++ .ci/s390x/filter_docker_s390x.sh | 51 +++++++++++++++++++++ .ci/s390x/filter_test_s390x.sh | 33 ++++++++++++++ .ci/s390x/lib_install_qemu_s390x.sh | 63 ++++++++++++++++++++++++++ .ci/s390x/lib_kata_image_s390x.sh | 23 ++++++++++ .ci/s390x/lib_setup_s390x.sh | 24 ++++++++++ .ci/setup.sh | 12 +++-- arch/s390x-options.mk | 10 ++++ integration/docker/docker.go | 3 -- integration/docker/docker_arch_base.go | 12 +++++ integration/docker/docker_s390x.go | 10 ++++ 14 files changed, 295 insertions(+), 28 deletions(-) create mode 100644 .ci/s390x/clean_up_s390x.sh create mode 100644 .ci/s390x/configuration_s390x.yaml create mode 100755 .ci/s390x/filter_docker_s390x.sh create mode 100755 .ci/s390x/filter_test_s390x.sh create mode 100755 .ci/s390x/lib_install_qemu_s390x.sh create mode 100644 .ci/s390x/lib_kata_image_s390x.sh create mode 100755 .ci/s390x/lib_setup_s390x.sh create mode 100755 arch/s390x-options.mk create mode 100644 integration/docker/docker_arch_base.go create mode 100644 integration/docker/docker_s390x.go diff --git a/.ci/install_go.sh b/.ci/install_go.sh index a5bac2224..2c990b374 100755 --- a/.ci/install_go.sh +++ b/.ci/install_go.sh @@ -108,7 +108,10 @@ case "$(arch)" in "ppc64le") goarch=ppc64le ;; - + + "s390x") + goarch=s390x + ;; "*") die "Arch $(arch) not supported" ;; diff --git a/.ci/install_qemu.sh b/.ci/install_qemu.sh index b22fd0851..b6a501c90 100755 --- a/.ci/install_qemu.sh +++ b/.ci/install_qemu.sh @@ -93,25 +93,31 @@ if [ -f "${cidir}/${QEMU_ARCH}/lib_install_qemu_${QEMU_ARCH}.sh" ]; then fi main() { - if [ "$QEMU_ARCH" == "x86_64" ]; then - packaged_qemu_commit=$(get_packaged_qemu_commit) - short_current_qemu_commit=${CURRENT_QEMU_COMMIT:0:10} - if [ "$packaged_qemu_commit" == "$short_current_qemu_commit" ]; then - # If installing packaged qemu from OBS fails, - # then build and install it from sources. - install_packaged_qemu || build_and_install_qemu - else - build_and_install_qemu - fi - elif [ "$QEMU_ARCH" == "aarch64" ] || [ "$QEMU_ARCH" == "ppc64le" ]; then - packaged_qemu_version=$(get_packaged_qemu_version) - short_current_qemu_version=${CURRENT_QEMU_VERSION#*-} - if [ "$packaged_qemu_version" == "$short_current_qemu_version" ] && [ -z "${CURRENT_QEMU_COMMIT}" ]; then - install_packaged_qemu || build_and_install_qemu - else - build_and_install_qemu - fi - fi + case "$QEMU_ARCH" in + "x86_64") + packaged_qemu_commit=$(get_packaged_qemu_commit) + short_current_qemu_commit=${CURRENT_QEMU_COMMIT:0:10} + if [ "$packaged_qemu_commit" == "$short_current_qemu_commit" ]; then + # If installing packaged qemu from OBS fails, + # then build and install it from sources. + install_packaged_qemu || build_and_install_qemu + else + build_and_install_qemu + fi + ;; + "aarch64"|"ppc64le"|"s390x") + packaged_qemu_version=$(get_packaged_qemu_version) + short_current_qemu_version=${CURRENT_QEMU_VERSION#*-} + if [ "$packaged_qemu_version" == "$short_current_qemu_version" ] && [ -z "${CURRENT_QEMU_COMMIT}" ] || [ "${QEMU_ARCH}" == "s390x" ]; then + install_packaged_qemu || build_and_install_qemu + else + build_and_install_qemu + fi + ;; + *) + die "Architecture $QEMU_ARCH not supported" + ;; + esac } main diff --git a/.ci/s390x/clean_up_s390x.sh b/.ci/s390x/clean_up_s390x.sh new file mode 100644 index 000000000..98954c2bd --- /dev/null +++ b/.ci/s390x/clean_up_s390x.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 +# + +lib_script="${GOPATH}/src/${tests_repo}/.ci/lib.sh" +source "${lib_script}" + +gen_clean_arch || info "Arch cleanup scripts failed" diff --git a/.ci/s390x/configuration_s390x.yaml b/.ci/s390x/configuration_s390x.yaml new file mode 100644 index 000000000..13a215204 --- /dev/null +++ b/.ci/s390x/configuration_s390x.yaml @@ -0,0 +1,22 @@ +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 +# +test: + - functional + - docker + +# Same model as is for aarch64 +docker: + Describe: + - CPUs and CPU set + - Update number of CPUs + - Hot plug CPUs + - Update CPU constraints + - Hotplug memory + - update memory constraints + Context: + - remove bind-mount source before container exits + - run container exceeding memory constraints + It: diff --git a/.ci/s390x/filter_docker_s390x.sh b/.ci/s390x/filter_docker_s390x.sh new file mode 100755 index 000000000..d664def3b --- /dev/null +++ b/.ci/s390x/filter_docker_s390x.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e + +GOPATH_LOCAL="${GOPATH%%:*}" +kata_dir="${GOPATH_LOCAL}/src/github.com/kata-containers" +test_dir="${kata_dir}/tests" +ci_dir="${test_dir}/.ci" +test_config_file="${ci_dir}/s390x/configuration_s390x.yaml" + +describe_skip_flag="docker.Describe" +context_skip_flag="docker.Context" +it_skip_flag="docker.It" + +# value for '-skip' in ginkgo +_skip_options=() + +filter_and_build() +{ + local dependency="$1" + local array_docker=$("${GOPATH_LOCAL}/bin/yq" read "${test_config_file}" "${dependency}") + [ "${array_docker}" = "null" ] && return + mapfile -t _array_docker <<< "${array_docker}" + for entry in "${_array_docker[@]}" + do + _skip_options+=("${entry#- }|") + done +} + +main() +{ + # build skip option based on Describe block + filter_and_build "${describe_skip_flag}" + + # build skip option based on context block + filter_and_build "${context_skip_flag}" + + # build skip option based on it block + filter_and_build "${it_skip_flag}" + + skip_options=$(IFS= ; echo "${_skip_options[*]}") + + echo "${skip_options%|}" +} + +main diff --git a/.ci/s390x/filter_test_s390x.sh b/.ci/s390x/filter_test_s390x.sh new file mode 100755 index 000000000..4e5458ede --- /dev/null +++ b/.ci/s390x/filter_test_s390x.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e + +GOPATH_LOCAL="${GOPATH%%:*}" +kata_dir="${GOPATH_LOCAL}/src/github.com/kata-containers" +test_dir="${kata_dir}/tests" +ci_dir="${test_dir}/.ci" +test_config_file="${ci_dir}/s390x/configuration_s390x.yaml" + +test_filter_flag="test" + +_test_union=() + +main() +{ + local array_test=$("${GOPATH_LOCAL}/bin/yq" read "${test_config_file}" "${test_filter_flag}") + [ "${array_test}" = "null" ] && return + mapfile -t _array_test <<< "${array_test}" + for entry in "${_array_test[@]}" + do + _test_union+=("${entry#- }") + done + test_union=$(IFS=" "; echo "${_test_union[*]}") + echo "${test_union}" +} + +main diff --git a/.ci/s390x/lib_install_qemu_s390x.sh b/.ci/s390x/lib_install_qemu_s390x.sh new file mode 100755 index 000000000..c29833f95 --- /dev/null +++ b/.ci/s390x/lib_install_qemu_s390x.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e + +CURRENT_QEMU_VERSION=$(get_version "assets.hypervisor.qemu.version") +PACKAGED_QEMU="qemu" + +[ "$ID" == "ubuntu" ] || die "Unsupported distro: $ID" + +get_packaged_qemu_version() { + if [ "$ID" == "ubuntu" ]; then + sudo apt-get update > /dev/null + qemu_version=$(apt-cache madison $PACKAGED_QEMU \ + | awk '{print $3}' | cut -d':' -f2 | cut -d'+' -f1 | head -n 1 ) + fi + + if [ -z "$qemu_version" ]; then + die "unknown qemu version" + else + echo "${qemu_version}" + fi +} + +install_packaged_qemu() { + sudo apt install -y "$PACKAGED_QEMU" +} + +build_and_install_qemu() { + QEMU_REPO=$(get_version "assets.hypervisor.qemu.url") + # Remove 'https://' from the repo url to be able to clone the repo using 'go get' + QEMU_REPO_PATH=${QEMU_REPO/https:\/\//} + + PACKAGING_REPO="github.com/kata-containers/packaging" + QEMU_CONFIG_SCRIPT="${GOPATH}/src/${PACKAGING_REPO}/scripts/configure-hypervisor.sh" + + if [ ! -d "${GOPATH}/src/${QEMU_REPO_PATH}" ]; then + mkdir -p "${GOPATH}/src/${QEMU_REPO_PATH}" + pushd "${GOPATH}/src/${QEMU_REPO_PATH}" + chronic git clone "${QEMU_REPO}" "." + popd + fi + + go get -d "$PACKAGING_REPO" || true + + pushd "${GOPATH}/src/${QEMU_REPO_PATH}" + git fetch + git checkout "$CURRENT_QEMU_VERSION" + [ -d "capstone" ] || git clone https://github.com/qemu/capstone.git capstone + [ -d "ui/keycodemapdb" ] || git clone https://github.com/qemu/keycodemapdb.git ui/keycodemapdb + + echo "Build Qemu" + "${QEMU_CONFIG_SCRIPT}" "qemu" | xargs ./configure + make -j $(nproc) + + echo "Install Qemu" + sudo -E make install + popd +} diff --git a/.ci/s390x/lib_kata_image_s390x.sh b/.ci/s390x/lib_kata_image_s390x.sh new file mode 100644 index 000000000..b536deea3 --- /dev/null +++ b/.ci/s390x/lib_kata_image_s390x.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e + +OSBUILDER_DISTRO="ubuntu" +AGENT_INIT="yes" + +#packaged kata agent haven't been supported in any mainstream distribution +get_packaged_agent_version() { + version="" + echo "$version" +} + +#packaged kata image haven't been supported in any mainstream distribution +install_packaged_image() { + info "installing packaged kata-image not supported in s390x" + return 1 +} diff --git a/.ci/s390x/lib_setup_s390x.sh b/.ci/s390x/lib_setup_s390x.sh new file mode 100755 index 000000000..a75323f8b --- /dev/null +++ b/.ci/s390x/lib_setup_s390x.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e + +filter_test_script="${cidir}/${arch}/filter_test_s390x.sh" + +check_test_union() +{ + local test_union=$(bash -f ${filter_test_script}) + flag="$1" + # regex match + [[ ${test_union} =~ ${flag} ]] && echo "true" + + echo "false" +} + +CRIO=$(check_test_union crio) +KUBERNETES=$(check_test_union kubernetes) +OPENSHIFT=$(check_test_union openshift) diff --git a/.ci/setup.sh b/.ci/setup.sh index 3c769cec8..aca8c61d1 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -57,12 +57,14 @@ install_docker() { enable_nested_virtualization() { case "$arch" in - x86_64) - if [ "$CI" == true ] && grep -q "N" /sys/module/kvm_intel/parameters/nested 2>/dev/null; then + x86_64 | s390x) + kvm_arch="kvm" + [ ${arch} == "x86_64" ] && kvm_arch="kvm_intel" + if [ "$CI" == true ] && grep -q "N" /sys/module/$kvm_arch/parameters/nested 2>/dev/null; then echo "enable Nested Virtualization" - sudo modprobe -r kvm_intel - sudo modprobe kvm_intel nested=1 - if grep -q "N" /sys/module/kvm_intel/parameters/nested 2>/dev/null; then + sudo modprobe -r $kvm_arch + sudo modprobe $kvm_arch nested=1 + if grep -q "N" /sys/module/$kvm_arch/parameters/nested 2>/dev/null; then die "Failed to find or enable Nested virtualization" fi fi diff --git a/arch/s390x-options.mk b/arch/s390x-options.mk new file mode 100755 index 000000000..04640c31e --- /dev/null +++ b/arch/s390x-options.mk @@ -0,0 +1,10 @@ +# +# Copyright (c) 2019 IBM +# +# SPDX-License-Identifier: Apache-2.0 + +# union for 'make test' +UNION := $(shell bash -f .ci/s390x/filter_test_s390x.sh) + +# skiped test suites for docker integration tests +SKIP := $(shell bash -f .ci/s390x/filter_docker_s390x.sh) diff --git a/integration/docker/docker.go b/integration/docker/docker.go index 4cee5a933..19e1292ea 100644 --- a/integration/docker/docker.go +++ b/integration/docker/docker.go @@ -36,9 +36,6 @@ const ( // FedoraImage is the fedora image FedoraImage = "fedora" - // CentosImage is the centos image - CentosImage = "centos" - // StressImage is the vish/stress image StressImage = "vish/stress" diff --git a/integration/docker/docker_arch_base.go b/integration/docker/docker_arch_base.go new file mode 100644 index 000000000..1efcc042a --- /dev/null +++ b/integration/docker/docker_arch_base.go @@ -0,0 +1,12 @@ +// +build !s390x + +// Copyright (c) 2019 IBM +// +// SPDX-License-Identifier: Apache-2.0 + +package docker + +const ( + // CentosImage is the centos image + CentosImage = "centos" +) diff --git a/integration/docker/docker_s390x.go b/integration/docker/docker_s390x.go new file mode 100644 index 000000000..4f0344082 --- /dev/null +++ b/integration/docker/docker_s390x.go @@ -0,0 +1,10 @@ +// Copyright (c) 2019 IBM +// +// SPDX-License-Identifier: Apache-2.0 + +package docker + +const ( + // On s390x there is not centos image, instead the clefos image could be used + CentosImage = "clefos" +) From c3528bc0e3868a858854a8eb21f7ca5aa7d2c7aa Mon Sep 17 00:00:00 2001 From: Salvador Fuentes Date: Wed, 23 Jan 2019 17:11:11 -0600 Subject: [PATCH 27/34] tests: Add kubernetes e2e conformance test script Add script that runs the heptio/sonobuoy e2e conformance tests. Fixes: #1081 Related: https://github.com/kata-containers/runtime/issues/1108 Signed-off-by: Salvador Fuentes --- integration/kubernetes/e2e_conformance/run.sh | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100755 integration/kubernetes/e2e_conformance/run.sh diff --git a/integration/kubernetes/e2e_conformance/run.sh b/integration/kubernetes/e2e_conformance/run.sh new file mode 100755 index 000000000..abb8d8910 --- /dev/null +++ b/integration/kubernetes/e2e_conformance/run.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# This script runs the Sonobuoy e2e Conformance tests. +# Run this script once your K8s cluster is running. +# WARNING: it is prefered to use containerd as the +# runtime interface instead of cri-o as we have seen +# errors with cri-o that still need to be debugged. + +set -o errexit +set -o nounset +set -o pipefail + +export KUBECONFIG=$HOME/.kube/config +SCRIPT_PATH=$(dirname "$(readlink -f "$0")") +source "${SCRIPT_PATH}/../../../lib/common.bash" + +RUNTIME="${RUNTIME:-kata-runtime}" + +# Check if Sonobuoy is still running every 5 minutes. +WAIT_TIME=300 + +# Add a global timeout of 2 hours to stop the execution +# in case Sonobuoy gets hanged. +GLOBAL_TIMEOUT=$((WAIT_TIME*24)) + +create_kata_webhook() { + pushd "${SCRIPT_PATH}/../../../kata-webhook" >> /dev/null + # Create certificates for the kata webhook + ./create-certs.sh + + # Apply kata-webhook deployment + kubectl apply -f deploy/ + popd +} + +run_sonobuoy() { + sonobuoy_repo="github.com/heptio/sonobuoy" + go get -u "$sonobuoy_repo" + + # Run Sonobuoy e2e tests + info "Starting sonobuoy execution." + info "When using kata as k8s runtime, the tests take around 2 hours to finish." + sonobuoy run + + start_time=$(date +%s) + estimated_end_time=$((start_time + GLOBAL_TIMEOUT)) + + # Wait for the sonobuoy pod to be running. + kubectl wait --for condition=Ready pod sonobuoy -n heptio-sonobuoy + + while sonobuoy status | grep -Eq "running|pending" && [ "$(date +%s)" -le "$estimated_end_time" ]; do + info "sonobuoy still running, sleeping $WAIT_TIME seconds" + sleep "$WAIT_TIME" + done + + # Retrieve results + e2e_result_dir="$(mktemp -d /tmp/kata_e2e_results.XXXXX)" + sonobuoy retrieve "$e2e_result_dir" || \ + die "Couldn't retrieve sonobuoy results, please check status using: sonobuoy status" + pushd "$e2e_result_dir" >> /dev/null + + # Uncompress results + ls | grep tar.gz | xargs tar -xvf + e2e_result_log="${e2e_result_dir}/plugins/e2e/results/e2e.log" + info "Results of the e2e tests can be found on: $e2e_result_log" + + # If on CI, display the e2e log on the console. + [ "$CI" == true ] && cat "$e2e_result_log" + + # Check for Success message on the logs. + grep -aq " 0 Failed" "$e2e_result_log" + grep -aq "SUCCESS" "$e2e_result_log" && \ + info " k8s e2e conformance using Kata runtime finished successfully" + popd +} + +cleanup() { + # Remove sonobuoy execution pods + sonobuoy delete + info "Results directory $e2e_result_dir will not be deleted" +} + +main() { + if [ "$RUNTIME" == "kata-runtime" ]; then + create_kata_webhook + fi + run_sonobuoy + cleanup +} + +main From 9ceaa7e430e8a3a6e7e61f525bb76c33dc832285 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Thu, 7 Feb 2019 01:56:40 -0600 Subject: [PATCH 28/34] test: Check KATA_HYPERVISOR variable at the Makefile Check the KATA_HYPERVISOR variable to avoid the message of `ci/hypervisors//filter_docker.sh: No such file`. Fixes #1158 Signed-off-by: Gabriela Cervantes --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ff9e0dc1a..06265ef9e 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,10 @@ TIMEOUT := 60 UNION := functional docker crio docker-compose network netmon docker-stability oci openshift kubernetes swarm vm-factory entropy ramdisk shimv2 # skipped test suites for docker integration tests -SKIP := $(shell bash -f .ci/hypervisors/$(KATA_HYPERVISOR)/filter_docker_$(KATA_HYPERVISOR).sh) +FILTER_FILE = .ci/hypervisors/$(KATA_HYPERVISOR)/filter_docker_$(KATA_HYPERVISOR).sh +ifneq ($(wildcard $(FILTER_FILE)),) + SKIP := $(shell bash -f $(FILTER_FILE)) +endif # get arch ARCH := $(shell bash -c '.ci/kata-arch.sh -d') From ca7affe9ce9bf4b7877068314d9086bb3e2ac59d Mon Sep 17 00:00:00 2001 From: Tuan Hoang Date: Tue, 12 Feb 2019 13:15:56 +0100 Subject: [PATCH 29/34] ci: add exec bit for s390x scripts These were missing in commit 76cf07e7572287e97fff0b001c3850821c5c06fa Fixes #1169 Signed-off-by: Tuan Hoang --- .ci/s390x/clean_up_s390x.sh | 0 .ci/s390x/lib_kata_image_s390x.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .ci/s390x/clean_up_s390x.sh mode change 100644 => 100755 .ci/s390x/lib_kata_image_s390x.sh diff --git a/.ci/s390x/clean_up_s390x.sh b/.ci/s390x/clean_up_s390x.sh old mode 100644 new mode 100755 diff --git a/.ci/s390x/lib_kata_image_s390x.sh b/.ci/s390x/lib_kata_image_s390x.sh old mode 100644 new mode 100755 From 251f451a05e64135de9ce5cc882f1a55abc170fa Mon Sep 17 00:00:00 2001 From: Salvador Fuentes Date: Tue, 12 Feb 2019 13:06:31 -0600 Subject: [PATCH 30/34] kubeadm: do not hardcode k8s version on config file Instead of hardcoding the k8s version that we are going to use in the kubeadm config file, lets get it from the versions.yaml file. Fixes: #1171. Signed-off-by: Salvador Fuentes --- integration/kubernetes/init.sh | 1 + integration/kubernetes/kubeadm/config.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/integration/kubernetes/init.sh b/integration/kubernetes/init.sh index 0814bdd23..3df60803e 100755 --- a/integration/kubernetes/init.sh +++ b/integration/kubernetes/init.sh @@ -50,6 +50,7 @@ kubeadm_config_template="${SCRIPT_PATH}/kubeadm/config.yaml" kubeadm_config_file="$(mktemp --tmpdir kubeadm_config.XXXXXX.yaml)" sed -e "s|CRI_RUNTIME_SOCKET|${cri_runtime_socket}|" "${kubeadm_config_template}" > "${kubeadm_config_file}" +sed -i "s|KUBERNETES_VERSION|v${kubernetes_version/-*}|" "${kubeadm_config_file}" if [ "${use_runtime_class}" == true ]; then echo "Add RuntimeClass feature for apiserver in kubeadm config file" diff --git a/integration/kubernetes/kubeadm/config.yaml b/integration/kubernetes/kubeadm/config.yaml index b90559dfd..0a7e0c5aa 100644 --- a/integration/kubernetes/kubeadm/config.yaml +++ b/integration/kubernetes/kubeadm/config.yaml @@ -5,7 +5,7 @@ nodeRegistration: --- apiVersion: kubeadm.k8s.io/v1alpha3 kind: ClusterConfiguration -kubernetesVersion: v1.12.0 +kubernetesVersion: KUBERNETES_VERSION networking: dnsDomain: cluster.local podSubnet: 10.244.0.0/16 From 13d9d57d56dfcbf6a9794926f2ae7ce0a85cac90 Mon Sep 17 00:00:00 2001 From: Penny Zheng Date: Fri, 15 Feb 2019 13:08:09 +0800 Subject: [PATCH 31/34] clean-up: fix 'target busy' error when umounting on aarch64 when umounting directory ../rootfs/ which has a few sub-directories mounted, we will face 'target busy' error. So we need to add '-R' flag to umount recursively. Fixes: #1190 Signed-off-by: Penny Zheng --- .ci/lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/lib.sh b/.ci/lib.sh index c1761dcfd..60cb770b8 100755 --- a/.ci/lib.sh +++ b/.ci/lib.sh @@ -220,7 +220,7 @@ delete_stale_docker_resource() local mount_point_union=$(mount | grep "${stale_docker_mount_point}" | awk '{print $3}') if [ -n "${mount_point_union}" ]; then while IFS='$\n' read mount_point; do - sudo umount "${mount_point}" + [ -n "$(grep "${mount_point}" "/proc/mounts")" ] && sudo umount -R "${mount_point}" done <<< "${mount_point_union}" fi done From 4089f398015f17386f2bbf9fc68ee1df663d6d4e Mon Sep 17 00:00:00 2001 From: Tuan Hoang Date: Wed, 20 Feb 2019 13:33:01 +0100 Subject: [PATCH 32/34] CI: correct memory test suite name on s390x The correct name should be 'memory constraints'. Fixes #1205 Signed-off-by: Tuan Hoang --- .ci/s390x/configuration_s390x.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/s390x/configuration_s390x.yaml b/.ci/s390x/configuration_s390x.yaml index 13a215204..4c04e675b 100644 --- a/.ci/s390x/configuration_s390x.yaml +++ b/.ci/s390x/configuration_s390x.yaml @@ -15,7 +15,7 @@ docker: - Hot plug CPUs - Update CPU constraints - Hotplug memory - - update memory constraints + - memory constraints Context: - remove bind-mount source before container exits - run container exceeding memory constraints From 34a15fd759b8687010e1e1f1b48b99a65b8f830b Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Tue, 26 Feb 2019 06:29:14 -0600 Subject: [PATCH 33/34] test: Skip memory constraints K8s test Currently the issue https://github.com/kata-containers/runtime/issues/1249 is blocking to run properly the memory constraints test. Fixes #1244 Signed-off-by: Gabriela Cervantes --- integration/kubernetes/k8s-memory.bats | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/integration/kubernetes/k8s-memory.bats b/integration/kubernetes/k8s-memory.bats index 9551be144..d5cd847de 100644 --- a/integration/kubernetes/k8s-memory.bats +++ b/integration/kubernetes/k8s-memory.bats @@ -8,9 +8,10 @@ load "${BATS_TEST_DIRNAME}/../../.ci/lib.sh" TEST_INITRD="${TEST_INITRD:-no}" issue="https://github.com/kata-containers/runtime/issues/1127" +memory_issue="https://github.com/kata-containers/runtime/issues/1249" setup() { - [ "${TEST_INITRD}" == "yes" ] && skip "test not working see: ${issue}" + skip "test not working see: ${issue}, ${memory_issue}" export KUBECONFIG=/etc/kubernetes/admin.conf pod_name="memory-test" @@ -23,7 +24,7 @@ setup() { } @test "Exceeding memory constraints" { - [ "${TEST_INITRD}" == "yes" ] && skip "test not working see: ${issue}" + skip "test not working see: ${issue}, ${memory_issue}" memory_limit_size="50Mi" allocated_size="250M" @@ -41,7 +42,7 @@ setup() { } @test "Running within memory constraints" { - [ "${TEST_INITRD}" == "yes" ] && skip "test not working see: ${issue}" + skip "test not working see: ${issue}, ${memory_issue}" memory_limit_size="200Mi" allocated_size="100M" From 1468206891a3437341c1f9271a3443456a0c320f Mon Sep 17 00:00:00 2001 From: Salvador Fuentes Date: Mon, 25 Feb 2019 17:00:54 -0600 Subject: [PATCH 34/34] k8s: skip Quality of Service tests These tests have been failing because a cgroup gets out of memory. For details, see: https://github.com/kata-containers/runtime/issues/1249 Skip them until we figure out what to do with this issue. Fixes: #1239. Signed-off-by: Salvador Fuentes --- integration/kubernetes/k8s-qos-pods.bats | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/integration/kubernetes/k8s-qos-pods.bats b/integration/kubernetes/k8s-qos-pods.bats index 4bfba7c03..092529b25 100644 --- a/integration/kubernetes/k8s-qos-pods.bats +++ b/integration/kubernetes/k8s-qos-pods.bats @@ -8,9 +8,10 @@ load "${BATS_TEST_DIRNAME}/../../.ci/lib.sh" TEST_INITRD="${TEST_INITRD:-no}" issue="https://github.com/kata-containers/runtime/issues/1127" +memory_issue="https://github.com/kata-containers/runtime/issues/1249" setup() { - [ "${TEST_INITRD}" == "yes" ] && skip "test not working see: ${issue}" + skip "test not working see: ${issue}, ${memory_issue}" export KUBECONFIG=/etc/kubernetes/admin.conf if sudo -E kubectl get runtimeclass | grep kata; then @@ -21,7 +22,7 @@ setup() { } @test "Guaranteed QoS" { - [ "${TEST_INITRD}" == "yes" ] && skip "test not working see: ${issue}" + skip "test not working see: ${issue}, ${memory_issue}" pod_name="qos-test" @@ -36,7 +37,7 @@ setup() { } @test "Burstable QoS" { - [ "${TEST_INITRD}" == "yes" ] && skip "test not working see: ${issue}" + skip "test not working see: ${issue}, ${memory_issue}" pod_name="burstable-test" @@ -51,7 +52,7 @@ setup() { } @test "BestEffort QoS" { - [ "${TEST_INITRD}" == "yes" ] && skip "test not working see: ${issue}" + skip "test not working see: ${issue}, ${memory_issue}" pod_name="besteffort-test" # Create pod @@ -65,6 +66,6 @@ setup() { } teardown() { - [ "${TEST_INITRD}" == "yes" ] && skip "test not working see: ${issue}" - sudo -E kubectl delete pod "$pod_name" + skip "test not working see: ${issue}, ${memory_issue}" + kubectl delete pod "$pod_name" }