Merge branch 'master' into disable_tld_per_domain

.semaphore/semaphore.yml

@@ -34,6 +34,7 @@ blocks:
           - docker run --rm --privileged multiarch/qemu-user-static:4.2.0-7 --reset -p yes
           - export PATH="$PATH:$HOME/.local/bin"
           - pip install --user .[ci]
+          - curl -Lo $HOME/.local/bin/kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64 && chmod +x $HOME/.local/bin/kind
           - tox -e codestyle,test
           # Build docker image
           - docker login --username "${DOCKER_USERNAME}" --password-stdin <<< "${DOCKER_PASSWORD}"

fiaas_deploy_daemon/crd/watcher.py

@@ -31,6 +31,7 @@
 from ..base_thread import DaemonThread
 from ..deployer import DeployerEvent
 from ..log_extras import set_extras
+from ..retry import retry_on_upsert_conflict
 from ..specs.factory import InvalidConfiguration
 
 LOG = logging.getLogger(__name__)
@@ -68,6 +69,7 @@ def create_custom_resource_definitions(cls):
         cls._create("ApplicationStatus", "application-statuses", ("status", "appstatus", "fs"), "fiaas.schibsted.io")
 
     @staticmethod
+    @retry_on_upsert_conflict
     def _create(kind, plural, short_names, group):
         name = "%s.%s" % (plural, group)
         metadata = ObjectMeta(name=name)

setup.py

@@ -68,7 +68,6 @@ def read(filename):
     'pytest == 3.10.1',
     'requests-file == 1.4.3',
     'callee == 0.3',
-    'docker == 4.0.2'
 ]
 
 DEV_TOOLS = [

tests/conftest.py

@@ -18,7 +18,7 @@
 import os
 import re
 import subprocess
-import uuid
+import uuid as uuidlib
 
 import pytest
 from xdist.scheduler import LoadScopeScheduling
@@ -28,6 +28,10 @@
 pytest_plugins = ['helpers_namespace']
 
 
+def uuid():
+    return str(uuidlib.uuid4())[:8]
+
+
 @pytest.fixture(autouse=True)
 def prometheus_registry():
     from prometheus_client.core import REGISTRY
@@ -157,23 +161,19 @@ def pytest_addoption(parser):
 @pytest.fixture(scope="session")
 def use_docker_for_e2e(request):
     def dockerize(test_request, cert_path, service_type, k8s_version, port, apiserver_ip):
-        container_name = "fdd_{}_{}_{}".format(service_type, k8s_version, str(uuid.uuid4()))
+        container_name = "fdd_{}_{}_{}".format(service_type, k8s_version, uuid())
         test_request.addfinalizer(lambda: subprocess.call(["docker", "stop", container_name]))
         args = [
             "docker", "run",
             "-i", "--rm",
             "-e", "NAMESPACE",
             "--name", container_name,
+            "--network=kind",
             "--publish", "{port}:{port}".format(port=port),
             "--mount", "type=bind,src={},dst={},ro".format(cert_path, cert_path),
             # make `kubernetes` resolve to the apiserver's IP to make it possible to validate its TLS cert
             "--add-host", "kubernetes:{}".format(apiserver_ip),
         ]
-        if not _is_macos():
-            # Linux needs host networking to make the fiaas-deploy-daemon port available on localhost when running it
-            # in a container. To do the same thing on Docker for mac it is enough to use --publish, and enabling host
-            # networking will make it impossible to connect to the port.
-            args += ["--network", "host"]
         return args + ["fiaas/fiaas-deploy-daemon:latest"]
 
     if request.config.getoption(DOCKER_FOR_E2E_OPTION):

tests/fiaas_deploy_daemon/conftest.py

@@ -191,11 +191,11 @@ def _mock_namespace_file_open(name, *args, **kwargs):
 
 @pytest.fixture(scope="session", params=(
     "v1.15.12",
-    "v1.16.13",
-    "v1.18.6",
-    "v1.19.4",
+    "v1.16.15",
+    "v1.18.19",
+    "v1.19.11",
     "v1.20.7",
-    pytest.param("v1.21.1", marks=pytest.mark.e2e_latest)
+    pytest.param("v1.21.2", marks=pytest.mark.e2e_latest)
 ))
 def k8s_version(request):
     yield request.param

tests/fiaas_deploy_daemon/test_bootstrap_e2e.py

@@ -14,13 +14,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import contextlib
 import os
 import os.path
 import subprocess
 import sys
-import uuid
-from datetime import datetime
 
 import pytest
 from k8s import config
@@ -36,7 +33,7 @@
 from fiaas_deploy_daemon.tools import merge_dicts
 from utils import wait_until, crd_available, crd_supported, \
     skip_if_crd_not_supported, read_yml, sanitize_resource_name, assert_k8s_resource_matches, get_unbound_port, \
-    KindWrapper
+    KindWrapper, uuid
 
 PATIENCE = 30
 TIMEOUT = 5
@@ -87,7 +84,7 @@ class TestBootstrapE2E(object):
     @pytest.fixture(scope="module")
     def kubernetes(self, k8s_version):
         try:
-            name = "_".join(("bootstrap", k8s_version, str(uuid.uuid4())))
+            name = 'kind-bootstrap-{}-{}'.format(k8s_version, uuid())
             kind = KindWrapper(k8s_version, name)
             try:
                 yield kind.start()
@@ -97,18 +94,6 @@ def kubernetes(self, k8s_version):
             msg = "Unable to run kind: %s"
             pytest.fail(msg % str(e))
 
-    @pytest.fixture
-    def kind_logger(self, kubernetes):
-        @contextlib.contextmanager
-        def wrapped():
-            start_time = datetime.now()
-            try:
-                yield
-            finally:
-                kubernetes["log_dumper"](since=start_time, until=datetime.now())
-
-        return wrapped
-
     @pytest.fixture(autouse=True)
     def k8s_client(self, kubernetes):
         Client.clear_session()
@@ -148,38 +133,37 @@ def custom_resource_definition_test_case(self, fiaas_path, namespace, labels, ex
         spec = FiaasApplicationSpec(application=name, image=IMAGE, config=fiaas_yml)
         return name, FiaasApplication(metadata=metadata, spec=spec), expected
 
-    def test_bootstrap_crd(self, request, kubernetes, k8s_version, use_docker_for_e2e, kind_logger):
-        with kind_logger():
-            skip_if_crd_not_supported(k8s_version)
+    def test_bootstrap_crd(self, request, kubernetes, k8s_version, use_docker_for_e2e):
+        skip_if_crd_not_supported(k8s_version)
 
-            CrdWatcher.create_custom_resource_definitions()
-            wait_until(crd_available(kubernetes, timeout=TIMEOUT), "CRD available", RuntimeError, patience=PATIENCE)
+        CrdWatcher.create_custom_resource_definitions()
+        wait_until(crd_available(kubernetes, timeout=TIMEOUT), "CRD available", RuntimeError, patience=PATIENCE)
 
-            def prepare_test_case(test_case):
-                name, fiaas_application, expected = self.custom_resource_definition_test_case(*test_case)
+        def prepare_test_case(test_case):
+            name, fiaas_application, expected = self.custom_resource_definition_test_case(*test_case)
 
-                ensure_resources_not_exists(name, expected, fiaas_application.metadata.namespace)
+            ensure_resources_not_exists(name, expected, fiaas_application.metadata.namespace)
 
-                fiaas_application.save()
+            fiaas_application.save()
 
-                return name, fiaas_application.metadata.namespace, fiaas_application.metadata.uid, expected
+            return name, fiaas_application.metadata.namespace, fiaas_application.metadata.uid, expected
 
-            expectations = [prepare_test_case(test_case) for test_case in TEST_CASES]
+        expectations = [prepare_test_case(test_case) for test_case in TEST_CASES]
 
-            exit_code = self.run_bootstrap(request, kubernetes, k8s_version, use_docker_for_e2e)
-            assert exit_code == 0
+        exit_code = self.run_bootstrap(request, kubernetes, k8s_version, use_docker_for_e2e)
+        assert exit_code == 0
 
-            def success():
-                all(deploy_successful(name, namespace, app_uid, expected) for name, namespace, app_uid, expected in expectations)
+        def success():
+            all(deploy_successful(name, namespace, app_uid, expected) for name, namespace, app_uid, expected in expectations)
 
-            wait_until(success, "CRD bootstrapping was successful", patience=PATIENCE)
+        wait_until(success, "CRD bootstrapping was successful", patience=PATIENCE)
 
-            for name, namespace, app_uid, expected in expectations:
-                for kind in expected.keys():
-                    try:
-                        kind.delete(name, namespace=namespace)
-                    except NotFound:
-                        pass  # already missing
+        for name, namespace, app_uid, expected in expectations:
+            for kind in expected.keys():
+                try:
+                    kind.delete(name, namespace=namespace)
+                except NotFound:
+                    pass  # already missing
 
 
 def ensure_resources_not_exists(name, expected, namespace):