Parallel local tests

tests/integration/local/common_utils.py

@@ -1,12 +1,16 @@
 # Common utils between local tests
 import logging
+import os
 import random
 import time
 
 LOG = logging.getLogger(__name__)
 
 START_WAIT_TIME_SECONDS = 300
 
+PYTEST_WORKER_COUNT = int(os.environ.get("PYTEST_XDIST_WORKER_COUNT", 4))
+PYTEST_WORKER_ID = os.environ.get("PYTEST_XDIST_WORKER", 0)
+
 
 class InvalidAddressException(Exception):
     pass
@@ -34,5 +38,18 @@ def wait_for_local_process(process, port, collect_output=False) -> str:
     return output
 
 
+def get_pytest_worker_id():
+    try:
+        return int(PYTEST_WORKER_ID[2:])
+    except TypeError:
+        return 0
+
+
 def random_port():
-    return random.randint(30000, 40000)
+    start_port = 30000
+    end_port = 40000
+
+    port_window = (end_port - start_port) / PYTEST_WORKER_COUNT
+    port_worker_start = int(start_port + (get_pytest_worker_id() * port_window))
+    port_worker_end = int(port_worker_start + port_window)
+    return random.randint(port_worker_start, port_worker_end)

tests/integration/local/start_api/test_start_api.py

@@ -2126,6 +2126,13 @@ def test_swagger_got_parsed_and_api_is_reachable_and_payload_version_is_2(self):
 
 
 class TestWarmContainersBaseClass(StartApiIntegBaseClass):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.mode_env_variable = str(uuid.uuid4())
+        cls.parameter_overrides = {"ModeEnvVariable": cls.mode_env_variable}
+        super().setUpClass()
+
     def setUp(self):
         self.url = "http://127.0.0.1:{}".format(self.port)
 
@@ -2147,8 +2154,6 @@ def count_running_containers(self):
 )
 class TestWarmContainers(TestWarmContainersBaseClass):
     container_mode = ContainersInitializationMode.EAGER.value
-    mode_env_variable = str(uuid.uuid4())
-    parameter_overrides = {"ModeEnvVariable": mode_env_variable}
 
     @pytest.mark.flaky(reruns=3)
     @pytest.mark.timeout(timeout=600, method="thread")
@@ -2167,8 +2172,6 @@ def test_can_invoke_lambda_function_successfully(self):
 )
 class TestWarmContainersInitialization(TestWarmContainersBaseClass):
     container_mode = ContainersInitializationMode.EAGER.value
-    mode_env_variable = str(uuid.uuid4())
-    parameter_overrides = {"ModeEnvVariable": mode_env_variable}
 
     @pytest.mark.flaky(reruns=3)
     @pytest.mark.timeout(timeout=600, method="thread")
@@ -2187,8 +2190,6 @@ def test_all_containers_are_initialized_before_any_invoke(self):
 )
 class TestWarmContainersMultipleInvoke(TestWarmContainersBaseClass):
     container_mode = ContainersInitializationMode.EAGER.value
-    mode_env_variable = str(uuid.uuid4())
-    parameter_overrides = {"ModeEnvVariable": mode_env_variable}
 
     @pytest.mark.flaky(reruns=3)
     @pytest.mark.timeout(timeout=600, method="thread")
@@ -2210,8 +2211,6 @@ def test_no_new_created_containers_after_lambda_function_invoke(self):
 )
 class TestLazyContainers(TestWarmContainersBaseClass):
     container_mode = ContainersInitializationMode.LAZY.value
-    mode_env_variable = str(uuid.uuid4())
-    parameter_overrides = {"ModeEnvVariable": mode_env_variable}
 
     @pytest.mark.flaky(reruns=3)
     @pytest.mark.timeout(timeout=600, method="thread")
@@ -2230,8 +2229,6 @@ def test_can_invoke_lambda_function_successfully(self):
 )
 class TestLazyContainersInitialization(TestWarmContainersBaseClass):
     container_mode = ContainersInitializationMode.LAZY.value
-    mode_env_variable = str(uuid.uuid4())
-    parameter_overrides = {"ModeEnvVariable": mode_env_variable}
 
     @pytest.mark.flaky(reruns=3)
     @pytest.mark.timeout(timeout=600, method="thread")
@@ -2251,8 +2248,6 @@ def test_no_container_is_initialized_before_any_invoke(self):
 )
 class TestLazyContainersMultipleInvoke(TestWarmContainersBaseClass):
     container_mode = ContainersInitializationMode.LAZY.value
-    mode_env_variable = str(uuid.uuid4())
-    parameter_overrides = {"ModeEnvVariable": mode_env_variable}
 
     @pytest.mark.flaky(reruns=3)
     @pytest.mark.timeout(timeout=600, method="thread")
@@ -3149,8 +3144,6 @@ def tearDownClass(self) -> None:
 class TestWarmContainersRemoteLayers(WarmContainersWithRemoteLayersBase):
     template_path = "/testdata/start_api/template-warm-containers-layers.yaml"
     container_mode = ContainersInitializationMode.EAGER.value
-    mode_env_variable = str(uuid.uuid4())
-    parameter_overrides = {"ModeEnvVariable": mode_env_variable}
 
     @pytest.mark.flaky(reruns=3)
     @pytest.mark.timeout(timeout=600, method="thread")
@@ -3170,8 +3163,6 @@ def test_can_invoke_lambda_layer_successfully(self):
 class TestWarmContainersRemoteLayersLazyInvoke(WarmContainersWithRemoteLayersBase):
     template_path = "/testdata/start_api/template-warm-containers-layers.yaml"
     container_mode = ContainersInitializationMode.LAZY.value
-    mode_env_variable = str(uuid.uuid4())
-    parameter_overrides = {"ModeEnvVariable": mode_env_variable}
 
     @pytest.mark.flaky(reruns=3)
     @pytest.mark.timeout(timeout=600, method="thread")

tests/integration/local/start_lambda/start_lambda_api_integ_base.py

@@ -1,5 +1,7 @@
 import shutil
+import time
 import uuid
+from shutil import rmtree
 from typing import Optional, Dict, List
 from unittest import TestCase, skipIf
 import threading
@@ -11,6 +13,7 @@
 import docker
 from docker.errors import APIError
 
+from samcli.lib.utils.osutils import copytree
 from tests.integration.local.common_utils import random_port, InvalidAddressException, wait_for_local_process
 from tests.testing_utils import (
     SKIP_DOCKER_TESTS,
@@ -45,6 +48,12 @@ class StartLambdaIntegBaseClass(TestCase):
     def setUpClass(cls):
         # This is the directory for tests/integration which will be used to file the testdata
         # files for integ tests
+        scratch_dir = Path(__file__).resolve().parent.joinpath(".tmp", str(uuid.uuid4()).replace("-", "")[:10], "testdata")
+        shutil.rmtree(scratch_dir, ignore_errors=True)
+        os.makedirs(scratch_dir)
+        copytree(str(Path(cls.integration_dir).joinpath("testdata")), str(scratch_dir))
+        cls.integration_dir = str(scratch_dir.parent)
+
         cls.template = cls.integration_dir + cls.template_path
         cls.working_dir = str(Path(cls.template).resolve().parents[0])
         cls.env_var_path = cls.integration_dir + "/testdata/invoke/vars.json"
@@ -91,6 +100,8 @@ def start_lambda_with_retry(cls, retries=3, input=None, env=None):
         if retry_count == retries:
             raise ValueError("Ran out of retries attempting to start lambda")
 
+        time.sleep(20)
+
     @classmethod
     def get_start_lambda_command(
         cls,
@@ -179,6 +190,7 @@ def tearDownClass(cls):
         # After all the tests run, we need to kill the start_lambda process.
         cls.stop_reading_thread = True
         kill_process(cls.start_lambda_process)
+        rmtree(cls.integration_dir)
 
 
 class WatchWarmContainersIntegBaseClass(StartLambdaIntegBaseClass):
@@ -190,16 +202,15 @@ class WatchWarmContainersIntegBaseClass(StartLambdaIntegBaseClass):
     @classmethod
     def setUpClass(cls):
         cls.temp_path = str(uuid.uuid4()).replace("-", "")[:10]
-        working_dir = str(Path(cls.integration_dir).resolve().joinpath(cls.temp_path))
+        working_dir = Path(cls.integration_dir).resolve().joinpath("testdata", ".tmp", cls.temp_path)
         if Path(working_dir).resolve().exists():
             shutil.rmtree(working_dir, ignore_errors=True)
-        os.mkdir(working_dir)
-        os.mkdir(Path(cls.integration_dir).resolve().joinpath(cls.temp_path).joinpath("dir"))
-        cls.template_path = f"/{cls.temp_path}/template.yaml"
-        cls.code_path = f"/{cls.temp_path}/main.py"
-        cls.code_path2 = f"/{cls.temp_path}/dir/main2.py"
-        cls.docker_file_path = f"/{cls.temp_path}/Dockerfile"
-        cls.docker_file_path2 = f"/{cls.temp_path}/Dockerfile2"
+        os.makedirs(working_dir.joinpath("dir"))
+        cls.template_path = f"/testdata/.tmp/{cls.temp_path}/template.yaml"
+        cls.code_path = f"/testdata/.tmp/{cls.temp_path}/main.py"
+        cls.code_path2 = f"/testdata/.tmp/{cls.temp_path}/dir/main2.py"
+        cls.docker_file_path = f"/testdata/.tmp/{cls.temp_path}/Dockerfile"
+        cls.docker_file_path2 = f"/testdata/.tmp/{cls.temp_path}/Dockerfile2"
 
         if cls.template_content:
             cls._write_file_content(cls.template_path, cls.template_content)

tests/integration/local/start_lambda/test_start_lambda.py

@@ -1,7 +1,10 @@
+import logging
 import uuid
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from time import time, sleep
 import json
+
+import docker
 from parameterized import parameterized, parameterized_class
 
 import pytest
@@ -15,6 +18,8 @@
 from samcli.commands.local.cli_common.invoke_context import ContainersInitializationMode
 from .start_lambda_api_integ_base import StartLambdaIntegBaseClass, WatchWarmContainersIntegBaseClass
 
+LOG = logging.getLogger(__name__)
+
 
 class TestParallelRequests(StartLambdaIntegBaseClass):
     template_path = "/testdata/invoke/template.yml"
@@ -296,6 +301,13 @@ def test_invoke_with_function_timeout(self, use_full_path):
 
 
 class TestWarmContainersBaseClass(StartLambdaIntegBaseClass):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.mode_env_variable = str(uuid.uuid4())
+        cls.parameter_overrides = {"ModeEnvVariable": cls.mode_env_variable}
+        super().setUpClass()
+
     def setUp(self):
         self.url = "http://127.0.0.1:{}".format(self.port)
         self.lambda_client = boto3.client(
@@ -310,9 +322,18 @@ def setUp(self):
     def count_running_containers(self):
         running_containers = 0
         for container in self.docker_client.containers.list():
-            _, output = container.exec_run(["bash", "-c", "'printenv'"])
-            if f"MODE={self.mode_env_variable}" in str(output):
+            if container.attrs.get("State", {}).get("Status", "") == "Running" and \
+                    container.attrs.get("Config", {}).get("Env", {}).get("MODE", "") == self.mode_env_variable:
                 running_containers += 1
+            # try:
+            #     _, output = container.exec_run(["bash", "-c", "'printenv'"])
+            #     if f"MODE={self.mode_env_variable}" in str(output):
+            #         running_containers += 1
+            # except (docker.errors.NotFound, docker.errors.APIError) as ex:
+            #     # running tests in parallel might might cause this issue since this container in the loop
+            #     # might be created by other tests and might be at the removal step now
+            #     LOG.error("Failed to ping container %s for test %s", container.id, self.id(), exc_info=ex)
+            #     pass
         return running_containers
 
 
@@ -325,8 +346,6 @@ def count_running_containers(self):
 )
 class TestWarmContainers(TestWarmContainersBaseClass):
     container_mode = ContainersInitializationMode.EAGER.value
-    mode_env_variable = str(uuid.uuid4())
-    parameter_overrides = {"ModeEnvVariable": mode_env_variable}
 
     @pytest.mark.flaky(reruns=3)
     @pytest.mark.timeout(timeout=600, method="thread")
@@ -348,8 +367,6 @@ def test_can_invoke_lambda_function_successfully(self):
 )
 class TestWarmContainersInitialization(TestWarmContainersBaseClass):
     container_mode = ContainersInitializationMode.EAGER.value
-    mode_env_variable = str(uuid.uuid4())
-    parameter_overrides = {"ModeEnvVariable": mode_env_variable}
 
     @pytest.mark.flaky(reruns=3)
     @pytest.mark.timeout(timeout=600, method="thread")
@@ -368,8 +385,6 @@ def test_all_containers_are_initialized_before_any_invoke(self):
 )
 class TestWarmContainersMultipleInvoke(TestWarmContainersBaseClass):
     container_mode = ContainersInitializationMode.EAGER.value
-    mode_env_variable = str(uuid.uuid4())
-    parameter_overrides = {"ModeEnvVariable": mode_env_variable}
 
     @pytest.mark.flaky(reruns=3)
     @pytest.mark.timeout(timeout=600, method="thread")
@@ -391,8 +406,6 @@ def test_no_new_created_containers_after_lambda_function_invoke(self):
 )
 class TestLazyContainers(TestWarmContainersBaseClass):
     container_mode = ContainersInitializationMode.LAZY.value
-    mode_env_variable = str(uuid.uuid4())
-    parameter_overrides = {"ModeEnvVariable": mode_env_variable}
 
     @pytest.mark.flaky(reruns=3)
     @pytest.mark.timeout(timeout=600, method="thread")
@@ -414,8 +427,6 @@ def test_can_invoke_lambda_function_successfully(self):
 )
 class TestLazyContainersInitialization(TestWarmContainersBaseClass):
     container_mode = ContainersInitializationMode.LAZY.value
-    mode_env_variable = str(uuid.uuid4())
-    parameter_overrides = {"ModeEnvVariable": mode_env_variable}
 
     @pytest.mark.flaky(reruns=3)
     @pytest.mark.timeout(timeout=600, method="thread")
@@ -435,8 +446,6 @@ def test_no_container_is_initialized_before_any_invoke(self):
 )
 class TestLazyContainersMultipleInvoke(TestWarmContainersBaseClass):
     container_mode = ContainersInitializationMode.LAZY.value
-    mode_env_variable = str(uuid.uuid4())
-    parameter_overrides = {"ModeEnvVariable": mode_env_variable}
 
     @pytest.mark.flaky(reruns=3)
     @pytest.mark.timeout(timeout=600, method="thread")