diff --git a/tests/integration/local/common_utils.py b/tests/integration/local/common_utils.py index 0dbb23aa7c..3ac26dd9f5 100644 --- a/tests/integration/local/common_utils.py +++ b/tests/integration/local/common_utils.py @@ -1,5 +1,6 @@ # Common utils between local tests import logging +import os import random import time @@ -7,6 +8,9 @@ START_WAIT_TIME_SECONDS = 300 +PYTEST_WORKER_COUNT = int(os.environ.get("PYTEST_XDIST_WORKER_COUNT", 4)) +PYTEST_WORKER_ID = os.environ.get("PYTEST_XDIST_WORKER", 0) + class InvalidAddressException(Exception): pass @@ -34,5 +38,18 @@ def wait_for_local_process(process, port, collect_output=False) -> str: return output +def get_pytest_worker_id(): + try: + return int(PYTEST_WORKER_ID[2:]) + except TypeError: + return 0 + + def random_port(): - return random.randint(30000, 40000) + start_port = 30000 + end_port = 40000 + + port_window = (end_port - start_port) / PYTEST_WORKER_COUNT + port_worker_start = int(start_port + (get_pytest_worker_id() * port_window)) + port_worker_end = int(port_worker_start + port_window) + return random.randint(port_worker_start, port_worker_end) diff --git a/tests/integration/local/start_api/test_start_api.py b/tests/integration/local/start_api/test_start_api.py index c43936031d..9ec7934d2c 100644 --- a/tests/integration/local/start_api/test_start_api.py +++ b/tests/integration/local/start_api/test_start_api.py @@ -2126,6 +2126,13 @@ def test_swagger_got_parsed_and_api_is_reachable_and_payload_version_is_2(self): class TestWarmContainersBaseClass(StartApiIntegBaseClass): + + @classmethod + def setUpClass(cls): + cls.mode_env_variable = str(uuid.uuid4()) + cls.parameter_overrides = {"ModeEnvVariable": cls.mode_env_variable} + super().setUpClass() + def setUp(self): self.url = "http://127.0.0.1:{}".format(self.port) @@ -2147,8 +2154,6 @@ def count_running_containers(self): ) class TestWarmContainers(TestWarmContainersBaseClass): container_mode = ContainersInitializationMode.EAGER.value - mode_env_variable = str(uuid.uuid4()) - parameter_overrides = {"ModeEnvVariable": mode_env_variable} @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=600, method="thread") @@ -2167,8 +2172,6 @@ def test_can_invoke_lambda_function_successfully(self): ) class TestWarmContainersInitialization(TestWarmContainersBaseClass): container_mode = ContainersInitializationMode.EAGER.value - mode_env_variable = str(uuid.uuid4()) - parameter_overrides = {"ModeEnvVariable": mode_env_variable} @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=600, method="thread") @@ -2187,8 +2190,6 @@ def test_all_containers_are_initialized_before_any_invoke(self): ) class TestWarmContainersMultipleInvoke(TestWarmContainersBaseClass): container_mode = ContainersInitializationMode.EAGER.value - mode_env_variable = str(uuid.uuid4()) - parameter_overrides = {"ModeEnvVariable": mode_env_variable} @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=600, method="thread") @@ -2210,8 +2211,6 @@ def test_no_new_created_containers_after_lambda_function_invoke(self): ) class TestLazyContainers(TestWarmContainersBaseClass): container_mode = ContainersInitializationMode.LAZY.value - mode_env_variable = str(uuid.uuid4()) - parameter_overrides = {"ModeEnvVariable": mode_env_variable} @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=600, method="thread") @@ -2230,8 +2229,6 @@ def test_can_invoke_lambda_function_successfully(self): ) class TestLazyContainersInitialization(TestWarmContainersBaseClass): container_mode = ContainersInitializationMode.LAZY.value - mode_env_variable = str(uuid.uuid4()) - parameter_overrides = {"ModeEnvVariable": mode_env_variable} @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=600, method="thread") @@ -2251,8 +2248,6 @@ def test_no_container_is_initialized_before_any_invoke(self): ) class TestLazyContainersMultipleInvoke(TestWarmContainersBaseClass): container_mode = ContainersInitializationMode.LAZY.value - mode_env_variable = str(uuid.uuid4()) - parameter_overrides = {"ModeEnvVariable": mode_env_variable} @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=600, method="thread") @@ -3149,8 +3144,6 @@ def tearDownClass(self) -> None: class TestWarmContainersRemoteLayers(WarmContainersWithRemoteLayersBase): template_path = "/testdata/start_api/template-warm-containers-layers.yaml" container_mode = ContainersInitializationMode.EAGER.value - mode_env_variable = str(uuid.uuid4()) - parameter_overrides = {"ModeEnvVariable": mode_env_variable} @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=600, method="thread") @@ -3170,8 +3163,6 @@ def test_can_invoke_lambda_layer_successfully(self): class TestWarmContainersRemoteLayersLazyInvoke(WarmContainersWithRemoteLayersBase): template_path = "/testdata/start_api/template-warm-containers-layers.yaml" container_mode = ContainersInitializationMode.LAZY.value - mode_env_variable = str(uuid.uuid4()) - parameter_overrides = {"ModeEnvVariable": mode_env_variable} @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=600, method="thread") diff --git a/tests/integration/local/start_lambda/start_lambda_api_integ_base.py b/tests/integration/local/start_lambda/start_lambda_api_integ_base.py index a6cdb4f838..87f8b368c7 100644 --- a/tests/integration/local/start_lambda/start_lambda_api_integ_base.py +++ b/tests/integration/local/start_lambda/start_lambda_api_integ_base.py @@ -1,5 +1,7 @@ import shutil +import time import uuid +from shutil import rmtree from typing import Optional, Dict, List from unittest import TestCase, skipIf import threading @@ -11,6 +13,7 @@ import docker from docker.errors import APIError +from samcli.lib.utils.osutils import copytree from tests.integration.local.common_utils import random_port, InvalidAddressException, wait_for_local_process from tests.testing_utils import ( SKIP_DOCKER_TESTS, @@ -45,6 +48,12 @@ class StartLambdaIntegBaseClass(TestCase): def setUpClass(cls): # This is the directory for tests/integration which will be used to file the testdata # files for integ tests + scratch_dir = Path(__file__).resolve().parent.joinpath(".tmp", str(uuid.uuid4()).replace("-", "")[:10], "testdata") + shutil.rmtree(scratch_dir, ignore_errors=True) + os.makedirs(scratch_dir) + copytree(str(Path(cls.integration_dir).joinpath("testdata")), str(scratch_dir)) + cls.integration_dir = str(scratch_dir.parent) + cls.template = cls.integration_dir + cls.template_path cls.working_dir = str(Path(cls.template).resolve().parents[0]) cls.env_var_path = cls.integration_dir + "/testdata/invoke/vars.json" @@ -91,6 +100,8 @@ def start_lambda_with_retry(cls, retries=3, input=None, env=None): if retry_count == retries: raise ValueError("Ran out of retries attempting to start lambda") + time.sleep(20) + @classmethod def get_start_lambda_command( cls, @@ -179,6 +190,7 @@ def tearDownClass(cls): # After all the tests run, we need to kill the start_lambda process. cls.stop_reading_thread = True kill_process(cls.start_lambda_process) + rmtree(cls.integration_dir) class WatchWarmContainersIntegBaseClass(StartLambdaIntegBaseClass): @@ -190,16 +202,15 @@ class WatchWarmContainersIntegBaseClass(StartLambdaIntegBaseClass): @classmethod def setUpClass(cls): cls.temp_path = str(uuid.uuid4()).replace("-", "")[:10] - working_dir = str(Path(cls.integration_dir).resolve().joinpath(cls.temp_path)) + working_dir = Path(cls.integration_dir).resolve().joinpath("testdata", ".tmp", cls.temp_path) if Path(working_dir).resolve().exists(): shutil.rmtree(working_dir, ignore_errors=True) - os.mkdir(working_dir) - os.mkdir(Path(cls.integration_dir).resolve().joinpath(cls.temp_path).joinpath("dir")) - cls.template_path = f"/{cls.temp_path}/template.yaml" - cls.code_path = f"/{cls.temp_path}/main.py" - cls.code_path2 = f"/{cls.temp_path}/dir/main2.py" - cls.docker_file_path = f"/{cls.temp_path}/Dockerfile" - cls.docker_file_path2 = f"/{cls.temp_path}/Dockerfile2" + os.makedirs(working_dir.joinpath("dir")) + cls.template_path = f"/testdata/.tmp/{cls.temp_path}/template.yaml" + cls.code_path = f"/testdata/.tmp/{cls.temp_path}/main.py" + cls.code_path2 = f"/testdata/.tmp/{cls.temp_path}/dir/main2.py" + cls.docker_file_path = f"/testdata/.tmp/{cls.temp_path}/Dockerfile" + cls.docker_file_path2 = f"/testdata/.tmp/{cls.temp_path}/Dockerfile2" if cls.template_content: cls._write_file_content(cls.template_path, cls.template_content) diff --git a/tests/integration/local/start_lambda/test_start_lambda.py b/tests/integration/local/start_lambda/test_start_lambda.py index 79393ce8eb..c625b5f2f5 100644 --- a/tests/integration/local/start_lambda/test_start_lambda.py +++ b/tests/integration/local/start_lambda/test_start_lambda.py @@ -1,7 +1,10 @@ +import logging import uuid from concurrent.futures import ThreadPoolExecutor, as_completed from time import time, sleep import json + +import docker from parameterized import parameterized, parameterized_class import pytest @@ -15,6 +18,8 @@ from samcli.commands.local.cli_common.invoke_context import ContainersInitializationMode from .start_lambda_api_integ_base import StartLambdaIntegBaseClass, WatchWarmContainersIntegBaseClass +LOG = logging.getLogger(__name__) + class TestParallelRequests(StartLambdaIntegBaseClass): template_path = "/testdata/invoke/template.yml" @@ -296,6 +301,13 @@ def test_invoke_with_function_timeout(self, use_full_path): class TestWarmContainersBaseClass(StartLambdaIntegBaseClass): + + @classmethod + def setUpClass(cls): + cls.mode_env_variable = str(uuid.uuid4()) + cls.parameter_overrides = {"ModeEnvVariable": cls.mode_env_variable} + super().setUpClass() + def setUp(self): self.url = "http://127.0.0.1:{}".format(self.port) self.lambda_client = boto3.client( @@ -310,9 +322,18 @@ def setUp(self): def count_running_containers(self): running_containers = 0 for container in self.docker_client.containers.list(): - _, output = container.exec_run(["bash", "-c", "'printenv'"]) - if f"MODE={self.mode_env_variable}" in str(output): + if container.attrs.get("State", {}).get("Status", "") == "Running" and \ + container.attrs.get("Config", {}).get("Env", {}).get("MODE", "") == self.mode_env_variable: running_containers += 1 + # try: + # _, output = container.exec_run(["bash", "-c", "'printenv'"]) + # if f"MODE={self.mode_env_variable}" in str(output): + # running_containers += 1 + # except (docker.errors.NotFound, docker.errors.APIError) as ex: + # # running tests in parallel might might cause this issue since this container in the loop + # # might be created by other tests and might be at the removal step now + # LOG.error("Failed to ping container %s for test %s", container.id, self.id(), exc_info=ex) + # pass return running_containers @@ -325,8 +346,6 @@ def count_running_containers(self): ) class TestWarmContainers(TestWarmContainersBaseClass): container_mode = ContainersInitializationMode.EAGER.value - mode_env_variable = str(uuid.uuid4()) - parameter_overrides = {"ModeEnvVariable": mode_env_variable} @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=600, method="thread") @@ -348,8 +367,6 @@ def test_can_invoke_lambda_function_successfully(self): ) class TestWarmContainersInitialization(TestWarmContainersBaseClass): container_mode = ContainersInitializationMode.EAGER.value - mode_env_variable = str(uuid.uuid4()) - parameter_overrides = {"ModeEnvVariable": mode_env_variable} @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=600, method="thread") @@ -368,8 +385,6 @@ def test_all_containers_are_initialized_before_any_invoke(self): ) class TestWarmContainersMultipleInvoke(TestWarmContainersBaseClass): container_mode = ContainersInitializationMode.EAGER.value - mode_env_variable = str(uuid.uuid4()) - parameter_overrides = {"ModeEnvVariable": mode_env_variable} @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=600, method="thread") @@ -391,8 +406,6 @@ def test_no_new_created_containers_after_lambda_function_invoke(self): ) class TestLazyContainers(TestWarmContainersBaseClass): container_mode = ContainersInitializationMode.LAZY.value - mode_env_variable = str(uuid.uuid4()) - parameter_overrides = {"ModeEnvVariable": mode_env_variable} @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=600, method="thread") @@ -414,8 +427,6 @@ def test_can_invoke_lambda_function_successfully(self): ) class TestLazyContainersInitialization(TestWarmContainersBaseClass): container_mode = ContainersInitializationMode.LAZY.value - mode_env_variable = str(uuid.uuid4()) - parameter_overrides = {"ModeEnvVariable": mode_env_variable} @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=600, method="thread") @@ -435,8 +446,6 @@ def test_no_container_is_initialized_before_any_invoke(self): ) class TestLazyContainersMultipleInvoke(TestWarmContainersBaseClass): container_mode = ContainersInitializationMode.LAZY.value - mode_env_variable = str(uuid.uuid4()) - parameter_overrides = {"ModeEnvVariable": mode_env_variable} @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=600, method="thread")