From 79c9bb0a6ca15652fadfe3b76ac6ab850079ae8d Mon Sep 17 00:00:00 2001 From: Lucas <12496191+lucashuy@users.noreply.github.com> Date: Mon, 4 Dec 2023 16:00:39 -0800 Subject: [PATCH] feat: Building projects in source (#5933) * feat: Added event tracking for feature (#5872) * Added event tracking for feature * Correctly pass event value and fix typing * feat: Resolve symlinks at root level and mount to container (#5870) * Resolve symlinks at root level and mount to container * Continue early instead of nesting in if statement * make format * Added integration test * make format * Use node project to test local deps and building in source * Removed old test data * Update test class name to reference testing symlinks * Use correct skip decorator from unittest * Changed mount mode to read only * Removed the removed mount parameter in tests * Remove kwarg in method call * feat: Build in source click option (#5951) * Added skeleton for build in source click option * Fix tests and renamed option * Added description and updated click option to include build * Disable build tests again until esbuild resolution fix * Revert template change * Make hook name option mutually exclusive with build in source * Fix integration test message assert * chore(tests): Enable build in souce build command tests (#6099) * chore(tests): Node build in source test (#6132) * Added nodejs tests * make format * Enable retries * Run build test inside of a scratch directory * make format * Updated Makefile test to properly use scratch directory and moved source copy to setup * Changed copytree call to in house version * Removed redundant parent method calling * make format * chore: Bump Nodejs version in AppVeyor job configuration (#6210) * Bump npm version to 10.2.3 for linux testing * Upgrade node to 18.18.2 * chore(tests): Build in source sync temp folder (#6166) * Enable and use new test data folder paths * Added nodejs tests * feat: Resolve root level symlinked folders in Layers (#6236) * Resolve Layer symlinks before mounting container * Use .get instead of if else * Added integration tests * Unprivate static method and make format * Create paths using pathlib * Remove debug flag * Revert "Resolve Layer symlinks before mounting container" This reverts commit e557067c7b0daa86a1591c2cacbea54bfd14a119. * Resolve links when creating tarfile for Docker build call * make format * Removed debug line * Updated arugment with optional typing * feat: Add additional default excludes and exclude option for sync (#6299) * Add additional default excludes and exclude option * Updated test and option definition * Updated option and added tests * Update resource match to consider nested stacks * Handle parameters saved in samconfig * fix: Fix sync tests not cleaning up correctly and Linux watch behaviour (#6372) * Use develop version of Lambda Builders * Run nodejs local dep command in scratch directory and clean up aws-sam folder * Exclude the folder of the modified file from trigger sync flows on Linux * Revert "Use develop version of Lambda Builders" This reverts commit fde143e48b53d53ac7db2c41e7011e60126b3088. * Add test validing ignoring Linux modified folder events * Fix test --- appveyor-linux-binary.yml | 4 +- appveyor-ubuntu.yml | 4 +- samcli/cli/types.py | 59 ++++++ samcli/commands/_utils/options.py | 80 +++++++- samcli/commands/build/build_context.py | 20 +- samcli/commands/build/command.py | 11 +- samcli/commands/build/core/options.py | 2 +- samcli/commands/sync/command.py | 17 +- samcli/commands/sync/core/options.py | 2 + samcli/lib/sync/watch_manager.py | 31 +++- samcli/lib/telemetry/event.py | 1 + samcli/lib/utils/code_trigger_factory.py | 14 +- samcli/lib/utils/resource_trigger.py | 17 +- samcli/lib/utils/tar.py | 21 ++- samcli/local/docker/container.py | 40 +++- samcli/local/docker/lambda_image.py | 2 +- schema/samcli.json | 22 ++- .../buildcmd/test_build_in_source.py | 112 +++++++---- ...uild_terraform_applications_other_cases.py | 2 +- .../local/invoke/invoke_integ_base.py | 12 +- .../invoke/test_invoke_build_in_source.py | 83 +++++++++ tests/integration/sync/sync_integ_base.py | 4 + .../sync/test_sync_build_in_source.py | 175 +++++++++++++++--- tests/integration/sync/test_sync_watch.py | 40 +++- .../layer_symlink/mycoolfunction/app.js | 10 + .../layer_symlink/mycoolfunction/package.json | 9 + .../layer_symlink/mycoollayer/package.json | 12 ++ .../layer_symlink/template.yaml | 25 +++ .../invoke/build-in-source/local-dep/index.js | 5 + .../build-in-source/local-dep/package.json | 8 + .../invoke/build-in-source/src/index.js | 5 + .../invoke/build-in-source/src/package.json | 10 + .../invoke/build-in-source/template.yaml | 10 + .../sync/code/after/nodejs_function/app.js | 2 +- .../after/nodejs_local_dep/local-dep/index.js | 5 + .../nodejs_local_dep/local-dep/package.json | 8 + .../code/after/nodejs_local_dep/src/index.js | 10 + .../after/nodejs_local_dep/src/package.json | 10 + .../code/before/nodejs_local_dep/src/index.js | 8 + .../before/nodejs_local_dep/src/package.json | 10 + .../before/template_nodejs_local_dep.yaml | 10 + tests/unit/cli/test_types.py | 32 +++- tests/unit/commands/_utils/test_options.py | 16 ++ .../commands/buildcmd/test_build_context.py | 32 ++++ .../unit/commands/samconfig/test_samconfig.py | 12 +- tests/unit/commands/sync/test_command.py | 6 + tests/unit/lib/sync/test_watch_manager.py | 22 ++- .../lib/utils/test_code_trigger_factory.py | 16 +- tests/unit/lib/utils/test_resource_trigger.py | 11 +- tests/unit/lib/utils/test_tar.py | 6 +- tests/unit/local/docker/test_container.py | 58 +++++- 51 files changed, 1017 insertions(+), 126 deletions(-) create mode 100644 tests/integration/local/invoke/test_invoke_build_in_source.py create mode 100644 tests/integration/testdata/invoke/build-in-source/layer_symlink/mycoolfunction/app.js create mode 100644 tests/integration/testdata/invoke/build-in-source/layer_symlink/mycoolfunction/package.json create mode 100644 tests/integration/testdata/invoke/build-in-source/layer_symlink/mycoollayer/package.json create mode 100644 tests/integration/testdata/invoke/build-in-source/layer_symlink/template.yaml create mode 100644 tests/integration/testdata/invoke/build-in-source/local-dep/index.js create mode 100644 tests/integration/testdata/invoke/build-in-source/local-dep/package.json create mode 100644 tests/integration/testdata/invoke/build-in-source/src/index.js create mode 100644 tests/integration/testdata/invoke/build-in-source/src/package.json create mode 100644 tests/integration/testdata/invoke/build-in-source/template.yaml create mode 100644 tests/integration/testdata/sync/code/after/nodejs_local_dep/local-dep/index.js create mode 100644 tests/integration/testdata/sync/code/after/nodejs_local_dep/local-dep/package.json create mode 100644 tests/integration/testdata/sync/code/after/nodejs_local_dep/src/index.js create mode 100644 tests/integration/testdata/sync/code/after/nodejs_local_dep/src/package.json create mode 100644 tests/integration/testdata/sync/code/before/nodejs_local_dep/src/index.js create mode 100644 tests/integration/testdata/sync/code/before/nodejs_local_dep/src/package.json create mode 100644 tests/integration/testdata/sync/code/before/template_nodejs_local_dep.yaml diff --git a/appveyor-linux-binary.yml b/appveyor-linux-binary.yml index 614e3443d0..64cb3dc63d 100644 --- a/appveyor-linux-binary.yml +++ b/appveyor-linux-binary.yml @@ -19,7 +19,7 @@ environment: PYTHON_HOME: "$HOME/venv3.11/bin" PYTHON_VERSION: '3.11' AWS_DEFAULT_REGION: us-east-1 - NODE_VERSION: "14.17.6" + NODE_VERSION: "18.18.2" AWS_S3: 'AWS_S3_TESTING' AWS_ECR: 'AWS_ECR_TESTING' CARGO_LAMBDA_VERSION: "v0.17.1" @@ -52,7 +52,7 @@ install: - sh: "docker info" - sh: "docker version" - sh: "nvm install ${NODE_VERSION}" - - sh: "npm install npm@7.24.2 -g" + - sh: "npm install npm@10.2.3 -g" - sh: "npm -v" # Install latest gradle diff --git a/appveyor-ubuntu.yml b/appveyor-ubuntu.yml index b15c68f23f..4efbfde8e2 100644 --- a/appveyor-ubuntu.yml +++ b/appveyor-ubuntu.yml @@ -20,7 +20,7 @@ environment: PYTHON_VERSION: '3.8' AWS_DEFAULT_REGION: us-east-1 SAM_CLI_DEV: 1 - NODE_VERSION: "14.17.6" + NODE_VERSION: "18.18.2" AWS_S3: 'AWS_S3_TESTING' AWS_ECR: 'AWS_ECR_TESTING' CARGO_LAMBDA_VERSION: "v0.17.1" @@ -52,7 +52,7 @@ install: - sh: "docker info" - sh: "docker version" - sh: "nvm install ${NODE_VERSION}" - - sh: "npm install npm@7.24.2 -g" + - sh: "npm install npm@10.2.3 -g" - sh: "npm -v" # Install latest gradle diff --git a/samcli/cli/types.py b/samcli/cli/types.py index 04f8f27446..a1488407d6 100644 --- a/samcli/cli/types.py +++ b/samcli/cli/types.py @@ -6,6 +6,7 @@ import logging import re from json import JSONDecodeError +from typing import Dict, List, Optional, Union import click @@ -474,3 +475,61 @@ def convert(self, value, param, ctx): LOG.debug("Converting provided %s option value to Enum", param.opts[0]) value = super().convert(value, param, ctx) return self.enum(value) + + +class SyncWatchExcludeType(click.ParamType): + """ + Custom parameter type to parse Key=Value pairs into dictionary mapping. + """ + + name = "list" + + WATCH_EXCLUDE_EXPECTED_LENGTH = 2 + WATCH_EXCLUDE_DELIMITER = "=" + + EXCEPTION_MESSAGE = "Argument must be a key, value pair in the format Key=Value." + + def convert( + self, value: Union[Dict[str, List[str]], str], param: Optional[click.Parameter], ctx: Optional[click.Context] + ) -> Dict[str, List[str]]: + """ + Parses the multiple provided values into a mapping of resources to a list of exclusions. + + Parameters + ---------- + value: Union[Dict[str, List[str]], str] + The passed in arugment + param: click.Parameter + The parameter that was provided + ctx: click.Context + The click context + + Returns + ------- + Dict[str, List[str]] + A key value pair of Logical/Resource Id and excluded file + """ + if isinstance(value, dict): + return value + + mapping_pair = value.split(self.WATCH_EXCLUDE_DELIMITER) + + if len(mapping_pair) != self.WATCH_EXCLUDE_EXPECTED_LENGTH: + raise click.BadParameter( + param=param, + param_hint=f"'{value}'", + ctx=ctx, + message=self.EXCEPTION_MESSAGE, + ) + + resource_id, excluded_path = mapping_pair + + if not (resource_id and excluded_path): + raise click.BadParameter( + param=param, + param_hint=f"'{value}'", + ctx=ctx, + message=self.EXCEPTION_MESSAGE, + ) + + return {resource_id: [excluded_path]} diff --git a/samcli/commands/_utils/options.py b/samcli/commands/_utils/options.py index f8f92146bb..4796610b56 100644 --- a/samcli/commands/_utils/options.py +++ b/samcli/commands/_utils/options.py @@ -5,7 +5,7 @@ import logging import os from functools import partial -from typing import List, Tuple +from typing import Dict, List, Tuple import click from click.types import FuncParamType @@ -18,7 +18,9 @@ ImageRepositoryType, RemoteInvokeBotoApiParameterType, SigningProfilesOptionType, + SyncWatchExcludeType, ) +from samcli.commands._utils.click_mutex import ClickMutex from samcli.commands._utils.constants import ( DEFAULT_BUILD_DIR, DEFAULT_BUILT_TEMPLATE_PATH, @@ -33,8 +35,17 @@ from samcli.lib.hook.hook_wrapper import get_available_hook_packages_ids from samcli.lib.observability.util import OutputOption from samcli.lib.utils.packagetype import IMAGE, ZIP +from samcli.local.docker.lambda_image import Runtime _TEMPLATE_OPTION_DEFAULT_VALUE = "template.[yaml|yml|json]" +SUPPORTED_BUILD_IN_SOURCE_WORKFLOWS = [ + Runtime.nodejs12x.value, + Runtime.nodejs14x.value, + Runtime.nodejs16x.value, + Runtime.nodejs18x.value, + "Makefile", + "esbuild", +] LOG = logging.getLogger(__name__) @@ -237,6 +248,39 @@ def skip_prepare_infra_callback(ctx, param, provided_value): raise click.BadOptionUsage(option_name=param.name, ctx=ctx, message="Missing option --hook-name") +def watch_exclude_option_callback( + ctx: click.Context, param: click.Option, values: Tuple[Dict[str, List[str]]] +) -> Dict[str, List[str]]: + """ + Parses the multiple provided values into a mapping of resources to a list of exclusions. + + Parameters + ---------- + ctx: click.Context + The click context + param: click.Option + The parameter that was provided + values: Tuple[Dict[str, List[str]]] + A list of values that was passed in + + Returns + ------- + Dict[str, List[str]] + A mapping of LogicalIds to a list of files and/or folders to exclude + from file change watches + """ + resource_exclude_mappings: Dict[str, List[str]] = {} + + for mappings in values: + for resource_id, excluded_files in mappings.items(): + current_excludes = resource_exclude_mappings.get(resource_id, []) + current_excludes.extend(excluded_files) + + resource_exclude_mappings[resource_id] = current_excludes + + return resource_exclude_mappings + + def template_common_option(f): """ Common ClI option for template @@ -915,3 +959,37 @@ def generate_next_command_recommendation(command_tuples: List[Tuple[str, str]]) """ command_list_txt = "\n".join(f"[*] {description}: {command}" for description, command in command_tuples) return template.format(command_list_txt) + + +def build_in_source_click_option(): + return click.option( + "--build-in-source/--no-build-in-source", + required=False, + is_flag=True, + help="Opts in to build project in the source folder. The following workflows support " + f"building in source: {SUPPORTED_BUILD_IN_SOURCE_WORKFLOWS}", + cls=ClickMutex, + incompatible_params=["use_container", "hook_name"], + ) + + +def build_in_source_option(f): + return build_in_source_click_option()(f) + + +def watch_exclude_option(f): + return watch_exclude_click_option()(f) + + +def watch_exclude_click_option(): + return click.option( + "--watch-exclude", + help="Excludes a file or folder from being observed for file changes. " + "Files and folders that are excluded will not trigger a sync workflow. " + "This option can be provided multiple times.\n\n" + "Examples:\n\nHelloWorldFunction=package-lock.json\n\n" + "ChildStackA/FunctionName=database.sqlite3", + multiple=True, + type=SyncWatchExcludeType(), + callback=watch_exclude_option_callback, + ) diff --git a/samcli/commands/build/build_context.py b/samcli/commands/build/build_context.py index a6d6b69102..7e2450a155 100644 --- a/samcli/commands/build/build_context.py +++ b/samcli/commands/build/build_context.py @@ -33,12 +33,12 @@ ) from samcli.lib.build.workflow_config import UnsupportedRuntimeException from samcli.lib.intrinsic_resolver.intrinsics_symbol_table import IntrinsicsSymbolTable -from samcli.lib.providers.provider import ResourcesToBuildCollector, Stack, Function, LayerVersion +from samcli.lib.providers.provider import ResourcesToBuildCollector, Stack, LayerVersion from samcli.lib.providers.sam_api_provider import SamApiProvider from samcli.lib.providers.sam_function_provider import SamFunctionProvider from samcli.lib.providers.sam_layer_provider import SamLayerProvider from samcli.lib.providers.sam_stack_provider import SamLocalStackProvider -from samcli.lib.telemetry.event import EventTracker +from samcli.lib.telemetry.event import EventTracker, UsedFeature, EventName from samcli.lib.utils.osutils import BUILD_DIR_PERMISSIONS from samcli.local.docker.manager import ContainerManager from samcli.local.lambdafn.exceptions import ( @@ -232,13 +232,15 @@ def __exit__(self, *args): def get_resources_to_build(self): return self.resources_to_build - def run(self): + def run(self) -> None: """Runs the building process by creating an ApplicationBuilder.""" if self._is_sam_template(): SamApiProvider.check_implicit_api_resource_ids(self.stacks) self._stacks = self._handle_build_pre_processing() + caught_exception: Optional[Exception] = None + try: # boolean value indicates if mount with write or not, defaults to READ ONLY mount_with_write = False @@ -276,7 +278,7 @@ def run(self): self._check_rust_cargo_experimental_flag() for f in self.get_resources_to_build().functions: - EventTracker.track_event("BuildFunctionRuntime", f.runtime) + EventTracker.track_event(EventName.BUILD_FUNCTION_RUNTIME.value, f.runtime) self._build_result = builder.build() @@ -306,6 +308,8 @@ def run(self): click.secho(msg, fg="yellow") except FunctionNotFound as function_not_found_ex: + caught_exception = function_not_found_ex + raise UserException( str(function_not_found_ex), wrapped_from=function_not_found_ex.__class__.__name__ ) from function_not_found_ex @@ -317,6 +321,8 @@ def run(self): InvalidBuildGraphException, ResourceNotFound, ) as ex: + caught_exception = ex + click.secho("\nBuild Failed", fg="red") # Some Exceptions have a deeper wrapped exception that needs to be surfaced @@ -324,6 +330,12 @@ def run(self): deep_wrap = getattr(ex, "wrapped_from", None) wrapped_from = deep_wrap if deep_wrap else ex.__class__.__name__ raise UserException(str(ex), wrapped_from=wrapped_from) from ex + finally: + if self.build_in_source: + exception_name = type(caught_exception).__name__ if caught_exception else None + EventTracker.track_event( + EventName.USED_FEATURE.value, UsedFeature.BUILD_IN_SOURCE.value, exception_name + ) def _is_sam_template(self) -> bool: """Check if a given template is a SAM template""" diff --git a/samcli/commands/build/command.py b/samcli/commands/build/command.py index 7f235a3c35..2f7cd00392 100644 --- a/samcli/commands/build/command.py +++ b/samcli/commands/build/command.py @@ -9,6 +9,7 @@ from samcli.cli.context import Context from samcli.commands._utils.options import ( + build_in_source_option, skip_prepare_infra_option, template_option_without_build, docker_common_options, @@ -74,10 +75,11 @@ @terraform_project_root_path_option @hook_name_click_option( force_prepare=True, - invalid_coexist_options=["t", "template-file", "template", "parameter-overrides"], + invalid_coexist_options=["t", "template-file", "template", "parameter-overrides", "build-in-source"], ) @skip_prepare_infra_option @use_container_build_option +@build_in_source_option @click.option( "--container-env-var", "-e", @@ -158,8 +160,9 @@ def cli( save_params: bool, hook_name: Optional[str], skip_prepare_infra: bool, - mount_with, + mount_with: str, terraform_project_root_path: Optional[str], + build_in_source: Optional[bool], ) -> None: """ `sam build` command entry point @@ -189,7 +192,7 @@ def cli( build_image, exclude, hook_name, - None, # TODO: replace with build_in_source once it's added as a click option + build_in_source, mount_with, ) # pragma: no cover @@ -216,7 +219,7 @@ def do_cli( # pylint: disable=too-many-locals, too-many-statements exclude: Optional[Tuple[str, ...]], hook_name: Optional[str], build_in_source: Optional[bool], - mount_with, + mount_with: str, ) -> None: """ Implementation of the ``cli`` method diff --git a/samcli/commands/build/core/options.py b/samcli/commands/build/core/options.py index a23af2f920..80f3b06ebb 100644 --- a/samcli/commands/build/core/options.py +++ b/samcli/commands/build/core/options.py @@ -27,7 +27,7 @@ EXTENSION_OPTIONS: List[str] = ["hook_name", "skip_prepare_infra"] -BUILD_STRATEGY_OPTIONS: List[str] = ["parallel", "exclude", "manifest", "cached"] +BUILD_STRATEGY_OPTIONS: List[str] = ["parallel", "exclude", "manifest", "cached", "build_in_source"] ARTIFACT_LOCATION_OPTIONS: List[str] = [ "build_dir", diff --git a/samcli/commands/sync/command.py b/samcli/commands/sync/command.py index b1c55d5659..dfafae7ea4 100644 --- a/samcli/commands/sync/command.py +++ b/samcli/commands/sync/command.py @@ -1,7 +1,7 @@ """CLI command for "sync" command.""" import logging import os -from typing import TYPE_CHECKING, List, Optional, Set, Tuple +from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple import click @@ -22,6 +22,7 @@ from samcli.commands._utils.options import ( base_dir_option, build_image_option, + build_in_source_option, capabilities_option, image_repositories_option, image_repository_option, @@ -36,6 +37,7 @@ tags_option, template_option_without_build, use_container_build_option, + watch_exclude_option, ) from samcli.commands.build.click_container import ContainerOptions from samcli.commands.build.command import _get_mode_value_from_envvar @@ -155,9 +157,11 @@ help="This option will skip the initial infrastructure deployment if it is not required" " by comparing the local template with the template deployed in cloud.", ) +@watch_exclude_option @stack_name_option(required=True) # pylint: disable=E1120 @base_dir_option @use_container_build_option +@build_in_source_option @build_image_option(cls=ContainerOptions) @image_repository_option @image_repositories_option @@ -209,6 +213,8 @@ def cli( config_file: str, config_env: str, build_image: Optional[Tuple[str]], + build_in_source: Optional[bool], + watch_exclude: Optional[Dict[str, List[str]]], ) -> None: """ `sam sync` command entry point @@ -244,7 +250,8 @@ def cli( build_image, config_file, config_env, - None, # TODO: replace with build_in_source once it's added as a click option + build_in_source, + watch_exclude, ) # pragma: no cover @@ -277,6 +284,7 @@ def do_cli( config_file: str, config_env: str, build_in_source: Optional[bool], + watch_exclude: Optional[Dict[str, List[str]]], ) -> None: """ Implementation of the ``cli`` method @@ -390,6 +398,8 @@ def do_cli( dependency_layer, build_context.build_dir, build_context.cache_dir, skip_deploy_sync ) as sync_context: if watch: + watch_excludes_filter = watch_exclude or {} + execute_watch( template=template_file, build_context=build_context, @@ -398,6 +408,7 @@ def do_cli( sync_context=sync_context, auto_dependency_layer=dependency_layer, disable_infra_syncs=code, + watch_exclude=watch_excludes_filter, ) elif code: execute_code_sync( @@ -516,6 +527,7 @@ def execute_watch( sync_context: "SyncContext", auto_dependency_layer: bool, disable_infra_syncs: bool, + watch_exclude: Dict[str, List[str]], ): """Start sync watch execution @@ -547,6 +559,7 @@ def execute_watch( sync_context, auto_dependency_layer, disable_infra_syncs, + watch_exclude, ) watch_manager.start() diff --git a/samcli/commands/sync/core/options.py b/samcli/commands/sync/core/options.py index bb7673318a..c03e6ef190 100644 --- a/samcli/commands/sync/core/options.py +++ b/samcli/commands/sync/core/options.py @@ -39,6 +39,8 @@ "resource_id", "resource", "base_dir", + "build_in_source", + "watch_exclude", ] OTHER_OPTIONS: List[str] = ["debug", "help"] diff --git a/samcli/lib/sync/watch_manager.py b/samcli/lib/sync/watch_manager.py index d26df09717..ca3935878e 100644 --- a/samcli/lib/sync/watch_manager.py +++ b/samcli/lib/sync/watch_manager.py @@ -2,12 +2,13 @@ WatchManager for Sync Watch Logic """ import logging +import platform import threading import time from pathlib import Path -from typing import TYPE_CHECKING, List, Optional, Set +from typing import TYPE_CHECKING, Dict, List, Optional, Set -from watchdog.events import EVENT_TYPE_OPENED, FileSystemEvent +from watchdog.events import EVENT_TYPE_MODIFIED, EVENT_TYPE_OPENED, FileSystemEvent from samcli.lib.providers.exceptions import InvalidTemplateFile, MissingCodeUri, MissingLocalDefinition from samcli.lib.providers.provider import ResourceIdentifier, Stack, get_all_resource_ids @@ -57,6 +58,7 @@ def __init__( sync_context: "SyncContext", auto_dependency_layer: bool, disable_infra_syncs: bool, + watch_exclude: Dict[str, List[str]], ): """Manager for sync watch execution logic. This manager will observe template and its code resources. @@ -92,6 +94,8 @@ def __init__( self._waiting_infra_sync = False self._color = Colored() + self._watch_exclude = watch_exclude + def queue_infra_sync(self) -> None: """Queue up an infra structure sync. A simple bool flag is suffice @@ -132,7 +136,10 @@ def _add_code_triggers(self) -> None: resource_ids = get_all_resource_ids(self._stacks) for resource_id in resource_ids: try: - trigger = self._trigger_factory.create_trigger(resource_id, self._on_code_change_wrapper(resource_id)) + additional_excludes = self._watch_exclude.get(str(resource_id), []) + trigger = self._trigger_factory.create_trigger( + resource_id, self._on_code_change_wrapper(resource_id), additional_excludes + ) except (MissingCodeUri, MissingLocalDefinition): LOG.warning( self._color.color_log( @@ -329,6 +336,24 @@ def on_code_change(event: Optional[FileSystemEvent] = None) -> None: LOG.debug("Ignoring file system OPENED event") return + if ( + platform.system().lower() == "linux" + and event + and event.event_type == EVENT_TYPE_MODIFIED + and event.is_directory + ): + # Linux machines appear to emit an additional event when + # a file gets updated; a folder modfied event + # If folder/file.txt gets updated, there will be two events: + # 1. file.txt modified event + # 2. folder modified event + # We want to ignore the second event + # + # It looks like the other way a folder modified event can happen + # is if the permissions of the folder were changed + LOG.debug(f"Ignoring file system MODIFIED event for folder {event.src_path}") + return + # sync flow factory should always exist, but guarding just incase if not self._sync_flow_factory: LOG.debug("Sync flow factory not defined, skipping trigger") diff --git a/samcli/lib/telemetry/event.py b/samcli/lib/telemetry/event.py index 9369a87f11..1e76e8eb83 100644 --- a/samcli/lib/telemetry/event.py +++ b/samcli/lib/telemetry/event.py @@ -41,6 +41,7 @@ class UsedFeature(Enum): INIT_WITH_APPLICATION_INSIGHTS = "InitWithApplicationInsights" CFNLint = "CFNLint" INVOKED_CUSTOM_LAMBDA_AUTHORIZERS = "InvokedLambdaAuthorizers" + BUILD_IN_SOURCE = "BuildInSource" class EventType: diff --git a/samcli/lib/utils/code_trigger_factory.py b/samcli/lib/utils/code_trigger_factory.py index a92c5a3c44..eb009d9aa5 100644 --- a/samcli/lib/utils/code_trigger_factory.py +++ b/samcli/lib/utils/code_trigger_factory.py @@ -44,10 +44,11 @@ def _create_lambda_trigger( resource_type: str, resource: Dict[str, Any], on_code_change: Callable, + watch_exclude: List[str], ): package_type = resource.get("Properties", dict()).get("PackageType", ZIP) if package_type == ZIP: - return LambdaZipCodeTrigger(resource_identifier, self._stacks, self.base_dir, on_code_change) + return LambdaZipCodeTrigger(resource_identifier, self._stacks, self.base_dir, on_code_change, watch_exclude) if package_type == IMAGE: return LambdaImageCodeTrigger(resource_identifier, self._stacks, self.base_dir, on_code_change) return None @@ -58,8 +59,9 @@ def _create_layer_trigger( resource_type: str, resource: Dict[str, Any], on_code_change: Callable, + watch_exclude: List[str], ): - return LambdaLayerCodeTrigger(resource_identifier, self._stacks, self.base_dir, on_code_change) + return LambdaLayerCodeTrigger(resource_identifier, self._stacks, self.base_dir, on_code_change, watch_exclude) def _create_definition_code_trigger( self, @@ -67,11 +69,13 @@ def _create_definition_code_trigger( resource_type: str, resource: Dict[str, Any], on_code_change: Callable, + watch_exclude: List[str], ): return DefinitionCodeTrigger(resource_identifier, resource_type, self._stacks, self.base_dir, on_code_change) GeneratorFunction = Callable[ - ["CodeTriggerFactory", ResourceIdentifier, str, Dict[str, Any], Callable], Optional[CodeResourceTrigger] + ["CodeTriggerFactory", ResourceIdentifier, str, Dict[str, Any], Callable, List[str]], + Optional[CodeResourceTrigger], ] GENERATOR_MAPPING: Dict[str, GeneratorFunction] = { AWS_LAMBDA_FUNCTION: _create_lambda_trigger, @@ -91,7 +95,7 @@ def _get_generator_mapping(self) -> Dict[str, GeneratorFunction]: # pylint: dis return CodeTriggerFactory.GENERATOR_MAPPING def create_trigger( - self, resource_identifier: ResourceIdentifier, on_code_change: Callable + self, resource_identifier: ResourceIdentifier, on_code_change: Callable, watch_exclude: List[str] ) -> Optional[CodeResourceTrigger]: """Create Trigger for the resource type @@ -113,5 +117,5 @@ def create_trigger( if not generator or not resource or not resource_type: return None return cast(CodeTriggerFactory.GeneratorFunction, generator)( - self, resource_identifier, resource_type, resource, on_code_change + self, resource_identifier, resource_type, resource, on_code_change, watch_exclude ) diff --git a/samcli/lib/utils/resource_trigger.py b/samcli/lib/utils/resource_trigger.py index a1f94d3fab..024a3b3caf 100644 --- a/samcli/lib/utils/resource_trigger.py +++ b/samcli/lib/utils/resource_trigger.py @@ -17,7 +17,7 @@ from samcli.lib.utils.resources import RESOURCES_WITH_LOCAL_PATHS from samcli.local.lambdafn.exceptions import FunctionNotFound, ResourceNotFound -AWS_SAM_FOLDER_REGEX = "^.*\\.aws-sam.*$" +DEFAULT_WATCH_IGNORED_RESOURCES = ["^.*\\.aws-sam.*$", "^.*node_modules.*$"] class OnChangeCallback(Protocol): @@ -155,6 +155,7 @@ def __init__( stacks: List[Stack], base_dir: Path, on_code_change: OnChangeCallback, + watch_exclude: Optional[List[str]] = None, ): """ Parameters @@ -182,6 +183,10 @@ def __init__( self._on_code_change = on_code_change self.base_dir = base_dir + self._watch_exclude = [*DEFAULT_WATCH_IGNORED_RESOURCES] + for exclude in watch_exclude or []: + self._watch_exclude.append(f"^.*{exclude}.*$") + class LambdaFunctionCodeTrigger(CodeResourceTrigger): _function: Function @@ -193,6 +198,7 @@ def __init__( stacks: List[Stack], base_dir: Path, on_code_change: OnChangeCallback, + watch_exclude: Optional[List[str]] = None, ): """ Parameters @@ -213,7 +219,7 @@ def __init__( MissingCodeUri raised when there is no CodeUri property in the function definition. """ - super().__init__(function_identifier, stacks, base_dir, on_code_change) + super().__init__(function_identifier, stacks, base_dir, on_code_change, watch_exclude) function = SamFunctionProvider(stacks).get(str(function_identifier)) if not function: raise FunctionNotFound() @@ -242,7 +248,7 @@ def get_path_handlers(self) -> List[PathHandler]: PathHandlers for the code folder associated with the function """ dir_path_handler = ResourceTrigger.get_dir_path_handler( - self.base_dir.joinpath(self._code_uri), ignore_regexes=[AWS_SAM_FOLDER_REGEX] + self.base_dir.joinpath(self._code_uri), ignore_regexes=self._watch_exclude ) dir_path_handler.self_create = self._on_code_change dir_path_handler.self_delete = self._on_code_change @@ -272,6 +278,7 @@ def __init__( stacks: List[Stack], base_dir: Path, on_code_change: OnChangeCallback, + watch_exclude: Optional[List[str]] = None, ): """ Parameters @@ -292,7 +299,7 @@ def __init__( MissingCodeUri raised when there is no CodeUri property in the function definition. """ - super().__init__(layer_identifier, stacks, base_dir, on_code_change) + super().__init__(layer_identifier, stacks, base_dir, on_code_change, watch_exclude) layer = SamLayerProvider(stacks).get(str(layer_identifier)) if not layer: raise ResourceNotFound() @@ -310,7 +317,7 @@ def get_path_handlers(self) -> List[PathHandler]: PathHandlers for the code folder associated with the layer """ dir_path_handler = ResourceTrigger.get_dir_path_handler( - self.base_dir.joinpath(self._code_uri), ignore_regexes=[AWS_SAM_FOLDER_REGEX] + self.base_dir.joinpath(self._code_uri), ignore_regexes=self._watch_exclude ) dir_path_handler.self_create = self._on_code_change dir_path_handler.self_delete = self._on_code_change diff --git a/samcli/lib/utils/tar.py b/samcli/lib/utils/tar.py index 351f31ac8c..1d51177510 100644 --- a/samcli/lib/utils/tar.py +++ b/samcli/lib/utils/tar.py @@ -5,22 +5,31 @@ import os import tarfile from contextlib import contextmanager +from pathlib import Path from tempfile import TemporaryFile -from typing import IO, Optional, Union +from typing import IO, Callable, Dict, Optional, Union @contextmanager -def create_tarball(tar_paths, tar_filter=None, mode="w"): +def create_tarball( + tar_paths: Dict[Union[str, Path], str], + tar_filter: Optional[Callable[[tarfile.TarInfo], Union[None, tarfile.TarInfo]]] = None, + mode: str = "w", + dereference: bool = False, +): """ Context Manger that creates the tarball of the Docker Context to use for building the image Parameters ---------- - tar_paths dict(str, str) + tar_paths: Dict[Union[str, Path], str] Key representing a full path to the file or directory and the Value representing the path within the tarball - - mode str + tar_filter: Optional[Callable[[tarfile.TarInfo], Union[None, tarfile.TarInfo]]] + A method that modifies the tar file entry before adding it to the archive. Default to `None` + mode: str The mode in which the tarfile is opened. Defaults to "w". + dereference: bool + Pass `True` to resolve symlinks before adding to archive. Otherwise, adds the symlink itself to the archive Yields ------ @@ -29,7 +38,7 @@ def create_tarball(tar_paths, tar_filter=None, mode="w"): """ tarballfile = TemporaryFile() - with tarfile.open(fileobj=tarballfile, mode=mode) as archive: + with tarfile.open(fileobj=tarballfile, mode=mode, dereference=dereference) as archive: for path_on_system, path_in_tarball in tar_paths.items(): archive.add(path_on_system, arcname=path_in_tarball, filter=tar_filter) diff --git a/samcli/local/docker/container.py b/samcli/local/docker/container.py index 44a7b78f52..ac060f082a 100644 --- a/samcli/local/docker/container.py +++ b/samcli/local/docker/container.py @@ -11,7 +11,7 @@ import tempfile import threading import time -from typing import Iterator, Optional, Tuple, Union +from typing import Dict, Iterator, Optional, Tuple, Union import docker import requests @@ -161,7 +161,8 @@ def create(self): # https://docs.docker.com/storage/bind-mounts "bind": self._working_dir, "mode": mount_mode, - } + }, + **self._create_mapped_symlink_files(), } kwargs = { @@ -228,6 +229,41 @@ def create(self): return self.id + def _create_mapped_symlink_files(self) -> Dict[str, Dict[str, str]]: + """ + Resolves any top level symlinked files and folders that are found on the + host directory and creates additional bind mounts to correctly map them + inside of the container. + + Returns + ------- + Dict[str, Dict[str, str]] + A dictonary representing the resolved file or directory and the bound path + on the container + """ + mount_mode = "ro,delegated" + additional_volumes = {} + + with os.scandir(self._host_dir) as directory_iterator: + for file in directory_iterator: + if not file.is_symlink(): + continue + + host_resolved_path = os.path.realpath(file.path) + container_full_path = pathlib.Path(self._working_dir, file.name).as_posix() + + additional_volumes[host_resolved_path] = { + "bind": container_full_path, + "mode": mount_mode, + } + + LOG.info( + "Mounting resolved symlink (%s -> %s) as %s:%s, inside runtime container" + % (file.path, host_resolved_path, container_full_path, mount_mode) + ) + + return additional_volumes + def stop(self, timeout=3): """ Stop a container, with a given number of seconds between sending SIGTERM and SIGKILL. diff --git a/samcli/local/docker/lambda_image.py b/samcli/local/docker/lambda_image.py index 58b633735e..c775a94f4f 100644 --- a/samcli/local/docker/lambda_image.py +++ b/samcli/local/docker/lambda_image.py @@ -336,7 +336,7 @@ def set_item_permission(tar_info): # Set only on Windows, unix systems will preserve the host permission into the tarball tar_filter = set_item_permission if platform.system().lower() == "windows" else None - with create_tarball(tar_paths, tar_filter=tar_filter) as tarballfile: + with create_tarball(tar_paths, tar_filter=tar_filter, dereference=True) as tarballfile: try: resp_stream = self.docker_client.api.build( fileobj=tarballfile, diff --git a/schema/samcli.json b/schema/samcli.json index d6b0c219f8..c057f3368a 100644 --- a/schema/samcli.json +++ b/schema/samcli.json @@ -248,7 +248,7 @@ "properties": { "parameters": { "title": "Parameters for the build command", - "description": "Available parameters for the build command:\n* terraform_project_root_path:\nUsed for passing the Terraform project root directory path. Current directory will be used as a default value, if this parameter is not provided.\n* hook_name:\nHook package id to extend AWS SAM CLI commands functionality. \n\nExample: `terraform` to extend AWS SAM CLI commands functionality to support terraform applications. \n\nAvailable Hook Names: ['terraform']\n* skip_prepare_infra:\nSkip preparation stage when there are no infrastructure changes. Only used in conjunction with --hook-name.\n* use_container:\nBuild functions within an AWS Lambda-like container.\n* container_env_var:\nEnvironment variables to be passed into build containers\nResource format (FuncName.VarName=Value) or Global format (VarName=Value).\n\n Example: --container-env-var Func1.VAR1=value1 --container-env-var VAR2=value2\n* container_env_var_file:\nEnvironment variables json file (e.g., env_vars.json) to be passed to build containers.\n* build_image:\nContainer image URIs for building functions/layers. You can specify for all functions/layers with just the image URI (--build-image public.ecr.aws/sam/build-nodejs18.x:latest). You can specify for each individual function with (--build-image FunctionLogicalID=public.ecr.aws/sam/build-nodejs18.x:latest). A combination of the two can be used. If a function does not have build image specified or an image URI for all functions, the default SAM CLI build images will be used.\n* exclude:\nName of the resource(s) to exclude from AWS SAM CLI build.\n* parallel:\nEnable parallel builds for AWS SAM template's functions and layers.\n* mount_with:\nSpecify mount mode for building functions/layers inside container. If it is mounted with write permissions, some files in source code directory may be changed/added by the build process. By default the source code directory is read only.\n* build_dir:\nDirectory to store build artifacts.Note: This directory will be first removed before starting a build.\n* cache_dir:\nDirectory to store cached artifacts. The default cache directory is .aws-sam/cache\n* base_dir:\nResolve relative paths to function's source code with respect to this directory. Use this if SAM template and source code are not in same enclosing folder. By default, relative paths are resolved with respect to the SAM template's location.\n* manifest:\nPath to a custom dependency manifest. Example: custom-package.json\n* cached:\nEnable cached builds.Reuse build artifacts that have not changed from previous builds. \n\nAWS SAM CLI evaluates if files in your project directory have changed. \n\nNote: AWS SAM CLI does not evaluate changes made to third party modules that the project depends on.Example: Python function includes a requirements.txt file with the following entry requests=1.x and the latest request module version changes from 1.1 to 1.2, AWS SAM CLI will not pull the latest version until a non-cached build is run.\n* template_file:\nAWS SAM template file.\n* parameter_overrides:\nString that contains AWS CloudFormation parameter overrides encoded as key=value pairs.\n* skip_pull_image:\nSkip pulling down the latest Docker image for Lambda runtime.\n* docker_network:\nName or ID of an existing docker network for AWS Lambda docker containers to connect to, along with the default bridge network. If not specified, the Lambda containers will only connect to the default bridge docker network.\n* beta_features:\nEnable/Disable beta features.\n* debug:\nTurn on debug logging to print debug message generated by AWS SAM CLI and display timestamps.\n* profile:\nSelect a specific profile from your credential file to get AWS credentials.\n* region:\nSet the AWS Region of the service. (e.g. us-east-1)\n* save_params:\nSave the parameters provided via the command line to the configuration file.", + "description": "Available parameters for the build command:\n* terraform_project_root_path:\nUsed for passing the Terraform project root directory path. Current directory will be used as a default value, if this parameter is not provided.\n* hook_name:\nHook package id to extend AWS SAM CLI commands functionality. \n\nExample: `terraform` to extend AWS SAM CLI commands functionality to support terraform applications. \n\nAvailable Hook Names: ['terraform']\n* skip_prepare_infra:\nSkip preparation stage when there are no infrastructure changes. Only used in conjunction with --hook-name.\n* use_container:\nBuild functions within an AWS Lambda-like container.\n* build_in_source:\nOpts in to build project in the source folder. The following workflows support building in source: ['nodejs12.x', 'nodejs14.x', 'nodejs16.x', 'nodejs18.x', 'Makefile', 'esbuild']\n* container_env_var:\nEnvironment variables to be passed into build containers\nResource format (FuncName.VarName=Value) or Global format (VarName=Value).\n\n Example: --container-env-var Func1.VAR1=value1 --container-env-var VAR2=value2\n* container_env_var_file:\nEnvironment variables json file (e.g., env_vars.json) to be passed to build containers.\n* build_image:\nContainer image URIs for building functions/layers. You can specify for all functions/layers with just the image URI (--build-image public.ecr.aws/sam/build-nodejs18.x:latest). You can specify for each individual function with (--build-image FunctionLogicalID=public.ecr.aws/sam/build-nodejs18.x:latest). A combination of the two can be used. If a function does not have build image specified or an image URI for all functions, the default SAM CLI build images will be used.\n* exclude:\nName of the resource(s) to exclude from AWS SAM CLI build.\n* parallel:\nEnable parallel builds for AWS SAM template's functions and layers.\n* mount_with:\nSpecify mount mode for building functions/layers inside container. If it is mounted with write permissions, some files in source code directory may be changed/added by the build process. By default the source code directory is read only.\n* build_dir:\nDirectory to store build artifacts.Note: This directory will be first removed before starting a build.\n* cache_dir:\nDirectory to store cached artifacts. The default cache directory is .aws-sam/cache\n* base_dir:\nResolve relative paths to function's source code with respect to this directory. Use this if SAM template and source code are not in same enclosing folder. By default, relative paths are resolved with respect to the SAM template's location.\n* manifest:\nPath to a custom dependency manifest. Example: custom-package.json\n* cached:\nEnable cached builds.Reuse build artifacts that have not changed from previous builds. \n\nAWS SAM CLI evaluates if files in your project directory have changed. \n\nNote: AWS SAM CLI does not evaluate changes made to third party modules that the project depends on.Example: Python function includes a requirements.txt file with the following entry requests=1.x and the latest request module version changes from 1.1 to 1.2, AWS SAM CLI will not pull the latest version until a non-cached build is run.\n* template_file:\nAWS SAM template file.\n* parameter_overrides:\nString that contains AWS CloudFormation parameter overrides encoded as key=value pairs.\n* skip_pull_image:\nSkip pulling down the latest Docker image for Lambda runtime.\n* docker_network:\nName or ID of an existing docker network for AWS Lambda docker containers to connect to, along with the default bridge network. If not specified, the Lambda containers will only connect to the default bridge docker network.\n* beta_features:\nEnable/Disable beta features.\n* debug:\nTurn on debug logging to print debug message generated by AWS SAM CLI and display timestamps.\n* profile:\nSelect a specific profile from your credential file to get AWS credentials.\n* region:\nSet the AWS Region of the service. (e.g. us-east-1)\n* save_params:\nSave the parameters provided via the command line to the configuration file.", "type": "object", "properties": { "terraform_project_root_path": { @@ -271,6 +271,11 @@ "type": "boolean", "description": "Build functions within an AWS Lambda-like container." }, + "build_in_source": { + "title": "build_in_source", + "type": "boolean", + "description": "Opts in to build project in the source folder. The following workflows support building in source: ['nodejs12.x', 'nodejs14.x', 'nodejs16.x', 'nodejs18.x', 'Makefile', 'esbuild']" + }, "container_env_var": { "title": "container_env_var", "type": "string", @@ -1520,7 +1525,7 @@ "properties": { "parameters": { "title": "Parameters for the sync command", - "description": "Available parameters for the sync command:\n* template_file:\nAWS SAM template file.\n* code:\nSync ONLY code resources. This includes Lambda Functions, API Gateway, and Step Functions.\n* watch:\nWatch local files and automatically sync with cloud.\n* resource_id:\nSync code for all the resources with the ID. To sync a resource within a nested stack, use the following pattern {ChildStack}/{logicalId}.\n* resource:\nSync code for all resources of the given resource type. Accepted values are ['AWS::Serverless::Function', 'AWS::Lambda::Function', 'AWS::Serverless::LayerVersion', 'AWS::Lambda::LayerVersion', 'AWS::Serverless::Api', 'AWS::ApiGateway::RestApi', 'AWS::Serverless::HttpApi', 'AWS::ApiGatewayV2::Api', 'AWS::Serverless::StateMachine', 'AWS::StepFunctions::StateMachine']\n* dependency_layer:\nSeparate dependencies of individual function into a Lambda layer for improved performance.\n* skip_deploy_sync:\nThis option will skip the initial infrastructure deployment if it is not required by comparing the local template with the template deployed in cloud.\n* stack_name:\nName of the AWS CloudFormation stack.\n* base_dir:\nResolve relative paths to function's source code with respect to this directory. Use this if SAM template and source code are not in same enclosing folder. By default, relative paths are resolved with respect to the SAM template's location.\n* use_container:\nBuild functions within an AWS Lambda-like container.\n* build_image:\nContainer image URIs for building functions/layers. You can specify for all functions/layers with just the image URI (--build-image public.ecr.aws/sam/build-nodejs18.x:latest). You can specify for each individual function with (--build-image FunctionLogicalID=public.ecr.aws/sam/build-nodejs18.x:latest). A combination of the two can be used. If a function does not have build image specified or an image URI for all functions, the default SAM CLI build images will be used.\n* image_repository:\nAWS ECR repository URI where artifacts referenced in the template are uploaded.\n* image_repositories:\nMapping of Function Logical ID to AWS ECR Repository URI.\n\nExample: Function_Logical_ID=ECR_Repo_Uri\nThis option can be specified multiple times.\n* s3_bucket:\nAWS S3 bucket where artifacts referenced in the template are uploaded.\n* s3_prefix:\nPrefix name that is added to the artifact's name when it is uploaded to the AWS S3 bucket.\n* kms_key_id:\nThe ID of an AWS KMS key that is used to encrypt artifacts that are at rest in the AWS S3 bucket.\n* role_arn:\nARN of an IAM role that AWS Cloudformation assumes when executing a deployment change set.\n* parameter_overrides:\nString that contains AWS CloudFormation parameter overrides encoded as key=value pairs.\n* beta_features:\nEnable/Disable beta features.\n* debug:\nTurn on debug logging to print debug message generated by AWS SAM CLI and display timestamps.\n* profile:\nSelect a specific profile from your credential file to get AWS credentials.\n* region:\nSet the AWS Region of the service. (e.g. us-east-1)\n* metadata:\nMap of metadata to attach to ALL the artifacts that are referenced in the template.\n* notification_arns:\nARNs of SNS topics that AWS Cloudformation associates with the stack.\n* tags:\nList of tags to associate with the stack.\n* capabilities:\nList of capabilities that one must specify before AWS Cloudformation can create certain stacks.\n\nAccepted Values: CAPABILITY_IAM, CAPABILITY_NAMED_IAM, CAPABILITY_RESOURCE_POLICY, CAPABILITY_AUTO_EXPAND.\n\nLearn more at: https://docs.aws.amazon.com/serverlessrepo/latest/devguide/acknowledging-application-capabilities.html\n* save_params:\nSave the parameters provided via the command line to the configuration file.", + "description": "Available parameters for the sync command:\n* template_file:\nAWS SAM template file.\n* code:\nSync ONLY code resources. This includes Lambda Functions, API Gateway, and Step Functions.\n* watch:\nWatch local files and automatically sync with cloud.\n* resource_id:\nSync code for all the resources with the ID. To sync a resource within a nested stack, use the following pattern {ChildStack}/{logicalId}.\n* resource:\nSync code for all resources of the given resource type. Accepted values are ['AWS::Serverless::Function', 'AWS::Lambda::Function', 'AWS::Serverless::LayerVersion', 'AWS::Lambda::LayerVersion', 'AWS::Serverless::Api', 'AWS::ApiGateway::RestApi', 'AWS::Serverless::HttpApi', 'AWS::ApiGatewayV2::Api', 'AWS::Serverless::StateMachine', 'AWS::StepFunctions::StateMachine']\n* dependency_layer:\nSeparate dependencies of individual function into a Lambda layer for improved performance.\n* skip_deploy_sync:\nThis option will skip the initial infrastructure deployment if it is not required by comparing the local template with the template deployed in cloud.\n* watch_exclude:\nExcludes a file or folder from being observed for file changes. Files and folders that are excluded will not trigger a sync workflow. This option can be provided multiple times.\n\nExamples:\n\nHelloWorldFunction=package-lock.json\n\nChildStackA/FunctionName=database.sqlite3\n* stack_name:\nName of the AWS CloudFormation stack.\n* base_dir:\nResolve relative paths to function's source code with respect to this directory. Use this if SAM template and source code are not in same enclosing folder. By default, relative paths are resolved with respect to the SAM template's location.\n* use_container:\nBuild functions within an AWS Lambda-like container.\n* build_in_source:\nOpts in to build project in the source folder. The following workflows support building in source: ['nodejs12.x', 'nodejs14.x', 'nodejs16.x', 'nodejs18.x', 'Makefile', 'esbuild']\n* build_image:\nContainer image URIs for building functions/layers. You can specify for all functions/layers with just the image URI (--build-image public.ecr.aws/sam/build-nodejs18.x:latest). You can specify for each individual function with (--build-image FunctionLogicalID=public.ecr.aws/sam/build-nodejs18.x:latest). A combination of the two can be used. If a function does not have build image specified or an image URI for all functions, the default SAM CLI build images will be used.\n* image_repository:\nAWS ECR repository URI where artifacts referenced in the template are uploaded.\n* image_repositories:\nMapping of Function Logical ID to AWS ECR Repository URI.\n\nExample: Function_Logical_ID=ECR_Repo_Uri\nThis option can be specified multiple times.\n* s3_bucket:\nAWS S3 bucket where artifacts referenced in the template are uploaded.\n* s3_prefix:\nPrefix name that is added to the artifact's name when it is uploaded to the AWS S3 bucket.\n* kms_key_id:\nThe ID of an AWS KMS key that is used to encrypt artifacts that are at rest in the AWS S3 bucket.\n* role_arn:\nARN of an IAM role that AWS Cloudformation assumes when executing a deployment change set.\n* parameter_overrides:\nString that contains AWS CloudFormation parameter overrides encoded as key=value pairs.\n* beta_features:\nEnable/Disable beta features.\n* debug:\nTurn on debug logging to print debug message generated by AWS SAM CLI and display timestamps.\n* profile:\nSelect a specific profile from your credential file to get AWS credentials.\n* region:\nSet the AWS Region of the service. (e.g. us-east-1)\n* metadata:\nMap of metadata to attach to ALL the artifacts that are referenced in the template.\n* notification_arns:\nARNs of SNS topics that AWS Cloudformation associates with the stack.\n* tags:\nList of tags to associate with the stack.\n* capabilities:\nList of capabilities that one must specify before AWS Cloudformation can create certain stacks.\n\nAccepted Values: CAPABILITY_IAM, CAPABILITY_NAMED_IAM, CAPABILITY_RESOURCE_POLICY, CAPABILITY_AUTO_EXPAND.\n\nLearn more at: https://docs.aws.amazon.com/serverlessrepo/latest/devguide/acknowledging-application-capabilities.html\n* save_params:\nSave the parameters provided via the command line to the configuration file.", "type": "object", "properties": { "template_file": { @@ -1573,6 +1578,14 @@ "description": "This option will skip the initial infrastructure deployment if it is not required by comparing the local template with the template deployed in cloud.", "default": true }, + "watch_exclude": { + "title": "watch_exclude", + "type": "array", + "description": "Excludes a file or folder from being observed for file changes. Files and folders that are excluded will not trigger a sync workflow. This option can be provided multiple times.\n\nExamples:\n\nHelloWorldFunction=package-lock.json\n\nChildStackA/FunctionName=database.sqlite3", + "items": { + "type": "string" + } + }, "stack_name": { "title": "stack_name", "type": "string", @@ -1588,6 +1601,11 @@ "type": "boolean", "description": "Build functions within an AWS Lambda-like container." }, + "build_in_source": { + "title": "build_in_source", + "type": "boolean", + "description": "Opts in to build project in the source folder. The following workflows support building in source: ['nodejs12.x', 'nodejs14.x', 'nodejs16.x', 'nodejs18.x', 'Makefile', 'esbuild']" + }, "build_image": { "title": "build_image", "type": "string", diff --git a/tests/integration/buildcmd/test_build_in_source.py b/tests/integration/buildcmd/test_build_in_source.py index 8c9958a912..6bb8d4ab43 100644 --- a/tests/integration/buildcmd/test_build_in_source.py +++ b/tests/integration/buildcmd/test_build_in_source.py @@ -1,32 +1,36 @@ import os -import shutil +from pathlib import Path import pytest import logging -from unittest import skip from parameterized import parameterized +from samcli.lib.utils import osutils -from tests.integration.buildcmd.build_integ_base import BuildIntegProvidedBase, BuildIntegEsbuildBase +from tests.integration.buildcmd.build_integ_base import ( + BuildIntegNodeBase, + BuildIntegProvidedBase, + BuildIntegEsbuildBase, +) +from tests.testing_utils import run_command LOG = logging.getLogger(__name__) -@skip("Building in source option is not exposed yet. Stop skipping once it is.") class TestBuildCommand_BuildInSource_Makefile(BuildIntegProvidedBase): template = "template.yaml" is_nested_parent = False - @classmethod - def setUpClass(cls): - super().setUpClass() - cls.code_uri = "provided_create_new_file" - cls.code_uri_path = os.path.join(cls.test_data_path, cls.code_uri) - cls.file_created_from_make_command = "file-created-from-make-command.txt" + def setUp(self): + super().setUp() - def tearDown(self): - super().tearDown() - new_file_in_codeuri_path = os.path.join(self.code_uri_path, self.file_created_from_make_command) - if os.path.isfile(new_file_in_codeuri_path): - os.remove(new_file_in_codeuri_path) + self.code_uri = "provided_create_new_file" + test_data_code_uri = Path(self.test_data_path, self.code_uri) + self.file_created_from_make_command = "file-created-from-make-command.txt" + + scratch_code_uri_path = Path(self.working_dir, self.code_uri) + self.code_uri_path = str(scratch_code_uri_path) + + # copy source code into temporary directory and update code uri to that scratch dir + osutils.copytree(test_data_code_uri, scratch_code_uri_path) @parameterized.expand( [ @@ -41,7 +45,7 @@ def test_builds_successfully_with_makefile(self, build_in_source, new_file_shoul runtime="provided.al2", use_container=False, manifest=None, - code_uri=self.code_uri, + code_uri=self.code_uri_path, build_in_source=build_in_source, ) @@ -50,19 +54,15 @@ def test_builds_successfully_with_makefile(self, build_in_source, new_file_shoul ) -@skip("Building in source option is not exposed yet. Stop skipping once it is.") class TestBuildCommand_BuildInSource_Esbuild(BuildIntegEsbuildBase): is_nested_parent = False + template = "template_with_metadata_esbuild.yaml" def setUp(self): super().setUp() - self.source_directories = [] - def tearDown(self): - super().tearDown() - # clean up dependencies installed in source directories - for source in self.source_directories: - shutil.rmtree(os.path.join(source, "node_modules"), ignore_errors=True) + source_files_path = Path(self.test_data_path, "Esbuild") + osutils.copytree(source_files_path, self.working_dir) @parameterized.expand( [ @@ -73,13 +73,11 @@ def tearDown(self): ) @pytest.mark.flaky(reruns=3) def test_builds_successfully_without_local_dependencies(self, build_in_source, dependencies_expected_in_source): - self.template_path = os.path.join(self.test_data_path, "template_with_metadata.yaml") - codeuri = os.path.join(self.test_data_path, "Esbuild", "Node") - self.source_directories = [codeuri] + codeuri = os.path.join(self.working_dir, "Node") self._test_with_default_package_json( build_in_source=build_in_source, - runtime="nodejs16.x", + runtime="nodejs18.x", code_uri=codeuri, handler="main.lambdaHandler", architecture="x86_64", @@ -92,10 +90,8 @@ def test_builds_successfully_without_local_dependencies(self, build_in_source, d @pytest.mark.flaky(reruns=3) def test_builds_successfully_with_local_dependency(self): - self.template_path = os.path.join(self.test_data_path, "template_with_metadata.yaml") - codeuri = os.path.join(self.test_data_path, "Esbuild", "NodeWithLocalDependency") - self.source_directories = [codeuri] - runtime = "nodejs16.x" + codeuri = os.path.join(self.working_dir, "NodeWithLocalDependency") + runtime = "nodejs18.x" architecture = "x86_64" self._test_with_default_package_json( @@ -110,3 +106,57 @@ def test_builds_successfully_with_local_dependency(self): # check whether dependencies were installed in source dir self.assertEqual(os.path.isdir(os.path.join(codeuri, "node_modules")), True) + + +class TestBuildCommand_BuildInSource_Nodejs(BuildIntegNodeBase): + is_nested_parent = False + template = "template.yaml" + + def setUp(self): + super().setUp() + + osutils.copytree(Path(self.test_data_path, "Esbuild"), self.working_dir) + + def tearDown(self): + super().tearDown() + + def validate_node_modules(self, is_build_in_source_behaviour: bool): + # validate if node modules exist in the built artifact dir + built_node_modules = Path(self.default_build_dir, "Function", "node_modules") + self.assertEqual(built_node_modules.is_dir(), True, "node_modules not found in artifact dir") + self.assertEqual(built_node_modules.is_symlink(), is_build_in_source_behaviour) + + # validate that node modules are suppose to exist in the source dir + source_node_modules = Path(self.codeuri_path, "node_modules") + self.assertEqual(source_node_modules.is_dir(), is_build_in_source_behaviour) + + @parameterized.expand( + [ + (True, True), # build in source + (False, False), # don't build in source + (None, False), # use default for workflow (don't build in source) + ] + ) + @pytest.mark.flaky(reruns=3) + def test_builds_successfully_without_local_dependencies(self, build_in_source, expected_built_in_source): + self.codeuri_path = Path(self.working_dir, "Node") + + overrides = self.get_override( + runtime="nodejs18.x", code_uri=self.codeuri_path, architecture="x86_64", handler="main.lambdaHandler" + ) + command_list = self.get_command_list(build_in_source=build_in_source, parameter_overrides=overrides, debug=True) + + run_command(command_list, self.working_dir) + self.validate_node_modules(expected_built_in_source) + + @pytest.mark.flaky(reruns=3) + def test_builds_successfully_with_local_dependency(self): + self.codeuri_path = Path(self.working_dir, "NodeWithLocalDependency") + + overrides = self.get_override( + runtime="nodejs18.x", code_uri=self.codeuri_path, architecture="x86_64", handler="main.lambdaHandler" + ) + command_list = self.get_command_list(build_in_source=True, parameter_overrides=overrides) + + run_command(command_list, self.working_dir) + self.validate_node_modules(True) diff --git a/tests/integration/buildcmd/test_build_terraform_applications_other_cases.py b/tests/integration/buildcmd/test_build_terraform_applications_other_cases.py index 7bd5675c0a..08c5653f39 100644 --- a/tests/integration/buildcmd/test_build_terraform_applications_other_cases.py +++ b/tests/integration/buildcmd/test_build_terraform_applications_other_cases.py @@ -26,7 +26,7 @@ def test_invalid_coexist_parameters(self): process_stderr = stderr.strip() self.assertRegex( process_stderr.decode("utf-8"), - "Error: Invalid value: Parameters hook-name, and t,template-file,template,parameter-overrides cannot " + "Error: Invalid value: Parameters hook-name, and t,template-file,template,parameter-overrides,build-in-source cannot " "be used together", ) self.assertNotEqual(return_code, 0) diff --git a/tests/integration/local/invoke/invoke_integ_base.py b/tests/integration/local/invoke/invoke_integ_base.py index dccc2b6583..f30cc8db96 100644 --- a/tests/integration/local/invoke/invoke_integ_base.py +++ b/tests/integration/local/invoke/invoke_integ_base.py @@ -96,6 +96,8 @@ def get_build_command_list( cached=None, parallel=None, use_container=None, + build_dir=None, + build_in_source=None, ): command_list = [self.cmd, "build"] @@ -111,10 +113,16 @@ def get_build_command_list( if use_container: command_list = command_list + ["-u"] + if build_dir: + command_list = command_list + ["-b", build_dir] + + if build_in_source: + command_list = command_list + ["--build-in-source"] + return command_list - def run_command(self, command_list, env=None): - process = Popen(command_list, stdout=PIPE, env=env) + def run_command(self, command_list, env=None, cwd=None): + process = Popen(command_list, stdout=PIPE, env=env, cwd=cwd) try: (stdout, stderr) = process.communicate(timeout=TIMEOUT) return stdout, stderr, process.returncode diff --git a/tests/integration/local/invoke/test_invoke_build_in_source.py b/tests/integration/local/invoke/test_invoke_build_in_source.py new file mode 100644 index 0000000000..8204cfadf3 --- /dev/null +++ b/tests/integration/local/invoke/test_invoke_build_in_source.py @@ -0,0 +1,83 @@ +from pathlib import Path +import shutil +import tempfile +import os + +from samcli.lib.utils import osutils +from tests.integration.local.invoke.invoke_integ_base import InvokeIntegBase + + +class BuildInSourceInvokeBase(InvokeIntegBase): + project_test_folder: str + + def setUp(self): + self.project_folder_path = Path(self.test_data_path, "invoke", self.project_test_folder) + self.test_project_folder = tempfile.mkdtemp() + self.build_dir = Path(self.test_project_folder, ".aws-sam", "build") + + osutils.copytree(self.project_folder_path, self.test_project_folder) + + self.template_path = Path(self.test_project_folder, "template.yaml") + self.built_template_path = Path(self.build_dir, "template.yaml") + + def tearDown(self): + try: + shutil.rmtree(self.test_project_folder, ignore_errors=True) + except: + pass + + +class TestInvokeBuildInSourceSymlinkedModules(BuildInSourceInvokeBase): + project_test_folder = "build-in-source" + + def _validate_modules_linked(self): + node_modules = Path(self.build_dir, "PrintLocalDep", "node_modules") + local_dep = Path(node_modules, "local-dep") + + # node modules folder should be a symlink + self.assertEqual(os.path.islink(node_modules), True) + + # local-deps folder should not if links were installed + self.assertEqual(os.path.islink(local_dep), False) + + def test_successful_invoke(self): + build_command = self.get_build_command_list( + template_path=self.template_path, build_dir=self.build_dir, build_in_source=True + ) + _, _, exit_code = self.run_command(build_command) + + self.assertEqual(exit_code, 0) + self._validate_modules_linked() + + invoke_command = self.get_command_list( + template_path=self.built_template_path, function_to_invoke="PrintLocalDep" + ) + stdout, _, exit_code = self.run_command(invoke_command) + + self.assertEqual(exit_code, 0) + self.assertEqual(stdout.decode("utf-8"), "123") + + +class TestInvokeBuildInSourceSymlinkedLayers(BuildInSourceInvokeBase): + project_test_folder = str(Path("build-in-source", "layer_symlink")) + + def test_successful_invoke(self): + build_command = self.get_build_command_list( + template_path=self.template_path, build_dir=self.build_dir, build_in_source=True + ) + + _, _, exit_code = self.run_command(build_command, cwd=self.test_project_folder) + + self.assertEqual(exit_code, 0) + + # check if layers is symlinked + layer_artifact_node_folder = Path(self.build_dir, "MyLayer", "nodejs", "node_modules") + self.assertEqual(os.path.islink(layer_artifact_node_folder), True) + + invoke_command = self.get_command_list( + template_path=self.built_template_path, function_to_invoke="HelloWorldFunction" + ) + stdout, _, exit_code = self.run_command(invoke_command, cwd=self.test_project_folder) + + self.assertEqual(exit_code, 0) + self.assertEqual(stdout.decode("utf-8"), '{"statusCode": 200, "body": "foo bar"}') diff --git a/tests/integration/sync/sync_integ_base.py b/tests/integration/sync/sync_integ_base.py index 575cfc1b00..7797bf11c7 100644 --- a/tests/integration/sync/sync_integ_base.py +++ b/tests/integration/sync/sync_integ_base.py @@ -244,6 +244,7 @@ def get_sync_command_list( debug=None, use_container=False, build_in_source=None, + watch_exclude=None, ): command_list = [get_sam_command(), "sync"] @@ -305,5 +306,8 @@ def get_sync_command_list( command_list += ["--use-container"] if build_in_source is not None: command_list += ["--build-in-source"] if build_in_source else ["--no-build-in-source"] + if watch_exclude: + for exclude in watch_exclude: + command_list += ["--watch-exclude", exclude] return command_list diff --git a/tests/integration/sync/test_sync_build_in_source.py b/tests/integration/sync/test_sync_build_in_source.py index 5af1a917ce..1c996b51a2 100644 --- a/tests/integration/sync/test_sync_build_in_source.py +++ b/tests/integration/sync/test_sync_build_in_source.py @@ -4,7 +4,6 @@ import pytest import logging import json -from unittest import skip from parameterized import parameterized, parameterized_class from tests.integration.sync.sync_integ_base import SyncIntegBase @@ -15,7 +14,6 @@ LOG = logging.getLogger(__name__) -@skip("Building in source option is not exposed yet. Stop skipping once it is.") class TestSyncInfra_BuildInSource_Makefile(SyncIntegBase): dependency_layer = False @@ -30,6 +28,7 @@ def setUp(self): def tearDown(self): super().tearDown() + for path in self.new_files_in_source: if os.path.isfile(path): os.remove(path) @@ -57,7 +56,7 @@ def test_sync_builds_and_deploys_successfully(self, build_in_source, new_file_sh tags="integ=true clarity=yes foo_bar=baz", build_in_source=build_in_source, ) - sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode()) + sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode(), cwd=self.test_data_path) self.assertEqual(sync_process_execute.process.returncode, 0) self.assertIn("Sync infra completed.", str(sync_process_execute.stderr)) @@ -74,7 +73,6 @@ def test_sync_builds_and_deploys_successfully(self, build_in_source, new_file_sh ) -@skip("Building in source option is not exposed yet. Stop skipping once it is.") class TestSyncCode_BuildInSource_Makefile(TestSyncCodeBase): dependency_layer = False folder = "code" @@ -87,10 +85,11 @@ def setUp(self): Path("makefile_layer_create_new_file", "file-created-from-makefile-layer.txt"), ] # When running tests, TestSyncCodeBase copies the source onto a temp directory - self.new_files_in_source = [TestSyncCodeBase.temp_dir.joinpath(path) for path in paths] + self.new_files_in_source = [Path(self.test_data_path, self.folder, "before", path) for path in paths] def tearDown(self): super().tearDown() + for path in self.new_files_in_source: if os.path.isfile(path): os.remove(path) @@ -105,11 +104,10 @@ def tearDown(self): def test_sync_code_builds_and_deploys_successfully(self, build_in_source, new_file_should_be_in_source): # update layer to trigger rebuild layer_path = "makefile_layer_create_new_file" - shutil.rmtree(TestSyncCodeBase.temp_dir.joinpath(layer_path), ignore_errors=True) - shutil.copytree( - self.test_data_path.joinpath(self.folder).joinpath("after").joinpath(layer_path), - TestSyncCodeBase.temp_dir.joinpath(layer_path), - ) + code_uri = Path(self.test_data_path, self.folder) + + shutil.rmtree(Path(code_uri, "before", layer_path), ignore_errors=True) + shutil.copytree(Path(code_uri, "after", layer_path), Path(code_uri, "before", layer_path)) # Run code sync sync_command_list = self.get_sync_command_list( @@ -123,7 +121,7 @@ def test_sync_code_builds_and_deploys_successfully(self, build_in_source, new_fi tags="integ=true clarity=yes foo_bar=baz", build_in_source=build_in_source, ) - sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode()) + sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode(), cwd=self.test_data_path) self.assertEqual(sync_process_execute.process.returncode, 0) # check whether the new files were created in the source directory @@ -139,16 +137,16 @@ def test_sync_code_builds_and_deploys_successfully(self, build_in_source, new_fi ) -@skip("Building in source option is not exposed yet. Stop skipping once it is.") -@parameterized_class([{"dependency_layer": True}, {"dependency_layer": False}]) class TestSyncInfra_BuildInSource_Esbuild(SyncIntegBase): + dependency_layer = False + def setUp(self): super().setUp() self.source_dependencies_paths = [] def tearDown(self): super().tearDown() - # clean up dependencies installed in source directories + for path in self.source_dependencies_paths: shutil.rmtree(path, ignore_errors=True) @@ -182,7 +180,7 @@ def test_sync_builds_successfully_without_local_dependencies( tags="integ=true clarity=yes foo_bar=baz", build_in_source=build_in_source, ) - sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode()) + sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode(), cwd=self.test_data_path) self.assertEqual(sync_process_execute.process.returncode, 0) self.assertIn("Sync infra completed.", str(sync_process_execute.stderr)) @@ -217,7 +215,7 @@ def test_sync_builds_successfully_with_local_dependency(self): tags="integ=true clarity=yes foo_bar=baz", build_in_source=True, ) - sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode()) + sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode(), cwd=self.test_data_path) self.assertEqual(sync_process_execute.process.returncode, 0) self.assertIn("Sync infra completed.", str(sync_process_execute.stderr)) @@ -232,8 +230,6 @@ def test_sync_builds_successfully_with_local_dependency(self): self.assertEqual(lambda_response.get("message"), "hello world") -@skip("Building in source option is not exposed yet. Stop skipping once it is.") -@parameterized_class([{"dependency_layer": True}, {"dependency_layer": False}]) class TestSyncCode_BuildInSource_Esbuild(TestSyncCodeBase): dependency_layer = False folder = "code" @@ -241,11 +237,12 @@ class TestSyncCode_BuildInSource_Esbuild(TestSyncCodeBase): def setUp(self): super().setUp() + self.source_dependencies_paths = [] def tearDown(self): super().tearDown() - # clean up dependencies installed in source directories + for path in self.source_dependencies_paths: shutil.rmtree(path, ignore_errors=True) @@ -259,7 +256,19 @@ def tearDown(self): def test_sync_code_builds_successfully_without_local_dependencies( self, build_in_source, dependencies_expected_in_source ): - self.source_dependencies_paths = [TestSyncCodeBase.temp_dir.joinpath("esbuild_function", "node_modules")] + code_folder = "esbuild_function" + + # make a code change (update message and add extra message in output) + shutil.rmtree(self.test_data_path.joinpath(self.folder, "before", code_folder)) + shutil.copytree( + self.test_data_path.joinpath(self.folder, "after", code_folder), + self.test_data_path.joinpath(self.folder, "before", code_folder), + dirs_exist_ok=True, + ) + + self.source_dependencies_paths = [ + Path.joinpath(self.test_data_path, self.folder, "before", code_folder, "node_modules") + ] # Run code sync sync_command_list = self.get_sync_command_list( @@ -273,7 +282,7 @@ def test_sync_code_builds_successfully_without_local_dependencies( tags="integ=true clarity=yes foo_bar=baz", build_in_source=build_in_source, ) - sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode()) + sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode(), cwd=self.test_data_path) self.assertEqual(sync_process_execute.process.returncode, 0) # check whether dependencies were installed in the source directory @@ -284,11 +293,12 @@ def test_sync_code_builds_successfully_without_local_dependencies( lambda_functions = stack_resources.get(AWS_LAMBDA_FUNCTION) for lambda_function in lambda_functions: lambda_response = json.loads(self._get_lambda_response(lambda_function)) - self.assertEqual(lambda_response.get("message"), "hello world") + self.assertEqual(lambda_response.get("message"), "Hello world!") + self.assertEqual(lambda_response.get("extra_message"), "banana") def test_sync_code_builds_successfully_with_local_dependencies(self): codeuri = "esbuild_function_with_local_dependency" - self.source_dependencies_paths = [TestSyncCodeBase.temp_dir.joinpath(codeuri, "node_modules")] + self.source_dependencies_paths = [Path(self.test_data_path, self.folder, "before", codeuri, "node_modules")] # Run code sync sync_command_list = self.get_sync_command_list( @@ -303,7 +313,7 @@ def test_sync_code_builds_successfully_with_local_dependencies(self): tags="integ=true clarity=yes foo_bar=baz", build_in_source=True, ) - sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode()) + sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode(), cwd=self.test_data_path) self.assertEqual(sync_process_execute.process.returncode, 0) # check whether dependencies were installed in the source directory @@ -315,3 +325,120 @@ def test_sync_code_builds_successfully_with_local_dependencies(self): for lambda_function in lambda_functions: lambda_response = json.loads(self._get_lambda_response(lambda_function)) self.assertEqual(lambda_response.get("message"), "hello world") + + +class TestSyncCode_BuildInSource_Nodejs_Without_Local_Dep(TestSyncCodeBase): + dependency_layer = False + folder = "code" + template = "template-nodejs.yaml" + + def tearDown(self): + super().tearDown() + shutil.rmtree(Path(self.test_data_path, ".aws-sam"), ignore_errors=True) + + for path in self.source_dependencies_paths: + shutil.rmtree(path, ignore_errors=True) + + @parameterized.expand( + [ + (True, True), # build in source + (False, False), # don't build in source + (None, False), # use default for workflow (don't build in source) + ] + ) + def test_sync_code_builds_successfully_without_local_dependencies( + self, build_in_source, dependencies_expected_in_source + ): + code_folder = "nodejs_function" + + # make a code change (calling faker api method) + shutil.rmtree(self.test_data_path.joinpath(self.folder, "before", code_folder), ignore_errors=True) + shutil.copytree( + self.test_data_path.joinpath(self.folder, "after", code_folder), + self.test_data_path.joinpath(self.folder, "before", code_folder), + dirs_exist_ok=True, + ) + + self.source_dependencies_paths = [ + Path.joinpath(self.test_data_path, self.folder, "before", code_folder, "node_modules") + ] + + # Run code sync + sync_command_list = self.get_sync_command_list( + template_file=TestSyncCodeBase.template_path, + stack_name=TestSyncCodeBase.stack_name, + code=True, + dependency_layer=self.dependency_layer, + image_repository=self.ecr_repo_name, + s3_prefix=self.s3_prefix, + kms_key_id=self.kms_key, + tags="integ=true clarity=yes foo_bar=baz", + build_in_source=build_in_source, + ) + sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode(), cwd=self.test_data_path) + self.assertEqual(sync_process_execute.process.returncode, 0) + + # check whether dependencies were installed in the source directory + for path in self.source_dependencies_paths: + self.assertEqual(os.path.isdir(path), dependencies_expected_in_source) + + stack_resources = self._get_stacks(TestSyncCodeBase.stack_name) + lambda_functions = stack_resources.get(AWS_LAMBDA_FUNCTION) + for lambda_function in lambda_functions: + lambda_response = json.loads(self._get_lambda_response(lambda_function)) + + self.assertEqual(lambda_response.get("message"), "Hello world!") + # extra message is new field containing random name + self.assertIn("extra_message", lambda_response) + + +class TestSyncCode_BuildInSource_Nodejs_Using_Local_Dep(TestSyncCodeBase): + dependency_layer = False + folder = "code" + template = "template_nodejs_local_dep.yaml" + + def tearDown(self): + super().tearDown() + + for path in self.source_dependencies_paths: + shutil.rmtree(path, ignore_errors=True) + + def test_sync_code_builds_successfully_with_local_dependencies(self): + code_folder = "nodejs_local_dep" + + # make a code change (installing local dep) + shutil.rmtree(self.test_data_path.joinpath(self.folder, "before", code_folder), ignore_errors=True) + shutil.copytree( + self.test_data_path.joinpath(self.folder, "after", code_folder), + self.test_data_path.joinpath(self.folder, "before", code_folder), + dirs_exist_ok=True, + ) + + self.source_dependencies_paths = [ + Path(self.test_data_path, self.folder, "before", code_folder, "src", "node_modules") + ] + + # Run code sync + sync_command_list = self.get_sync_command_list( + template_file=TestSyncCodeBase.template_path, + stack_name=TestSyncCodeBase.stack_name, + code=True, + dependency_layer=self.dependency_layer, + image_repository=self.ecr_repo_name, + s3_prefix=self.s3_prefix, + kms_key_id=self.kms_key, + tags="integ=true clarity=yes foo_bar=baz", + build_in_source=True, + ) + sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode(), cwd=self.test_data_path) + self.assertEqual(sync_process_execute.process.returncode, 0) + + # check whether dependencies were installed in the source directory + for path in self.source_dependencies_paths: + self.assertEqual(os.path.isdir(path), True) + + stack_resources = self._get_stacks(TestSyncCodeBase.stack_name) + lambda_functions = stack_resources.get(AWS_LAMBDA_FUNCTION) + for lambda_function in lambda_functions: + lambda_response = json.loads(self._get_lambda_response(lambda_function)) + self.assertEqual(lambda_response.get("message"), 123) diff --git a/tests/integration/sync/test_sync_watch.py b/tests/integration/sync/test_sync_watch.py index f1e2ebfc26..f450060cb9 100644 --- a/tests/integration/sync/test_sync_watch.py +++ b/tests/integration/sync/test_sync_watch.py @@ -7,7 +7,7 @@ import logging import json from pathlib import Path -from typing import Dict +from typing import Dict, List from unittest import skipIf import pytest @@ -55,6 +55,7 @@ class TestSyncWatchBase(SyncIntegBase): template_before = "" parameter_overrides: Dict[str, str] = {} + watch_exclude: List[str] = [] def setUp(self): # set up clean testing folder @@ -161,6 +162,7 @@ def _setup_verify_infra(self): s3_prefix=self.s3_prefix, kms_key_id=self.kms_key, tags="integ=true clarity=yes foo_bar=baz", + watch_exclude=self.watch_exclude, ) self.watch_process = start_persistent_process(sync_command_list, cwd=self.test_data_path) @@ -826,3 +828,39 @@ def test_sync_watch_infra(self): # Updated Infra Validation self.run_initial_infra_validation() + + +class TestSyncWatchCodeWatchExclude(TestSyncWatchEsbuildBase): + dependency_layer = False + template_before = str(Path("code", "before", "template-esbuild.yaml")) + watch_exclude = ["HelloWorldFunction=app.ts"] + + def test_sync_watch_code_excludes(self): + self.stack_resources = self._get_stacks(self.stack_name) + + # Test Lambda Function + lambda_functions = self.stack_resources.get(AWS_LAMBDA_FUNCTION) + for lambda_function in lambda_functions: + lambda_response = json.loads(self._get_lambda_response(lambda_function)) + self.assertNotIn("extra_message", lambda_response) + self.assertEqual(lambda_response.get("message"), "hello world") + + self.update_file( + self.test_data_path.joinpath("code", "after", "esbuild_function", "app.ts"), + self.test_data_path.joinpath("code", "before", "esbuild_function", "app.ts"), + ) + + try: + # wait a couple of seconds to see if a sync flow starts + read_until_string(self.watch_process, "", timeout=3) + self.fail("Sync started syncflow when app.ts file update was ignored") + except TimeoutError: + # got timeout error, this is expected since there isn't + # suppose to be a sync + pass + + lambda_functions = self.stack_resources.get(AWS_LAMBDA_FUNCTION) + for lambda_function in lambda_functions: + lambda_response = json.loads(self._get_lambda_response(lambda_function)) + self.assertNotIn("extra_message", lambda_response) + self.assertEqual(lambda_response.get("message"), "hello world") diff --git a/tests/integration/testdata/invoke/build-in-source/layer_symlink/mycoolfunction/app.js b/tests/integration/testdata/invoke/build-in-source/layer_symlink/mycoolfunction/app.js new file mode 100644 index 0000000000..fe0a362ce6 --- /dev/null +++ b/tests/integration/testdata/invoke/build-in-source/layer_symlink/mycoolfunction/app.js @@ -0,0 +1,10 @@ +const faker = require("@faker-js/faker"); + +exports.handler = async (event, context) => { + const name = faker.faker.name.firstName(); + + return { + 'statusCode': 200, + 'body': "foo bar" + } +} \ No newline at end of file diff --git a/tests/integration/testdata/invoke/build-in-source/layer_symlink/mycoolfunction/package.json b/tests/integration/testdata/invoke/build-in-source/layer_symlink/mycoolfunction/package.json new file mode 100644 index 0000000000..2429ba224b --- /dev/null +++ b/tests/integration/testdata/invoke/build-in-source/layer_symlink/mycoolfunction/package.json @@ -0,0 +1,9 @@ +{ + "name": "hello_world", + "version": "1.0.0", + "description": "hello world sample for NodeJS", + "author": "SAM CLI", + "license": "MIT", + "dependencies": { + } +} \ No newline at end of file diff --git a/tests/integration/testdata/invoke/build-in-source/layer_symlink/mycoollayer/package.json b/tests/integration/testdata/invoke/build-in-source/layer_symlink/mycoollayer/package.json new file mode 100644 index 0000000000..9a53f32b35 --- /dev/null +++ b/tests/integration/testdata/invoke/build-in-source/layer_symlink/mycoollayer/package.json @@ -0,0 +1,12 @@ +{ + "name": "hello_world", + "version": "1.0.0", + "description": "hello world sample for NodeJS", + "main": "app.js", + "author": "SAM CLI", + "license": "MIT", + "dependencies": { + "axios": ">=0.21.1", + "@faker-js/faker": "7.1.0" + } +} \ No newline at end of file diff --git a/tests/integration/testdata/invoke/build-in-source/layer_symlink/template.yaml b/tests/integration/testdata/invoke/build-in-source/layer_symlink/template.yaml new file mode 100644 index 0000000000..b18c3cf634 --- /dev/null +++ b/tests/integration/testdata/invoke/build-in-source/layer_symlink/template.yaml @@ -0,0 +1,25 @@ +AWSTemplateFormatVersion: "2010-09-09" +Transform: AWS::Serverless-2016-10-31 + +Globals: + Function: + Timeout: 10 + +Resources: + HelloWorldFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: mycoolfunction + Handler: app.handler + Runtime: nodejs18.x + Layers: + - Ref: MyLayer + MyLayer: + Type: AWS::Lambda::LayerVersion + Properties: + LayerName: mycoollayer + Content: mycoollayer + CompatibleRuntimes: + - nodejs18.x + Metadata: + BuildMethod: nodejs18.x \ No newline at end of file diff --git a/tests/integration/testdata/invoke/build-in-source/local-dep/index.js b/tests/integration/testdata/invoke/build-in-source/local-dep/index.js new file mode 100644 index 0000000000..3744416445 --- /dev/null +++ b/tests/integration/testdata/invoke/build-in-source/local-dep/index.js @@ -0,0 +1,5 @@ +const foo = 123; + +exports.exported = () => { + return foo; +} diff --git a/tests/integration/testdata/invoke/build-in-source/local-dep/package.json b/tests/integration/testdata/invoke/build-in-source/local-dep/package.json new file mode 100644 index 0000000000..3979d77b27 --- /dev/null +++ b/tests/integration/testdata/invoke/build-in-source/local-dep/package.json @@ -0,0 +1,8 @@ +{ + "name": "local-dep", + "version": "1.0.0", + "description": "", + "main": "index.js", + "author": "" + } + \ No newline at end of file diff --git a/tests/integration/testdata/invoke/build-in-source/src/index.js b/tests/integration/testdata/invoke/build-in-source/src/index.js new file mode 100644 index 0000000000..7c4a39b82a --- /dev/null +++ b/tests/integration/testdata/invoke/build-in-source/src/index.js @@ -0,0 +1,5 @@ +const localDep = require("local-dep"); + +exports.handler = async (event, context) => { + return localDep.exported(); +} diff --git a/tests/integration/testdata/invoke/build-in-source/src/package.json b/tests/integration/testdata/invoke/build-in-source/src/package.json new file mode 100644 index 0000000000..e8a8127b29 --- /dev/null +++ b/tests/integration/testdata/invoke/build-in-source/src/package.json @@ -0,0 +1,10 @@ +{ + "name": "src", + "version": "1.0.0", + "description": "", + "main": "index.js", + "author": "", + "dependencies": { + "local-dep": "file:../local-dep" + } +} diff --git a/tests/integration/testdata/invoke/build-in-source/template.yaml b/tests/integration/testdata/invoke/build-in-source/template.yaml new file mode 100644 index 0000000000..450d2bba35 --- /dev/null +++ b/tests/integration/testdata/invoke/build-in-source/template.yaml @@ -0,0 +1,10 @@ +AWSTemplateFormatVersion : '2010-09-09' +Transform: AWS::Serverless-2016-10-31 + +Resources: + PrintLocalDep: + Type: AWS::Serverless::Function + Properties: + Handler: index.handler + Runtime: nodejs18.x + CodeUri: src diff --git a/tests/integration/testdata/sync/code/after/nodejs_function/app.js b/tests/integration/testdata/sync/code/after/nodejs_function/app.js index 4a2983dffd..75d5a69266 100644 --- a/tests/integration/testdata/sync/code/after/nodejs_function/app.js +++ b/tests/integration/testdata/sync/code/after/nodejs_function/app.js @@ -1,4 +1,4 @@ -import * as faker from '@faker-js/faker'; +const faker = require("@faker-js/faker"); const name = faker.faker.name.firstName(); let response; diff --git a/tests/integration/testdata/sync/code/after/nodejs_local_dep/local-dep/index.js b/tests/integration/testdata/sync/code/after/nodejs_local_dep/local-dep/index.js new file mode 100644 index 0000000000..3744416445 --- /dev/null +++ b/tests/integration/testdata/sync/code/after/nodejs_local_dep/local-dep/index.js @@ -0,0 +1,5 @@ +const foo = 123; + +exports.exported = () => { + return foo; +} diff --git a/tests/integration/testdata/sync/code/after/nodejs_local_dep/local-dep/package.json b/tests/integration/testdata/sync/code/after/nodejs_local_dep/local-dep/package.json new file mode 100644 index 0000000000..3979d77b27 --- /dev/null +++ b/tests/integration/testdata/sync/code/after/nodejs_local_dep/local-dep/package.json @@ -0,0 +1,8 @@ +{ + "name": "local-dep", + "version": "1.0.0", + "description": "", + "main": "index.js", + "author": "" + } + \ No newline at end of file diff --git a/tests/integration/testdata/sync/code/after/nodejs_local_dep/src/index.js b/tests/integration/testdata/sync/code/after/nodejs_local_dep/src/index.js new file mode 100644 index 0000000000..e6b0828cd8 --- /dev/null +++ b/tests/integration/testdata/sync/code/after/nodejs_local_dep/src/index.js @@ -0,0 +1,10 @@ +const localDep = require("local-dep"); + +exports.handler = async (event, context) => { + return { + 'statusCode': 200, + 'body': JSON.stringify({ + message: localDep.exported(), + }) + }; +} diff --git a/tests/integration/testdata/sync/code/after/nodejs_local_dep/src/package.json b/tests/integration/testdata/sync/code/after/nodejs_local_dep/src/package.json new file mode 100644 index 0000000000..e8a8127b29 --- /dev/null +++ b/tests/integration/testdata/sync/code/after/nodejs_local_dep/src/package.json @@ -0,0 +1,10 @@ +{ + "name": "src", + "version": "1.0.0", + "description": "", + "main": "index.js", + "author": "", + "dependencies": { + "local-dep": "file:../local-dep" + } +} diff --git a/tests/integration/testdata/sync/code/before/nodejs_local_dep/src/index.js b/tests/integration/testdata/sync/code/before/nodejs_local_dep/src/index.js new file mode 100644 index 0000000000..0260c2696d --- /dev/null +++ b/tests/integration/testdata/sync/code/before/nodejs_local_dep/src/index.js @@ -0,0 +1,8 @@ +exports.handler = async (event, context) => { + return { + 'statusCode': 200, + 'body': JSON.stringify({ + message: 'no local dep', + }) + } +} diff --git a/tests/integration/testdata/sync/code/before/nodejs_local_dep/src/package.json b/tests/integration/testdata/sync/code/before/nodejs_local_dep/src/package.json new file mode 100644 index 0000000000..9de668ad9d --- /dev/null +++ b/tests/integration/testdata/sync/code/before/nodejs_local_dep/src/package.json @@ -0,0 +1,10 @@ +{ + "name": "src", + "version": "1.0.0", + "description": "", + "main": "index.js", + "author": "", + "dependencies": { + "axios": "^0.27.2" + } +} diff --git a/tests/integration/testdata/sync/code/before/template_nodejs_local_dep.yaml b/tests/integration/testdata/sync/code/before/template_nodejs_local_dep.yaml new file mode 100644 index 0000000000..a919e7f9fb --- /dev/null +++ b/tests/integration/testdata/sync/code/before/template_nodejs_local_dep.yaml @@ -0,0 +1,10 @@ +AWSTemplateFormatVersion : '2010-09-09' +Transform: AWS::Serverless-2016-10-31 + +Resources: + PrintLocalDep: + Type: AWS::Serverless::Function + Properties: + Handler: index.handler + Runtime: nodejs18.x + CodeUri: nodejs_local_dep/src diff --git a/tests/unit/cli/test_types.py b/tests/unit/cli/test_types.py index 68a285b2ac..5db9225001 100644 --- a/tests/unit/cli/test_types.py +++ b/tests/unit/cli/test_types.py @@ -1,5 +1,5 @@ from unittest import TestCase -from unittest.mock import Mock, ANY +from unittest.mock import MagicMock, Mock, ANY from click import BadParameter from parameterized import parameterized @@ -12,6 +12,7 @@ ImageRepositoriesType, RemoteInvokeBotoApiParameterType, RemoteInvokeOutputFormatType, + SyncWatchExcludeType, ) from samcli.cli.types import CfnMetadataType from samcli.lib.remote_invoke.remote_invoke_executors import RemoteInvokeOutputFormat @@ -516,3 +517,32 @@ def test_must_fail_on_invalid_values(self, input): def test_successful_parsing(self, input, expected): result = self.param_type.convert(input, self.mock_param, None) self.assertEqual(result, expected) + + +class TestSyncWatchExcludeType(TestCase): + def setUp(self): + self.exclude_type = SyncWatchExcludeType() + + @parameterized.expand( + [ + ("HelloWorldFunction=file.txt", {"HelloWorldFunction": ["file.txt"]}), + ({"HelloWorldFunction": ["file.txt"]}, {"HelloWorldFunction": ["file.txt"]}), + ] + ) + def test_convert_parses_input(self, input, expected): + result = self.exclude_type.convert(input, MagicMock(), MagicMock()) + + self.assertEqual(result, expected) + + @parameterized.expand( + [ + ("not a key value pair",), + ("",), + ("key=",), + ("=value",), + ("key=value=foo=bar",), + ] + ) + def test_convert_fails_parse_input(self, input): + with self.assertRaises(BadParameter): + self.exclude_type.convert(input, MagicMock(), MagicMock()) diff --git a/tests/unit/commands/_utils/test_options.py b/tests/unit/commands/_utils/test_options.py index f5cfe6e490..7fd33528d6 100644 --- a/tests/unit/commands/_utils/test_options.py +++ b/tests/unit/commands/_utils/test_options.py @@ -25,6 +25,7 @@ skip_prepare_infra_callback, generate_next_command_recommendation, terraform_project_root_path_callback, + watch_exclude_option_callback, ) from samcli.commands._utils.parameterized_option import parameterized_option from samcli.commands.package.exceptions import PackageResolveS3AndS3SetError, PackageResolveS3AndS3NotSetError @@ -597,3 +598,18 @@ def test_generate_next_command_recommendation(self): [*] Deploy: sam deploy --guided """ self.assertEqual(output, expectedOutput) + + +class TestWatchExcludeOption(TestCase): + @parameterized.expand( + [ + ( + ({"hello": ["world"]}, {"hello": ["mars"]}, {"foo": ["bar"]}), + {"hello": ["world", "mars"], "foo": ["bar"]}, + ), + ((), {}), + ] + ) + def test_merging_values(self, input, expected): + results = watch_exclude_option_callback(Mock(), Mock(), input) + self.assertEqual(results, expected) diff --git a/tests/unit/commands/buildcmd/test_build_context.py b/tests/unit/commands/buildcmd/test_build_context.py index 417ce473e3..5d48139d4b 100644 --- a/tests/unit/commands/buildcmd/test_build_context.py +++ b/tests/unit/commands/buildcmd/test_build_context.py @@ -18,6 +18,7 @@ from samcli.lib.build.bundler import EsbuildBundlerManager from samcli.lib.build.workflow_config import UnsupportedRuntimeException from samcli.lib.providers.provider import Function, get_function_build_info +from samcli.lib.telemetry.event import EventName, UsedFeature from samcli.lib.utils.osutils import BUILD_DIR_PERMISSIONS from samcli.lib.utils.packagetype import ZIP, IMAGE from samcli.local.lambdafn.exceptions import FunctionNotFound @@ -1245,6 +1246,37 @@ def test_must_catch_function_not_found_exception( self.assertEqual(str(ctx.exception), "Function Not Found") + @patch("samcli.commands.build.build_context.BuildContext._is_sam_template") + @patch("samcli.commands.build.build_context.BuildContext.get_resources_to_build") + @patch("samcli.commands.build.build_context.BuildContext._check_exclude_warning") + @patch("samcli.commands.build.build_context.BuildContext._check_rust_cargo_experimental_flag") + @patch("samcli.lib.build.app_builder.ApplicationBuilder.build") + @patch("samcli.lib.telemetry.event.EventTracker.track_event") + def test_build_in_source_event_sent( + self, mock_track_event, mock_builder, mock_rust, mock_warning, mock_get_resources, mock_is_sam_template + ): + mock_builder.side_effect = [FunctionNotFound()] + + context = BuildContext( + resource_identifier="", + template_file="template_file", + base_dir="base_dir", + build_dir="build_dir", + cache_dir="cache_dir", + cached=False, + clean=False, + parallel=False, + mode="mode", + build_in_source=True, + ) + + with self.assertRaises(UserException): + context.run() + + mock_track_event.assert_called_with( + EventName.USED_FEATURE.value, UsedFeature.BUILD_IN_SOURCE.value, "FunctionNotFound" + ) + class TestBuildContext_is_sam_template(TestCase): @parameterized.expand( diff --git a/tests/unit/commands/samconfig/test_samconfig.py b/tests/unit/commands/samconfig/test_samconfig.py index a006cc8047..eaeaa54b8a 100644 --- a/tests/unit/commands/samconfig/test_samconfig.py +++ b/tests/unit/commands/samconfig/test_samconfig.py @@ -158,7 +158,7 @@ def test_build(self, do_cli_mock): ("",), ("",), None, - None, + False, "READ", ) @@ -218,7 +218,7 @@ def test_build_with_no_cached_override(self, do_cli_mock): ("",), ("",), None, - None, + False, "READ", ) @@ -275,7 +275,7 @@ def test_build_with_container_env_vars(self, do_cli_mock): (), (), None, - None, + False, "READ", ) @@ -331,7 +331,7 @@ def test_build_with_build_images(self, do_cli_mock): ("Function1=image_1", "image_2"), (), None, - None, + False, "READ", ) @@ -970,6 +970,7 @@ def test_sync( "confirm_changeset": True, "region": "myregion", "signing_profiles": "function=profile:owner", + "watch_exclude": {"HelloWorld": ["file.txt", "other.txt"], "HelloMars": ["single.file"]}, } with samconfig_parameters(["sync"], self.scratch_dir, **config_values) as config_path: @@ -1013,7 +1014,8 @@ def test_sync( (), "samconfig.toml", "default", - None, + False, + {"HelloWorld": ["file.txt", "other.txt"], "HelloMars": ["single.file"]}, ) diff --git a/tests/unit/commands/sync/test_command.py b/tests/unit/commands/sync/test_command.py index eee29f5fef..a4ce01c752 100644 --- a/tests/unit/commands/sync/test_command.py +++ b/tests/unit/commands/sync/test_command.py @@ -146,6 +146,7 @@ def test_infra_must_succeed_sync( self.config_file, self.config_env, build_in_source=False, + watch_exclude={}, ) if use_container and auto_dependency_layer: @@ -305,6 +306,7 @@ def test_watch_must_succeed_sync( self.config_file, self.config_env, build_in_source=False, + watch_exclude={}, ) BuildContextMock.assert_called_with( @@ -377,6 +379,7 @@ def test_watch_must_succeed_sync( sync_context=sync_context_mock, auto_dependency_layer=auto_dependency_layer, disable_infra_syncs=disable_infra_syncs, + watch_exclude={}, ) @parameterized.expand([(True, False, True, True, False), (True, False, False, False, True)]) @@ -452,6 +455,7 @@ def test_code_must_succeed_sync( self.config_file, self.config_env, build_in_source=None, + watch_exclude={}, ) execute_code_sync_mock.assert_called_once_with( template=self.template_file, @@ -787,6 +791,7 @@ def test_execute_watch( self.sync_context, auto_dependency_layer, disable_infra_syncs, + {}, ) watch_manager_mock.assert_called_once_with( @@ -797,6 +802,7 @@ def test_execute_watch( self.sync_context, auto_dependency_layer, disable_infra_syncs, + {}, ) watch_manager_mock.return_value.start.assert_called_once_with() diff --git a/tests/unit/lib/sync/test_watch_manager.py b/tests/unit/lib/sync/test_watch_manager.py index 61283d9ab9..d50ed72633 100644 --- a/tests/unit/lib/sync/test_watch_manager.py +++ b/tests/unit/lib/sync/test_watch_manager.py @@ -32,6 +32,7 @@ def setUp(self) -> None: self.sync_context, False, False, + {}, ) def tearDown(self) -> None: @@ -92,8 +93,8 @@ def test_add_code_triggers(self, get_all_resource_ids_mock, patched_log): self.watch_manager._add_code_triggers() - trigger_factory.create_trigger.assert_any_call(resource_ids[0], on_code_change_wrapper_mock.return_value) - trigger_factory.create_trigger.assert_any_call(resource_ids[1], on_code_change_wrapper_mock.return_value) + trigger_factory.create_trigger.assert_any_call(resource_ids[0], on_code_change_wrapper_mock.return_value, []) + trigger_factory.create_trigger.assert_any_call(resource_ids[1], on_code_change_wrapper_mock.return_value, []) on_code_change_wrapper_mock.assert_any_call(resource_ids[0]) on_code_change_wrapper_mock.assert_any_call(resource_ids[1]) @@ -367,6 +368,23 @@ def test_on_code_change_wrapper_opened_event_not_called(self): factory_mock.create_sync_flow.assert_not_called() + @patch("samcli.lib.sync.watch_manager.platform.system") + def test_on_code_change_wrapper_opened_event_not_called_linux_folder(self, platform_mock): + flow1 = MagicMock() + resource_id_mock = MagicMock() + factory_mock = MagicMock() + event_mock = MagicMock() + event_mock.event_type = "modified" + event_mock.is_directory = True + platform_mock.return_value = "linux" + + self.watch_manager._sync_flow_factory = factory_mock + factory_mock.create_sync_flow.return_value = flow1 + + self.watch_manager._on_code_change_wrapper(resource_id_mock)(event_mock) + + factory_mock.create_sync_flow.assert_not_called() + def test_on_code_change_wrapper_missing_factory_sync_not_called(self): resource_id_mock = MagicMock() diff --git a/tests/unit/lib/utils/test_code_trigger_factory.py b/tests/unit/lib/utils/test_code_trigger_factory.py index 8a87e608e0..e8351a54cc 100644 --- a/tests/unit/lib/utils/test_code_trigger_factory.py +++ b/tests/unit/lib/utils/test_code_trigger_factory.py @@ -16,16 +16,16 @@ def test_create_zip_function_trigger(self, trigger_mock): on_code_change_mock = MagicMock() resource_identifier = ResourceIdentifier("Function1") resource = {"Properties": {"PackageType": "Zip"}} - result = self.factory._create_lambda_trigger(resource_identifier, "Type", resource, on_code_change_mock) + result = self.factory._create_lambda_trigger(resource_identifier, "Type", resource, on_code_change_mock, []) self.assertEqual(result, trigger_mock.return_value) - trigger_mock.assert_called_once_with(resource_identifier, self.stacks, self.base_dir, on_code_change_mock) + trigger_mock.assert_called_once_with(resource_identifier, self.stacks, self.base_dir, on_code_change_mock, []) @patch("samcli.lib.utils.code_trigger_factory.LambdaImageCodeTrigger") def test_create_image_function_trigger(self, trigger_mock): on_code_change_mock = MagicMock() resource_identifier = ResourceIdentifier("Function1") resource = {"Properties": {"PackageType": "Image"}} - result = self.factory._create_lambda_trigger(resource_identifier, "Type", resource, on_code_change_mock) + result = self.factory._create_lambda_trigger(resource_identifier, "Type", resource, on_code_change_mock, []) self.assertEqual(result, trigger_mock.return_value) trigger_mock.assert_called_once_with(resource_identifier, self.stacks, self.base_dir, on_code_change_mock) @@ -33,9 +33,9 @@ def test_create_image_function_trigger(self, trigger_mock): def test_create_layer_trigger(self, trigger_mock): on_code_change_mock = MagicMock() resource_identifier = ResourceIdentifier("Layer1") - result = self.factory._create_layer_trigger(resource_identifier, "Type", {}, on_code_change_mock) + result = self.factory._create_layer_trigger(resource_identifier, "Type", {}, on_code_change_mock, []) self.assertEqual(result, trigger_mock.return_value) - trigger_mock.assert_called_once_with(resource_identifier, self.stacks, self.base_dir, on_code_change_mock) + trigger_mock.assert_called_once_with(resource_identifier, self.stacks, self.base_dir, on_code_change_mock, []) @patch("samcli.lib.utils.code_trigger_factory.DefinitionCodeTrigger") def test_create_definition_trigger(self, trigger_mock): @@ -43,7 +43,7 @@ def test_create_definition_trigger(self, trigger_mock): resource_identifier = ResourceIdentifier("API1") resource_type = "AWS::Serverless::Api" result = self.factory._create_definition_code_trigger( - resource_identifier, resource_type, {}, on_code_change_mock + resource_identifier, resource_type, {}, on_code_change_mock, [] ) self.assertEqual(result, trigger_mock.return_value) trigger_mock.assert_called_once_with( @@ -67,9 +67,9 @@ def test_create_trigger(self, get_resource_by_id_mock, parent_get_resource_by_id get_generator_function_mock.return_value = generator_mock self.factory._get_generator_function = get_generator_function_mock - result = self.factory.create_trigger(resource_identifier, on_code_change_mock) + result = self.factory.create_trigger(resource_identifier, on_code_change_mock, []) self.assertEqual(result, code_trigger) generator_mock.assert_called_once_with( - self.factory, resource_identifier, "AWS::Serverless::Api", get_resource_by_id, on_code_change_mock + self.factory, resource_identifier, "AWS::Serverless::Api", get_resource_by_id, on_code_change_mock, [] ) diff --git a/tests/unit/lib/utils/test_resource_trigger.py b/tests/unit/lib/utils/test_resource_trigger.py index 6ff95d6ca5..f21ff65e5b 100644 --- a/tests/unit/lib/utils/test_resource_trigger.py +++ b/tests/unit/lib/utils/test_resource_trigger.py @@ -1,8 +1,9 @@ import re from parameterized import parameterized from unittest.case import TestCase -from unittest.mock import MagicMock, patch, ANY +from unittest.mock import MagicMock, Mock, patch, ANY from samcli.lib.utils.resource_trigger import ( + DEFAULT_WATCH_IGNORED_RESOURCES, CodeResourceTrigger, DefinitionCodeTrigger, LambdaFunctionCodeTrigger, @@ -114,6 +115,14 @@ def test_init_invalid(self, get_resource_by_id_mock): with self.assertRaises(ResourceNotFound): CodeResourceTrigger(ResourceIdentifier("A"), stacks, base_dir, on_code_change_mock) + @patch.multiple(CodeResourceTrigger, __abstractmethods__=set()) + @patch("samcli.lib.utils.resource_trigger.get_resource_by_id") + def test_init_with_ignored_resources(self, get_resource_by_id_mock): + trigger = CodeResourceTrigger(ResourceIdentifier("A"), Mock(), Mock(), Mock(), ["first_path", "second"]) + expected_watches = [*DEFAULT_WATCH_IGNORED_RESOURCES, "^.*first_path.*$", "^.*second.*$"] + + self.assertEqual(trigger._watch_exclude, expected_watches) + class TestLambdaFunctionCodeTrigger(TestCase): @patch.multiple(LambdaFunctionCodeTrigger, __abstractmethods__=set()) diff --git a/tests/unit/lib/utils/test_tar.py b/tests/unit/lib/utils/test_tar.py index ec3e2a305f..47cfd88d37 100644 --- a/tests/unit/lib/utils/test_tar.py +++ b/tests/unit/lib/utils/test_tar.py @@ -31,7 +31,7 @@ def test_generating_tarball(self, temporary_file_patch, tarfile_open_patch): temp_file_mock.flush.assert_called_once() temp_file_mock.seek.assert_called_once_with(0) temp_file_mock.close.assert_called_once() - tarfile_open_patch.assert_called_once_with(fileobj=temp_file_mock, mode="w") + tarfile_open_patch.assert_called_once_with(fileobj=temp_file_mock, mode="w", dereference=False) @patch("samcli.lib.utils.tar.tarfile.open") @patch("samcli.lib.utils.tar.TemporaryFile") @@ -57,7 +57,7 @@ def test_generating_tarball_with_gzip(self, temporary_file_patch, tarfile_open_p temp_file_mock.flush.assert_called_once() temp_file_mock.seek.assert_called_once_with(0) temp_file_mock.close.assert_called_once() - tarfile_open_patch.assert_called_once_with(fileobj=temp_file_mock, mode="w:gz") + tarfile_open_patch.assert_called_once_with(fileobj=temp_file_mock, mode="w:gz", dereference=False) @patch("samcli.lib.utils.tar.tarfile.open") @patch("samcli.lib.utils.tar.TemporaryFile") @@ -89,7 +89,7 @@ def tar_filter(tar_info): temp_file_mock.flush.assert_called_once() temp_file_mock.seek.assert_called_once_with(0) temp_file_mock.close.assert_called_once() - tarfile_open_patch.assert_called_once_with(fileobj=temp_file_mock, mode="w") + tarfile_open_patch.assert_called_once_with(fileobj=temp_file_mock, mode="w", dereference=False) @patch("samcli.lib.utils.tar.tarfile.open") @patch("samcli.lib.utils.tar._is_within_directory") diff --git a/tests/unit/local/docker/test_container.py b/tests/unit/local/docker/test_container.py index 1ab3092dba..402fa305f9 100644 --- a/tests/unit/local/docker/test_container.py +++ b/tests/unit/local/docker/test_container.py @@ -3,7 +3,7 @@ """ import json from unittest import TestCase -from unittest.mock import Mock, call, patch, ANY +from unittest.mock import MagicMock, Mock, call, patch, ANY from parameterized import parameterized import docker @@ -81,7 +81,8 @@ def setUp(self): self.mock_docker_client.networks = Mock() self.mock_docker_client.networks.get = Mock() - def test_must_create_container_with_required_values(self): + @patch("samcli.local.docker.container.Container._create_mapped_symlink_files") + def test_must_create_container_with_required_values(self, mock_resolve_symlinks): """ Create a container with only required values. Optional values are not provided :return: @@ -119,7 +120,8 @@ def test_must_create_container_with_required_values(self): ) self.mock_docker_client.networks.get.assert_not_called() - def test_must_create_container_including_all_optional_values(self): + @patch("samcli.local.docker.container.Container._create_mapped_symlink_files") + def test_must_create_container_including_all_optional_values(self, mock_resolve_symlinks): """ Create a container with required and optional values. :return: @@ -174,7 +176,8 @@ def test_must_create_container_including_all_optional_values(self): self.mock_docker_client.networks.get.assert_not_called() @patch("samcli.local.docker.utils.os") - def test_must_create_container_translate_volume_path(self, os_mock): + @patch("samcli.local.docker.container.Container._create_mapped_symlink_files") + def test_must_create_container_translate_volume_path(self, mock_resolve_symlinks, os_mock): """ Create a container with required and optional values, with windows style volume mount. :return: @@ -233,7 +236,8 @@ def test_must_create_container_translate_volume_path(self, os_mock): ) self.mock_docker_client.networks.get.assert_not_called() - def test_must_connect_to_network_on_create(self): + @patch("samcli.local.docker.container.Container._create_mapped_symlink_files") + def test_must_connect_to_network_on_create(self, mock_resolve_symlinks): """ Create a container with only required values. Optional values are not provided :return: @@ -271,7 +275,8 @@ def test_must_connect_to_network_on_create(self): self.mock_docker_client.networks.get.assert_called_with(network_id) network_mock.connect.assert_called_with(container_id) - def test_must_connect_to_host_network_on_create(self): + @patch("samcli.local.docker.container.Container._create_mapped_symlink_files") + def test_must_connect_to_host_network_on_create(self, mock_resolve_symlinks): """ Create a container with only required values. Optional values are not provided :return: @@ -973,3 +978,44 @@ def test_real_container_is_running_return_true(self): real_container_mock.status = "running" self.mock_client.containers.get.return_value = real_container_mock self.assertTrue(self.container.is_created()) + + +class TestContainer_create_mapped_symlink_files(TestCase): + def setUp(self): + self.container = Container(Mock(), Mock(), Mock(), Mock(), docker_client=Mock()) + + self.mock_symlinked_file = MagicMock() + self.mock_symlinked_file.is_symlink.return_value = True + + self.mock_regular_file = MagicMock() + self.mock_regular_file.is_symlink.return_value = False + + @patch("samcli.local.docker.container.os.scandir") + def test_no_symlinks_returns_empty(self, mock_scandir): + mock_context = MagicMock() + mock_context.__enter__ = Mock(return_value=[self.mock_regular_file]) + mock_scandir.return_value = mock_context + + volumes = self.container._create_mapped_symlink_files() + + self.assertEqual(volumes, {}) + + @patch("samcli.local.docker.container.os.scandir") + @patch("samcli.local.docker.container.os.path.realpath") + @patch("samcli.local.docker.container.pathlib.Path") + def test_resolves_symlink(self, mock_path, mock_realpath, mock_scandir): + host_path = Mock() + container_path = Mock() + + mock_realpath.return_value = host_path + mock_as_posix = Mock() + mock_as_posix.as_posix = Mock(return_value=container_path) + mock_path.return_value = mock_as_posix + + mock_context = MagicMock() + mock_context.__enter__ = Mock(return_value=[self.mock_symlinked_file]) + mock_scandir.return_value = mock_context + + volumes = self.container._create_mapped_symlink_files() + + self.assertEqual(volumes, {host_path: {"bind": container_path, "mode": ANY}})