feat: Building projects in source (#5933)

* feat: Added event tracking for feature (#5872)

* Added event tracking for feature

* Correctly pass event value and fix typing

* feat: Resolve symlinks at root level and mount to container (#5870)

* Resolve symlinks at root level and mount to container

* Continue early instead of nesting in if statement

* make format

* Added integration test

* make format

* Use node project to test local deps and building in source

* Removed old test data

* Update test class name to reference testing symlinks

* Use correct skip decorator from unittest

* Changed mount mode to read only

* Removed the removed mount parameter in tests

* Remove kwarg in method call

* feat: Build in source click option (#5951)

* Added skeleton for build in source click option

* Fix tests and renamed option

* Added description and updated click option to include build

* Disable build tests again until esbuild resolution fix

* Revert template change

* Make hook name option mutually exclusive with build in source

* Fix integration test message assert

* chore(tests): Enable build in souce build command tests (#6099)

* chore(tests): Node build in source test (#6132)

* Added nodejs tests

* make format

* Enable retries

* Run build test inside of a scratch directory

* make format

* Updated Makefile test to properly use scratch directory and moved source copy to setup

* Changed copytree call to in house version

* Removed redundant parent method calling

* make format

* chore: Bump Nodejs version in AppVeyor job configuration (#6210)

* Bump npm version to 10.2.3 for linux testing

* Upgrade node to 18.18.2

* chore(tests): Build in source sync temp folder (#6166)

* Enable and use new test data folder paths

* Added nodejs tests

* feat: Resolve root level symlinked folders in Layers (#6236)

* Resolve Layer symlinks before mounting container

* Use .get instead of if else

* Added integration tests

* Unprivate static method and make format

* Create paths using pathlib

* Remove debug flag

* Revert "Resolve Layer symlinks before mounting container"

This reverts commit e557067.

* Resolve links when creating tarfile for Docker build call

* make format

* Removed debug line

* Updated arugment with optional typing

* feat: Add additional default excludes and exclude option for sync (#6299)

* Add additional default excludes and exclude option

* Updated test and option definition

* Updated option and added tests

* Update resource match to consider nested stacks

* Handle parameters saved in samconfig

* fix: Fix sync tests not cleaning up correctly and Linux watch behaviour (#6372)

* Use develop version of Lambda Builders

* Run nodejs local dep command in scratch directory and clean up aws-sam folder

* Exclude the folder of the modified file from trigger sync flows on Linux

* Revert "Use develop version of Lambda Builders"

This reverts commit fde143e.

* Add test validing ignoring Linux modified folder events

* Fix test

appveyor-linux-binary.yml

@@ -19,7 +19,7 @@ environment:
   PYTHON_HOME: "$HOME/venv3.11/bin"
   PYTHON_VERSION: '3.11'
   AWS_DEFAULT_REGION: us-east-1
-  NODE_VERSION: "14.17.6"
+  NODE_VERSION: "18.18.2"
   AWS_S3: 'AWS_S3_TESTING'
   AWS_ECR: 'AWS_ECR_TESTING'
   CARGO_LAMBDA_VERSION: "v0.17.1"
@@ -52,7 +52,7 @@ install:
   - sh: "docker info"
   - sh: "docker version"
   - sh: "nvm install ${NODE_VERSION}"
-  - sh: "npm install npm@7.24.2 -g"
+  - sh: "npm install npm@10.2.3 -g"
   - sh: "npm -v"
 
   # Install latest gradle

appveyor-ubuntu.yml

@@ -20,7 +20,7 @@ environment:
   PYTHON_VERSION: '3.8'
   AWS_DEFAULT_REGION: us-east-1
   SAM_CLI_DEV: 1
-  NODE_VERSION: "14.17.6"
+  NODE_VERSION: "18.18.2"
   AWS_S3: 'AWS_S3_TESTING'
   AWS_ECR: 'AWS_ECR_TESTING'
   CARGO_LAMBDA_VERSION: "v0.17.1"
@@ -52,7 +52,7 @@ install:
   - sh: "docker info"
   - sh: "docker version"
   - sh: "nvm install ${NODE_VERSION}"
-  - sh: "npm install npm@7.24.2 -g"
+  - sh: "npm install npm@10.2.3 -g"
   - sh: "npm -v"
 
   # Install latest gradle

samcli/cli/types.py

@@ -6,6 +6,7 @@
 import logging
 import re
 from json import JSONDecodeError
+from typing import Dict, List, Optional, Union
 
 import click
 
@@ -474,3 +475,61 @@ def convert(self, value, param, ctx):
         LOG.debug("Converting provided %s option value to Enum", param.opts[0])
         value = super().convert(value, param, ctx)
         return self.enum(value)
+
+
+class SyncWatchExcludeType(click.ParamType):
+    """
+    Custom parameter type to parse Key=Value pairs into dictionary mapping.
+    """
+
+    name = "list"
+
+    WATCH_EXCLUDE_EXPECTED_LENGTH = 2
+    WATCH_EXCLUDE_DELIMITER = "="
+
+    EXCEPTION_MESSAGE = "Argument must be a key, value pair in the format Key=Value."
+
+    def convert(
+        self, value: Union[Dict[str, List[str]], str], param: Optional[click.Parameter], ctx: Optional[click.Context]
+    ) -> Dict[str, List[str]]:
+        """
+        Parses the multiple provided values into a mapping of resources to a list of exclusions.
+
+        Parameters
+        ----------
+        value: Union[Dict[str, List[str]], str]
+            The passed in arugment
+        param: click.Parameter
+            The parameter that was provided
+        ctx: click.Context
+            The click context
+
+        Returns
+        -------
+        Dict[str, List[str]]
+            A key value pair of Logical/Resource Id and excluded file
+        """
+        if isinstance(value, dict):
+            return value
+
+        mapping_pair = value.split(self.WATCH_EXCLUDE_DELIMITER)
+
+        if len(mapping_pair) != self.WATCH_EXCLUDE_EXPECTED_LENGTH:
+            raise click.BadParameter(
+                param=param,
+                param_hint=f"'{value}'",
+                ctx=ctx,
+                message=self.EXCEPTION_MESSAGE,
+            )
+
+        resource_id, excluded_path = mapping_pair
+
+        if not (resource_id and excluded_path):
+            raise click.BadParameter(
+                param=param,
+                param_hint=f"'{value}'",
+                ctx=ctx,
+                message=self.EXCEPTION_MESSAGE,
+            )
+
+        return {resource_id: [excluded_path]}

samcli/commands/_utils/options.py

@@ -5,7 +5,7 @@
 import logging
 import os
 from functools import partial
-from typing import List, Tuple
+from typing import Dict, List, Tuple
 
 import click
 from click.types import FuncParamType
@@ -18,7 +18,9 @@
     ImageRepositoryType,
     RemoteInvokeBotoApiParameterType,
     SigningProfilesOptionType,
+    SyncWatchExcludeType,
 )
+from samcli.commands._utils.click_mutex import ClickMutex
 from samcli.commands._utils.constants import (
     DEFAULT_BUILD_DIR,
     DEFAULT_BUILT_TEMPLATE_PATH,
@@ -33,8 +35,17 @@
 from samcli.lib.hook.hook_wrapper import get_available_hook_packages_ids
 from samcli.lib.observability.util import OutputOption
 from samcli.lib.utils.packagetype import IMAGE, ZIP
+from samcli.local.docker.lambda_image import Runtime
 
 _TEMPLATE_OPTION_DEFAULT_VALUE = "template.[yaml|yml|json]"
+SUPPORTED_BUILD_IN_SOURCE_WORKFLOWS = [
+    Runtime.nodejs12x.value,
+    Runtime.nodejs14x.value,
+    Runtime.nodejs16x.value,
+    Runtime.nodejs18x.value,
+    "Makefile",
+    "esbuild",
+]
 
 LOG = logging.getLogger(__name__)
 
@@ -237,6 +248,39 @@ def skip_prepare_infra_callback(ctx, param, provided_value):
         raise click.BadOptionUsage(option_name=param.name, ctx=ctx, message="Missing option --hook-name")
 
 
+def watch_exclude_option_callback(
+    ctx: click.Context, param: click.Option, values: Tuple[Dict[str, List[str]]]
+) -> Dict[str, List[str]]:
+    """
+    Parses the multiple provided values into a mapping of resources to a list of exclusions.
+
+    Parameters
+    ----------
+    ctx: click.Context
+        The click context
+    param: click.Option
+        The parameter that was provided
+    values: Tuple[Dict[str, List[str]]]
+        A list of values that was passed in
+
+    Returns
+    -------
+    Dict[str, List[str]]
+        A mapping of LogicalIds to a list of files and/or folders to exclude
+        from file change watches
+    """
+    resource_exclude_mappings: Dict[str, List[str]] = {}
+
+    for mappings in values:
+        for resource_id, excluded_files in mappings.items():
+            current_excludes = resource_exclude_mappings.get(resource_id, [])
+            current_excludes.extend(excluded_files)
+
+            resource_exclude_mappings[resource_id] = current_excludes
+
+    return resource_exclude_mappings
+
+
 def template_common_option(f):
     """
     Common ClI option for template
@@ -915,3 +959,37 @@ def generate_next_command_recommendation(command_tuples: List[Tuple[str, str]])
 """
     command_list_txt = "\n".join(f"[*] {description}: {command}" for description, command in command_tuples)
     return template.format(command_list_txt)
+
+
+def build_in_source_click_option():
+    return click.option(
+        "--build-in-source/--no-build-in-source",
+        required=False,
+        is_flag=True,
+        help="Opts in to build project in the source folder. The following workflows support "
+        f"building in source: {SUPPORTED_BUILD_IN_SOURCE_WORKFLOWS}",
+        cls=ClickMutex,
+        incompatible_params=["use_container", "hook_name"],
+    )
+
+
+def build_in_source_option(f):
+    return build_in_source_click_option()(f)
+
+
+def watch_exclude_option(f):
+    return watch_exclude_click_option()(f)
+
+
+def watch_exclude_click_option():
+    return click.option(
+        "--watch-exclude",
+        help="Excludes a file or folder from being observed for file changes. "
+        "Files and folders that are excluded will not trigger a sync workflow. "
+        "This option can be provided multiple times.\n\n"
+        "Examples:\n\nHelloWorldFunction=package-lock.json\n\n"
+        "ChildStackA/FunctionName=database.sqlite3",
+        multiple=True,
+        type=SyncWatchExcludeType(),
+        callback=watch_exclude_option_callback,
+    )

samcli/commands/build/build_context.py

@@ -33,12 +33,12 @@
 )
 from samcli.lib.build.workflow_config import UnsupportedRuntimeException
 from samcli.lib.intrinsic_resolver.intrinsics_symbol_table import IntrinsicsSymbolTable
-from samcli.lib.providers.provider import ResourcesToBuildCollector, Stack, Function, LayerVersion
+from samcli.lib.providers.provider import ResourcesToBuildCollector, Stack, LayerVersion
 from samcli.lib.providers.sam_api_provider import SamApiProvider
 from samcli.lib.providers.sam_function_provider import SamFunctionProvider
 from samcli.lib.providers.sam_layer_provider import SamLayerProvider
 from samcli.lib.providers.sam_stack_provider import SamLocalStackProvider
-from samcli.lib.telemetry.event import EventTracker
+from samcli.lib.telemetry.event import EventTracker, UsedFeature, EventName
 from samcli.lib.utils.osutils import BUILD_DIR_PERMISSIONS
 from samcli.local.docker.manager import ContainerManager
 from samcli.local.lambdafn.exceptions import (
@@ -232,13 +232,15 @@ def __exit__(self, *args):
     def get_resources_to_build(self):
         return self.resources_to_build
 
-    def run(self):
+    def run(self) -> None:
         """Runs the building process by creating an ApplicationBuilder."""
         if self._is_sam_template():
             SamApiProvider.check_implicit_api_resource_ids(self.stacks)
 
         self._stacks = self._handle_build_pre_processing()
 
+        caught_exception: Optional[Exception] = None
+
         try:
             # boolean value indicates if mount with write or not, defaults to READ ONLY
             mount_with_write = False
@@ -276,7 +278,7 @@ def run(self):
             self._check_rust_cargo_experimental_flag()
 
             for f in self.get_resources_to_build().functions:
-                EventTracker.track_event("BuildFunctionRuntime", f.runtime)
+                EventTracker.track_event(EventName.BUILD_FUNCTION_RUNTIME.value, f.runtime)
 
             self._build_result = builder.build()
 
@@ -306,6 +308,8 @@ def run(self):
 
                 click.secho(msg, fg="yellow")
         except FunctionNotFound as function_not_found_ex:
+            caught_exception = function_not_found_ex
+
             raise UserException(
                 str(function_not_found_ex), wrapped_from=function_not_found_ex.__class__.__name__
             ) from function_not_found_ex
@@ -317,13 +321,21 @@ def run(self):
             InvalidBuildGraphException,
             ResourceNotFound,
         ) as ex:
+            caught_exception = ex
+
             click.secho("\nBuild Failed", fg="red")
 
             # Some Exceptions have a deeper wrapped exception that needs to be surfaced
             # from deeper than just one level down.
             deep_wrap = getattr(ex, "wrapped_from", None)
             wrapped_from = deep_wrap if deep_wrap else ex.__class__.__name__
             raise UserException(str(ex), wrapped_from=wrapped_from) from ex
+        finally:
+            if self.build_in_source:
+                exception_name = type(caught_exception).__name__ if caught_exception else None
+                EventTracker.track_event(
+                    EventName.USED_FEATURE.value, UsedFeature.BUILD_IN_SOURCE.value, exception_name
+                )
 
     def _is_sam_template(self) -> bool:
         """Check if a given template is a SAM template"""

samcli/commands/build/command.py

@@ -9,6 +9,7 @@
 
 from samcli.cli.context import Context
 from samcli.commands._utils.options import (
+    build_in_source_option,
     skip_prepare_infra_option,
     template_option_without_build,
     docker_common_options,
@@ -74,10 +75,11 @@
 @terraform_project_root_path_option
 @hook_name_click_option(
     force_prepare=True,
-    invalid_coexist_options=["t", "template-file", "template", "parameter-overrides"],
+    invalid_coexist_options=["t", "template-file", "template", "parameter-overrides", "build-in-source"],
 )
 @skip_prepare_infra_option
 @use_container_build_option
+@build_in_source_option
 @click.option(
     "--container-env-var",
     "-e",
@@ -158,8 +160,9 @@ def cli(
     save_params: bool,
     hook_name: Optional[str],
     skip_prepare_infra: bool,
-    mount_with,
+    mount_with: str,
     terraform_project_root_path: Optional[str],
+    build_in_source: Optional[bool],
 ) -> None:
     """
     `sam build` command entry point
@@ -189,7 +192,7 @@ def cli(
         build_image,
         exclude,
         hook_name,
-        None,  # TODO: replace with build_in_source once it's added as a click option
+        build_in_source,
         mount_with,
     )  # pragma: no cover
 
@@ -216,7 +219,7 @@ def do_cli(  # pylint: disable=too-many-locals, too-many-statements
     exclude: Optional[Tuple[str, ...]],
     hook_name: Optional[str],
     build_in_source: Optional[bool],
-    mount_with,
+    mount_with: str,
 ) -> None:
     """
     Implementation of the ``cli`` method

samcli/commands/build/core/options.py

@@ -27,7 +27,7 @@
 
 EXTENSION_OPTIONS: List[str] = ["hook_name", "skip_prepare_infra"]
 
-BUILD_STRATEGY_OPTIONS: List[str] = ["parallel", "exclude", "manifest", "cached"]
+BUILD_STRATEGY_OPTIONS: List[str] = ["parallel", "exclude", "manifest", "cached", "build_in_source"]
 
 ARTIFACT_LOCATION_OPTIONS: List[str] = [
     "build_dir",