Merge branch 'develop' into feat/build-in-source

aws · Oct 17, 2023 · 1785a36 · 1785a36
2 parents 65c18df + 02f754d
commit 1785a36
Show file tree

Hide file tree

Showing 52 changed files with 2,517 additions and 1,646 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -88,13 +88,15 @@ jobs:
         shell: bash
 
   integration-tests:
-    name: Integration Tests / ${{ matrix.os }} / ${{ matrix.python }} / ${{ matrix.tests_folder }}
+    name: Integ / ${{ matrix.os }} / ${{ matrix.python }} / ${{ matrix.tests_config }}
     if: github.repository_owner == 'aws'
     runs-on: ${{ matrix.os }}
     env:
       AWS_DEFAULT_REGION: us-east-1
       SAM_CLI_DEV: "1"
       CARGO_LAMBDA_VERSION: 0.17.1
+      CARGO_INCREMENTAL: 0 # disable incremental compilation
+      CARGO_TERM_COLOR: always # enable colors in Cargo output
       CI: true
     defaults:
       run:
@@ -110,24 +112,24 @@ jobs:
           - "3.8"
           - "3.9"
         # folders that is commented below requires credentials, no need to spare time to run them
-        tests_folder:
-          - "buildcmd"
+        tests_config:
+          - "-n 2 --reruns 3 tests/integration/buildcmd"
           #- "delete"
           #- "deploy"
-          - "init"
+          - "tests/integration/init"
           #- "list"
-          - "local"
+          - "tests/integration/local"
           #- "logs"
           #- "package"
-          - "pipeline"
+          - "tests/integration/pipeline"
           #- "publish"
-          - "root"
-          - "scripts"
+          - "tests/integration/root"
+          - "tests/integration/scripts"
           #- "sync"
-          - "telemetry"
+          - "tests/integration/telemetry"
           #- "traces"
           #- "validate"
-          - "docs"
+          - "tests/integration/docs"
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
@@ -156,33 +158,24 @@ jobs:
         with:
           distribution: 'corretto'
           java-version: '17'
-       # Install and configure Rust
-      - name: Install rustup
+       # Install and configure Rust & Cargo Lambda
+      - name: Install and configure Rust & Cargo Lambda
+        if: ${{ matrix.os == 'ubuntu-latest' }}
         run: |
           : install rustup if needed
           if ! command -v rustup &> /dev/null ; then
             curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused -fsSL "https://sh.rustup.rs" | sh -s -- --default-toolchain none -y
             echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> $GITHUB_PATH
           fi
-        if: ${{ matrix.os == 'ubuntu-latest' }}
-      - name: rustup toolchain install stable
-        run: rustup toolchain install stable --profile minimal --no-self-update
-      - run: rustup default stable
-      - run: |
-          : disable incremental compilation
-          echo CARGO_INCREMENTAL=0 >> $GITHUB_ENV
-      - run: |
-          : enable colors in Cargo output
-          echo CARGO_TERM_COLOR=always >> $GITHUB_ENV
-      # Install and configure Cargo Lambda
-      - name: Install Cargo Lambda
-        run: pip install cargo-lambda==$CARGO_LAMBDA_VERSION
-      - run: echo "$HOME/.local/bin" >> $GITHUB_PATH
+          rustup toolchain install stable --profile minimal --no-self-update
+          rustup default stable
+          pip install cargo-lambda==$CARGO_LAMBDA_VERSION
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
 
       - name: Init samdev
         run: make init
-      - name: Run integration tests for ${{ matrix.tests_folder }}
-        run: pytest -vv tests/integration/${{ matrix.tests_folder }}
+      - name: Run integration tests for ${{ matrix.tests_config }}
+        run: pytest -vv ${{ matrix.tests_config }}
 
   smoke-and-functional-tests:
     name: Functional & Smoke Tests / ${{ matrix.os }} / ${{ matrix.python }}

diff --git a/Makefile b/Makefile
@@ -79,7 +79,7 @@ update-reproducible-mac-reqs:
 update-reproducible-win-reqs:
 	python -m venv venv-update-reproducible-win
 	.\venv-update-reproducible-win\Scripts\activate
-	pip install --upgrade pip-tools pip
+	python.exe -m pip install --upgrade pip-tools pip
 	pip install -r requirements\base.txt
 	pip-compile --generate-hashes --allow-unsafe -o requirements\reproducible-win.txt
 

diff --git a/appveyor-linux-binary.yml b/appveyor-linux-binary.yml
@@ -4,6 +4,9 @@ image:
 
 configuration:
   - BuildIntegTesting
+  - BuildIntegTestingJavaPythonProvided
+  - BuildIntegTestingArm64
+  - BuildIntegTestingArm64Java
   - AllTerraformBuildTesting
   - DeployIntegTesting
   - PackageIntegTesting
@@ -47,7 +50,7 @@ install:
   - sh: "mvn --version"
 
   - sh: "source ${HOME}/venv${PYTHON_VERSION}/bin/activate"
-  - sh: "rvm use 2.7.2"
+  - sh: "rvm use 3.2.2"
   - sh: "docker info"
   - sh: "docker version"
   - sh: "nvm install ${NODE_VERSION}"
@@ -169,11 +172,29 @@ for:
         - configuration: BuildIntegTesting
 
     test_script:
-      - sh: "pytest -vv tests/integration/buildcmd --ignore=tests/integration/buildcmd/test_build_terraform_applications.py --ignore=tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json"
+      - sh: "pytest -vv -n 2 --reruns 3 tests/integration/buildcmd -m 'not java and not python and not provided' --ignore=tests/integration/buildcmd/test_build_cmd_arm64.py --ignore=tests/integration/buildcmd/test_build_terraform_applications.py --ignore=tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json"
 
-      # Set JAVA_HOME to java11
-      - sh: "JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64"
-      - sh: "pytest -vv tests/integration/buildcmd/test_build_cmd.py -k test_building_java11_in_process --json-report --json-report-file=TEST_REPORT-integration-buildcmd-java11.json"
+  - matrix:
+      only:
+        - configuration: BuildIntegTestingJavaPythonProvided
+
+    test_script:
+      - sh: "pytest -vv -n 2 --reruns 3 tests/integration/buildcmd -m 'java or python or provided' --ignore=tests/integration/buildcmd/test_build_cmd_arm64.py --ignore=tests/integration/buildcmd/test_build_terraform_applications.py --ignore=tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd-java-python-provided.json"
+
+  # Integ testing build arm64 functions
+  - matrix:
+      only:
+        - configuration: BuildIntegTestingArm64
+
+    test_script:
+      - sh: "pytest -vv --reruns 3 tests/integration/buildcmd/test_build_cmd_arm64.py -m 'not java' --json-report --json-report-file=TEST_REPORT-integration-buildcmd-arm64.json"
+
+  - matrix:
+      only:
+        - configuration: BuildIntegTestingArm64Java
+
+    test_script:
+      - sh: "pytest -vv --reruns 3 tests/integration/buildcmd/test_build_cmd_arm64.py -m 'java' --json-report --json-report-file=TEST_REPORT-integration-buildcmd-arm64-java.json"
 
   # Integ testing Terraform build
   -

diff --git a/appveyor-ubuntu.yml b/appveyor-ubuntu.yml
@@ -4,10 +4,12 @@ image:
 
 configuration:
   - BuildIntegTesting
+  - BuildIntegTestingJavaPythonProvided
+  - BuildIntegTestingArm64
+  - BuildIntegTestingArm64Java
   - AllTerraformBuildTesting
   - DeployIntegTesting
-  - PackageIntegTesting
-  - DeleteIntegTesting
+  - PackageAndDeleteIntegTesting
   - SyncIntegTesting
   - LocalIntegTesting
   - EndToEndTesting
@@ -47,7 +49,7 @@ install:
   - sh: "mvn --version"
 
   - sh: "source ${HOME}/venv${PYTHON_VERSION}/bin/activate"
-  - sh: "rvm use 2.7.2"
+  - sh: "rvm use 3.2.2"
   - sh: "docker info"
   - sh: "docker version"
   - sh: "nvm install ${NODE_VERSION}"
@@ -157,11 +159,30 @@ for:
         - configuration: BuildIntegTesting
 
     test_script:
-      - sh: "pytest -vv tests/integration/buildcmd --ignore=tests/integration/buildcmd/test_build_terraform_applications.py --ignore=tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json"
+      - sh: "pytest -vv -n 2 --reruns 3 tests/integration/buildcmd -m 'not java and not python and not provided' --ignore=tests/integration/buildcmd/test_build_cmd_arm64.py --ignore=tests/integration/buildcmd/test_build_terraform_applications.py --ignore=tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json"
 
-      # Set JAVA_HOME to java11
-      - sh: "JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64"
-      - sh: "pytest -vv tests/integration/buildcmd/test_build_cmd.py -k test_building_java11_in_process --json-report --json-report-file=TEST_REPORT-integration-buildcmd-java11.json"
+  - matrix:
+      only:
+        - configuration: BuildIntegTestingJavaPythonProvided
+
+    test_script:
+      - sh: "pytest -vv -n 2 --reruns 3 tests/integration/buildcmd -m 'java or python or provided' --ignore=tests/integration/buildcmd/test_build_cmd_arm64.py --ignore=tests/integration/buildcmd/test_build_terraform_applications.py --ignore=tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd-java-python-provided.json"
+
+  # Integ testing build arm64 functions
+  - 
+    matrix:
+      only:
+        - configuration: BuildIntegTestingArm64
+
+    test_script:
+      - sh: "pytest -vv --reruns 3 tests/integration/buildcmd/test_build_cmd_arm64.py -m 'not java' --json-report --json-report-file=TEST_REPORT-integration-buildcmd-arm64.json"
+
+  - matrix:
+      only:
+        - configuration: BuildIntegTestingArm64Java
+
+    test_script:
+      - sh: "pytest -vv --reruns 3 tests/integration/buildcmd/test_build_cmd_arm64.py -m 'java' --json-report --json-report-file=TEST_REPORT-integration-buildcmd-arm64-java.json"
 
   # Integ testing Terraform build
   -
@@ -190,23 +211,14 @@ for:
     test_script:
       - sh: "pytest -vv tests/integration/deploy -n 4 --reruns 4 --dist=loadgroup --json-report --json-report-file=TEST_REPORT-integration-deploy.json"
 
-  # Integ testing package
-  -
-    matrix:
-      only:
-        - configuration: PackageIntegTesting
-
-    test_script:
-      - sh: "pytest -vv tests/integration/package -n 4 --reruns 4 --json-report --json-report-file=TEST_REPORT-integration-package.json"
-
-  # Integ testing delete
+  # Integ testing package & delete
   -
     matrix:
       only:
-        - configuration: DeleteIntegTesting
+        - configuration: PackageAndDeleteIntegTesting
 
     test_script:
-      - sh: "pytest -vv tests/integration/delete -n 4 --reruns 4 --json-report --json-report-file=TEST_REPORT-integration-delete.json"
+      - sh: "pytest -vv tests/integration/package tests/integration/delete -n 4 --reruns 4 --json-report --json-report-file=TEST_REPORT-integration-package-delete.json"
 
   # Integ testing sync
   -

diff --git a/appveyor-windows-binary.yml b/appveyor-windows-binary.yml
@@ -6,6 +6,8 @@ clone_folder: C:\source
 
 configuration:
   - BuildIntegTesting
+  - BuildIntegTestingJavaPythonProvided
+  - BuildIntegTestingArm64
   - AllTerraformBuildTesting
   - DeployIntegTesting
   - PackageIntegTesting
@@ -66,7 +68,7 @@ install:
 
   # Make sure the temp directory exists for Python to use.
   - ps: "mkdir -Force C:\\tmp"
-  - 'set PATH=%PYTHON_HOME%;C:\Ruby27-x64\bin;%PATH%;C:\Python37-x64;C:\Python39-x64;C:\Python310-x64;C:\Python311-x64'
+  - 'set PATH=%PYTHON_HOME%;C:\Ruby32-x64\bin;%PATH%;C:\Python37-x64;C:\Python39-x64;C:\Python310-x64;C:\Python311-x64'
   - "echo %PYTHON_HOME%"
   - "echo %PATH%"
   - "python --version"
@@ -174,11 +176,18 @@ on_finish:
 
 
 for:
-  #Integ testing build, regression
+  #Integ testing build
   - matrix:
       only:
         - configuration: BuildIntegTesting
 
+    test_script:
+      - ps: "pytest -vv -n 2 --reruns 3 tests/integration/buildcmd -m 'not java and not python and not provided' --ignore=tests/integration/buildcmd/test_build_cmd_arm64.py --ignore=tests/integration/buildcmd/test_build_terraform_applications.py --ignore=tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json"
+
+  - matrix:
+      only:
+        - configuration: BuildIntegTestingJavaPythonProvided
+
     build_script:
       # install Rust in build_script to not override the default "install" actions
       - appveyor-retry appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
@@ -197,7 +206,15 @@ for:
       - cargo lambda -V
 
     test_script:
-      - ps: "pytest -vv tests/integration/buildcmd --ignore tests/integration/buildcmd/test_build_terraform_applications.py --ignore=tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json"
+      - ps: "pytest -vv -n 2 --reruns 3 tests/integration/buildcmd -m 'java or python or provided' --ignore=tests/integration/buildcmd/test_build_cmd_arm64.py --ignore tests/integration/buildcmd/test_build_terraform_applications.py --ignore=tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json"
+
+  #Integ testing build arm64
+  - matrix:
+      only:
+        - configuration: BuildIntegTestingArm64
+
+    test_script:
+      - ps: "pytest -vv --reruns 3 tests/integration/buildcmd/test_build_cmd_arm64.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd-arm64.json"
 
   #Integ testing Terraform build
   - matrix:

diff --git a/appveyor-windows.yml b/appveyor-windows.yml
@@ -6,10 +6,11 @@ clone_folder: C:\source
 
 configuration:
   - BuildIntegTesting
+  - BuildIntegTestingJavaPythonProvided
+  - BuildIntegTestingArm64
   - AllTerraformBuildTesting
   - DeployIntegTesting
-  - PackageIntegTesting
-  - DeleteIntegTesting
+  - PackageAndDeleteIntegTesting
   - SyncIntegTesting
   - LocalIntegTesting
   - EndToEndTesting
@@ -65,7 +66,7 @@ install:
 
   # Make sure the temp directory exists for Python to use.
   - ps: "mkdir -Force C:\\tmp"
-  - 'set PATH=%PYTHON_HOME%;C:\Ruby27-x64\bin;%PATH%;C:\Python37-x64;C:\Python39-x64;C:\Python310-x64;C:\Python311-x64'
+  - 'set PATH=%PYTHON_HOME%;C:\Ruby32-x64\bin;%PATH%;C:\Python37-x64;C:\Python39-x64;C:\Python310-x64;C:\Python311-x64'
   - "echo %PYTHON_HOME%"
   - "echo %PATH%"
   - "python --version"
@@ -165,11 +166,43 @@ on_finish:
 
 
 for:
-  #Integ testing build, regression
+  #Integ testing build
   - matrix:
       only:
         - configuration: BuildIntegTesting
 
+    test_script:
+      - ps: "pytest -vv -n 2 --reruns 3 tests/integration/buildcmd -m 'not java and not python and not provided' --ignore=tests/integration/buildcmd/test_build_cmd_arm64.py --ignore=tests/integration/buildcmd/test_build_terraform_applications.py --ignore=tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json"
+
+  - matrix:
+      only:
+        - configuration: BuildIntegTestingJavaPythonProvided
+
+    build_script:
+      # install Rust in build_script to not override the default "install" actions
+      - appveyor-retry appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
+      - rustup-init.exe -y --default-host x86_64-pc-windows-msvc --default-toolchain stable
+      - set PATH=%PATH%;C:\Users\appveyor\.cargo\bin
+      - set RUST_BACKTRACE=1
+      - rustup toolchain install stable --profile minimal --no-self-update
+      - rustup default stable
+      - rustup target add x86_64-unknown-linux-gnu --toolchain stable
+      - rustup target add aarch64-unknown-linux-gnu --toolchain stable
+      - ps: "choco install zig"
+      - ps: Invoke-WebRequest -Uri https://github.com/cargo-lambda/cargo-lambda/releases/download/$env:CARGO_LAMBDA_VERSION/cargo-lambda-$env:CARGO_LAMBDA_VERSION.windows-x64.zip -OutFile C:\Users\appveyor\cargo-lambda.zip
+      - ps: Expand-Archive -DestinationPath C:\Users\appveyor\.cargo\bin C:\Users\appveyor\cargo-lambda.zip
+      - rustc -V
+      - cargo -V
+      - cargo lambda -V
+
+    test_script:
+      - ps: "pytest -vv -n 2 --reruns 3 tests/integration/buildcmd -m 'java or python or provided' --ignore=tests/integration/buildcmd/test_build_cmd_arm64.py --ignore=tests/integration/buildcmd/test_build_terraform_applications.py --ignore=tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd-java-python-provided.json"
+
+  #Integ testing build arm64
+  - matrix:
+      only:
+        - configuration: BuildIntegTestingArm64
+
     build_script:
       # install Rust in build_script to not override the default "install" actions
       - appveyor-retry appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
@@ -188,7 +221,7 @@ for:
       - cargo lambda -V
 
     test_script:
-      - ps: "pytest -vv tests/integration/buildcmd --ignore tests/integration/buildcmd/test_build_terraform_applications.py --ignore=tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json"
+      - ps: "pytest -vv --reruns 3 tests/integration/buildcmd/test_build_cmd_arm64.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd-arm64.json"
 
   #Integ testing Terraform build
   - matrix:
@@ -203,21 +236,13 @@ for:
       - ps: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json"
       - ps: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json"
 
-  # Integ testing package
-  - matrix:
-      only:
-        - configuration: PackageIntegTesting
-
-    test_script:
-      - ps: "pytest -vv tests/integration/package -n 4 --reruns 4 --json-report --json-report-file=TEST_REPORT-integration-package.json"
-
-  # Integ testing delete
+  # Integ testing package & delete
   - matrix:
       only:
-        - configuration: DeleteIntegTesting
+        - configuration: PackageAndDeleteIntegTesting
 
     test_script:
-      - ps: "pytest -vv tests/integration/delete -n 4 --reruns 4 --json-report --json-report-file=TEST_REPORT-integration-delete.json"
+      - ps: "pytest -vv tests/integration/package tests/integration/delete -n 4 --reruns 4 --json-report --json-report-file=TEST_REPORT-integration-package-delete.json"
 
   # Integ testing sync
   - matrix: