Test fix ups

tests/utils/shippable/check_matrix.py

@@ -26,7 +26,7 @@
 def main():  # type: () -> None
     """Main entry point."""
     repo_full_name = os.environ['REPO_FULL_NAME']
-    required_repo_full_name = 'ansible-collections/amazon-core'
+    required_repo_full_name = 'ansible-collections/amazon.aws'
 
     if repo_full_name != required_repo_full_name:
         sys.stderr.write('Skipping matrix check on repo "%s" which is not "%s".\n' % (repo_full_name, required_repo_full_name))

tests/utils/shippable/shippable.sh

@@ -34,20 +34,20 @@ export PATH="${PWD}/bin:${PATH}"
 export PYTHONIOENCODING='utf-8'
 
 if [ "${JOB_TRIGGERED_BY_NAME:-}" == "nightly-trigger" ]; then
-#    COVERAGE=yes
+    COVERAGE=yes
     COMPLETE=yes
 fi
 
-#if [ -n "${COVERAGE:-}" ]; then
-#    # on-demand coverage reporting triggered by setting the COVERAGE environment variable to a non-empty value
-#    export COVERAGE="--coverage"
-#elif [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]]; then
-#    # on-demand coverage reporting triggered by having 'ci_coverage' in the latest commit message
-#    export COVERAGE="--coverage"
-#else
-#    # on-demand coverage reporting disabled (default behavior, always-on coverage reporting remains enabled)
-#    export COVERAGE="--coverage-check"
-#fi
+if [ -n "${COVERAGE:-}" ]; then
+    # on-demand coverage reporting triggered by setting the COVERAGE environment variable to a non-empty value
+    export COVERAGE="--coverage"
+elif [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]]; then
+    # on-demand coverage reporting triggered by having 'ci_coverage' in the latest commit message
+    export COVERAGE="--coverage"
+else
+    # on-demand coverage reporting disabled (default behavior, always-on coverage reporting remains enabled)
+    export COVERAGE="--coverage-check"
+fi
 
 if [ -n "${COMPLETE:-}" ]; then
     # disable change detection triggered by setting the COMPLETE environment variable to a non-empty value
@@ -68,11 +68,6 @@ else
     export UNSTABLE=""
 fi
 
-virtualenv --python /usr/bin/python3.7 ~/ansible-venv
-set +ux
-. ~/ansible-venv/bin/activate
-set -ux
-
 pip install setuptools==44.1.0
 pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check
 
@@ -92,6 +87,7 @@ ansible-galaxy collection install ansible.netcommon
 cd "${cwd}"
 
 export ANSIBLE_COLLECTIONS_PATHS="${HOME}/.ansible/"
+SHIPPABLE_RESULT_DIR="$(pwd)/shippable"
 TEST_DIR="${HOME}/.ansible/collections/ansible_collections/amazon/aws/"
 mkdir -p "${TEST_DIR}"
 cp -aT "${SHIPPABLE_BUILD_DIR}" "${TEST_DIR}"
@@ -108,22 +104,58 @@ function cleanup
                 stub=""
             fi
 
+            # use python 3.7 for coverage to avoid running out of memory during coverage xml processing
+            # only use it for coverage to avoid the additional overhead of setting up a virtual environment for a potential no-op job
+            virtualenv --python /usr/bin/python3.7 ~/ansible-venv
+            set +ux
+            . ~/ansible-venv/bin/activate
+            set -ux
+            pip install codecov
+
             # shellcheck disable=SC2086
             ansible-test coverage xml --color -v --requirements --group-by command --group-by version ${stub:+"$stub"}
-            cp -a tests/output/reports/coverage=*.xml shippable/codecoverage/
+            cp -a tests/output/reports/coverage=*.xml "$SHIPPABLE_RESULT_DIR/codecoverage/"
+
+            # analyze and capture code coverage aggregated by integration test target
+            ansible-test coverage analyze targets generate -v "$SHIPPABLE_RESULT_DIR/testresults/coverage-analyze-targets.json"
+
+            # upload coverage report to codecov.io only when using complete on-demand coverage
+            if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ]; then
+                for file in test/results/reports/coverage=*.xml; do
+                    flags="${file##*/coverage=}"
+                    flags="${flags%-powershell.xml}"
+                    flags="${flags%.xml}"
+                    # remove numbered component from stub files when converting to tags
+                    flags="${flags//stub-[0-9]*/stub}"
+                    flags="${flags//=/,}"
+                    flags="${flags//[^a-zA-Z0-9_,]/_}"
+
+                    codecov \
+                        -f "${file}" \
+                        -F "${flags}" \
+                        -n "${test}" \
+                        -t bc371da7-e5d2-4743-93b5-309f81d457a4
+                        -X pycov \
+                        -X gcov \
+                        -X fix \
+                        -X search \
+                        -X xcode \
+                    || echo "Failed to upload code coverage report to codecov.io: ${file}"
+                done
+            fi
         fi
     fi
 
     if [ -d  tests/output/junit/ ]; then
-      cp -aT tests/output/junit/ shippable/testresults/
+      cp -aT tests/output/junit/ "$SHIPPABLE_RESULT_DIR/testresults/"
     fi
 
     if [ -d tests/output/data/ ]; then
-      cp -a tests/output/data/ shippable/testresults/
+      cp -a tests/output/data "$SHIPPABLE_RESULT_DIR/testresults/"
     fi
 
     if [ -d  tests/output/bot/ ]; then
-      cp -aT tests/output/bot/ shippable/testresults/
+      cp -aT tests/output/bot "$SHIPPABLE_RESULT_DIR/testresults/"
     fi
 }