chore(ci): extract package signing from release-draft MONGOSH-1746 (#…

…1932)

* chore(ci): remove now-unnecessary evergreen flag

* chore(ci): resolve TODO about using `download_compiled_artifact` for packaging task

* chore(build): remove now-unnecessary extra level of indirection

* chore(ci): split package task into package and sign tasks

* chore(ci): download packages to disk as part of the publishing task

.evergreen/evergreen.yml.in

@@ -1,7 +1,3 @@
-# Regenerate using `npm run update-evergreen-config`
-# https://jira.mongodb.org/browse/EVG-20276
-unset_function_vars: true
-
 <%
 const path = require('path');
 
@@ -119,7 +115,8 @@ post:
 #   test_connectivity - Runs extra connectivity tests.
 #   test_apistrict - Runs shell API and CLI tests with --apiStrict --apiDeprecationErrors.
 #   compile_artifact - Compile the release binary.
-#   package_and_upload_artifact - Upload the release binary to S3.
+#   package_artifact - Upload the release binary together with other files to S3.
+#   sign_artifact - Get a package file from S3, sign it, put it back into S3.
 #   test_linux_artifact - Test that the built artifact works where we expect it to.
 #                         We use this to verify that e.g. the Ubuntu-built release
 #                         binary also works on RHEL and Debian.
@@ -453,51 +450,60 @@ functions:
   ###
   # PACKAGING AND UPLOADING
   #
-  # package_and_upload_artifact generates a distributable package out of the compiled artifact,
+  # package_artifact generates a distributable package out of the compiled artifact,
   # i.e. it expects it to have been downloaded already. The distributable package will be uploaded
   # to a specifc Evergreen S3 bucket for later use.
+  # sign_artifact signs and/or notarizes the distributable package as appropriate.
   #
   # The URL to download the distributable package can be retrieved by get_artifact_url.
   #
-  # package_and_upload_artifact expects the following arguments:
+  # package_artifact expects the following arguments:
   # - distro_id
   # - package_variant
   # - executable_os_id
   #
-  # get_artifact_url expects the following arguments:
+  # get_artifact_url and put_artifact_url expects the following arguments:
   # - package_variant
+  # - signature_tag (either 'signed' or 'unsigned')
   ###
-  package_and_upload_artifact:
+  package_artifact:
     - command: expansions.write
       type: setup
       params:
         file: tmp/expansions.yaml
         redacted: true
-        # TODO: REPLACE WITH CALLING download_compiled_artifact BEFORE
-    - command: s3.get
+    - command: subprocess.exec
+      params:
+        working_dir: src
+        binary: bash .evergreen/package-and-upload-artifact.sh
+        env:
+          NODE_JS_VERSION: ${node_js_version}
+          DISTRO_ID_OVERRIDE: ${distro_id}
+          PACKAGE_VARIANT: ${package_variant}
+          ARTIFACT_URL_EXTRA_TAG: unsigned
+  sign_artifact:
+    - command: expansions.write
       type: setup
       params:
-        aws_key: ${aws_key}
-        aws_secret: ${aws_secret}
-        local_file: src/dist.tgz
-        remote_file: mongosh/binaries/${revision}/${revision_order_id}/mongosh-${executable_os_id}.tgz
-        bucket: mciuploads
+        file: tmp/expansions.yaml
+        redacted: true
     - command: subprocess.exec
       params:
         working_dir: src
-        binary: bash .evergreen/package-and-upload-artifact.sh
+        binary: bash .evergreen/sign-packaged-artifact.sh
         env:
           NODE_JS_VERSION: ${node_js_version}
           DISTRO_ID_OVERRIDE: ${distro_id}
           PACKAGE_VARIANT: ${package_variant}
           MACOS_NOTARY_KEY: ${macos_notary_key}
           MACOS_NOTARY_SECRET: ${macos_notary_secret}
+  put_artifact_url:
     - command: s3.put
       params:
         aws_key: ${aws_key}
         aws_secret: ${aws_secret}
-        local_file: src/artifact-url.txt
-        remote_file: mongosh/binaries/${revision}/${revision_order_id}/artifact-url-${package_variant}.txt
+        local_file: artifact-url.txt
+        remote_file: mongosh/binaries/${revision}/${revision_order_id}/artifact-url-${package_variant}-${signature_tag}.txt
         bucket: mciuploads
         permissions: public-read
         content_type: application/x-gzip
@@ -508,7 +514,7 @@ functions:
         aws_key: ${aws_key}
         aws_secret: ${aws_secret}
         local_file: artifact-url.txt
-        remote_file: mongosh/binaries/${revision}/${revision_order_id}/artifact-url-${source_package_variant}.txt
+        remote_file: mongosh/binaries/${revision}/${revision_order_id}/artifact-url-${package_variant}-${signature_tag}.txt
         bucket: mciuploads
 
   write_preload_script:
@@ -789,6 +795,33 @@ functions:
           npm run evergreen-release draft
           }
 
+  release_publish_download_and_list_artifacts:
+    - command: expansions.write
+      type: system
+      params:
+        file: tmp/expansions.yaml
+        redacted: true
+    - command: shell.exec
+      params:
+        working_dir: src
+        shell: bash
+        env:
+          node_js_version: ${node_js_version}
+        script: |
+          set -e
+          export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD="true"
+          .evergreen/run-evergreen-release.sh download-and-list-artifacts
+    - command: shell.exec
+      params:
+        working_dir: src
+        shell: bash
+        script: |
+          set -e
+          set -x
+          ls -lh dist/
+          cat -n dist/SHASUMS1.txt
+          cat -n dist/SHASUMS256.txt
+
   release_publish_dry_run:
     - command: expansions.write
       type: system
@@ -806,7 +839,7 @@ functions:
         script: |
           set -e
           export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD="true"
-          .evergreen/run-evergreen-release.sh -- --dry-run
+          .evergreen/run-evergreen-release.sh publish -- --dry-run
 
   release_publish:
     - command: expansions.write
@@ -825,7 +858,7 @@ functions:
         script: |
           set -e
           export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD="true"
-          .evergreen/run-evergreen-release.sh
+          .evergreen/run-evergreen-release.sh publish
 
   run_perf_tests:
     - command: shell.exec
@@ -857,7 +890,8 @@ functions:
 #   test_connectivity - Runs extra connectivity tests.
 #   test_apistrict - Runs shell API and CLI tests with --apiStrict --apiDeprecationErrors.
 #   compile_artifact - Compile the release binary.
-#   package_and_upload_artifact - Upload the release binary to S3.
+#   package_artifact - Upload the release binary together with other files to S3.
+#   sign_artifact - Get a package file from S3, sign it, put it back into S3.
 #   test_linux_artifact - Test that the built artifact works where we expect it to.
 #   release_publish - Publishes the npm packages and uploads the tarballs.
 #   generate_license_and_vulnerability_report - Generates a report of vulnerabilities affecting the bundled application.
@@ -1067,7 +1101,7 @@ tasks:
   ###
   <% for (const { executableOsId, compileBuildVariant, packages } of RELEASE_PACKAGE_MATRIX) {
        for (const { name: packageVariant } of packages) { %>
-  - name: package_and_upload_artifact_<% out(packageVariant.replace(/-/g, '_')) %>
+  - name: package_artifact_<% out(packageVariant.replace(/-/g, '_')) %>
     depends_on:
       - name: compile_artifact
         variant: <% out(compileBuildVariant) %>
@@ -1076,11 +1110,39 @@ tasks:
       - func: install
         vars:
           node_js_version: "<% out(NODE_JS_VERSION_20) %>"
-      - func: package_and_upload_artifact
+      - func: download_compiled_artifact
+        vars:
+          executable_os_id: <% out(executableOsId) %>
+      - func: package_artifact
         vars:
           node_js_version: "<% out(NODE_JS_VERSION_20) %>"
           package_variant: <% out(packageVariant) %>
           executable_os_id: <% out(executableOsId) %>
+      - func: put_artifact_url
+        vars:
+          package_variant: <% out(packageVariant) %>
+          signature_tag: unsigned
+  - name: sign_artifact_<% out(packageVariant.replace(/-/g, '_')) %>
+    depends_on:
+      - name: package_artifact_<% out(packageVariant.replace(/-/g, '_')) %>
+        variant: "*"
+    commands:
+      - func: checkout
+      - func: install
+        vars:
+          node_js_version: "<% out(NODE_JS_VERSION_20) %>"
+      - func: get_artifact_url
+        vars:
+          package_variant: <% out(packageVariant) %>
+          signature_tag: unsigned
+      - func: sign_artifact
+        vars:
+          node_js_version: "<% out(NODE_JS_VERSION_20) %>"
+          package_variant: <% out(packageVariant) %>
+      - func: put_artifact_url
+        vars:
+          package_variant: <% out(packageVariant) %>
+          signature_tag: signed
   <% } } %>
 
   ###
@@ -1096,13 +1158,14 @@ tasks:
   - name: <% out(taskName) %>
     tags: ["smoke-test"]
     depends_on:
-      - name: package_and_upload_artifact_<% out(name.replace(/-/g, '_')) %>
-        variant: <% out(packageOn) %>
+      - name: sign_artifact_<% out(name.replace(/-/g, '_')) %>
+        variant: "*"
     commands:
       - func: checkout
       - func: get_artifact_url
         vars:
-          source_package_variant: <% out(name) %>
+          package_variant: <% out(name) %>
+          signature_tag: signed
       - func: write_preload_script
     <% switch (smokeTestKind) {
       case 'ssh': { %>
@@ -1172,6 +1235,9 @@ tasks:
       - func: install
         vars:
           node_js_version: "<% out(NODE_JS_VERSION_20) %>"
+      - func: release_publish_download_and_list_artifacts
+        vars:
+          node_js_version: "<% out(NODE_JS_VERSION_20) %>"
       - func: release_publish_dry_run
         vars:
           node_js_version: "<% out(NODE_JS_VERSION_20) %>"
@@ -1187,6 +1253,9 @@ tasks:
       - func: install
         vars:
           node_js_version: "<% out(NODE_JS_VERSION_20) %>"
+      - func: release_publish_download_and_list_artifacts
+        vars:
+          node_js_version: "<% out(NODE_JS_VERSION_20) %>"
       - func: release_publish
         vars:
           node_js_version: "<% out(NODE_JS_VERSION_20) %>"
@@ -1211,8 +1280,10 @@ buildvariants:
     tasks:
       - name: compile_artifact
       - name: e2e_tests_darwin_x64
-      - name: package_and_upload_artifact_darwin_x64
-      - name: package_and_upload_artifact_darwin_arm64
+      - name: package_artifact_darwin_x64
+      - name: sign_artifact_darwin_x64
+      - name: package_artifact_darwin_arm64
+      - name: sign_artifact_darwin_arm64
   - name: darwin_arm64
     display_name: "MacOS Big Sur (arm64)"
     run_on: macos-1100-arm64
@@ -1223,7 +1294,7 @@ buildvariants:
       - name: e2e_tests_darwin_arm64
 
   - name: linux_unit
-    display_name: "Ubuntu 18.04 x64 (Unit tests)"
+    display_name: "Ubuntu 20.04 x64 (Unit tests)"
     run_on: ubuntu2004-small
     tags: ["nightly-driver"]
     tasks:
@@ -1241,15 +1312,18 @@ buildvariants:
     tasks:
       - name: check_coverage
   - name: linux_package
-    display_name: "Ubuntu 18.04 x64 (Packaging)"
+    display_name: "Ubuntu 20.04 x64 (Packaging)"
     run_on: ubuntu2004-small
     tags: ["nightly-driver"]
     tasks:
       <% for (const { executableOsId, packages } of RELEASE_PACKAGE_MATRIX) {
           for (const { name: packageVariant } of packages) {
             if (executableOsId.startsWith('linux')) { %>
-      - name: package_and_upload_artifact_<% out(packageVariant.replace(/-/g, '_')) %>
+      - name: package_artifact_<% out(packageVariant.replace(/-/g, '_')) %>
+      - name: sign_artifact_<% out(packageVariant.replace(/-/g, '_')) %>
       <% } } } %>
+      - name: sign_artifact_win32_x64
+      - name: sign_artifact_win32msi_x64
   - name: linux_x64_build
     display_name: "RHEL 7.0 x64 (build)"
     run_on: rhel70-build
@@ -1517,8 +1591,8 @@ buildvariants:
       executable_os_id: win32
     tasks:
       - name: e2e_tests_win32
-      - name: package_and_upload_artifact_win32_x64
-      - name: package_and_upload_artifact_win32msi_x64
+      - name: package_artifact_win32_x64
+      - name: package_artifact_win32msi_x64
   - name: win32_build
     display_name: "Windows VS 2019 (build)"
     run_on: windows-64-vs2019-build

.evergreen/package-and-upload-artifact.sh

@@ -17,63 +17,19 @@ if [ "$(uname)" == Linux ]; then
     -e EVERGREEN_EXPANSIONS_PATH=/tmp/build/tmp/expansions.yaml \
     -e NODE_JS_VERSION \
     -e PACKAGE_VARIANT \
+    -e ARTIFACT_URL_EXTRA_TAG \
     -e ARTIFACT_URL_FILE="/tmp/build/artifact-url.txt" \
     --rm -v $PWD:/tmp/build --network host rocky8-package \
     -c 'cd /tmp/build && npm run evergreen-release package && npm run evergreen-release upload'
 else
-  if [[ "$OS" == "Windows_NT" && "$PACKAGE_VARIANT" == "win32msi-x64" ]]; then
-    # We have to setup a python venv for the notary client to work
-    # in order to sign the MSI
-    export PATH="/cygdrive/c/Python27:/cygdrive/c/python/Python27:$PATH"
-    # python --version prints to stderr...
-    if [[ ! "$(python --version 2>&1 | tr -d '\r')" =~ "2.7" ]]; then
-      echo "FAIL: could not properly setup Python 2.7"
-      exit 1
-    fi
-
-    # These packages have to be installed for the notary client to work
-    python -m virtualenv .venv
-    cd .venv
-    # Activating venv on Windows is a little different
-    source Scripts/activate
-    pip install requests
-    pip install poster
-    # pycrypto does not install on Windows so we use pycryptodome
-    pip install pycryptodome
-    cd ..
-  fi
-
   npm run evergreen-release package
   ls -lh dist/
 
-  if [ "$(uname)" == Darwin ]; then
-    # https://wiki.corp.mongodb.com/display/BUILD/How+to+use+MacOS+notary+service
-    # download macnotary client
-    curl -LO https://macos-notary-1628249594.s3.amazonaws.com/releases/client/latest/darwin_amd64.zip
-    unzip darwin_amd64.zip
-    chmod +x ./darwin_amd64/macnotary
-    ./darwin_amd64/macnotary -v
-
-    FILE=$(echo ./dist/*.zip)
-    echo "notarizing $FILE ..."
-
-    # notarize the client
-    ./darwin_amd64/macnotary \
-      -f "$FILE" \
-      -m notarizeAndSign -u https://dev.macos-notary.build.10gen.cc/api \
-      -b com.mongodb.mongosh \
-      -e config/macos-entitlements.xml \
-      -o "$FILE-signed.zip"
-    mv -v "$FILE-signed.zip" "$FILE"
-
-    # Verify signing
-    unzip "$FILE"
-    spctl -a -vvv -t install mongosh-*/bin/mongosh
-  fi
-
   if [ "$OS" == "Windows_NT" ]; then
     # Fix absolute path before handing over to node
     export ARTIFACT_URL_FILE="$(cygpath -w "$ARTIFACT_URL_FILE")"
   fi
   npm run evergreen-release upload
 fi
+
+cp -v $PWD/artifact-url.txt $PWD/../artifact-url.txt

.evergreen/run-evergreen-release.sh

@@ -6,4 +6,4 @@ set -x
 export NODE_JS_VERSION=${node_js_version}
 source .evergreen/setup-env.sh
 export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD="true"
-npm run evergreen-release publish $@
+npm run evergreen-release $@