Merge remote-tracking branch 'upstream/main'

# Conflicts: # inputs/values/dockers.json
Genometric · Mar 6, 2023 · ddd4ce3 · ddd4ce3
2 parents f7f000e + 0d67128
commit ddd4ce3
Show file tree

Hide file tree

Showing 58 changed files with 1,657 additions and 545 deletions.
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
@@ -56,7 +56,7 @@ specifically:
 
 
 3. `Publish`. This job is triggered when a PR is merged to a commit 
-  is pushed to the `master` branch. Similar to the `Test Images Build` job,
+  is pushed to the `main` branch. Similar to the `Test Images Build` job,
   this job builds Docker images and fails if the build process was 
   unsuccessful. However, in addition, this job pushes the built images
   to GCR. To authorize access to GCR, this job assumes a GCP service 
@@ -133,7 +133,7 @@ In order to set up the `Deploy` environment, you may take the following steps:
 
 Once the `Deploy` environment is set up, and the `Required reviewers`
 option under the section `Environment protection rules` is checked, 
-with every push to the `master` branch (e.g., merging a PR), the
+with every push to the `main` branch (e.g., merging a PR), the
 DIW execution will pause at the `Publish` job with the following 
 message: 
 

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -2,46 +2,41 @@ name: documentation
 
 on:
   pull_request:
-    branches: [master]
+    branches: [main]
+    paths:
+      - 'website/**'
   push:
-    branches: [master]
+    branches: [main]
+    paths:
+      - 'website/**'
 
 jobs:
-  checks:
+  Test:
     if: github.event_name != 'push'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
-            lfs: 'true'
-      - name: checkoutLFS
-        uses: actions/checkout@v2
-        run: git lfs pull
-      - uses: actions/setup-node@v1
+          lfs: 'true'
+      - uses: actions/setup-node@v3
         with:
           node-version: '18.x'
       - name: Test Build
         run: |
             cd website
-            if [ -e yarn.lock ]; then
-            yarn install --frozen-lockfile
-            elif [ -e package-lock.json ]; then
-            npm ci
-            else
-            npm i
+            if [ -e yarn.lock ]; then yarn install --frozen-lockfile;
+            elif [ -e package-lock.json ]; then npm ci;
+            else npm i;
             fi
             npm run build
-  gh-release:
+  Deploy:
     if: github.event_name != 'pull_request'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
-            lfs: 'true'
-      - name: checkoutLFS
-        uses: actions/checkout@v2
-        run: git lfs pull
-      - uses: actions/setup-node@v1
+          lfs: 'true'
+      - uses: actions/setup-node@v3
         with:
           node-version: '18.x'
       - name: Release to GitHub Pages
@@ -52,12 +47,9 @@ jobs:
           git config --global user.email "[email protected]"
           git config --global user.name "gh-actions"
           cd website
-          if [ -e yarn.lock ]; then
-          yarn install --frozen-lockfile
-          elif [ -e package-lock.json ]; then
-          npm ci
-          else
-          npm i
+          if [ -e yarn.lock ]; then yarn install --frozen-lockfile;
+          elif [ -e package-lock.json ]; then npm ci;
+          else npm i;
           fi
           npx docusaurus build
       - name: Deploy

diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml
@@ -2,9 +2,9 @@ name: Test Python Scripts
 
 on:
   push:
-    branches: [ master ]
+    branches: [ main ]
   pull_request:
-    branches: [ master ]
+    branches: [ main ]
 
 jobs:
   linting:

diff --git a/.github/workflows/sv_pipeline_docker.yml b/.github/workflows/sv_pipeline_docker.yml
@@ -3,15 +3,15 @@ name: Docker Images
 on:
   push:
     branches:
-      - master
+      - main
     paths:
       - 'src/**'
       - 'dockerfiles/**'
       - 'scripts/docker/build_docker.py'
       - '.github/workflows/sv_pipeline_docker.yml'
   pull_request:
     branches:
-      - master
+      - main
     paths:
       - 'src/**'
       - 'dockerfiles/**'
@@ -43,15 +43,15 @@ jobs:
         id: commit_sha
         # This action determines the SHA of two commits:
         # - BASE (BASE_SHA): The commit SHA of the base branch (e.g.,
-        #   broadinstitute/gatk-sv:master) which the feature branch targets.
+        #   broadinstitute/gatk-sv:main) which the feature branch targets.
         # - HEAD (HEAD_SHA): The commit SHA of the latest commit on the
         #   feature branch.
         #
         # In the following example, BASE_SHA=B and HEAD_SHA=Z
         #
         #     	    X---Y---Z     feature
         #          /         \
-        #     A---B---C---D---E   master
+        #     A---B---C---D---E   main
         #
         # 'E' is the merge commit (e.g., 'Merge pull request #0').
         #
@@ -155,6 +155,9 @@ jobs:
     strategy:
       matrix:
         python-version: ['3.8']
+    env:
+      DOCKERS_AZURE: "./inputs/values/dockers_azure.json"
+      DOCKERS_GCP: "./inputs/values/dockers.json"
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
@@ -203,24 +206,45 @@ jobs:
           sudo mv "$tmp" /etc/docker/daemon.json
           sudo systemctl restart docker.service
 
-      - name: Build and Publish Docker Images
+      - name: Build and Publish Docker Images to GCR
         id: build_and_publish
         run: |
           python ./scripts/docker/build_docker.py \
             --base-git-commit ${{ needs.build_args_job.outputs.base_sha }} \
             --current-git-commit ${{ needs.build_args_job.outputs.head_sha }} \
-            --gcr-project ${{ secrets.GCP_PROJECT_ID }}/gatk-sv \
+            --docker-repo us.gcr.io/${{ secrets.GCP_PROJECT_ID }}/gatk-sv \
             --image-tag ${{ needs.build_args_job.outputs.image_tag }} \
-            --prune-after-each-image
-          CHANGED=$(git diff --quiet ./inputs/values/dockers.json || echo True)
+            --input-json $DOCKERS_GCP \
+            --output-json $DOCKERS_GCP
+          
+          CHANGED=$(git diff --quiet $DOCKERS_GCP || echo True)
           echo "::set-output name=CHANGED::$CHANGED"
 
-      - name: Commit Changes to dockers.json
+      - name: Azure login
+        uses: azure/docker-login@v1
+        with:
+          login-server: ${{ secrets.AZ_CR }}
+          username: ${{ secrets.AZ_USERNAME }}
+          password: ${{ secrets.AZ_PASSWORD }}
+
+      - name: Build and Publish Docker Images to ACR
+        run: |
+          python ./scripts/docker/build_docker.py \
+            --base-git-commit ${{ needs.build_args_job.outputs.base_sha }} \
+            --current-git-commit ${{ needs.build_args_job.outputs.head_sha }} \
+            --docker-repo ${{ secrets.AZ_CR }} \
+            --image-tag ${{ needs.build_args_job.outputs.image_tag }} \
+            --input-json $DOCKERS_AZURE \
+            --output-json $DOCKERS_AZURE \
+            --prune-after-each-image \
+            --disable-git-protect
+
+      - name: Commit Changes to dockers_*.json
         if: steps.build_and_publish.outputs.CHANGED
         run: |
           COMMIT_SHA=${{ needs.build_args_job.outputs.head_sha }}
           git config --global user.name 'gatk-sv-bot'
           git config --global user.email '[email protected]'
-          git commit ./inputs/values/dockers.json -m "Update docker images list, triggered by "${COMMIT_SHA::8}
-          git pull --rebase origin master
+          git commit $DOCKERS_AZURE $DOCKERS_GCP -m "Update docker images list, triggered by "${COMMIT_SHA::8}
+          git pull --rebase origin main
           git push
diff --git a/.github/workflows/testwdls.yaml b/.github/workflows/testwdls.yaml
@@ -3,38 +3,58 @@ name: Test WDLs
 on:
   push:
     branches:
-      - master
+      - main
     paths:
       - 'wdl/**'
   pull_request:
     branches:
-      - master
+      - main
     paths:
       - 'wdl/**'
 
 jobs:
-  miniwdl_job:
+  syntax_test_job:
     runs-on: ubuntu-22.04
-    name: miniwdl validation
+    name: Check WDL Syntax
     strategy:
       matrix:
         python-version: ['3.8']
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install dependencies
+      - name: Setup Java
+        uses: actions/setup-java@v3
+        with:
+          # See https://github.com/actions/setup-java for the available options.
+          distribution: 'temurin'
+          java-version: '17'
+
+      - name: Setup
         run: |
           python -m pip install --upgrade pip
+          # Setup for running miniwdl
           pip install miniwdl
+          
+          # Setup for running womtool
+          pip install jinja2==3.1.2
+          wget -O womtool.jar https://github.com/broadinstitute/cromwell/releases/download/84/womtool-84.jar
+          echo \
+            '{ "google_project_id": "my-google-project-id", "terra_billing_project_id": "my-terra-billing-project" }' \
+            > inputs/values/google_cloud.my_project.json
+          scripts/inputs/build_default_inputs.sh -d . -c google_cloud.my_project
 
-      - name: Run Tests
+      - name: Test with Miniwdl
         run: |
           python scripts/test/miniwdl_validation.py \
             --imports-dir wdl \
             wdl/*.wdl
+
+      - name: Test with WOMtool
+        run: |
+          scripts/test/validate.sh -d . -j womtool.jar
diff --git a/carrot/README.md b/carrot/README.md
@@ -133,7 +133,7 @@ a publicly accessible GitHub repository. Therefore, in order to define/update
 tests, `carrot_helper` requires to know the GitHub repository and the git 
 branch where the test and evaluation WDLs are available. If you want to run 
 existing tests, you may use `https://github.com/broadinstitute/gatk-sv` and
-`master` for repository and branch respectively. If you are developing 
+`main` for repository and branch respectively. If you are developing 
 a carrot test for a WDL, then you may set the repository to your fork
 of `github.com/broadinstitute/gatk-sv` and set the branch to your feature
 branch.
@@ -181,7 +181,7 @@ and any necessary mapping between them) in the `.carrot_pipelines.json`.
 
 The `.carrot_pipelines.json` file tracked on git contains metadata belonging
 to the `carrot` resources defined for tests and WDLs available from the 
-`master` branch of the 
+`main` branch of the 
 [`github.com/broadinstitute/gatk-sv`](https://github.com/broadinstitute/gatk-sv) 
 repository on a `carrot` server maintained for internal use at the Broad 
 institute. You may use this file to run and updated (read the following)

diff --git a/carrot/carrot_helper.py b/carrot/carrot_helper.py
@@ -792,7 +792,7 @@ def init_config():
     print("\nPlease enter the branch that contains the test and evaluation"
           "WDLs and their input JSON files. If you want to run the "
           "existing tests on broadinstitute/gatk-sv, this can be "
-          "`master`, or the name of your feature branch.")
+          "`main`, or the name of your feature branch.")
     branch = None
     while not branch or not is_url_accessible(f"{repo}/tree/{branch}"):
         branch = input("\tPlease enter branch name: ")

diff --git a/dockerfiles/samtools-cloud/Dockerfile b/dockerfiles/samtools-cloud/Dockerfile
@@ -2,47 +2,43 @@
 # so we base off the sv-base-mini image, which has samtools installed, 
 # and only install GCloud SDK here for authentication purpose
 
-# Start with the barebones image that has samtools (a version that must support NIO) installed 
+# Start with the barebones image that has samtools (a version that must support NIO) installed
 ARG MINIBASE_IMAGE=sv-base-mini:latest
 ARG VIRTUAL_ENV_IMAGE=samtoolscloud-virtual-env:latest
 # available gsutil versions here: https://cloud.google.com/sdk/docs/release-notes
-ARG CLOUD_SDK_VERSION=389.0.0
+ARG CLOUD_SDK_VERSION=414.0.0-0
 ARG CONDA_ENV_NAME="gatk-sv"
 ARG CONDA_INSTALL_DIR=/opt/conda
-ARG GOOGLE_CLOUD_INSTALL_DIR=/opt/google-cloud-sdk
-
 ################## install google-cloud-cli into a copy of the virtual image
 FROM $VIRTUAL_ENV_IMAGE as prep_virtual_env
-
 # get prerequisites for installing google-cloud-cli
 ARG DEBIAN_FRONTEND=noninteractive
 RUN apt-get -qqy update --fix-missing && \
     apt-get -qqy install --no-upgrade --no-install-recommends \
-                 apt-transport-https ca-certificates wget libssl3
+                 apt-transport-https ca-certificates wget curl gnupg2 libssl3 \
+                 libfontconfig1-dev
 
-# install google-cloud-cli
-# NOTE --override-components to avoid installing bq (BigQuery)
-# NOTE: despite this being a "versioned" package, the installer will insist on upgrading at install time, so this
-#       is difficult to fully pin
+# Google cloud SDK is installed using the documentation in the following link.
+# https://cloud.google.com/sdk/docs/install#deb
 ARG CLOUD_SDK_VERSION
-ARG GOOGLE_CLOUD_INSTALL_DIR
-ENV CLOUDSDK_CONFIG=$GOOGLE_CLOUD_INSTALL_DIR
-ENV PATH=$GOOGLE_CLOUD_INSTALL_DIR/bin:$PATH
-RUN wget -q https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \
-    tar xzf google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \
-    mkdir -p $(basename $GOOGLE_CLOUD_INSTALL_DIR) && \
-    mv google-cloud-sdk $GOOGLE_CLOUD_INSTALL_DIR && \
-    cd $GOOGLE_CLOUD_INSTALL_DIR && \
-    ./install.sh --usage-reporting false --override-components core gsutil && \
+
+RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" \
+    | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
+    curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | \
+    tee /usr/share/keyrings/cloud.google.gpg && \
+    apt-get update -y && \
+    apt-get install google-cloud-sdk=$CLOUD_SDK_VERSION -y && \
     gcloud config set core/disable_usage_reporting true && \
     gcloud config set component_manager/disable_update_check true && \
     gcloud config set metrics/environment github_docker_image
 
+ENV GOOGLE_CLOUD_INSTALL_DIR=/usr/lib/google-cloud-sdk
+ENV CLOUDSDK_CONFIG=$GOOGLE_CLOUD_INSTALL_DIR
+
 # clean up some stuff we don't need
 #    -anthoscli  (Anthos)
 RUN rm $GOOGLE_CLOUD_INSTALL_DIR/bin/anthoscli
-
-########## Copy results to final image
+# Copy results to final image
 FROM $MINIBASE_IMAGE
 ARG CONDA_INSTALL_DIR
 ARG CONDA_ENV_NAME
@@ -53,7 +49,16 @@ COPY --from=prep_virtual_env $CONDA_ENV_PATH $CONDA_ENV_PATH
 ARG GOOGLE_CLOUD_INSTALL_DIR
 COPY --from=prep_virtual_env $GOOGLE_CLOUD_INSTALL_DIR $GOOGLE_CLOUD_INSTALL_DIR
 ENV PATH=/opt/google-cloud-sdk/bin:$CONDA_ENV_PATH/bin:$PATH
-
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get -qqy update --fix-missing && \
+    apt-get -qqy install --no-upgrade --no-install-recommends libfontconfig1-dev && \
+    apt-get -qqy clean && \
+    rm -rf /tmp/* \
+           /var/tmp/* \
+           /var/cache/apt/* \
+           /var/lib/apt/lists/* \
+           /usr/share/man/?? \
+           /usr/share/man/??_*
 # show the google packages run
 RUN gcloud --version
 RUN gsutil --version
diff --git a/dockerfiles/sv-base/Dockerfile b/dockerfiles/sv-base/Dockerfile
@@ -24,9 +24,8 @@ RUN apt-get -qqy update --fix-missing && \
     apt-get -qqy install --no-upgrade --no-install-recommends \
                  $GATK_BUILD_DEP $GATK_RUN_DEP
 # clone requested GATK_COMMIT
-RUN git clone https://github.com/broadinstitute/gatk.git && \
+RUN GIT_LFS_SKIP_SMUDGE=1 git clone https://github.com/broadinstitute/gatk.git && \
     cd gatk && \
-    git lfs install && \
     git checkout ${GATK_COMMIT}
 # build GATK_JAR
 RUN cd gatk && \