build.yaml

steps:
  - kind: buildImage2
    name: git_make_bash_image
    publishAs: git-make-bash
    dockerFile:
      inline: |
        FROM {{ global.docker_root_image }}
        RUN apt-get update && apt-get install -y git make bash
  - kind: runImage
    name: merge_code
    image:
      valueFrom: git_make_bash_image.image
    script: |
      set -ex
      cd /io
      mkdir repo
      cd repo
      {{ code.checkout_script }}
      make -C hail python-version-info
      git rev-parse HEAD > git_version
    outputs:
      - from: /io/repo/
        to: /repo
      - from: /io/repo/hail/python/hail/hail_version
        to: /hail_version
      - from: /io/repo/hail/python/hail/hail_pip_version
        to: /hail_pip_version
      - from: /io/repo/git_version
        to: /git_version
    dependsOn:
      - git_make_bash_image
  - kind: buildImage2
    name: hail_ubuntu_image
    dockerFile: /io/hail-ubuntu/Dockerfile
    contextPath: /io/hail-ubuntu
    publishAs: hail-ubuntu
    inputs:
      - from: /repo/docker/hail-ubuntu
        to: /io/hail-ubuntu
    dependsOn:
      - merge_code
  - kind: createNamespace
    name: default_ns
    namespaceName: default
    public: true
    secrets:
      - name: auth-oauth2-client-secret
      - name: registry-push-credentials
      - name: hail-ci-0-1-github-oauth-token
      - name: test-gsa-key
      - name: test-aws-key
        clouds:
          - gcp
      - name: test-azure-key
        clouds:
          - gcp
      - name: zulip-config
      - name: billing-monitor-gsa-key
        clouds:
          - gcp
      - name: hail-ci-0-1-service-account-key
      - name: test-dataproc-service-account-key
        clouds:
          - gcp
      - name: batch-worker-ssh-public-key
        clouds:
          - azure
  - kind: deploy
    name: deploy_batch_sa
    namespace:
      valueFrom: default_ns.name
    config: batch/service-account.yaml
    dependsOn:
      - default_ns
  - kind: deploy
    name: deploy_ci_agent
    namespace:
      valueFrom: default_ns.name
    config: ci/ci-agent.yaml
    dependsOn:
      - default_ns
  - kind: deploy
    name: deploy_test_batch_sa
    namespace:
      valueFrom: default_ns.name
    config: batch/test-sa.yaml
    dependsOn:
      - default_ns
  - kind: buildImage2
    name: base_image
    dockerFile: /io/repo/docker/Dockerfile.base
    contextPath: /io/repo
    publishAs: base
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
    inputs:
      - from: /repo
        to: /io/repo
    dependsOn:
      - hail_ubuntu_image
      - merge_code
  - kind: buildImage2
    name: base_spark_3_2
    dockerFile: /io/repo/docker/Dockerfile.base-with-spark-3-2
    contextPath: /io/repo
    publishAs: base_spark_3_2
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
    inputs:
      - from: /repo
        to: /io/repo
    dependsOn:
      - base_image
      - merge_code
  - kind: runImage
    name: check_pip_dependencies
    image:
      valueFrom: base_image.image
    script: |
      set -ex
      pip install pip-tools
      cd /io/repo
      chmod 755 ./check_pip_requirements.sh
      make check-pip-dependencies
    inputs:
      - from: /repo
        to: /io/repo
    dependsOn:
      - base_image
      - merge_code
  - kind: runImage
    name: check_hail
    image:
      valueFrom: base_image.image
    script: |
      set -ex
      cd /io/repo
      make check-hail
    inputs:
      - from: /repo
        to: /io/repo
    dependsOn:
      - base_image
      - merge_code
  - kind: buildImage2
    name: service_base_image
    dockerFile: /io/repo/docker/Dockerfile.service-base
    contextPath: /io/repo
    publishAs: service-base
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
    dependsOn:
      - base_image
      - merge_code
    inputs:
      - from: /hail_version
        to: /io/repo/hail_version
      - from: /repo
        to: /io/repo
  - kind: runImage
    name: check_services
    image:
      valueFrom: service_base_image.image
    script: |
      set -ex
      cd /io/repo
      {% if 'target_sha' in code %}
      export HAIL_TARGET_SHA={{ code.target_sha }}
      {% endif %}
      make -k check-services
    inputs:
      - from: /repo
        to: /io/repo
    dependsOn:
      - service_base_image
      - merge_code
  - kind: buildImage2
    name: create_certs_image
    dockerFile: /io/tls/Dockerfile
    contextPath: /io/tls
    publishAs: create_certs_image
    inputs:
      - from: /repo/tls
        to: /io/tls
    dependsOn:
      - hail_ubuntu_image
      - merge_code
  - kind: runImage
    name: create_ssl_config_hail_root
    image:
      valueFrom: create_certs_image.image
    script: |
      openssl req -new -x509 -subj /CN=hail-root -nodes -newkey rsa:4096 -keyout hail-root-key.pem -out hail-root-cert.pem
      until kubectl get secret -n {{ default_ns.name }} ssl-config-hail-root
      do
          kubectl create secret generic -n {{ default_ns.name }} ssl-config-hail-root \
                  --from-file=hail-root-key.pem \
                  --from-file=hail-root-cert.pem
      done
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    scopes:
      - test
      - dev
    dependsOn:
      - default_ns
      - create_certs_image
  - kind: runImage
    name: create_certs
    image:
      valueFrom: create_certs_image.image
    script: |
      set -ex
      python3 create_certs.py \
              {{ default_ns.name }} \
              config.yaml \
              /ssl-config-hail-root/hail-root-key.pem \
              /ssl-config-hail-root/hail-root-cert.pem
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    secrets:
      - name: ssl-config-hail-root
        namespace:
          valueFrom: default_ns.name
        mountPath: /ssl-config-hail-root
    dependsOn:
      - default_ns
      - create_certs_image
      - create_ssl_config_hail_root
  - kind: createDatabase
    name: test_database_instance
    databaseName: test-instance
    migrations: []
    namespace:
      valueFrom: default_ns.name
    scopes:
      - test
    dependsOn:
      - default_ns
  - kind: runImage
    name: create_database_server_config
    image:
      valueFrom: base_image.image
    script: |
     kubectl -n {{ default_ns.name }} get -o json secret {{ test_database_instance.admin_secret_name }} | jq '{apiVersion, kind, type, data, metadata: {name: "database-server-config"}}' | kubectl -n {{ default_ns.name }} apply -f -
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    scopes:
      - test
    dependsOn:
      - default_ns
      - base_image
      - test_database_instance
  - kind: runImage
    name: delete_auth_tables
    image:
      valueFrom: service_base_image.image
    script: |
      set -ex
      mysql --defaults-extra-file=/sql-config/sql-config.cnf < /io/sql/delete-auth-tables.sql
    inputs:
      - from: /repo/auth/sql
        to: /io/sql
    secrets:
      - name: database-server-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /sql-config
    runIfRequested: true
    scopes:
      - dev
    dependsOn:
      - default_ns
      - service_base_image
      - merge_code
      - create_database_server_config
  - kind: createDatabase
    name: auth_database
    databaseName: auth
    migrations:
      - name: initial
        script: /io/sql/initial.sql
      - name: copy-paste-tokens
        script: /io/sql/copy-paste-tokens.sql
      - name: drop-bucket
        script: /io/sql/drop-bucket.sql
      - name: add-trial-billing-project
        script: /io/sql/add-trial-billing-project.sql
      - name: add-roles
        script: /io/sql/add-roles.sql
      - name: rename-gsa-identity
        script: /io/sql/rename-gsa-identity.sql
      - name: support-azure-oauth
        script: /io/sql/support-azure-oauth.sql
    inputs:
      - from: /repo/auth/sql
        to: /io/sql
    namespace:
      valueFrom: default_ns.name
    shutdowns:
      - kind: Deployment
        namespace:
          valueFrom: default_ns.name
        name: auth
    dependsOn:
      - default_ns
      - merge_code
      - delete_auth_tables
  - kind: runImage
    name: create_deploy_config
    image:
      valueFrom: base_image.image
    script: |
      set -ex

      # k8s deploy config
      cat > deploy-config.json <<EOF
      {"location":"k8s","default_namespace":"{{ default_ns.name }}","domain":"{{ global.domain }}"}
      EOF
      kubectl -n {{ default_ns.name }} create secret generic deploy-config \
              --from-file=./deploy-config.json \
              --save-config --dry-run=client -o yaml \
          | kubectl -n {{ default_ns.name }} apply -f -

      # worker deploy config
      cat > deploy-config.json <<EOF
      {"location":"gce","default_namespace":"{{ default_ns.name }}","domain":"{{ global.domain }}"}
      EOF
      kubectl -n {{ default_ns.name }} create secret generic worker-deploy-config \
              --from-file=./deploy-config.json \
              --save-config --dry-run=client -o yaml \
          | kubectl -n {{ default_ns.name }} apply -f -
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    dependsOn:
      - default_ns
      - base_image
  - kind: deploy
    name: deploy_admin_pod
    namespace:
      valueFrom: default_ns.name
    config: admin-pod/admin-pod.yaml
    scopes:
      - deploy
      - dev
    dependsOn:
      - default_ns
      - service_base_image
      - create_deploy_config
      - create_certs
  - kind: runImage
    name: create_session_key
    image:
      valueFrom: base_image.image
    script: |
      set -ex
      # create session keys
      N=$(kubectl -n {{ default_ns.name }} get secret --ignore-not-found=true --no-headers session-secret-key | wc -l | tr -d '[:space:]')
      if [[ $N != 0 ]]; then
        exit
      fi
      mkdir /session-secret-key
      cat > generate-session-key.py <<EOF
      import base64
      from cryptography import fernet
      with open('/session-secret-key/session-secret-key', 'wb') as f:
          f.write(base64.urlsafe_b64decode(fernet.Fernet.generate_key()))
      EOF
      python3 generate-session-key.py
      kubectl -n {{ default_ns.name }} create secret generic session-secret-key --from-file=/session-secret-key/session-secret-key
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    dependsOn:
      - default_ns
      - base_image
  - kind: runImage
    name: create_test_gsa_keys
    image:
      valueFrom: service_base_image.image
    script: |
      kubectl -n {{ default_ns.name }} get -o json secret test-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "auth-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -
      kubectl -n {{ default_ns.name }} get -o json secret test-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "batch-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -
      kubectl -n {{ default_ns.name }} get -o json secret test-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "ci-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -
      kubectl -n {{ default_ns.name }} get -o json secret test-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "test-dev-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -
      kubectl -n {{ default_ns.name }} get -o json secret test-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "grafana-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -
    scopes:
      - test
      - dev
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    dependsOn:
      - default_ns
      - service_base_image
  - kind: buildImage2
    name: auth_image
    dockerFile: /io/repo/auth/Dockerfile
    contextPath: /io/repo
    publishAs: auth
    inputs:
      - from: /repo
        to: /io/repo
    dependsOn:
      - service_base_image
      - merge_code
  - kind: deploy
    name: deploy_auth_driver_service_account
    namespace:
      valueFrom: default_ns.name
    config: auth/auth-driver-service-account.yaml
    dependsOn:
      - default_ns
  - kind: runImage
    name: create_accounts
    image:
      valueFrom: auth_image.image
    script: |
      set -ex
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_SCOPE={{ scope }}
      python3 /io/bootstrap_create_accounts.py
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    secrets:
      - name:
          valueFrom: auth_database.user_secret_name
        namespace:
          valueFrom: default_ns.name
        mountPath: /sql-config
      - name: database-server-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /database-server-config
      - name: worker-deploy-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /deploy-config
      - name: auth-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /auth-gsa-key
      - name: global-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /global-config
    inputs:
      - from: /repo/ci/bootstrap_create_accounts.py
        to: /io/bootstrap_create_accounts.py
    dependsOn:
      - default_ns
      - deploy_test_batch_sa
      - auth_database
      - auth_image
      - create_deploy_config
      - deploy_auth_driver_service_account
      - create_test_gsa_keys
      - create_database_server_config
  - kind: runImage
    name: create_initial_user
    runIfRequested: true
    image:
      valueFrom: service_base_image.image
    script: |
      set -ex
      export NAMESPACE={{ default_ns.name }}
      export CLOUD={{ global.cloud }}
      python3 /io/create_initial_account.py {{ code.username }} {{ code.login_id }}
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    secrets:
      - name:
          valueFrom: auth_database.user_secret_name
        namespace:
          valueFrom: default_ns.name
        mountPath: /sql-config
    inputs:
      - from: /repo/ci/create_initial_account.py
        to: /io/create_initial_account.py
    dependsOn:
      - default_ns
      - service_base_image
      - merge_code
      - auth_database
  - kind: buildImage2
    name: hail_build_image
    dockerFile: /io/hail/Dockerfile.hail-build
    contextPath: /io/hail
    publishAs: hail-build
    inputs:
      - from: /repo/hail
        to: /io/hail
    dependsOn:
      - base_image
      - merge_code
  - kind: buildImage2
    name: hail_run_image
    dockerFile: /io/repo/hail/Dockerfile.hail-run
    contextPath: /io/repo
    publishAs: hail-run
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
    inputs:
      - from: /repo
        to: /io/repo
    dependsOn:
      - service_base_image
      - merge_code
  - kind: buildImage2
    name: hail_run_tests_image
    dockerFile: /io/hail/Dockerfile.hail-run-tests
    contextPath: /io/hail
    publishAs: hail-run-tests
    inputs:
      - from: /repo/hail
        to: /io/hail
    dependsOn:
      - hail_run_image
      - merge_code
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
  - kind: buildImage2
    name: monitoring_image
    dockerFile: /io/repo/monitoring/Dockerfile
    contextPath: /io/repo
    publishAs: monitoring
    inputs:
      - from: /repo
        to: /io/repo
    dependsOn:
      - service_base_image
      - merge_code
  - kind: buildImage2
    name: batch_driver_nginx_image
    dockerFile: /io/batch/Dockerfile.driver-nginx
    contextPath: /io/batch
    publishAs: batch-driver-nginx
    inputs:
      - from: /repo/batch
        to: /io/batch
    dependsOn:
      - hail_ubuntu_image
      - merge_code
  - kind: buildImage2
    name: batch_image
    dockerFile: /io/batch/Dockerfile
    contextPath: /io/batch/
    publishAs: batch
    inputs:
      - from: /repo/batch
        to: /io/batch
    dependsOn:
      - service_base_image
      - merge_code
  - kind: runImage
    name: render_grafana_nginx_conf
    image:
      valueFrom: service_base_image.image
    script: |
      set -ex
      cd /io/repo/grafana
      {% if deploy %}
      DEPLOY=true
      {% else %}
      DEPLOY=false
      {% endif %}
      python3 ../ci/jinja2_render.py '{"deploy": '${DEPLOY}', "global": {"domain": "{{ global.domain }}"}, "default_ns": {"name": "{{ default_ns.name }}"}}' nginx.conf nginx.conf.out
    inputs:
      - from: /repo
        to: /io/repo
    outputs:
      - from: /io/repo/grafana/nginx.conf.out
        to: /grafana/nginx.conf.out
    dependsOn:
      - default_ns
      - service_base_image
      - merge_code
  - kind: buildImage2
    name: grafana_nginx_image
    dockerFile: /io/grafana/Dockerfile.nginx
    contextPath: /io/grafana
    publishAs: grafana
    inputs:
      - from: /repo/grafana
        to: /io/grafana
      - from: /grafana/nginx.conf.out
        to: /io/grafana/nginx.conf.out
    dependsOn:
      - hail_ubuntu_image
      - render_grafana_nginx_conf
      - merge_code
  - kind: buildImage2
    name: test_ci_image
    dockerFile: /io/repo/ci/Dockerfile.test
    contextPath: /io/repo
    publishAs: test-ci
    scopes:
      - test
      - dev
    inputs:
      - from: /repo
        to: /io/repo
      - from: /hail_version
        to: /io/repo/hail_version
    dependsOn:
      - base_image
      - merge_code
  - kind: buildImage2
    name: test_monitoring_image
    dockerFile: /io/repo/monitoring/Dockerfile.test
    contextPath: /io/repo
    publishAs: test-monitoring
    inputs:
      - from: /repo
        to: /io/repo
    dependsOn:
      - service_base_image
      - merge_code
  - kind: buildImage2
    name: ci_image
    dockerFile: /io/repo/ci/Dockerfile
    contextPath: /io/repo
    publishAs: ci
    inputs:
      - from: /repo
        to: /io/repo
    dependsOn:
      - service_base_image
      - merge_code
  - kind: buildImage2
    name: hail_buildkit_image
    dockerFile: /io/repo/ci/buildkit/Dockerfile
    contextPath: /io/repo/ci
    publishAs: hail-buildkit
    inputs:
      - from: /repo/ci
        to: /io/repo/ci
    dependsOn:
      - service_base_image
      - merge_code
  - kind: buildImage2
    name: ci_utils_image
    dockerFile: /io/ci/Dockerfile.ci-utils
    contextPath: /io/ci
    publishAs: ci-utils
    resources:
      storage: 20Gi
      cpu: "2"
      memory: standard
    inputs:
      - from: /repo/ci
        to: /io/ci
    dependsOn:
      - service_base_image
      - merge_code
  - kind: runImage
    name: build_hail_jar_and_wheel_only
    image:
      valueFrom: hail_build_image.image
    resources:
      memory: standard
      cpu: '4'
    script: |
      set -ex
      cd /io/repo/hail
      chmod 755 ./gradlew
      time retry ./gradlew --version
      time retry make shadowJar wheel
      (cd build/deploy/dist/ && tar -cvf wheel-container.tar hail-*-py3-none-any.whl)
    inputs:
      - from: /repo
        to: /io/repo
    outputs:
      - from: /io/repo/hail/build/libs/hail-all-spark.jar
        to: /just-jar/hail.jar
      - from: /io/repo/hail/build/deploy/dist/wheel-container.tar
        to: /just-wheel/wheel-container.tar
    dependsOn:
      - hail_build_image
      - merge_code
  - kind: runImage
    name: build_hail_jar_and_wheel_only_spark_3_2
    image:
      valueFrom: hail_build_image.image
    resources:
      memory: standard
      cpu: '4'
    script: |
      set -ex
      cd /io/repo/hail
      chmod 755 ./gradlew
      time retry ./gradlew --version
      export SPARK_VERSION="3.2.1" SCALA_VERSION="2.12.12"
      time retry make jars wheel
      (cd build/deploy/dist/ && tar -cvf wheel-container.tar hail-*-py3-none-any.whl)
    inputs:
      - from: /repo
        to: /io/repo
    outputs:
      - from: /io/repo/hail/build/libs/hail-all-spark.jar
        to: /just-jar/spark-32/hail.jar
      - from: /io/repo/hail/build/libs/hail-all-spark-test.jar
        to: /just-jar/spark-32/hail-test.jar
      - from: /io/repo/hail/build/deploy/dist/wheel-container.tar
        to: /just-wheel/spark-32/wheel-container.tar
    dependsOn:
      - hail_build_image
      - merge_code
  - kind: runImage
    name: build_hail
    image:
      valueFrom: hail_build_image.image
    resources:
      memory: standard
      cpu: '4'
    script: |
      set -ex
      cd /io/repo/hail
      chmod 755 ./gradlew
      time retry ./gradlew --version
      time retry make jars wheel HAIL_DEBUG_MODE=1
      (cd build/deploy/dist/ && tar -cvf debug-wheel-container.tar hail-*-py3-none-any.whl)
      cd /io/repo/hail
      mkdir build/debug_libs
      mv build/libs/hail-all-spark.jar build/debug_libs/
      mv build/libs/hail-all-spark-test.jar build/debug_libs/
      mv build/deploy/dist/debug-wheel-container.tar build/debug_libs
      time retry make jars python-version-info wheel

      # Check wheel size is small enough for pypi (< 100 MB)
      HAIL_PIP_VERSION=$(cat python/hail/hail_pip_version)
      WHEEL_PATH="build/deploy/dist/hail-$HAIL_PIP_VERSION-py3-none-any.whl"
      du -h $WHEEL_PATH
      $(python3 -c "import os; exit(1) if (os.path.getsize('$WHEEL_PATH')) > (200 * 1024 * 1024) else exit(0)")

      time (cd python && zip -r hail.zip hail hailtop)
      time tar czf test.tar.gz -C python test
      time tar czf resources.tar.gz -C src/test resources
      time tar czf data.tar.gz -C python/hail/docs data
      (cd .. && time tar czf hail/website-src.tar.gz website)
      time tar czf cluster-tests.tar.gz python/cluster-tests
      (cd build/deploy/dist/ && tar -cvf wheel-container.tar hail-*-py3-none-any.whl)
      time TESTNG_SPLITS=5 python3 generate_splits.py
      time tar czf splits.tar.gz testng-splits-*.xml
    inputs:
      - from: /repo
        to: /io/repo
    outputs:
      - from: /io/repo/hail/build/libs/hail-all-spark.jar
        to: /hail.jar
      - from: /io/repo/hail/build/libs/hail-all-spark-test.jar
        to: /hail-test.jar
      - from: /io/repo/hail/build/debug_libs/hail-all-spark-test.jar
        to: /hail-debug-test.jar
      - from: /io/repo/hail/testng-build.xml
        to: /testng-build.xml
      - from: /io/repo/hail/testng-services.xml
        to: /testng-services.xml
      - from: /io/repo/hail/testng-fs.xml
        to: /testng-fs.xml
      - from: /io/repo/hail/python/hail.zip
        to: /hail.zip
      - from: /io/repo/hail/build/deploy/dist/wheel-container.tar
        to: /wheel-container.tar
      - from: /io/repo/hail/build/debug_libs/debug-wheel-container.tar
        to: /debug-wheel-container.tar
      - from: /io/repo/hail/test.tar.gz
        to: /test.tar.gz
      - from: /io/repo/hail/resources.tar.gz
        to: /resources.tar.gz
      - from: /io/repo/hail/splits.tar.gz
        to: /splits.tar.gz
      - from: /io/repo/hail/data.tar.gz
        to: /data.tar.gz
      - from: /io/repo/hail/website-src.tar.gz
        to: /website-src.tar.gz
      - from: /io/repo/hail/cluster-tests.tar.gz
        to: /cluster-tests.tar.gz
    dependsOn:
      - hail_build_image
      - merge_code
  - kind: runImage
    name: build_hail_spark2.4.8
    image:
      valueFrom: hail_build_image.image
    resources:
      memory: standard
      cpu: '4'
    script: |
      set -ex
      cd /io/repo/hail
      chmod 755 ./gradlew
      time retry ./gradlew --version
      export SPARK_VERSION="2.4.8" SCALA_VERSION="2.12.12"
      time retry make jars python-version-info wheel
    inputs:
      - from: /repo
        to: /io/repo
    dependsOn:
      - hail_build_image
      - merge_code
  - kind: runImage
    name: build_hail_spark3.0.1
    image:
      valueFrom: hail_build_image.image
    resources:
      memory: standard
      cpu: '4'
    script: |
      set -ex
      cd /io/repo/hail
      chmod 755 ./gradlew
      time retry ./gradlew --version
      export SPARK_VERSION="3.0.1" SCALA_VERSION="2.12.12"
      time retry make jars python-version-info wheel
    inputs:
      - from: /repo
        to: /io/repo
    dependsOn:
      - hail_build_image
      - merge_code
  - kind: runImage
    name: build_wheel_for_azure
    image:
      valueFrom: hail_build_image.image
    resources:
      memory: standard
      cpu: '4'
    script: |
      set -ex
      cd /io/repo/hail
      chmod 755 ./gradlew
      time retry ./gradlew --version
      export SPARK_VERSION="3.0.2" SCALA_VERSION="2.12.10"
      time retry make wheel
      (cd build/deploy/dist/ && tar -cvf wheel-container.tar hail-*-py3-none-any.whl)
    inputs:
      - from: /repo
        to: /io/repo
    outputs:
      - from: /io/repo/hail/build/deploy/dist/wheel-container.tar
        to: /wheel-for-azure-container.tar
    dependsOn:
      - hail_build_image
      - merge_code
  - kind: runImage
    name: compile_batch_worker_jvm_entryway
    image:
      valueFrom: base_image.image
    script: |
      set -ex

      cd /io

      curl -sSfL https://github.com/kohlschutter/junixsocket/releases/download/junixsocket-parent-2.3.3/junixsocket-selftest-2.3.3-jar-with-dependencies.jar \
           > junixsocket-selftest-2.3.3-jar-with-dependencies.jar

      javac -cp junixsocket-selftest-2.3.3-jar-with-dependencies.jar \
            batch/src/main/java/is/hail/JVMEntryway.java
    inputs:
      - from: /repo/batch
        to: /io/batch
    outputs:
      - from: /io/junixsocket-selftest-2.3.3-jar-with-dependencies.jar
        to: /junixsocket-selftest-2.3.3-jar-with-dependencies.jar
      - from: /io/batch/src/main/java/is/hail
        to: /batch-jvm-entryway-classes
    dependsOn:
      - base_image
      - merge_code
  - kind: buildImage2
    name: batch_worker_image
    dockerFile: /io/repo/batch/Dockerfile.worker
    contextPath: /io/repo
    publishAs: batch-worker
    inputs:
      - from: /repo
        to: /io/repo
      - from: /batch-jvm-entryway-classes
        to: /io/repo/batch/src/main/java/is/hail
      - from: /junixsocket-selftest-2.3.3-jar-with-dependencies.jar
        to: /io/repo/batch/jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar
      - from: /hail_version
        to: /io/repo/hail_version
    dependsOn:
      - merge_code
      - hail_ubuntu_image
      - compile_batch_worker_jvm_entryway
  - kind: buildImage2
    name: service_java_run_base_image
    dockerFile: /io/repo/docker/Dockerfile.service-java-run-base
    contextPath: /io/repo
    publishAs: service-java-run-base
    inputs:
      - from: /repo
        to: /io/repo
      - from: /hail_version
        to: /io/repo/hail_version
    dependsOn:
      - hail_ubuntu_image
      - merge_code
  - kind: buildImage2
    name: memory_image
    dockerFile: /io/repo/memory/Dockerfile
    contextPath: /io/repo
    publishAs: memory
    inputs:
      - from: /repo
        to: /io/repo
    dependsOn:
      - service_base_image
      - merge_code
  - kind: deploy
    name: deploy_memory_sa
    namespace:
      valueFrom: default_ns.name
    config: memory/service-account.yaml
    dependsOn:
      - default_ns
  - kind: buildImage2
    name: hail_pip_installed_image
    dockerFile: /io/hail/Dockerfile.hail-pip-installed
    contextPath: /io/hail
    publishAs: hail-run
    inputs:
      - from: /repo/hail
        to: /io/hail
      - from: /just-wheel/wheel-container.tar
        to: /io/hail/wheel-container.tar
    dependsOn:
      - service_base_image
      - build_hail_jar_and_wheel_only
      - merge_code
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
  - kind: runImage
    name: upload_test_resources_to_blob_storage
    image:
      valueFrom: hail_pip_installed_image.image
    script: |
      set -ex
      cd /io/repo/hail/
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      export AZURE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      python3 -m hailtop.aiotools.copy 'null' '[
      {"from": "src/test/resources",
       "to":   "{{ global.test_storage_uri }}/{{ token }}/test/resources"},
      {"from": "python/hail/docs/data",
       "to":   "{{ global.test_storage_uri }}/{{ token }}/doctest/data"}
      ]'
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    inputs:
      - from: /repo/hail/src/test/resources
        to: /io/repo/hail/src/test/resources
      - from: /repo/hail/python/hail/docs/data
        to: /io/repo/hail/python/hail/docs/data
    dependsOn:
      - default_ns
      - hail_pip_installed_image
      - merge_code
  - kind: runImage
    name: test_hail_java
    numSplits: 5
    image:
      valueFrom: hail_run_tests_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io
      mkdir -p src/test
      tar xzf resources.tar.gz -C src/test
      tar xzf splits.tar.gz
      export HAIL_TEST_SKIP_R=1
      java -cp hail-test.jar:$SPARK_HOME/jars/* org.testng.TestNG -listener is.hail.LogTestListener testng-splits-$HAIL_RUN_IMAGE_SPLIT_INDEX.xml
    inputs:
      - from: /resources.tar.gz
        to: /io/resources.tar.gz
      - from: /hail-debug-test.jar
        to: /io/hail-test.jar
      - from: /splits.tar.gz
        to: /io/splits.tar.gz
    outputs:
      - from: /io/test-output
        to: /test-output
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - default_ns
      - hail_run_tests_image
      - build_hail
  - kind: buildImage2
    name: hail_base_image
    dockerFile: /io/repo/hail/Dockerfile.hail-base
    contextPath: /io/repo
    publishAs: hail-base
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
    inputs:
      - from: /repo
        to: /io/repo
      - from: /hail.jar
        to: /io/repo/hail.jar
      - from: /hail.zip
        to: /io/repo/hail.zip
    dependsOn:
      - hail_run_image
      - build_hail
      - merge_code
  - kind: buildImage2
    name: hail_pip_installed_python37_image
    dockerFile: /io/repo/hail/Dockerfile.hail-pip-installed-python37
    contextPath: /io/repo
    publishAs: hail-pip-installed-python37
    inputs:
      - from: /repo
        to: /io/repo
      - from: /just-wheel/wheel-container.tar
        to: /io/repo/wheel-container.tar
    dependsOn:
      - build_hail_jar_and_wheel_only
      - hail_ubuntu_image
      - merge_code
  - kind: buildImage2
    name: hail_pip_installed_python38_image
    dockerFile: /io/repo/hail/Dockerfile.hail-pip-installed-python38
    contextPath: /io/repo
    publishAs: hail-pip-installed-python38
    inputs:
      - from: /repo
        to: /io/repo
      - from: /just-wheel/wheel-container.tar
        to: /io/repo/wheel-container.tar
    dependsOn:
      - build_hail_jar_and_wheel_only
      - hail_ubuntu_image
      - merge_code
  - kind: runImage
    name: check_hail_python37
    image:
      valueFrom: hail_pip_installed_python37_image.image
    script: |
      set -x
      SITE_PACKAGES=$(pip3 show hail | grep Location | sed 's/Location: //')

      exit_status=0

      python3 -m flake8 --config setup.cfg $SITE_PACKAGES/hail $SITE_PACKAGES/hailtop || exit_status=$?
      python3 -m pylint --rcfile pylintrc hailtop || exit_status=$?
      # working around this: https://github.com/python/mypy/issues/7087
      ln -s $SITE_PACKAGES/hailtop hailtop
      mypy --config-file /setup.cfg -p hailtop || exit_status=$?

      exit $exit_status
    dependsOn:
      - hail_pip_installed_python37_image
  - kind: runImage
    name: check_hail_python38
    image:
      valueFrom: hail_pip_installed_python38_image.image
    script: |
      set -x
      SITE_PACKAGES=$(pip3 show hail | grep Location | sed 's/Location: //')

      exit_status=0

      python3 -m flake8 --config setup.cfg $SITE_PACKAGES/hail $SITE_PACKAGES/hailtop || exit_status=$?
      python3 -m pylint --rcfile pylintrc hailtop || exit_status=$?
      # working around this: https://github.com/python/mypy/issues/7087
      ln -s $SITE_PACKAGES/hailtop hailtop
      mypy --config-file /setup.cfg -p hailtop || exit_status=$?

      exit $exit_status
    dependsOn:
      - hail_pip_installed_python38_image
  - kind: buildImage2
    name: notebook_image
    dockerFile: /io/repo/notebook/Dockerfile
    contextPath: /io/repo/
    publishAs: notebook
    inputs:
      - from: /repo
        to: /io/repo
    dependsOn:
      - service_base_image
      - merge_code
  - kind: runImage
    name: render_notebook_nginx_conf
    image:
      valueFrom: service_base_image.image
    script: |
      set -ex
      cd /io/repo/notebook
      {% if deploy %}
      DEPLOY=true
      {% else %}
      DEPLOY=false
      {% endif %}
      python3 ../ci/jinja2_render.py '{"deploy": '${DEPLOY}', "default_ns": {"name": "{{ default_ns.name }}"}}' nginx.conf nginx.conf.out
    inputs:
      - from: /repo
        to: /io/repo
    outputs:
      - from: /io/repo/notebook/nginx.conf.out
        to: /notebook/nginx.conf.out
    dependsOn:
      - default_ns
      - service_base_image
      - merge_code
  - kind: buildImage2
    name: notebook_nginx_image
    dockerFile: /io/notebook/Dockerfile.nginx
    contextPath: /io/notebook
    publishAs: notebook_nginx
    inputs:
      - from: /repo/notebook
        to: /io/notebook
      - from: /notebook/nginx.conf.out
        to: /io/notebook/nginx.conf.out
    dependsOn:
      - hail_ubuntu_image
      - render_notebook_nginx_conf
      - merge_code
  - kind: runImage
    name: check_notebook
    image:
      valueFrom: notebook_image.image
    script: |
      set -ex
      SITE_PACKAGES=$(pip3 show notebook | grep Location | sed 's/Location: //')
      python3 -m flake8 $SITE_PACKAGES/notebook
      python3 -m pylint --rcfile pylintrc notebook
    dependsOn:
      - notebook_image
  - kind: runImage
    name: test_hail_python
    numSplits: 7
    image:
      valueFrom: hail_run_tests_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io
      tar xzf test.tar.gz
      tar xzf resources.tar.gz
      tar xzf data.tar.gz
      tar xvf debug-wheel-container.tar
      python3 -m pip install --no-dependencies hail-*-py3-none-any.whl

      # pyspark/conf/core-site.xml already points at /gsa-key/key.json
      mv /test-gsa-key/key.json /gsa-key/key.json

      export HAIL_TEST_RESOURCES_DIR=./resources
      export HAIL_DOCTEST_DATA_DIR=./data
      export HAIL_TEST_STORAGE_URI={{ global.test_storage_uri }}/{{ token }}
      export PYSPARK_SUBMIT_ARGS="--driver-memory 6g pyspark-shell"
      python3 -m pytest \
              --ignore=test/hailtop/batch/ \
              --ignore=test/hailtop/inter_cloud \
              --log-cli-level=INFO \
              -s \
              -vv \
              --instafail \
              --durations=50 \
              test
    inputs:
      - from: /debug-wheel-container.tar
        to: /io/debug-wheel-container.tar
      - from: /test.tar.gz
        to: /io/test.tar.gz
      - from: /resources.tar.gz
        to: /io/resources.tar.gz
      - from: /data.tar.gz
        to: /io/data.tar.gz
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - default_ns
      - hail_run_tests_image
      - build_hail
    clouds:
      - gcp
  - kind: runImage
    name: test_hail_python_fs
    numSplits: 5
    image:
      valueFrom: hail_run_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io
      tar xzf test.tar.gz
      tar xvf debug-wheel-container.tar
      python3 -m pip install --no-dependencies hail-*-py3-none-any.whl

      export HAIL_TEST_GCS_BUCKET={{ global.hail_test_gcs_bucket }}
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json

      export HAIL_TEST_S3_BUCKET=hail-test-dy5rg
      export AWS_SHARED_CREDENTIALS_FILE=/test-aws-key/credentials

      export HAIL_TEST_AZURE_ACCOUNT=hailtest
      export HAIL_TEST_AZURE_CONTAINER=hail-test-4nxei
      export AZURE_APPLICATION_CREDENTIALS=/test-azure-key/credentials.json

      python3 -m pytest \
              --log-cli-level=INFO \
              -s \
              -vv \
              --instafail \
              --durations=50 \
              test/hailtop/inter_cloud
    inputs:
      - from: /debug-wheel-container.tar
        to: /io/debug-wheel-container.tar
      - from: /test.tar.gz
        to: /io/test.tar.gz
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
      - name: test-aws-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-aws-key
      - name: test-azure-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-azure-key
    dependsOn:
      - default_ns
      - hail_run_image
      - build_hail
    clouds:
      - gcp
  - kind: runImage
    name: test_hail_python_unchecked_allocator
    image:
      valueFrom: hail_run_tests_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io
      tar xzf test.tar.gz
      tar xzf resources.tar.gz
      tar xzf data.tar.gz
      tar xvf wheel-container.tar
      python3 -m pip install --no-dependencies hail-*-py3-none-any.whl

      # pyspark/conf/core-site.xml already points at /gsa-key/key.json
      mv /test-gsa-key/key.json /gsa-key/key.json

      export HAIL_TEST_RESOURCES_DIR=./resources
      export HAIL_DOCTEST_DATA_DIR=./data
      export HAIL_TEST_STORAGE_URI={{ global.test_storage_uri }}/{{ token }}
      export PYSPARK_SUBMIT_ARGS="--driver-memory 6g pyspark-shell"
      python3 -m pytest \
              -m unchecked_allocator \
              --ignore=test/hailtop/batch/ \
              --ignore=test/hailtop/inter_cloud \
              --log-cli-level=INFO \
              -s \
              -vv \
              --instafail \
              --durations=50 \
              test
    inputs:
      - from: /wheel-container.tar
        to: /io/wheel-container.tar
      - from: /test.tar.gz
        to: /io/test.tar.gz
      - from: /resources.tar.gz
        to: /io/resources.tar.gz
      - from: /data.tar.gz
        to: /io/data.tar.gz
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - default_ns
      - hail_run_tests_image
      - build_hail
    clouds:
      - gcp
  - kind: runImage
    name: test_hail_python_local_backend
    numSplits: 7
    image:
      valueFrom: hail_run_tests_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io
      tar xzf test.tar.gz
      tar xzf resources.tar.gz
      tar xzf data.tar.gz
      tar xvf debug-wheel-container.tar
      python3 -m pip install --no-dependencies hail-*-py3-none-any.whl
      mkdir -p /io/tmp

      # The test should use the test credentials, not CI's credentials
      sed -i 's/gsa-key/test-gsa-key/g' ${SPARK_HOME}/conf/core-site.xml
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json

      export HAIL_CLOUD={{ global.cloud }}
      export HAIL_TEST_STORAGE_URI=/io/tmp/
      export HAIL_TEST_RESOURCES_DIR=./resources
      export HAIL_DOCTEST_DATA_DIR=./data

      hailctl config set query/backend local

      python3 -m pytest \
              --ignore=test/hailtop/ \
              --log-cli-level=INFO \
              -s \
              -vv \
              --instafail \
              --durations=50 \
              test
    inputs:
      - from: /debug-wheel-container.tar
        to: /io/debug-wheel-container.tar
      - from: /test.tar.gz
        to: /io/test.tar.gz
      - from: /resources.tar.gz
        to: /io/resources.tar.gz
      - from: /data.tar.gz
        to: /io/data.tar.gz
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - default_ns
      - hail_run_tests_image
      - build_hail
  - kind: runImage
    name: test_python_docs
    image:
      valueFrom: hail_base_image.image
    script: |
      set -ex
      cd /hail/python/hail
      python3 -m pytest --instafail \
        --doctest-modules \
        --doctest-glob='*.rst' \
        --ignore=docs/conf.py \
        --ignore=docs/doctest_write_data.py
    dependsOn:
      - hail_base_image
    clouds:
      - gcp
  - kind: runImage
    name: make_docs
    image:
      valueFrom: hail_base_image.image
    script: |
      set -ex
      set -o pipefail
      export HAIL_SHORT_VERSION='0.2'
      export SPHINXOPTS='-tgenerate_notebook_outputs'

      mkdir -p {{ token }}/python
      cd {{ token }}
      unzip -q -d python /io/hail.zip

      tar xzf /io/website-src.tar.gz
      export HAIL_WEBSITE_DIR=$(pwd)/website/website/

      sed -E "s/\(hail\#([0-9]+)\)/(\[#\1](https:\/\/github.com\/hail-is\/hail\/pull\/\1))/g" \
        < python/hail/docs/change_log.md \
        | pandoc -o python/hail/docs/change_log.rst --fail-if-warnings --verbose

      make -C python/hail/docs BUILDDIR=_build clean html
      make -C python/hailtop/batch/docs BUILDDIR=_build clean html

      mkdir -p www/docs
      mv python/hail/docs/_build/html www/docs/0.2
      mv python/hailtop/batch/docs/_build/html www/docs/batch

      HAIL_CACHE_VERSION=$(cat python/hail/hail_version)
      find www -iname *.html -type f -exec sed -i -e "s/\.css/\.css\?v\=$HAIL_CACHE_VERSION/" {} +;

      tar czf /io/www.tar.gz www
    resources:
      memory: standard
      cpu: '2'
    inputs:
      - from: /hail.zip
        to: /io/hail.zip
      - from: /website-src.tar.gz
        to: /io/website-src.tar.gz
    outputs:
      - from: /io/www.tar.gz
        to: /www.tar.gz
    dependsOn:
      - hail_base_image
      - build_hail
  - kind: runImage
    name: get_pip_versioned_docs
    image:
      valueFrom: hail_pip_installed_image.image
    script: |
      set -ex

      cd /io/repo
      make -C hail python/hail/hail_pip_version

      # dev deploy elides the hail-is remote, add it and retrieve the tags
      git remote add hail-is https://github.com/hail-is/hail.git

      export HAIL_PIP_VERSION=$(cat hail/python/hail/hail_pip_version)

      if git ls-remote --exit-code --tags hail-is $HAIL_PIP_VERSION
      then
        # In this case, we want to get the docs from Google Storage.
        python3 -m hailtop.aiotools.copy 'null' '[
        {"from": "gs://hail-common/website/'$HAIL_PIP_VERSION'/www.tar.gz",
        "to": "www.tar.gz"}
        ]'
      else
        cp /io/www.tar.gz .
      fi

      mkdir -p hail/build/www/docs/0.2
      tar -xvf www.tar.gz -C hail/build/www/docs --strip-components 2 --no-same-owner

      python3 -m hailtop.aiotools.copy 'null' '[
      {"from": "gs://hail-common/builds/0.1/docs/hail-0.1-docs-5a6778710097.tar.gz",
       "to": "hail-0.1-docs-5a6778710097.tar.gz"}
      ]'

      mkdir -p hail/build/www/docs/0.1
      tar -xvf hail-0.1-docs-5a6778710097.tar.gz -C hail/build/www/docs/0.1 --strip-components 2 --no-same-owner

      tar czf /io/docs.tar.gz -C hail/build/www .
    resources:
      memory: standard
      cpu: '2'
    inputs:
      - from: /repo
        to: /io/repo
      - from: /www.tar.gz
        to: /io/www.tar.gz
    outputs:
      - from: /io/docs.tar.gz
        to: /docs.tar.gz
    dependsOn:
      - hail_pip_installed_image
      - merge_code
      - make_docs
    clouds:
      - gcp
  - kind: deploy
    name: deploy_grafana
    namespace:
      valueFrom: default_ns.name
    config: grafana/deployment.yaml
    scopes:
      - deploy
      - test
      - dev
    dependsOn:
      - default_ns
      - base_image
      - grafana_nginx_image
      - create_certs
  - kind: runImage
    name: render_prom_nginx_conf
    image:
      valueFrom: service_base_image.image
    script: |
      set -ex
      cd /io/repo/prometheus
      {% if deploy %}
      DEPLOY=true
      {% else %}
      DEPLOY=false
      {% endif %}
      python3 ../ci/jinja2_render.py '{"deploy": '${DEPLOY}', "default_ns": {"name": "{{ default_ns.name }}"}}' nginx.conf nginx.conf.out
    inputs:
      - from: /repo
        to: /io/repo
    outputs:
      - from: /io/repo/prometheus/nginx.conf.out
        to: /prometheus/nginx.conf.out
    dependsOn:
      - default_ns
      - service_base_image
      - merge_code
  - kind: buildImage2
    name: prom_nginx_image
    dockerFile: /io/prometheus/Dockerfile.nginx
    contextPath: /io/prometheus
    publishAs: prometheus
    inputs:
      - from: /repo/prometheus
        to: /io/prometheus
      - from: /prometheus/nginx.conf.out
        to: /io/prometheus/nginx.conf.out
    dependsOn:
      - hail_ubuntu_image
      - render_prom_nginx_conf
      - merge_code
  - kind: deploy
    name: deploy_prometheus
    namespace:
      valueFrom: default_ns.name
    config: prometheus/prometheus.yaml
    scopes:
      - deploy
      - dev
    dependsOn:
      - default_ns
      - prom_nginx_image
      - create_certs
  - kind: runImage
    name: create_dummy_oauth2_client_secret
    image:
      valueFrom: base_image.image
    script: |
      set -ex
      kubectl -n {{ default_ns.name }} create secret generic auth-oauth2-client-secret || true
    scopes:
      - test
      - dev
    dependsOn:
      - default_ns
      - base_image
  - kind: deploy
    name: deploy_auth
    namespace:
      valueFrom: default_ns.name
    config: auth/deployment.yaml
    wait:
      - kind: Service
        name: auth
        for: alive
      - kind: Service
        name: auth-driver
        for: alive
    dependsOn:
      - default_ns
      - create_deploy_config
      - create_session_key
      - auth_database
      - auth_image
      - create_dummy_oauth2_client_secret
      - create_certs
      - create_accounts
  - kind: runImage
    name: check_monitoring
    image:
      valueFrom: monitoring_image.image
    script: |
      set -ex
      SITE_PACKAGES=$(pip3 show monitoring | grep Location | sed 's/Location: //')
      python3 -m flake8 $SITE_PACKAGES/monitoring
      python3 -m pylint --rcfile pylintrc monitoring
    dependsOn:
      - monitoring_image
  - kind: runImage
    name: delete_monitoring_tables
    image:
      valueFrom: service_base_image.image
    script: |
      set -ex
      mysql --defaults-extra-file=/sql-config/sql-config.cnf < /io/sql/delete-monitoring-tables.sql
    inputs:
      - from: /repo/monitoring/sql
        to: /io/sql
    secrets:
      - name: database-server-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /sql-config
    runIfRequested: true
    scopes:
      - dev
    dependsOn:
      - default_ns
      - service_base_image
      - merge_code
      - create_database_server_config
  - kind: createDatabase
    name: monitoring_database
    databaseName: monitoring
    migrations:
      - name: initial
        script: /io/sql/initial.sql
    inputs:
      - from: /repo/monitoring/sql
        to: /io/sql
    namespace:
      valueFrom: default_ns.name
    shutdowns:
      - kind: Deployment
        namespace:
          valueFrom: default_ns.name
        name: monitoring
    dependsOn:
      - default_ns
      - merge_code
      - delete_monitoring_tables
  - kind: deploy
    name: deploy_monitoring
    namespace:
      valueFrom: default_ns.name
    config: monitoring/deployment.yaml
    wait:
      - kind: Service
        name: monitoring
        for: alive
    dependsOn:
      - default_ns
      - monitoring_image
      - monitoring_database
      - deploy_auth
      - create_certs
      - create_accounts
    clouds:
      - gcp
  - kind: runImage
    name: test_monitoring
    image:
      valueFrom: test_monitoring_image.image
    script: |
      set -ex
      python3 -m pytest --log-cli-level=INFO -s -vv --instafail --durations=50 /test/
    secrets:
      - name: worker-deploy-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /deploy-config
      - name: test-dev-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /user-tokens
      - name: ssl-config-monitoring-tests
        namespace:
          valueFrom: default_ns.name
        mountPath: /ssl-config
    timeout: 300
    dependsOn:
      - test_monitoring_image
      - create_deploy_config
      - create_accounts
      - default_ns
      - create_certs
      - deploy_monitoring
    clouds:
      - gcp
  - kind: runImage
    name: test_auth_copy_paste_login
    image:
      valueFrom: service_base_image.image
    script: |
      set -ex

      export HAIL_DEPLOY_CONFIG_FILE=/deploy-config/deploy-config.json

      COPY_PASTE_TOKEN=$(hailctl curl {{ default_ns.name }} \
                         auth /api/v1alpha/copy-paste-token \
                         -fsSL \
                         --retry 3 \
                         --retry-delay 5 \
                         -XPOST)
      hailctl auth copy-paste-login "$COPY_PASTE_TOKEN" --namespace {{ default_ns.name }}

      if hailctl auth copy-paste-login "$COPY_PASTE_TOKEN" --namespace {{ default_ns.name }}
      then
          echo "reusing a token should not work, but did"
          exit 1
      fi

      COPY_PASTE_TOKEN=$(hailctl curl {{ default_ns.name }} \
                         auth /api/v1alpha/copy-paste-token \
                         -fsSL \
                         --retry 3 \
                         --retry-delay 5 \
                         -XPOST)
      python3 -c '
      from hailtop.auth import copy_paste_login;
      copy_paste_login("'$COPY_PASTE_TOKEN'", "{{ default_ns.name }}")
      '

      python3 -c '
      from hailtop.auth import copy_paste_login;
      import aiohttp
      try:
          copy_paste_login("'$COPY_PASTE_TOKEN'", "{{ default_ns.name }}")
          print("reusing a token should not work, but did")
          sys.exit(1)
      except aiohttp.client_exceptions.ClientResponseError as exc:
          assert(exc.status == 401)
      '
    secrets:
      - name: test-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /user-tokens
      - name: worker-deploy-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /deploy-config
    dependsOn:
      - default_ns
      - create_accounts
      - deploy_auth
      - create_deploy_config
      - service_base_image
  - kind: runImage
    name: test_auth_copy_paste_login_timeout
    image:
      valueFrom: service_base_image.image
    script: |
      set -ex

      export HAIL_DEPLOY_CONFIG_FILE=/deploy-config/deploy-config.json

      COPY_PASTE_TOKEN=$(hailctl curl {{ default_ns.name }} \
                         auth /api/v1alpha/copy-paste-token \
                         -fsSL \
                         --retry 3 \
                         --retry-delay 5 \
                         -XPOST)
      sleep $(( 5 * 60 + 1))
      if hailctl auth copy-paste-login "$COPY_PASTE_TOKEN" --namespace {{ default_ns.name }}
      then
          echo "using an expired token should not work, but did"
          exit 1
      fi

      python3 -c '
      import aiohttp
      from hailtop.auth import copy_paste_login;
      try:
          copy_paste_login("'$COPY_PASTE_TOKEN'", "{{ default_ns.name }}")
          print("using an expired token should not work, but did")
          sys.exit(1)
      except aiohttp.client_exceptions.ClientResponseError as exc:
          assert(exc.status == 401)
      '
    secrets:
      - name: test-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /user-tokens
      - name: worker-deploy-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /deploy-config
    dependsOn:
      - default_ns
      - create_accounts
      - deploy_auth
      - create_deploy_config
      - service_base_image
  - kind: runImage
    name: delete_batch_tables
    image:
      valueFrom: service_base_image.image
    script: |
      set -ex
      mysql --defaults-extra-file=/sql-config/sql-config.cnf < /io/sql/delete-batch-tables.sql
    inputs:
      - from: /repo/batch/sql
        to: /io/sql
    secrets:
      - name: database-server-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /sql-config
    runIfRequested: true
    scopes:
      - dev
    dependsOn:
      - default_ns
      - service_base_image
      - merge_code
      - create_database_server_config
  - kind: runImage
    name: delete_ci_tables
    image:
      valueFrom: service_base_image.image
    script: |
      set -ex
      mysql --defaults-extra-file=/sql-config/sql-config.cnf < /io/sql/delete-ci-tables.sql
    inputs:
      - from: /repo/ci/sql
        to: /io/sql
    secrets:
      - name: database-server-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /sql-config
    runIfRequested: true
    scopes:
      - dev
    dependsOn:
      - default_ns
      - service_base_image
      - merge_code
      - create_database_server_config
  - kind: createDatabase
    name: ci_database
    databaseName: ci
    migrations:
      - name: initial
        script: /io/sql/initial.sql
      - name: invalidated-batches
        script: /io/sql/invalidated-batches.sql
      - name: add-frozen-mode
        script: /io/sql/add-frozen-mode.sql
        online: true
      - name: active-namespaces
        script: /io/sql/active-namespaces.sql
        online: true
    inputs:
      - from: /repo/ci/sql
        to: /io/sql
    namespace:
      valueFrom: default_ns.name
    shutdowns:
      - kind: Deployment
        namespace:
          valueFrom: default_ns.name
        name: ci
    dependsOn:
      - default_ns
      - merge_code
      - delete_ci_tables
  - kind: createDatabase
    name: batch_database
    databaseName: batch
    migrations:
      - name: initial
        script: /io/sql/initial.sql
      - name: insert-globals
        script: /io/sql/insert_globals.py
      - name: create-billing-projects
        script: /io/sql/create-billing-projects.sql
      - name: increase-spec-size
        script: /io/sql/increase-spec-size.sql
      - name: remove-fk-batches-user-resources
        script: /io/sql/remove-fk-batches-user-resources.sql
      - name: change-ready-cores-bigint
        script: /io/sql/change-ready-cores-bigint.sql
      - name: add-user-resources-ready-cores-token
        script: /io/sql/add-user-resources-ready-cores-token.sql
      - name: add-lock-in-share-mode
        script: /io/sql/add-lock-in-share-mode.sql
      - name: add-batch-create-token
        script: /io/sql/add-batch-create-token.sql
      - name: improve-cancel
        script: /io/sql/improve-cancel.sql
      - name: store-specs-in-gcs
        script: /io/sql/store-specs-in-gcs.sql
      - name: add-batches-time-closed
        script: /io/sql/add-batches-time-closed.sql
      - name: add-instance-zone
        script: /io/sql/add-instance-zone.sql
      - name: add-batches-time-created-index
        script: /io/sql/add-batches-time-created-index.sql
      - name: change-test-worker-disk-size
        script: /io/sql/change_test_worker_disk_size.py
      - name: add-resources
        script: /io/sql/add-resources.sql
      - name: insert-resources
        script: /io/sql/insert_resources.py
      - name: insert-standing-worker-globals
        script: /io/sql/insert_standing_worker_globals.py
      - name: insert-local-ssd-resource
        script: /io/sql/insert_local_ssd_resource.py
      - name: fix-mark-job-complete-on-error
        script: /io/sql/fix-mark-job-complete-on-error.sql
      - name: add-worker-pd-ssd-data-disk
        script: /io/sql/add-worker-pd-ssd-data-disk.sql
      - name: add-aggregated-batch-resources-tokens
        script: /io/sql/add-aggregated-batch-resources-tokens.sql
      - name: change-test-worker-pool-size
        script: /io/sql/change_test_worker_pool_size.py
      - name: add-test-dev-to-billing-project-users
        script: /io/sql/add-test-dev-to-billing-project-users.sql
      - name: add-status-flag-billing-projects
        script: /io/sql/add-status-flag-billing-projects.sql
      - name: add-aggregated-billing-project-resources
        script: /io/sql/add-aggregated-billing-project-resources.sql
      - name: add-enable-standing-worker
        script: /io/sql/add-enable-standing-worker.py
      - name: add-inst-colls
        script: /io/sql/add-inst-colls.sql
      - name: add-creating-job-state
        script: /io/sql/add-creating-job-state.sql
      - name: add-job-private-inst-coll
        script: /io/sql/add-job-private-inst-coll.sql
      - name: insert-nonpreemptible-resources
        script: /io/sql/insert_nonpreemptible_resources.py
      - name: fix-schedule-job
        script: /io/sql/fix-schedule-job.sql
      - name: increase-test-and-dev-pool-sizes
        script: /io/sql/increase-test-and-dev-pool-sizes.py
      - name: fix-provisioning-bug
        script: /io/sql/fix-provisioning-bug.sql
      - name: do-not-lock-entire-batch
        script: /io/sql/do-not-lock-entire-batch.sql
      - name: add-fail-fast
        script: /io/sql/add-fail-fast.sql
      - name: add-frozen-mode
        script: /io/sql/add-frozen-mode.sql
      - name: add-instance-config
        script: /io/sql/add-instance-config.sql
      - name: cloud-agnostic
        script: /io/sql/cloud-agnostic.sql
      - name: support-azure
        script: /io/sql/support-azure.sql
      - name: add-azure-tables
        script: /io/sql/add_azure_tables.py
      - name: change-azure-test-highcpu-pool
        script: /io/sql/change_azure_test_highcpu_pool.py
      - name: add-latest-product-versions
        script: /io/sql/add-latest-product-versions.sql
      - name: add-products-and-versions
        script: /io/sql/add_products_and_versions.py
      - name: kill-more-deadlocks
        script: /io/sql/kill-more-deadlocks.sql
      - name: kill-more-deadlocks2
        script: /io/sql/kill-more-deadlocks2.sql
      - name: no-locks-add-attempt
        script: /io/sql/no-locks-add-attempt.sql
      - name: fix-n-cancelled-creating-jobs
        script: /io/sql/fix-n-cancelled-creating-jobs.sql
      - name: big-test-instances
        script: /io/sql/big-test-instances.py
      - name: kill-mjc-deadlocks
        script: /io/sql/kill-mjc-deadlocks.sql
      - name: add-nonpreemptible-pools
        script: /io/sql/add-nonpreemptible-pools.sql
      - name: set-test-and-dev-pools-to-8-core-max-2
        script: /io/sql/set-test-and-dev-pools-to-8-core-max-2.py
      - name: set-test-and-dev-jpim-to-max-5
        script: /io/sql/set-test-and-dev-jpim-to-max-5.py
        online: true
      - name: minimize-deadlock-errors
        script: /io/sql/minimize-deadlock-errors.sql
        online: true
      - name: add-resource-ids-sql
        script: /io/sql/add-resource-ids.sql
        online: true
      - name: add-resource-ids-python
        script: /io/sql/add_resource_ids.py
        online: true
      - name: modify-resource-id-trigger
        script: /io/sql/modify-resource-id-trigger.sql
        online: true
      - name: rm-resource-names-agg-resources
        script: /io/sql/rm-resource-names-agg-resources.sql
        online: true
      - name: rm-resource-foreign-keys
        script: /io/sql/rm-resource-foreign-keys.py
        online: true
      - name: rm-resource-names-agg-resources-pt-2
        script: /io/sql/rm-resource-names-agg-resources-pt-2.sql
        online: true
      - name: add-att-resources-format-version-lt-3
        script: /io/sql/add-att-resources-format-version-lt-3.sql
        online: true
      - name: insert-attempt-resources-format-version-lt-3
        script: /io/sql/insert_attempt_resources_format_version_lt_3.py
        online: true
      - name: revert-attempt-resources-trigger-back-compat
        script: /io/sql/revert-attempt-resources-trigger-back-compat.sql
        online: true
      - name: add-agg-billing-by-date
        script: /io/sql/add-agg-billing-by-date.sql
        online: true
      - name: populate-agg-billing-by-date
        script: /io/sql/populate_agg_billing_by_date.py
        online: true
      - name: revert-temp-agg-by-date-infra
        script: /io/sql/revert-temp-agg-by-date-infra.sql
        online: true
      - name: rename-timestamp-to-date
        script: /io/sql/rename-timestamp-to-date.sql
        online: true
      - name: add-real-time-billing
        script: /io/sql/add-real-time-billing.sql
        online: true
      - name: add-batch-updates
        script: /io/sql/add-batch-updates.sql
        online: true
      - name: populate-batch-updates
        script: /io/sql/populate_batch_updates.py
        online: true
      - name: cleanup-add-batch-updates
        script: /io/sql/cleanup-add-batch-updates.sql
        online: true
      - name: add-jobs-update-id-index
        script: /io/sql/add-jobs-update-id-index.sql
        online: true
      - name: no-dev-standing-workers-by-default
        script: /io/sql/no_dev_standing_workers_by_default.py
        online: true
      - name: add-commit-updates
        script: /io/sql/add-commit-updates.sql
        online: true
      - name: add-job-regions
        script: /io/sql/add-job-regions.sql
        online: true
    inputs:
      - from: /repo/batch/sql
        to: /io/sql
    namespace:
      valueFrom: default_ns.name
    shutdowns:
      - kind: Deployment
        namespace:
          valueFrom: default_ns.name
        name: batch
      - kind: Deployment
        namespace:
          valueFrom: default_ns.name
        name: batch-driver
    dependsOn:
      - default_ns
      - merge_code
      - delete_batch_tables
  - kind: deploy
    name: deploy_batch
    namespace:
      valueFrom: default_ns.name
    config: batch/deployment.yaml
    wait:
      - kind: Service
        name: batch
        for: alive
      - kind: Service
        name: batch-driver
        for: alive
    dependsOn:
      - default_ns
      - deploy_batch_sa
      - create_accounts
      - batch_image
      - batch_worker_image
      - batch_driver_nginx_image
      - batch_database
      - deploy_auth
      - create_certs
  - kind: runImage
    name: create_billing_projects
    image:
      valueFrom: service_base_image.image
    script: |
      set -ex
      # create billing projects and add users to them
      cat > create-billing-projects.py <<EOF
      import sys
      import json
      import asyncio
      import aiohttp
      from hailtop.utils import async_to_blocking
      from hailtop.batch_client.aioclient import BatchClient
      async def create(billing_project, user, limit=None):
          bc = await BatchClient.create(None)
          try:
              try:
                  await bc.create_billing_project(billing_project)
              except aiohttp.ClientResponseError as e:
                  if e.status != 403 or 'already exists' not in e.message:
                       raise

              try:
                  await bc.add_user(user, billing_project)
              except aiohttp.ClientResponseError as e:
                  if e.status != 403 or 'already member of billing project' not in e.message:
                       raise

              await bc.edit_billing_limit(billing_project, limit)
          finally:
              await bc.close()
      async def main():
          {% if deploy %}
          benchmark_limit = 25
          {% else %}
          benchmark_limit = 5
          {% endif %}

          await asyncio.gather(*[
              create(billing_project="benchmark", user="benchmark", limit=benchmark_limit)
          ])
      async_to_blocking(main())
      EOF
      python3 create-billing-projects.py
    secrets:
      - name: test-dev-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /user-tokens
      - name: worker-deploy-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /deploy-config
    dependsOn:
      - default_ns
      - service_base_image
      - create_deploy_config
      - create_accounts
      - deploy_batch
  - kind: runImage
    name: upload_query_jar
    image:
      valueFrom: hail_pip_installed_image.image
    script: |
      set -ex
      export GOOGLE_APPLICATION_CREDENTIALS=/batch-gsa-key/key.json
      export AZURE_APPLICATION_CREDENTIALS=/batch-gsa-key/key.json

      {% if scope == "deploy" %}
      HAIL_QUERY_JAR_URL={{ global.query_storage_uri }}
      {% elif scope == "test" or scope == "dev" %}
      HAIL_QUERY_JAR_URL={{ global.test_storage_uri }}/{{ default_ns.name }}
      {% else %}
      echo "!!! unexpected scope {{ scope }} !!!"
      exit 1
      {% endif %}
      HAIL_QUERY_JAR_URL=${HAIL_QUERY_JAR_URL}/jars/$(cat /io/git_version).jar

      python3 -m hailtop.aiotools.copy 'null' '[{"from": "/io/hail.jar", "to": "'${HAIL_QUERY_JAR_URL}'"}]'
    secrets:
      - name: batch-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /batch-gsa-key
    inputs:
      - from: /just-jar/spark-32/hail.jar
        to: /io/hail.jar
      - from: /git_version
        to: /io/git_version
    dependsOn:
      - default_ns
      - deploy_batch
      - hail_pip_installed_image
      - build_hail_jar_and_wheel_only_spark_3_2
      - merge_code
      - create_test_gsa_keys
  - kind: deploy
    name: deploy_memory
    namespace:
      valueFrom: default_ns.name
    config: memory/deployment.yaml
    wait:
      - kind: Service
        name: memory
        for: alive
    dependsOn:
      - default_ns
      - memory_image
      - deploy_memory_sa
      - create_certs
  - kind: runImage
    name: test_hail_python_service_backend
    numSplits: 5
    image:
      valueFrom: hail_run_tests_image.image
    resources:
      cpu: '1'
    script: |
      set -ex
      tar xvf /io/wheel-container.tar
      python3 -m pip install --no-dependencies hail-*-py3-none-any.whl

      cd /io/repo/hail/python

      export HAIL_CLOUD={{ global.cloud }}
      export HAIL_TEST_STORAGE_URI={{ global.test_storage_uri }}/{{ token }}
      export HAIL_TEST_RESOURCES_DIR="{{ global.test_storage_uri }}/{{ upload_test_resources_to_blob_storage.token }}/test/resources/"
      export HAIL_DOCTEST_DATA_DIR="{{ global.test_storage_uri }}/{{ upload_test_resources_to_blob_storage.token }}/doctest/data/"
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json

      export HAIL_SHUFFLE_MAX_BRANCH=4
      export HAIL_SHUFFLE_CUTOFF=1000000

      hailctl config set query/backend batch
      hailctl config set batch/billing_project test
      hailctl config set batch/remote_tmpdir {{ global.test_storage_uri }}

      python3 -m pytest \
              --ignore=test/hailtop/ \
              --log-cli-level=INFO \
              -s \
              -vv \
              --instafail \
              --durations=50 \
              -n 4 \
              test
    timeout: 5400
    inputs:
      - from: /just-wheel/spark-32/wheel-container.tar
        to: /io/wheel-container.tar
      - from: /repo/hail/python/test
        to: /io/repo/hail/python/test
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
      - name: worker-deploy-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /deploy-config
      - name: test-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /user-tokens
      - name: ssl-config-batch-tests
        namespace:
          valueFrom: default_ns.name
        mountPath: /ssl-config
    dependsOn:
      - default_ns
      - merge_code
      - deploy_batch
      - deploy_memory
      - create_deploy_config
      - create_accounts
      - hail_run_tests_image
      - upload_query_jar
      - upload_test_resources_to_blob_storage
      - build_hail_jar_and_wheel_only_spark_3_2
  - kind: buildImage2
    name: netcat_ubuntu_image
    publishAs: netcat
    dockerFile:
      inline: |
        FROM {{ hail_ubuntu_image.image }}
        RUN hail-apt-get-install netcat
    dependsOn:
      - hail_ubuntu_image
  - kind: buildImage2
    name: volume_image
    publishAs: volume
    dockerFile:
      inline: |
        FROM {{ hail_ubuntu_image.image }}
        VOLUME ["/data"]
    dependsOn:
      - hail_ubuntu_image
  - kind: buildImage2
    name: workdir_image
    publishAs: workdir
    dockerFile:
      inline: |
        FROM {{ hail_ubuntu_image.image }}
        WORKDIR ["/work"]
    dependsOn:
      - hail_ubuntu_image
  - kind: buildImage2
    name: curl_image
    publishAs: curl
    dockerFile:
      inline: |
        FROM {{ hail_ubuntu_image.image }}
        RUN hail-apt-get-install curl
    dependsOn:
      - hail_ubuntu_image
  - kind: runImage
    name: test_memory
    image:
      valueFrom: memory_image.image
    resources:
      memory: 3.75Gi
      cpu: '1'
    script: |
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      hailctl config set batch/remote_tmpdir {{ global.test_storage_uri }}/batch/
      python3 -m pytest --log-cli-level=INFO -s -vv --instafail --durations=50 /io/test/
    timeout: 600
    secrets:
      - name: worker-deploy-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /deploy-config
      - name: test-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /user-tokens
      - name: ssl-config-memory-tests
        namespace:
          valueFrom: default_ns.name
        mountPath: /ssl-config
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
      - name: global-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /global-config
    inputs:
      - from: /repo/memory/test
        to: /io/test
    dependsOn:
      - create_deploy_config
      - create_accounts
      - default_ns
      - merge_code
      - memory_image
      - deploy_memory
    clouds:
      - gcp
  - kind: runImage
    name: test_batch
    numSplits: 5
    image:
      valueFrom: batch_image.image
    script: |
      set -ex
      export HAIL_GSA_KEY_FILE=/test-gsa-key/key.json
      export HAIL_BASE_IMAGE={{ base_image.image }}
      export CI_UTILS_IMAGE={{ ci_utils_image.image }}
      export HAIL_CURL_IMAGE={{ curl_image.image }}
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_NETCAT_UBUNTU_IMAGE={{ netcat_ubuntu_image.image }}
      export HAIL_VOLUME_IMAGE={{ volume_image.image }}
      export HAIL_WORKDIR_IMAGE={{ workdir_image.image }}
      export HAIL_HAIL_BASE_IMAGE={{ hail_base_image.image }}
      export DOCKER_PREFIX="{{ global.docker_prefix }}"
      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      export HAIL_TEST_TOKEN_FILE=/user-tokens/tokens.json
      export HAIL_TEST_DEV_TOKEN_FILE=/dev-tokens/tokens.json
      export HAIL_TOKEN="{{ token }}"
      export HAIL_SCOPE="{{ scope }}"
      export HAIL_CLOUD="{{ global.cloud }}"
      export HAIL_DOMAIN="{{ global.domain }}"
      hailctl config set batch/remote_tmpdir {{ global.test_storage_uri }}/batch/
      python3 -m pytest \
              --asyncio-mode=auto \
              --log-date-format="%Y-%m-%dT%H:%M:%S" \
              --log-format="%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)d:%(funcName)s %(message)s" \
              --log-cli-level=INFO \
              -s \
              -vv \
              --instafail \
              -k "not test_scale and not test_invariants" \
              --durations=0 \
              /io/test/
    inputs:
      - from: /repo/batch/test
        to: /io/test
    port: 5000
    timeout: 1200
    secrets:
      - name: worker-deploy-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /deploy-config
      - name: test-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /user-tokens
      - name: test-dev-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /dev-tokens
      - name: ssl-config-batch-tests
        namespace:
          valueFrom: default_ns.name
        mountPath: /ssl-config
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - create_deploy_config
      - create_accounts
      - default_ns
      - merge_code
      - base_image
      - hail_base_image
      - batch_image
      - ci_utils_image
      - deploy_batch
      - netcat_ubuntu_image
      - volume_image
      - workdir_image
      - curl_image
  - kind: runImage
    name: delete_test_billing_projects
    image:
      valueFrom: batch_image.image
    script: |
      set -ex
      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      export HAIL_TEST_DEV_TOKEN_FILE=/dev-tokens/tokens.json
      export HAIL_TOKEN="{{ test_batch.token }}"
      cd /io/test
      python3 -c '
      import billing_projects
      import asyncio
      asyncio.get_event_loop().run_until_complete(billing_projects.delete_all_test_billing_projects())
      '
    inputs:
      - from: /repo/batch/test
        to: /io/test
    secrets:
      - name: worker-deploy-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /deploy-config
      - name: test-dev-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /dev-tokens
      - name: ssl-config-batch-tests
        namespace:
          valueFrom: default_ns.name
        mountPath: /ssl-config
    alwaysRun: true
    dependsOn:
      - create_deploy_config
      - create_accounts
      - default_ns
      - merge_code
      - batch_image
      - deploy_batch
      - test_batch
  - kind: runImage
    name: create_ci_test_repo
    image:
      valueFrom: base_image.image
    script: |
      set -e
      TOKEN=$(cat /secret/ci-secrets/user1)
      REPO_NAME="ci-test-{{token}}"

      echo creating $REPO_NAME...
      curl -XPOST \
        -i \
        -fsSL \
        https://api.github.com/orgs/hail-ci-test/repos \
        -H "Authorization: token ${TOKEN}" \
        -d "{ \"name\" : \"$REPO_NAME\" }"

      # checkout new ci repo
      # Repeated clones should happen in fresh directories so we mv to the final
      # destination on successful clone
      set +x
      clone() {
        tmp_dir=$(mktemp -d) && git clone https://$1@github.com/hail-ci-test/$2.git $tmp_dir && mv $tmp_dir $2
      }
      typeset -fx clone
      retry clone $TOKEN $REPO_NAME
      set -x
      cd $REPO_NAME

      mkdir -p ./ci/test ./hail/
      cp /io/repo/ci/test/resources/build.yaml ./
      cp -R /io/repo/ci/* ./ci/
      cp /io/repo/tls/Dockerfile ./ci/test/resources/Dockerfile.certs
      cp /io/repo/tls/create_certs.py ./ci/test/resources/
      cp /io/repo/pylintrc ./
      cp /io/repo/setup.cfg ./
      cp -R /io/repo/docker ./
      cp -R /io/repo/gear ./
      cp -R /io/repo/hail/python ./hail/
      cp /io/repo/hail/Makefile ./hail/
      cp /io/repo/hail/env_var.mk ./hail/
      cp -R /io/repo/web_common ./
      cp /io/repo/hail/python/setup-hailtop.py ./hail/python/

      git config user.name ci
      git config user.email ci@hail.is
      git add * && git commit -m "setup repo"
      retry git push
    secrets:
      - name: hail-ci-0-1-service-account-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /secret/ci-secrets
    scopes:
      - test
      - dev
    inputs:
      - from: /repo
        to: /io/repo
    dependsOn:
      - default_ns
      - base_image
      - merge_code
  - kind: runImage
    name: create_ci_config
    image:
      valueFrom: base_image.image
    script: |
      kubectl -n {{ default_ns.name }} create secret generic ci-config \
          --from-literal=github_context="ci-test" \
          --from-literal=storage_uri="{{ global.test_storage_uri }}" \
          --from-literal=deploy_steps="[]" \
          --from-literal=watched_branches="[[\"hail-ci-test/ci-test-{{create_ci_test_repo.token}}:master\", true, true]]" \
          --save-config --dry-run=client -o yaml \
        | kubectl -n {{ default_ns.name }} apply -f -
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    scopes:
      - test
      - dev
    dependsOn:
      - default_ns
      - base_image
      - create_ci_test_repo
  - kind: deploy
    name: deploy_ci
    namespace:
      valueFrom: default_ns.name
    config: ci/deployment.yaml
    wait:
      - kind: Service
        name: ci
        for: alive
    dependsOn:
      - default_ns
      - create_database_server_config
      - ci_image
      - ci_utils_image
      - create_accounts
      - ci_database
      - deploy_auth
      - deploy_batch
      - create_ci_test_repo
      - create_ci_config
      - deploy_ci_agent
      - create_certs
      - hail_buildkit_image
  - kind: runImage
    name: test_ci
    image:
      valueFrom: test_ci_image.image
    script: |
      set -ex
      export ORGANIZATION=hail-ci-test
      export REPO_NAME=ci-test-"{{ create_ci_test_repo.token }}"
      export NAMESPACE="{{ default_ns.name }}"
      python3 -m pytest --log-cli-level=INFO -s -vv --instafail --durations=50 /test/
    secrets:
      - name: worker-deploy-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /deploy-config
      - name: test-dev-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /user-tokens
      - name: hail-ci-0-1-service-account-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /secret/ci-secrets
      - name: ssl-config-ci-tests
        namespace:
          valueFrom: default_ns.name
        mountPath: /ssl-config
    timeout: 5400
    scopes:
      - test
      - dev
    dependsOn:
      - create_deploy_config
      - create_accounts
      - default_ns
      - create_certs
      - deploy_ci
      - test_ci_image
      - create_ci_test_repo
  - kind: buildImage2
    name: hailgenetics_hail_image
    dockerFile: /io/docker/hailgenetics/hail/Dockerfile
    contextPath: /io/docker/hailgenetics/hail
    publishAs: hailgenetics/hail
    inputs:
      - from: /repo/docker/hailgenetics/hail
        to: /io/docker/hailgenetics/hail
      - from: /just-wheel/wheel-container.tar
        to: /io/docker/hailgenetics/hail/wheel-container.tar
    dependsOn:
      - merge_code
      - build_hail_jar_and_wheel_only
      - hail_ubuntu_image
  - kind: runImage
    name: test_hailgenetics_hail_image
    image:
      valueFrom: hailgenetics_hail_image.image
    script: |
      set -ex
      python3 -c 'import hail as hl; hl.balding_nichols_model(3, 100, 100)._force_count_rows()'
      python3 -c 'import numpy; import pandas; import sklearn; import matplotlib; import scipy'
      gsutil --version
    dependsOn:
      - hailgenetics_hail_image
  - kind: runImage
    name: test_hailtop_batch
    numSplits: 5
    image:
      valueFrom: service_base_image.image
    script: |
      cd /io/hailtop
      set -ex
      export HAIL_CLOUD={{ global.cloud }}
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      export DOCKER_PREFIX="{{ global.docker_prefix }}"
      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      export HAIL_GENETICS_HAIL_IMAGE="{{ hailgenetics_hail_image.image }}"
      hailctl config set batch/billing_project test
      hailctl config set batch/remote_tmpdir {{ global.test_storage_uri }}/batch/
      python3 -m pytest \
              --durations=0 \
              --log-cli-level=INFO \
              -s \
              -vv \
              --instafail \
              /io/test/hailtop/batch/
    inputs:
      - from: /repo/hail/python/test
        to: /io/test
      - from: /repo/hail/python/hailtop
        to: /io/hailtop
    timeout: 1200
    secrets:
      - name: worker-deploy-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /deploy-config
      - name: test-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /user-tokens
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
      - name: ssl-config-batch-tests
        namespace:
          valueFrom: default_ns.name
        mountPath: /ssl-config
    dependsOn:
      - hailgenetics_hail_image
      - create_deploy_config
      - create_accounts
      - default_ns
      - merge_code
      - service_base_image
      - deploy_batch
  - kind: runImage
    name: test_batch_docs
    image:
      valueFrom: service_base_image.image
    script: |
      set -ex
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      cd /io/hailtop/batch
      hailctl config set batch/billing_project test
      hailctl config set batch/remote_tmpdir {{ global.test_storage_uri }}/batch/
      python3 -m pytest --instafail \
        --doctest-modules \
        --doctest-glob='*.rst' \
        --ignore=docs/cookbook/files/ \
        --ignore=docs/conf.py
    secrets:
      - name: worker-deploy-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /deploy-config
      - name: test-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /user-tokens
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - create_deploy_config
      - create_accounts
      - default_ns
      - service_base_image
      - merge_code
      - deploy_batch
    timeout: 1200
    inputs:
      - from: /repo/hail/python/hailtop
        to: /io/hailtop
  - kind: createDatabase
    name: notebook_database
    databaseName: notebook
    migrations:
      - name: initial
        script: /io/sql/initial.sql
    inputs:
      - from: /repo/notebook/sql
        to: /io/sql
    namespace:
      valueFrom: default_ns.name
    shutdowns:
      - kind: Deployment
        namespace:
          valueFrom: default_ns.name
        name: notebook
    dependsOn:
      - default_ns
      - merge_code
  - kind: deploy
    name: deploy_notebook
    namespace:
      valueFrom: default_ns.name
    config: notebook/deployment.yaml
    dependsOn:
      - default_ns
      - create_session_key
      - notebook_image
      - notebook_nginx_image
      - deploy_auth
      - notebook_database
      - create_certs
    wait:
      - kind: Service
        name: notebook
        for: alive
  - kind: runImage
    name: cleanup_ci_test_repo
    image:
      valueFrom: base_image.image
    script: |
      set -e
      TOKEN=$(cat /secret/ci-secrets/user1)
      echo deleting ci-test-{{ create_ci_test_repo.token }}...
      curl -XDELETE \
        -ifsSL \
        https://api.github.com/repos/hail-ci-test/ci-test-{{ create_ci_test_repo.token }} \
        -H "Authorization: token ${TOKEN}"
    secrets:
      - name: hail-ci-0-1-service-account-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /secret/ci-secrets
    alwaysRun: true
    scopes:
      - test
      - dev
    dependsOn:
      - default_ns
      - base_image
      - create_ci_test_repo
      - deploy_ci
      - test_ci
  - kind: buildImage2
    name: blog_nginx_image
    dockerFile: /io/blog/Dockerfile.nginx
    contextPath: /io/blog
    publishAs: blog_nginx
    inputs:
      - from: /repo/blog
        to: /io/blog
    dependsOn:
      - hail_ubuntu_image
      - merge_code
  - kind: deploy
    name: deploy_blog
    namespace:
      valueFrom: default_ns.name
    config: blog/deployment.yaml
    wait:
      - kind: Service
        name: blog
        for: alive
        resource_type: statefulset
    dependsOn:
      - default_ns
      - blog_nginx_image
      - create_certs
  - kind: runImage
    name: test_dataproc-37
    image:
      valueFrom: ci_utils_image.image
    script: |
      set -ex

      cd /io/repo

      gcloud auth activate-service-account --key-file=/test-dataproc-service-account-key/test-dataproc-service-account-key.json
      gcloud config set project hail-vdc
      gcloud config set dataproc/region us-central1

      if git ls-remote --exit-code --tags origin $(cat /io/hail_pip_version)
      then
          echo "tag $HAIL_PIP_VERSION already exists"
          exit 0
      fi

      cd hail
      chmod 755 ./gradlew
      time retry ./gradlew --version
      make test-dataproc-37 DEV_CLARIFIER=ci_test_dataproc
    dependsOn:
      - ci_utils_image
      - default_ns
      - merge_code
    inputs:
      - from: /hail_pip_version
        to: /io/hail_pip_version
      - from: /repo
        to: /io/repo
    secrets:
      - name: test-dataproc-service-account-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dataproc-service-account-key
    scopes:
      - deploy
      - dev
    clouds:
      - gcp
  - kind: runImage
    name: test_dataproc-38
    image:
      valueFrom: ci_utils_image.image
    script: |
      set -ex

      cd /io/repo

      gcloud auth activate-service-account --key-file=/test-dataproc-service-account-key/test-dataproc-service-account-key.json
      gcloud config set project hail-vdc
      gcloud config set dataproc/region us-central1

      if git ls-remote --exit-code --tags origin $(cat /io/hail_pip_version)
      then
          echo "tag $HAIL_PIP_VERSION already exists"
          exit 0
      fi

      cd hail
      chmod 755 ./gradlew
      time retry ./gradlew --version
      make test-dataproc-38 DEV_CLARIFIER=ci_test_dataproc
    dependsOn:
      - ci_utils_image
      - default_ns
      - merge_code
    inputs:
      - from: /hail_pip_version
        to: /io/hail_pip_version
      - from: /repo
        to: /io/repo
    secrets:
      - name: test-dataproc-service-account-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dataproc-service-account-key
    scopes:
      - deploy
      - dev
    clouds:
      - gcp
  - kind: runImage
    name: deploy
    image:
      valueFrom: ci_utils_image.image
    script: |
      set -ex
      cd /io

      gcloud auth activate-service-account --key-file=/ci-deploy-0-1--hail-is-hail/ci-deploy-0-1--hail-is-hail.json

      gcloud auth -q configure-docker {{ global.docker_prefix.split('/')[0] }}
      cat /docker-hub-hailgenetics/password | skopeo login --username hailgenetics --password-stdin docker.io

      cp /pypi-credentials/pypirc $HOME/.pypirc
      printf 'Authorization: token ' > github-oauth
      cat /hail-ci-0-1-github-oauth-token/oauth-token >>github-oauth
      printf '#!/bin/bash\necho ' > git-askpass
      cat /hail-ci-0-1-github-oauth-token/oauth-token >>git-askpass
      chmod 755 git-askpass
      export GIT_ASKPASS=/io/git-askpass

      cd /io/repo/hail

      if git ls-remote --exit-code --tags origin $(cat /io/hail_pip_version)
      then
          echo "tag $HAIL_PIP_VERSION already exists"
          exit 0
      fi

      chmod 755 ./gradlew
      time retry ./gradlew --version
      make wheel upload-artifacts DEPLOY_REMOTE=origin

      (mkdir /io/wheel-for-azure && cd /io/wheel-for-azure && tar xvf /io/wheel-for-azure-container.tar)

      bash scripts/deploy.sh $(cat /io/hail_pip_version) \
                             $(cat /io/hail_version) \
                             $(cat /io/git_version) \
                             origin \
                             /io/repo/hail/build/deploy/dist/hail-*-py3-none-any.whl \
                             /io/github-oauth \
                             docker://{{ hailgenetics_hail_image.image }} \
                             /io/wheel-for-azure/hail-*-py3-none-any.whl \
                             /io/www.tar.gz
    inputs:
      - from: /hail_version
        to: /io/hail_version
      - from: /hail_pip_version
        to: /io/hail_pip_version
      - from: /git_version
        to: /io/git_version
      - from: /repo
        to: /io/repo
      - from: /wheel-for-azure-container.tar
        to: /io/wheel-for-azure-container.tar
      - from: /www.tar.gz
        to: /io/www.tar.gz
    secrets:
      - name: pypi-credentials
        namespace:
          valueFrom: default_ns.name
        mountPath: /pypi-credentials
      - name: ci-deploy-0-1--hail-is-hail
        namespace:
          valueFrom: default_ns.name
        mountPath: /ci-deploy-0-1--hail-is-hail
      - name: docker-hub-hailgenetics
        namespace:
          valueFrom: default_ns.name
        mountPath: /docker-hub-hailgenetics
      - name: hail-ci-0-1-github-oauth-token
        namespace:
          valueFrom: default_ns.name
        mountPath: /hail-ci-0-1-github-oauth-token
    scopes:
      - deploy
      - dev
    dependsOn:
      - test_dataproc-37
      - test_dataproc-38
      - default_ns
      - ci_utils_image
      - build_hail
      - merge_code
      - hailgenetics_hail_image
      - build_wheel_for_azure
      - make_docs
    clouds:
      - gcp
  - kind: buildImage2
    name: website_image
    dockerFile: /io/website/Dockerfile
    contextPath: /io/website
    publishAs: website
    inputs:
      - from: /repo/website
        to: /io/website
      - from: /docs.tar.gz
        to: /io/website/docs.tar.gz
    dependsOn:
      - service_base_image
      - get_pip_versioned_docs
    clouds:
      - gcp
    resources:
      storage: 20Gi
      cpu: "2"
      memory: standard
  - kind: deploy
    name: deploy_website
    namespace:
      valueFrom: default_ns.name
    config: website/deployment.yaml
    wait:
      - kind: Service
        name: website
        for: alive
    dependsOn:
      - default_ns
      - website_image
      - create_certs
      - create_session_key
      - create_deploy_config
    clouds:
      - gcp
  - kind: runImage
    name: test_website
    image:
      valueFrom: service_base_image.image
    script: |
      set -ex
      hailctl curl {{ default_ns.name }} www / \
              -vvv \
              -fsSL \
              --retry 3 \
              --retry-delay 5
    secrets:
      - name: test-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /user-tokens
      - name: worker-deploy-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /deploy-config
    dependsOn:
      - default_ns
      - create_accounts
      - service_base_image
      - deploy_website
    clouds:
      - gcp
  - kind: runImage
    name: test_hail_scala_fs
    image:
      valueFrom: base_spark_3_2.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io
      mkdir -p src/test
      tar xzf resources.tar.gz -C src/test

      export HAIL_CLOUD={{ global.cloud }}
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      export AZURE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      export HAIL_FS_TEST_CLOUD_RESOURCES_URI={{ global.test_storage_uri }}/{{ upload_test_resources_to_blob_storage.token }}/test/resources/fs
      export HAIL_TEST_STORAGE_URI={{ global.test_storage_uri }}

      set +e
      java -Xms7500M -Xmx7500M \
           -cp hail-test.jar:$SPARK_HOME/jars/* \
           org.testng.TestNG \
           -listener is.hail.LogTestListener \
           testng-fs.xml
      exit_code=$?
      set -e
      if [[ $exit_code -eq 2 ]]
      then
          echo "some tests were skipped, but exiting success anyway"
          exit 0
      else
          exit $exit_code
      fi
    inputs:
      - from: /resources.tar.gz
        to: /io/resources.tar.gz
      - from: /just-jar/spark-32/hail-test.jar
        to: /io/hail-test.jar
      - from: /testng-fs.xml
        to: /io/testng-fs.xml
    secrets:
      - name: test-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /user-tokens
      - name: worker-deploy-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /deploy-config
      - name: ssl-config-services-java-tests
        namespace:
          valueFrom: default_ns.name
        mountPath: /ssl-config
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    timeout: 1200
    dependsOn:
      - default_ns
      - create_certs
      - create_accounts
      - base_spark_3_2
      - build_hail
      - build_hail_jar_and_wheel_only_spark_3_2
      - upload_test_resources_to_blob_storage
  - kind: runImage
    name: test_hail_services_java
    image:
      valueFrom: hail_run_tests_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io
      mkdir -p src/test
      tar xzf resources.tar.gz -C src/test
      java -Xms7500M -Xmx7500M \
           -cp hail-test.jar:$SPARK_HOME/jars/* \
           org.testng.TestNG \
           -listener is.hail.LogTestListener \
           testng-services.xml
    inputs:
      - from: /resources.tar.gz
        to: /io/resources.tar.gz
      - from: /hail-test.jar
        to: /io/hail-test.jar
      - from: /testng-services.xml
        to: /io/testng-services.xml
    secrets:
      - name: test-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /user-tokens
      - name: worker-deploy-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /deploy-config
      - name: ssl-config-services-java-tests
        namespace:
          valueFrom: default_ns.name
        mountPath: /ssl-config
    timeout: 1200
    dependsOn:
      - default_ns
      - create_certs
      - create_accounts
      - hail_run_tests_image
      - build_hail
      - deploy_batch
  - kind: runImage
    name: cancel_all_running_test_batches
    image:
      valueFrom: service_base_image.image
    script: |
      cat >cancel_all_running_test_batches.py <<'EOF'
      from hailtop.batch_client.aioclient import BatchClient
      import asyncio

      async def cancel_all(query):
          bc = await BatchClient.create('test')
          async for b in bc.list_batches(query):
              status = await b.last_known_status()
              print(status)
              await b.cancel()

      asyncio.get_event_loop().run_until_complete(cancel_all('user:test running'))
      EOF

      python3 cancel_all_running_test_batches.py
    secrets:
      - name: worker-deploy-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /deploy-config
      - name: test-dev-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /user-tokens
      - name: ssl-config-batch-tests
        namespace:
          valueFrom: default_ns.name
        mountPath: /ssl-config
    alwaysRun: true
    timeout: 300
    dependsOn:
      - create_deploy_config
      - create_accounts
      - default_ns
      - service_base_image
      - deploy_batch
      - test_batch
      - test_ci
      - test_hailtop_batch
      - test_hail_python_service_backend
  - kind: runImage
    name: test_batch_invariants
    image:
      valueFrom: batch_image.image
    script: |
      export DOCKER_PREFIX="{{ global.docker_prefix }}"
      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      python3 -m pytest --log-date-format="%Y-%m-%dT%H:%M:%S" --log-format="%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)d:%(funcName)s %(message)s" --log-cli-level=INFO -s -vv --instafail -k "test_invariants" /io/test/
    inputs:
      - from: /repo/batch/test
        to: /io/test
    timeout: 300
    secrets:
      - name: worker-deploy-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /deploy-config
      - name: test-dev-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /user-tokens
      - name: ssl-config-batch-tests
        namespace:
          valueFrom: default_ns.name
        mountPath: /ssl-config
    scopes:
      - test
      - dev
    dependsOn:
      - create_deploy_config
      - create_accounts
      - default_ns
      - merge_code
      - batch_image
      - deploy_batch
      - test_batch
      - test_ci
      - test_hailtop_batch
      - test_hail_python_service_backend
      - cancel_all_running_test_batches
  - kind: runImage
    name: delete_gcp_batch_instances
    image:
      valueFrom: base_image.image
    alwaysRun: true
    script: |
      set -ex
      gcloud -q auth activate-service-account --key-file=/test-gsa-key/key.json
      set +e
      kubectl -n {{ default_ns.name }} scale deployment batch-driver --replicas=0
      gcloud -q compute instances list \
          --filter 'tags.items=batch2-agent AND labels.namespace={{ default_ns.name }}' \
          --format="table[no-heading](zone.basename(), name)" \
          --project {{ global.gcp_project }} \
        | xargs -n2 -r sh -c 'gcloud -q compute instances delete --zone "$1" --project {{ global.gcp_project }} "$2" || true' argv0
      gcloud -q compute disks list \
          --filter 'labels.batch=1 AND labels.namespace={{ default_ns.name }}' \
          --format="table[no-heading](zone.basename(), name)" \
          --project {{ global.gcp_project }} \
        | xargs -n2 -r sh -c 'gcloud -q compute disks delete --zone "$1" --project {{ global.gcp_project }} "$2" || true' argv0
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    scopes:
      - dev
      - test
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    clouds:
      - gcp
    dependsOn:
      - default_ns
      - base_image
      - test_batch_invariants
      - test_batch
      - test_ci
      - test_hailtop_batch
      - test_hail_python_service_backend
      - cancel_all_running_test_batches
  - kind: runImage
    name: delete_azure_batch_instances
    image: mcr.microsoft.com/azure-cli
    alwaysRun: true
    script: |
      set -e -o pipefail
      AZURE_USERNAME=$(jq -r '.appId' /test-gsa-key/key.json)
      AZURE_PASSWORD=$(jq -r '.password' /test-gsa-key/key.json)
      AZURE_TENANT_ID=$(jq -r '.tenant' /test-gsa-key/key.json)
      az login --service-principal -u $AZURE_USERNAME -p $AZURE_PASSWORD --tenant $AZURE_TENANT_ID
      set +e
      set -x
      export AZURE_RESOURCE_GROUP="{{ global.azure_resource_group }}"
      az aks install-cli
      kubectl -n {{ default_ns.name }} scale deployment batch-driver --replicas=0
      az vm list --resource-group $AZURE_RESOURCE_GROUP -o tsv \
          --query "[?tags.namespace == '{{ default_ns.name }}'].name" \
        | xargs -n1 -r sh -c 'az vm delete --force-deletion true --resource-group $AZURE_RESOURCE_GROUP --name "$1" --yes || true' argv0
      az network nic list --resource-group $AZURE_RESOURCE_GROUP -o tsv \
          --query "[?tags.namespace == '{{ default_ns.name }}'].name" \
        | xargs -n1 -r sh -c 'az network nic delete --resource-group $AZURE_RESOURCE_GROUP --name "$1" || true' argv0
      az network public-ip list --resource-group $AZURE_RESOURCE_GROUP -o tsv \
          --query "[?tags.namespace == '{{ default_ns.name }}'].name" \
        | xargs -n1 -r sh -c 'az network public-ip delete --resource-group $AZURE_RESOURCE_GROUP --name "$1" || true' argv0
      az deployment group list --resource-group $AZURE_RESOURCE_GROUP -o tsv \
          --query "[?tags.namespace == '{{ default_ns.name }}'].name" \
        | xargs -n1 -r sh -c 'az deployment delete --name "$1" || true' argv0
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    scopes:
      - dev
      - test
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    clouds:
      - azure
    dependsOn:
      - default_ns
      - base_image
      - test_batch_invariants
      - test_batch
      - test_ci
      - test_hailtop_batch
      - cancel_all_running_test_batches