Merge pull request #334 from populationgenomics/upstream-14419

Merge upstream HEAD(6a6c38d, 2024-05-21) Expose HAIL_CI_STORAGE_URI as ci_storage_uri in CI Steps

Author: milo-hyben

Makefile

@@ -39,19 +39,21 @@ check-all: check-hail check-services
 
 .PHONY: check-hail-fast
 check-hail-fast:
-	ruff check hail/python/hail
-	ruff check hail/python/hailtop
+	ruff check hail
 	ruff format hail --diff
 	$(PYTHON) -m pyright hail/python/hailtop
 
+	ruff check hail/python/test/hailtop/batch
+	$(PYTHON) -m pyright hail/python/test/hailtop/batch
+
 .PHONY: pylint-hailtop
 pylint-hailtop:
 	# pylint on hail is still a work in progress
 	$(PYTHON) -m pylint --rcfile pylintrc hail/python/hailtop --score=n
 
 .PHONY: check-hail
 check-hail: check-hail-fast pylint-hailtop
-	cd hail && sh millw __.checkFormat
+	cd hail && sh millw __.checkFormat + __.fix --check
 
 .PHONY: check-services
 check-services: $(CHECK_SERVICES_MODULES)

auth/auth/auth.py

@@ -36,7 +36,6 @@
 from hailtop.auth import AzureFlow, Flow, GoogleFlow, IdentityProvider
 from hailtop.config import get_deploy_config
 from hailtop.hail_logging import AccessLogger
-from hailtop.tls import internal_server_ssl_context
 from hailtop.utils import secret_alnum_string
 from web_common import render_template, set_message, setup_aiohttp_jinja2, setup_common_static_routes
 
@@ -905,5 +904,5 @@ def run():
         host='0.0.0.0',
         port=443,
         access_log_class=AuthAccessLogger,
-        ssl_context=internal_server_ssl_context(),
+        ssl_context=deploy_config.server_ssl_context(),
     )

batch/batch/cloud/driver.py

@@ -1,10 +1,13 @@
+import os
+
 from gear import Database
 from gear.cloud_config import get_global_config
 
 from ..driver.driver import CloudDriver
 from ..inst_coll_config import InstanceCollectionConfigs
 from .azure.driver.driver import AzureDriver
 from .gcp.driver.driver import GCPDriver
+from .terra.azure.driver.driver import TerraAzureDriver
 
 
 async def get_cloud_driver(
@@ -17,6 +20,8 @@ async def get_cloud_driver(
     cloud = get_global_config()['cloud']
 
     if cloud == 'azure':
+        if os.environ.get('HAIL_TERRA'):
+            return await TerraAzureDriver.create(app, db, machine_name_prefix, namespace, inst_coll_configs)
         return await AzureDriver.create(app, db, machine_name_prefix, namespace, inst_coll_configs)
 
     assert cloud == 'gcp', cloud