[lint] replaces black formatting with ruff (#14132)

`ruff` is faster than `black` when formatting Python `files`, so this
change replaces the formatting checks for our files with calls to `ruff`
instead of `black`, and runs `ruff format` on all files that would cause
those checks to fail.

Author: iris-garden

.pre-commit-config.yaml

@@ -9,6 +9,12 @@ repos:
         language: system
         types: [python]
         require_serial: true
+      - id: ruff-format
+        name: ruff-format
+        entry: ruff format
+        language: system
+        types: [python]
+        require_serial: true
       - id: pyright
         name: pyright
         entry: pyright
@@ -23,11 +29,6 @@ repos:
     hooks:
       - id: end-of-file-fixer
       - id: trailing-whitespace
-  - repo: https://github.com/psf/black
-    rev: 22.3.0
-    hooks:
-      - id: black
-        language_version: python3
   - repo: https://github.com/thibaudcolas/curlylint
     rev: v0.13.1
     hooks:

Makefile

@@ -41,8 +41,8 @@ check-all: check-hail check-services
 check-hail-fast:
 	ruff check hail/python/hail
 	ruff check hail/python/hailtop
+	ruff format hail --check
 	$(PYTHON) -m pyright hail/python/hailtop
-	$(PYTHON) -m black hail --check --diff
 
 .PHONY: pylint-hailtop
 pylint-hailtop:
@@ -62,8 +62,8 @@ pylint-%:
 .PHONY: check-%-fast
 check-%-fast:
 	ruff check $*
+	ruff format $* --check
 	$(PYTHON) -m pyright $*
-	$(PYTHON) -m black $* --check --diff
 	curlylint $*
 	cd $* && bash ../check-sql.sh
 

auth/auth/auth.py

@@ -164,10 +164,10 @@ async def _insert(tx):
             return False
 
         await tx.execute_insertone(
-            '''
+            """
 INSERT INTO users (state, username, login_id, is_developer, is_service_account, hail_identity, hail_credentials_secret_name)
 VALUES (%s, %s, %s, %s, %s, %s, %s);
-''',
+""",
             (
                 'creating',
                 username,
@@ -482,9 +482,11 @@ async def rest_login(request: web.Request) -> web.Response:
     flow_data['callback_uri'] = callback_uri
 
     # keeping authorization_url and state for backwards compatibility
-    return json_response(
-        {'flow': flow_data, 'authorization_url': flow_data['authorization_url'], 'state': flow_data['state']}
-    )
+    return json_response({
+        'flow': flow_data,
+        'authorization_url': flow_data['authorization_url'],
+        'state': flow_data['state'],
+    })
 
 
 @routes.get('/api/v1alpha/oauth2-client')
@@ -511,10 +513,10 @@ async def post_create_role(request: web.Request, _) -> NoReturn:
     name = str(post['name'])
 
     role_id = await db.execute_insertone(
-        '''
+        """
 INSERT INTO `roles` (`name`)
 VALUES (%s);
-''',
+""",
         (name),
     )
 
@@ -564,10 +566,10 @@ async def rest_get_users(request: web.Request, userdata: UserData) -> web.Respon
         raise web.HTTPUnauthorized()
 
     db = request.app[AppKeys.DB]
-    _query = '''
+    _query = """
 SELECT id, username, login_id, state, is_developer, is_service_account, hail_identity
 FROM users;
-'''
+"""
     users = [x async for x in db.select_and_fetchall(_query)]
     return json_response(users)
 
@@ -579,10 +581,10 @@ async def rest_get_user(request: web.Request, _) -> web.Response:
     username = request.match_info['user']
 
     user = await db.select_and_fetchone(
-        '''
+        """
 SELECT id, username, login_id, state, is_developer, is_service_account, hail_identity FROM users
 WHERE username = %s;
-''',
+""",
         (username,),
     )
     if user is None:
@@ -599,11 +601,11 @@ async def _delete_user(db: Database, username: str, id: Optional[str]):
         where_args.append(id)
 
     n_rows = await db.execute_update(
-        f'''
+        f"""
 UPDATE users
 SET state = 'deleting'
 WHERE {' AND '.join(where_conditions)};
-''',
+""",
         where_args,
     )
 
@@ -743,11 +745,11 @@ async def get_userinfo_from_login_id_or_hail_identity_id(
     users = [
         x
         async for x in db.select_and_fetchall(
-            '''
+            """
 SELECT users.*
 FROM users
 WHERE (users.login_id = %s OR users.hail_identity_uid = %s) AND users.state = 'active'
-''',
+""",
             (login_id_or_hail_idenity_uid, login_id_or_hail_idenity_uid),
         )
     ]
@@ -767,12 +769,12 @@ async def get_userinfo_from_hail_session_id(request: web.Request, session_id: st
     users = [
         x
         async for x in db.select_and_fetchall(
-            '''
+            """
 SELECT users.*
 FROM users
 INNER JOIN sessions ON users.id = sessions.user_id
 WHERE users.state = 'active' AND sessions.session_id = %s AND (ISNULL(sessions.max_age_secs) OR (NOW() < TIMESTAMPADD(SECOND, sessions.max_age_secs, sessions.created)));
-''',
+""",
             session_id,
             'get_userinfo',
         )

auth/auth/driver/driver.py

@@ -94,10 +94,10 @@ async def delete(self):
             return
 
         await self.db.just_execute(
-            '''
+            """
 DELETE FROM sessions
 WHERE session_id = %s;
-''',
+""",
             (self.session_id,),
         )
         self.session_id = None
@@ -430,11 +430,11 @@ async def _create_user(app, user, skip_trial_bp, cleanup):
             updates['trial_bp_name'] = billing_project_name
 
     n_rows = await db.execute_update(
-        f'''
+        f"""
 UPDATE users
 SET {', '.join([f'{k} = %({k})s' for k in updates])}
 WHERE id = %(id)s AND state = 'creating';
-''',
+""",
         {'id': user['id'], **updates},
     )
     if n_rows != 1:
@@ -502,10 +502,10 @@ async def delete_user(app, user):
         await bp.delete()
 
     await db.just_execute(
-        '''
+        """
 DELETE FROM sessions WHERE user_id = %s;
 UPDATE users SET state = 'deleted' WHERE id = %s;
-''',
+""",
         (user['id'], user['id']),
     )
 
@@ -523,11 +523,11 @@ async def resolve_identity_uid(app, hail_identity):
         hail_identity_uid = await sp.get_service_principal_object_id()
 
     await db.just_execute(
-        '''
+        """
 UPDATE users
 SET hail_identity_uid = %s
 WHERE hail_identity = %s
-''',
+""",
         (hail_identity_uid, hail_identity),
     )
 

batch/batch/batch.py

@@ -112,11 +112,11 @@ async def cancel_batch_in_db(db, batch_id):
     @transaction(db)
     async def cancel(tx):
         record = await tx.execute_and_fetchone(
-            '''
+            """
 SELECT `state` FROM batches
 WHERE id = %s AND NOT deleted
 FOR UPDATE;
-''',
+""",
             (batch_id,),
         )
         if not record:

batch/batch/cloud/azure/driver/create_instance.py

@@ -92,7 +92,7 @@ def create_vm_config(
 
     jvm_touch_command = '\n'.join(touch_commands)
 
-    startup_script = r'''#cloud-config
+    startup_script = r"""#cloud-config
 
 mounts:
   - [ ephemeral0, null ]
@@ -123,10 +123,10 @@ def create_vm_config(
 
 runcmd:
   - sh /startup.sh
-'''
+"""
     startup_script = base64.b64encode(startup_script.encode('utf-8')).decode('utf-8')
 
-    run_script = f'''
+    run_script = f"""
 #!/bin/bash
 set -x
 
@@ -302,7 +302,7 @@ def create_vm_config(
 az vm delete -g $RESOURCE_GROUP -n $NAME --yes
 sleep 1
 done
-'''
+"""
 
     user_data = {
         'run_script': run_script,

batch/batch/cloud/azure/driver/driver.py

@@ -37,10 +37,10 @@ async def create(
 
         region_args = [(r,) for r in regions]
         await db.execute_many(
-            '''
+            """
 INSERT INTO regions (region) VALUES (%s)
 ON DUPLICATE KEY UPDATE region = region;
-''',
+""",
             region_args,
         )
 

batch/batch/cloud/azure/instance_config.py

@@ -35,16 +35,14 @@ def create(
         else:
             data_disk_resource = AzureStaticSizedDiskResource.create(product_versions, 'P', data_disk_size_gb, location)
 
-        resources: List[AzureResource] = filter_none(
-            [
-                AzureVMResource.create(product_versions, machine_type, preemptible, location),
-                AzureStaticSizedDiskResource.create(product_versions, 'E', boot_disk_size_gb, location),
-                data_disk_resource,
-                AzureDynamicSizedDiskResource.create(product_versions, 'P', location),
-                AzureIPFeeResource.create(product_versions, 1024),
-                AzureServiceFeeResource.create(product_versions),
-            ]
-        )
+        resources: List[AzureResource] = filter_none([
+            AzureVMResource.create(product_versions, machine_type, preemptible, location),
+            AzureStaticSizedDiskResource.create(product_versions, 'E', boot_disk_size_gb, location),
+            data_disk_resource,
+            AzureDynamicSizedDiskResource.create(product_versions, 'P', location),
+            AzureIPFeeResource.create(product_versions, 1024),
+            AzureServiceFeeResource.create(product_versions),
+        ])
 
         return AzureSlimInstanceConfig(
             machine_type=machine_type,

batch/batch/cloud/azure/worker/worker_api.py

@@ -66,14 +66,14 @@ def instance_config_from_config_dict(self, config_dict: Dict[str, str]) -> Azure
     def _blobfuse_credentials(self, credentials: Dict[str, str], account: str, container: str) -> str:
         credentials = orjson.loads(base64.b64decode(credentials['key.json']).decode())
         # https://github.com/Azure/azure-storage-fuse
-        return f'''
+        return f"""
 accountName {account}
 authType SPN
 servicePrincipalClientId {credentials["appId"]}
 servicePrincipalClientSecret {credentials["password"]}
 servicePrincipalTenantId {credentials["tenant"]}
 containerName {container}
-'''
+"""
 
     def _write_blobfuse_credentials(
         self,

batch/batch/cloud/gcp/driver/activity_logs.py

@@ -95,14 +95,14 @@ async def process_activity_log_events_since(
     project: str,
     mark: str,
 ) -> str:
-    filter = f'''
+    filter = f"""
 (logName="projects/{project}/logs/cloudaudit.googleapis.com%2Factivity" OR
 logName="projects/{project}/logs/cloudaudit.googleapis.com%2Fsystem_event"
 ) AND
 resource.type=gce_instance AND
 protoPayload.resourceName:"{machine_name_prefix}" AND
 timestamp >= "{mark}"
-'''
+"""
 
     body = {
         'resourceNames': [f'projects/{project}'],

batch/batch/cloud/gcp/driver/create_instance.py

@@ -85,12 +85,10 @@ def scheduling() -> dict:
         }
 
         if preemptible:
-            result.update(
-                {
-                    'provisioningModel': 'SPOT',
-                    'instanceTerminationAction': 'DELETE',
-                }
-            )
+            result.update({
+                'provisioningModel': 'SPOT',
+                'instanceTerminationAction': 'DELETE',
+            })
 
         return result
 
@@ -129,7 +127,7 @@ def scheduling() -> dict:
             'items': [
                 {
                     'key': 'startup-script',
-                    'value': '''
+                    'value': """
 #!/bin/bash
 set -x
 
@@ -150,11 +148,11 @@ def scheduling() -> dict:
 curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/run_script"  >./run.sh
 
 nohup /bin/bash run.sh >run.log 2>&1 &
-    ''',
+    """,
                 },
                 {
                     'key': 'run_script',
-                    'value': rf'''
+                    'value': rf"""
 #!/bin/bash
 set -x
 
@@ -346,18 +344,18 @@ def scheduling() -> dict:
 gcloud -q compute instances delete $NAME --zone=$ZONE
 sleep 1
 done
-''',
+""",
                 },
                 {
                     'key': 'shutdown-script',
-                    'value': '''
+                    'value': """
 set -x
 
 INSTANCE_ID=$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/instance_id")
 NAME=$(curl -s http://metadata.google.internal/computeMetadata/v1/instance/name -H 'Metadata-Flavor: Google')
 
 journalctl -u docker.service > dockerd.log
-''',
+""",
                 },
                 {'key': 'activation_token', 'value': activation_token},
                 {'key': 'batch_worker_image', 'value': BATCH_WORKER_IMAGE},