use ruff instead of tox

.github/workflows/tox.yaml

@@ -10,15 +10,17 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: [3.9]
+        python: [3.11]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
+
       - name: Install Tox and any other packages
         run: pip install tox
+
       - name: Run Tox
         # Run tox using the version of Python in `PATH`
         run: tox -e py

.pre-commit-config.yaml

@@ -14,3 +14,10 @@ repos:
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.4.4
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+      - id: ruff-format

integration_tests/cdk/app.py

@@ -1,28 +1,19 @@
-from config import build_app_config, AppConfig
-from aws_cdk import (
-    Stack,
-    aws_ec2,
-    aws_rds,
-    App,
-    RemovalPolicy
-)
+from aws_cdk import App, RemovalPolicy, Stack, aws_ec2, aws_rds
+from config import AppConfig, build_app_config
 from constructs import Construct
 from eoapi_cdk import (
     PgStacApiLambda,
     PgStacDatabase,
-    TitilerPgstacApiLambda,
     TiPgApiLambda,
+    TitilerPgstacApiLambda,
 )
 
 
 class VpcStack(Stack):
-    def __init__(self, scope: Construct, app_config: AppConfig, id: str, **kwargs) -> None:
-        super().__init__(
-            scope,
-            id=id,
-            tags=app_config.tags,
-            **kwargs
-        )
+    def __init__(
+        self, scope: Construct, app_config: AppConfig, id: str, **kwargs
+    ) -> None:
+        super().__init__(scope, id=id, tags=app_config.tags, **kwargs)
 
         self.vpc = aws_ec2.Vpc(
             self,
@@ -31,7 +22,7 @@ def __init__(self, scope: Construct, app_config: AppConfig, id: str, **kwargs) -
                 aws_ec2.SubnetConfiguration(
                     name="ingress", subnet_type=aws_ec2.SubnetType.PUBLIC, cidr_mask=24
                 ),
-            ]
+            ],
         )
 
         self.vpc.add_interface_endpoint(
@@ -88,7 +79,7 @@ def __init__(
             ),
             allocated_storage=app_config.db_allocated_storage,
             instance_type=aws_ec2.InstanceType(app_config.db_instance_type),
-            removal_policy=RemovalPolicy.DESTROY
+            removal_policy=RemovalPolicy.DESTROY,
         )
 
         pgstac_db.db.connections.allow_default_port_from_any_ipv4()
@@ -101,7 +92,7 @@ def __init__(
                 "description": f"{app_config.stage} STAC API",
             },
             db=pgstac_db.db,
-            db_secret=pgstac_db.pgstac_secret
+            db_secret=pgstac_db.pgstac_secret,
         )
 
         TitilerPgstacApiLambda(
@@ -145,10 +136,7 @@ def __init__(
 pgstac_infra_stack_id = f"pgstac{app_config.project_id}"
 
 pgstac_infra_stack = pgStacInfraStack(
-    scope=app,
-    vpc=vpc_stack.vpc,
-    app_config=app_config,
-    id=pgstac_infra_stack_id
+    scope=app, vpc=vpc_stack.vpc, app_config=app_config, id=pgstac_infra_stack_id
 )
 
 app.synth()

integration_tests/cdk/config.py

@@ -7,15 +7,9 @@
 
 
 class AppConfig(BaseSettings):
-    model_config = SettingsConfigDict(
-        env_file=".env"
-    )
-    aws_default_account: str = pydantic.Field(
-        description="AWS account ID"
-    )
-    project_id: str = pydantic.Field(
-        description="Project ID", default="eoapicdk"
-    )
+    model_config = SettingsConfigDict(env_file=".env")
+    aws_default_account: str = pydantic.Field(description="AWS account ID")
+    project_id: str = pydantic.Field(description="Project ID", default="eoapicdk")
     stage: str = pydantic.Field(description="Stage of deployment", default="test")
     # because of its validator, `tags` should always come after `project_id` and `stage`
     tags: Dict[str, str] | None = pydantic.Field(

lib/database/bootstrapper_runtime/handler.py

@@ -217,9 +217,7 @@ def handler(event, context):
         )
         with psycopg.connect(eoapi_db_admin_conninfo, autocommit=True) as conn:
             with conn.cursor() as cur:
-                print(
-                    f"Registering Extension in '{eoapi_params['dbname']}' database..."
-                )
+                print(f"Registering Extension in '{eoapi_params['dbname']}' database...")
                 register_extensions(cursor=cur)
 
             print("Starting PgSTAC Migration ")
@@ -247,18 +245,18 @@ def handler(event, context):
                 customization(cursor=cur, params=params)
 
         # Make sure the user can access the database
-        eoapi_user_dsn = (
-            "postgresql://{user}:{password}@{host}:{port}/{dbname}".format(
-                dbname=eoapi_params["dbname"],
-                user=eoapi_params["username"],
-                password=eoapi_params["password"],
-                host=admin_params["host"],
-                port=admin_params["port"],
-            )
+        eoapi_user_dsn = "postgresql://{user}:{password}@{host}:{port}/{dbname}".format(
+            dbname=eoapi_params["dbname"],
+            user=eoapi_params["username"],
+            password=eoapi_params["password"],
+            host=admin_params["host"],
+            port=admin_params["port"],
         )
         print("Checking eoAPI user access to the PgSTAC database...")
         with PgstacDB(dsn=eoapi_user_dsn, debug=True) as pgdb:
-            print(f"    OK - User has access to pgstac db, pgstac schema version: {pgdb.version}")
+            print(
+                f"    OK - User has access to pgstac db, pgstac schema version: {pgdb.version}"
+            )
 
     except Exception as e:
         print(f"Unable to bootstrap database with exception={e}")

lib/ingestor-api/runtime/dev_requirements.txt

@@ -1,2 +1,3 @@
 httpx
 moto[dynamodb, ssm]>=4.0.9,<5.0
+pytest

lib/ingestor-api/runtime/src/dependencies.py

@@ -52,9 +52,9 @@ def decode_token(
 
         claims.validate()
         return claims
-    except errors.JoseError:  #
+    except errors.JoseError as e:
         logger.exception("Unable to decode token")
-        raise HTTPException(status_code=403, detail="Bad auth token")
+        raise HTTPException(status_code=403, detail="Bad auth token") from e
 
 
 def get_username_from_token(
@@ -88,7 +88,7 @@ def fetch_ingestion(
 ):
     try:
         return db.fetch_one(username=username, ingestion_id=ingestion_id)
-    except services.NotInDb:
+    except services.NotInDb as e:
         raise HTTPException(
             status_code=404, detail="No ingestion found with provided ID"
-        )
+        ) from e

lib/ingestor-api/runtime/src/main.py

@@ -9,9 +9,7 @@
 )
 
 
-@app.get(
-    "/ingestions", response_model=schemas.ListIngestionResponse, tags=["Ingestion"]
-)
+@app.get("/ingestions", response_model=schemas.ListIngestionResponse, tags=["Ingestion"])
 async def list_ingestions(
     list_request: schemas.ListIngestionRequest = Depends(),
     db: services.Database = Depends(dependencies.get_db),
@@ -78,8 +76,7 @@ def cancel_ingestion(
         raise HTTPException(
             status_code=400,
             detail=(
-                "Unable to delete ingestion if status is not "
-                f"{schemas.Status.queued}"
+                "Unable to delete ingestion if status is not " f"{schemas.Status.queued}"
             ),
         )
     return ingestion.cancel(db)
@@ -100,7 +97,7 @@ def publish_collection(collection: schemas.StacCollection):
         raise HTTPException(
             status_code=400,
             detail=(f"Unable to publish collection: {e}"),
-        )
+        ) from e
 
 
 @app.delete(
@@ -114,7 +111,7 @@ def delete_collection(collection_id: str):
         return {f"Successfully deleted: {collection_id}"}
     except Exception as e:
         print(e)
-        raise HTTPException(status_code=400, detail=(f"{e}"))
+        raise HTTPException(status_code=400, detail=(f"{e}")) from e
 
 
 @app.get("/auth/me")

lib/ingestor-api/runtime/src/schemas.py

@@ -116,7 +116,7 @@ def __post_init_post_parse__(self) -> None:
 
         try:
             self.next = json.loads(base64.b64decode(self.next))
-        except (UnicodeDecodeError, binascii.Error):
+        except (UnicodeDecodeError, binascii.Error) as e:
             raise RequestValidationError(
                 [
                     error_wrappers.ErrorWrapper(
@@ -126,7 +126,7 @@ def __post_init_post_parse__(self) -> None:
                         "query.next",
                     )
                 ]
-            )
+            ) from e
 
 
 class ListIngestionResponse(BaseModel):

lib/ingestor-api/runtime/src/services.py

@@ -22,8 +22,8 @@ def fetch_one(self, username: str, ingestion_id: str):
         )
         try:
             return schemas.Ingestion.parse_obj(response["Item"])
-        except KeyError:
-            raise NotInDb("Record not found")
+        except KeyError as e:
+            raise NotInDb("Record not found") from e
 
     def fetch_many(
         self, status: str, next: dict = None, limit: int = None

lib/ingestor-api/runtime/src/validators.py

@@ -37,7 +37,7 @@ def s3_object_is_accessible(bucket: str, key: str):
     except client.exceptions.ClientError as e:
         raise ValueError(
             f"Asset not accessible: {e.__dict__['response']['Error']['Message']}"
-        )
+        ) from e
 
 
 def url_is_accessible(href: str):
@@ -49,7 +49,7 @@ def url_is_accessible(href: str):
     except requests.exceptions.HTTPError as e:
         raise ValueError(
             f"Asset not accessible: {e.response.status_code} {e.response.reason}"
-        )
+        ) from e
 
 
 @functools.cache

lib/ingestor-api/runtime/tests/test_ingestor.py

@@ -52,7 +52,7 @@ def test_handler(
     ingestor.handler(dynamodb_stream_event, {})
     load_items.assert_called_once_with(
         creds="",
-        ingestions=list([example_ingestion]),
+        ingestions=[example_ingestion],
     )
     response = mock_table.get_item(
         Key={"created_by": example_ingestion.created_by, "id": example_ingestion.id}

lib/ingestor-api/runtime/tests/test_registration.py

@@ -74,9 +74,7 @@ def test_create(self, client_authenticated, collection_exists, asset_exists):
         )
 
         assert response.status_code == 201
-        assert collection_exists.called_once_with(
-            self.example_ingestion.item.collection
-        )
+        assert collection_exists.called_once_with(self.example_ingestion.item.collection)
 
         stored_data = self.db.fetch_many(status="queued")["items"]
         assert len(stored_data) == 1
@@ -119,10 +117,8 @@ def test_validates_missing_assets(
         assert response.status_code == 422, "should get validation error"
         for asset_type in self.example_ingestion.item.assets.keys():
             assert any(
-                [
-                    err["loc"] == ["body", "assets", asset_type, "href"]
-                    for err in response.json()["detail"]
-                ]
+                err["loc"] == ["body", "assets", asset_type, "href"]
+                for err in response.json()["detail"]
             ), "should reference asset type in validation error response"
         assert (
             len(self.db.fetch_many(status="queued")["items"]) == 0

lib/ingestor-api/runtime/tests/test_utils.py

@@ -28,7 +28,7 @@ def dbcreds():
 def test_load_items(loader, pgstacdb, example_ingestion, dbcreds):
     import src.utils as utils
 
-    utils.load_items(dbcreds, list([example_ingestion]))
+    utils.load_items(dbcreds, [example_ingestion])
     loader.return_value.load_items.assert_called_once_with(
         file=jsonable_encoder([example_ingestion.item]),
         insert_mode=Methods.upsert,

lib/tipg-api/runtime/src/handler.py

@@ -15,9 +15,11 @@
 
 # skipping linting rule that wants all imports at the top
 from tipg.main import app  # noqa: E402
-from tipg.settings import CustomSQLSettings  # noqa: E402
-from tipg.settings import DatabaseSettings  # noqa: E402
-from tipg.settings import PostgresSettings  # noqa: E402; noqa: E402
+from tipg.settings import (
+    CustomSQLSettings,  # noqa: E402
+    DatabaseSettings,  # noqa: E402
+    PostgresSettings,  # noqa: E402; noqa: E402
+)
 
 postgres_settings = PostgresSettings()
 db_settings = DatabaseSettings()

ruff.toml

@@ -0,0 +1,23 @@
+line-length = 90
+
+[lint]
+select = [
+    "F",    # flake8
+    "C",    # flake8-comprehensions
+    "B",    # flake8-bugbear
+    "I001", # isort
+]
+
+ignore = [
+    "E203",
+    "E266",
+    "F403",
+    "E231",
+    "E501",  # line too long, handled by black
+    "B008",  # do not perform function calls in argument defaults
+    "B905",  # ignore zip() without an explicit strict= parameter, only support with python >3.10
+]
+
+
+[lint.mccabe]
+max-complexity = 18