diff --git a/api/api/views/audio_views.py b/api/api/views/audio_views.py index dcde7266b92..9960ddb524e 100644 --- a/api/api/views/audio_views.py +++ b/api/api/views/audio_views.py @@ -15,9 +15,9 @@ source_collection, stats, tag_collection, + waveform, ) from api.docs.audio_docs import thumbnail as thumbnail_docs -from api.docs.audio_docs import waveform from api.models import Audio from api.serializers.audio_serializers import ( AudioCollectionRequestSerializer, diff --git a/api/conf/settings/security.py b/api/conf/settings/security.py index a5159b9b701..b2e26c64b50 100644 --- a/api/conf/settings/security.py +++ b/api/conf/settings/security.py @@ -1,10 +1,10 @@ from socket import gethostbyname, gethostname -from decouple import config +from django.core.exceptions import ImproperlyConfigured -from conf.settings.base import INSTALLED_APPS, MIDDLEWARE, ENVIRONMENT +from decouple import config -from django.core.exceptions import ImproperlyConfigured +from conf.settings.base import ENVIRONMENT, INSTALLED_APPS, MIDDLEWARE # Quick-start development settings - unsuitable for production diff --git a/api/test/factory/models/audio.py b/api/test/factory/models/audio.py index 9735b2c2c1a..7f0ca49a728 100644 --- a/api/test/factory/models/audio.py +++ b/api/test/factory/models/audio.py @@ -1,10 +1,9 @@ -from test.factory.faker import Faker -from test.factory.models.media import IdentifierFactory, MediaFactory - import factory from factory.django import DjangoModelFactory from api.models.audio import Audio, AudioAddOn, AudioReport, MatureAudio +from test.factory.faker import Faker +from test.factory.models.media import IdentifierFactory, MediaFactory class MatureAudioFactory(DjangoModelFactory): diff --git a/api/test/factory/models/image.py b/api/test/factory/models/image.py index dc034ed3db0..e0758b6a6a6 100644 --- a/api/test/factory/models/image.py +++ b/api/test/factory/models/image.py @@ -1,9 +1,8 @@ -from test.factory.models.media import MediaFactory, MediaReportFactory - import factory from factory.django import DjangoModelFactory from api.models.image import Image, ImageReport, MatureImage +from test.factory.models.media import MediaFactory, MediaReportFactory class MatureImageFactory(DjangoModelFactory): diff --git a/api/test/factory/models/media.py b/api/test/factory/models/media.py index 2df54763b8d..a6e451fa825 100644 --- a/api/test/factory/models/media.py +++ b/api/test/factory/models/media.py @@ -1,4 +1,3 @@ -from test.factory.faker import Faker from uuid import uuid4 from django.conf import settings @@ -11,6 +10,7 @@ from api.constants.licenses import ALL_LICENSES from api.models.media import AbstractMedia +from test.factory.faker import Faker CREATED_BY_FIXTURE_MARKER = "__created_by_test_fixture" diff --git a/api/test/factory/models/oauth2.py b/api/test/factory/models/oauth2.py index 6a749654e39..22b3a697446 100644 --- a/api/test/factory/models/oauth2.py +++ b/api/test/factory/models/oauth2.py @@ -1,5 +1,3 @@ -from test.factory.faker import Faker - from django.utils import timezone import factory @@ -11,6 +9,7 @@ OAuth2Verification, ThrottledApplication, ) +from test.factory.faker import Faker class ThrottledApplicationFactory(DjangoModelFactory): diff --git a/api/test/media_integration.py b/api/test/media_integration.py index 0a4563344ae..34b2328977a 100644 --- a/api/test/media_integration.py +++ b/api/test/media_integration.py @@ -6,10 +6,11 @@ import json import re -from test.constants import API_URL import requests +from test.constants import API_URL + def search(fixture): """Return results for test query.""" diff --git a/api/test/test_audio_integration.py b/api/test/test_audio_integration.py index a172a1275bb..6bf1796d682 100644 --- a/api/test/test_audio_integration.py +++ b/api/test/test_audio_integration.py @@ -6,6 +6,12 @@ """ import json + +import pytest +import requests +from django_redis import get_redis_connection + +from api.utils.check_dead_links import CACHE_PREFIX from test.constants import API_URL from test.media_integration import ( creator_collection, @@ -28,12 +34,6 @@ uuid_validation, ) -import pytest -import requests -from django_redis import get_redis_connection - -from api.utils.check_dead_links import CACHE_PREFIX - @pytest.fixture def force_result_validity(): diff --git a/api/test/test_auth.py b/api/test/test_auth.py index 117193bbaeb..752c848fed4 100644 --- a/api/test/test_auth.py +++ b/api/test/test_auth.py @@ -1,6 +1,6 @@ import time import uuid -from test.constants import API_URL +from unittest.mock import patch from django.urls import reverse from django.utils.http import urlencode @@ -9,7 +9,8 @@ from oauth2_provider.models import AccessToken from api.models import OAuth2Verification, ThrottledApplication -from unittest.mock import patch +from test.constants import API_URL + cache_availability_params = pytest.mark.parametrize( "is_cache_reachable, cache_name", diff --git a/api/test/test_backwards_compat.py b/api/test/test_backwards_compat.py index fe1ebe9c843..4250aa5d049 100644 --- a/api/test/test_backwards_compat.py +++ b/api/test/test_backwards_compat.py @@ -6,10 +6,11 @@ """ import uuid -from test.constants import API_URL import requests +from test.constants import API_URL + def test_old_stats_endpoint(): response = requests.get( diff --git a/api/test/test_dead_link_filter.py b/api/test/test_dead_link_filter.py index f767023b3c0..caa4600fb7a 100644 --- a/api/test/test_dead_link_filter.py +++ b/api/test/test_dead_link_filter.py @@ -1,4 +1,3 @@ -from test.constants import API_URL from unittest.mock import patch from uuid import uuid4 @@ -8,6 +7,7 @@ import requests from api.controllers.elasticsearch.helpers import DEAD_LINK_RATIO +from test.constants import API_URL @pytest.fixture diff --git a/api/test/test_examples.py b/api/test/test_examples.py index 203b04d6ca5..e7ff9d648e1 100644 --- a/api/test/test_examples.py +++ b/api/test/test_examples.py @@ -1,10 +1,11 @@ import json import os import subprocess -from test.constants import API_URL import pytest +from test.constants import API_URL + os.environ["AUDIO_REQ_TOKEN"] = "" os.environ["AUDIO_REQ_ORIGIN"] = API_URL diff --git a/api/test/test_image_integration.py b/api/test/test_image_integration.py index 3e2df768bf0..f2709015ced 100644 --- a/api/test/test_image_integration.py +++ b/api/test/test_image_integration.py @@ -6,6 +6,11 @@ """ import json +from urllib.parse import urlencode + +import pytest +import requests + from test.constants import API_URL from test.media_integration import ( creator_collection, @@ -26,10 +31,6 @@ tag_collection, uuid_validation, ) -from urllib.parse import urlencode - -import pytest -import requests identifier = "cdbd3bf6-1745-45bb-b399-61ee149cd58a" diff --git a/api/test/test_v1_integration.py b/api/test/test_v1_integration.py index 752584292fc..b29c57f4a8f 100644 --- a/api/test/test_v1_integration.py +++ b/api/test/test_v1_integration.py @@ -6,7 +6,6 @@ """ import json -from test.constants import API_URL import pytest import requests @@ -14,6 +13,7 @@ from api.constants.licenses import LICENSE_GROUPS from api.models import Image from api.utils.watermark import watermark +from test.constants import API_URL @pytest.fixture diff --git a/api/test/unit/conftest.py b/api/test/unit/conftest.py index 213dafd0ca9..ad70cd77965 100644 --- a/api/test/unit/conftest.py +++ b/api/test/unit/conftest.py @@ -1,10 +1,4 @@ from dataclasses import dataclass -from test.factory import models as model_factories -from test.factory.models.media import ( - CREATED_BY_FIXTURE_MARKER, - MediaFactory, - MediaReportFactory, -) from unittest.mock import MagicMock from rest_framework.test import APIClient, APIRequestFactory @@ -37,6 +31,12 @@ MediaSearchRequestSerializer, MediaSerializer, ) +from test.factory import models as model_factories +from test.factory.models.media import ( + CREATED_BY_FIXTURE_MARKER, + MediaFactory, + MediaReportFactory, +) @pytest.fixture diff --git a/api/test/unit/controllers/elasticsearch/test_related.py b/api/test/unit/controllers/elasticsearch/test_related.py index 8da59714ecf..23370f3dd0b 100644 --- a/api/test/unit/controllers/elasticsearch/test_related.py +++ b/api/test/unit/controllers/elasticsearch/test_related.py @@ -1,9 +1,3 @@ -from test.factory.es_http import ( - MOCK_LIVE_RESULT_URL_PREFIX, - create_mock_es_http_image_response_with_identifier, - create_mock_es_http_image_search_response, -) -from test.factory.models import ImageFactory from unittest import mock import pook @@ -14,6 +8,12 @@ FILTERED_PROVIDERS_CACHE_KEY, FILTERED_PROVIDERS_CACHE_VERSION, ) +from test.factory.es_http import ( + MOCK_LIVE_RESULT_URL_PREFIX, + create_mock_es_http_image_response_with_identifier, + create_mock_es_http_image_search_response, +) +from test.factory.models import ImageFactory pytestmark = pytest.mark.django_db diff --git a/api/test/unit/controllers/test_search_controller.py b/api/test/unit/controllers/test_search_controller.py index 652c43ea0f6..f64d3fa3919 100644 --- a/api/test/unit/controllers/test_search_controller.py +++ b/api/test/unit/controllers/test_search_controller.py @@ -4,12 +4,6 @@ import re from collections.abc import Callable from enum import Enum, auto -from test.factory.es_http import ( - MOCK_DEAD_RESULT_URL_PREFIX, - MOCK_LIVE_RESULT_URL_PREFIX, - create_mock_es_http_image_search_response, -) -from test.factory.models.content_provider import ContentProviderFactory from unittest import mock from unittest.mock import patch from uuid import uuid4 @@ -26,6 +20,12 @@ from api.utils import tallies from api.utils.dead_link_mask import get_query_hash, save_query_mask from api.utils.search_context import SearchContext +from test.factory.es_http import ( + MOCK_DEAD_RESULT_URL_PREFIX, + MOCK_LIVE_RESULT_URL_PREFIX, + create_mock_es_http_image_search_response, +) +from test.factory.models.content_provider import ContentProviderFactory pytestmark = pytest.mark.django_db diff --git a/api/test/unit/management/commands/test_generatewaveforms.py b/api/test/unit/management/commands/test_generatewaveforms.py index bc12a9669d0..c7ca58c6c42 100644 --- a/api/test/unit/management/commands/test_generatewaveforms.py +++ b/api/test/unit/management/commands/test_generatewaveforms.py @@ -1,7 +1,5 @@ import subprocess from io import StringIO -from test.factory.faker import WaveformProvider -from test.factory.models.audio import AudioAddOnFactory, AudioFactory from unittest import mock from django.core.management import call_command @@ -12,6 +10,8 @@ from psycopg.errors import NotNullViolation from api.models.audio import Audio, AudioAddOn +from test.factory.faker import WaveformProvider +from test.factory.models.audio import AudioAddOnFactory, AudioFactory @mock.patch("api.models.audio.generate_peaks") diff --git a/api/test/unit/models/test_audio.py b/api/test/unit/models/test_audio.py index 6a1b0e8c546..3372c6b0e98 100644 --- a/api/test/unit/models/test_audio.py +++ b/api/test/unit/models/test_audio.py @@ -1,10 +1,10 @@ import uuid -from test.factory.faker import WaveformProvider from unittest import mock import pytest from api.models.audio import Audio, AudioAddOn +from test.factory.faker import WaveformProvider @pytest.fixture diff --git a/api/test/unit/serializers/test_media_serializers.py b/api/test/unit/serializers/test_media_serializers.py index d48798c40cb..b6819e3c43d 100644 --- a/api/test/unit/serializers/test_media_serializers.py +++ b/api/test/unit/serializers/test_media_serializers.py @@ -1,6 +1,5 @@ import random import uuid -from test.factory.models.oauth2 import AccessTokenFactory from unittest.mock import MagicMock, patch from django.conf import settings @@ -14,6 +13,7 @@ from api.serializers.audio_serializers import AudioSearchRequestSerializer from api.serializers.image_serializers import ImageSearchRequestSerializer from api.serializers.media_serializers import MediaSearchRequestSerializer +from test.factory.models.oauth2 import AccessTokenFactory @pytest.fixture diff --git a/api/test/unit/utils/test_image_proxy.py b/api/test/unit/utils/test_image_proxy.py index 5cf0e2833a9..cc0c9871545 100644 --- a/api/test/unit/utils/test_image_proxy.py +++ b/api/test/unit/utils/test_image_proxy.py @@ -1,7 +1,6 @@ import asyncio import itertools from dataclasses import replace -from test.factory.models.image import ImageFactory from unittest.mock import patch from urllib.parse import urlencode @@ -23,6 +22,7 @@ ) from api.utils.image_proxy import get as _photon_get from api.utils.tallies import get_monthly_timestamp +from test.factory.models.image import ImageFactory TEST_IMAGE_DOMAIN = "subdomain.example.com" diff --git a/api/test/unit/utils/test_throttle.py b/api/test/unit/utils/test_throttle.py index 531aecf441e..1afdaff5325 100644 --- a/api/test/unit/utils/test_throttle.py +++ b/api/test/unit/utils/test_throttle.py @@ -1,6 +1,3 @@ -from test.factory.models.image import ImageFactory -from test.factory.models.oauth2 import AccessTokenFactory - from django.http import HttpResponse from rest_framework.settings import api_settings from rest_framework.test import force_authenticate @@ -10,6 +7,8 @@ from api.utils import throttle from api.views.media_views import MediaViewSet +from test.factory.models.image import ImageFactory +from test.factory.models.oauth2 import AccessTokenFactory cache_availability_params = pytest.mark.parametrize( diff --git a/api/test/unit/views/test_image_views.py b/api/test/unit/views/test_image_views.py index 235c56d4464..0fe9aa82aa9 100644 --- a/api/test/unit/views/test_image_views.py +++ b/api/test/unit/views/test_image_views.py @@ -1,6 +1,5 @@ import json from pathlib import Path -from test.factory.models.image import ImageFactory from unittest.mock import patch import pook @@ -8,6 +7,7 @@ from PIL import UnidentifiedImageError from api.views.image_views import ImageViewSet +from test.factory.models.image import ImageFactory _MOCK_IMAGE_PATH = Path(__file__).parent / ".." / ".." / "factory" diff --git a/automations/python/models/label.py b/automations/python/models/label.py index 0340cd97e60..edbed65a4db 100644 --- a/automations/python/models/label.py +++ b/automations/python/models/label.py @@ -1,6 +1,7 @@ -from models.label_group import LabelGroup from shared.data import get_data +from models.label_group import LabelGroup + class Label: diff --git a/automations/python/shared/labels.py b/automations/python/shared/labels.py index 07dd412364b..04532e123f2 100644 --- a/automations/python/shared/labels.py +++ b/automations/python/shared/labels.py @@ -2,6 +2,7 @@ from models.label import Label from models.label_group import LabelGroup + from shared.data import get_data diff --git a/catalog/dags/database/delete_records/delete_records.py b/catalog/dags/database/delete_records/delete_records.py index 8cdc383f9b1..ba08db72543 100644 --- a/catalog/dags/database/delete_records/delete_records.py +++ b/catalog/dags/database/delete_records/delete_records.py @@ -7,7 +7,7 @@ from common import slack from common.constants import POSTGRES_CONN_ID from common.sql import RETURN_ROW_COUNT, PostgresHook -from common.storage.columns import DELETED_ON, Column, PROVIDER, FOREIGN_ID +from common.storage.columns import DELETED_ON, FOREIGN_ID, PROVIDER, Column from common.storage.db_columns import ( setup_db_columns_for_media_type, setup_deleted_db_columns_for_media_type, diff --git a/catalog/dags/elasticsearch_cluster/create_new_es_index/create_new_es_index_dag.py b/catalog/dags/elasticsearch_cluster/create_new_es_index/create_new_es_index_dag.py index 39f23f0d92d..ad4c3229c3a 100644 --- a/catalog/dags/elasticsearch_cluster/create_new_es_index/create_new_es_index_dag.py +++ b/catalog/dags/elasticsearch_cluster/create_new_es_index/create_new_es_index_dag.py @@ -110,6 +110,7 @@ CreateNewIndex, ) + logger = logging.getLogger(__name__) diff --git a/catalog/dags/popularity/popularity_refresh_dag_factory.py b/catalog/dags/popularity/popularity_refresh_dag_factory.py index ae7a11720dc..d8f05b4336e 100644 --- a/catalog/dags/popularity/popularity_refresh_dag_factory.py +++ b/catalog/dags/popularity/popularity_refresh_dag_factory.py @@ -28,16 +28,16 @@ from airflow import DAG from airflow.decorators import task from airflow.operators.trigger_dagrun import TriggerDagRunOperator + +from common import slack +from common.constants import DAG_DEFAULT_ARGS, POSTGRES_CONN_ID +from database.batched_update.constants import DAG_ID as BATCHED_UPDATE_DAG_ID from popularity import sql from popularity.popularity_refresh_types import ( POPULARITY_REFRESH_CONFIGS, PopularityRefresh, ) -from common import slack -from common.constants import DAG_DEFAULT_ARGS, POSTGRES_CONN_ID -from database.batched_update.constants import DAG_ID as BATCHED_UPDATE_DAG_ID - logger = logging.getLogger(__name__) diff --git a/catalog/dags/popularity/recreate_popularity_calculation_dag_factory.py b/catalog/dags/popularity/recreate_popularity_calculation_dag_factory.py index 31261ec38fc..f8662b06b02 100644 --- a/catalog/dags/popularity/recreate_popularity_calculation_dag_factory.py +++ b/catalog/dags/popularity/recreate_popularity_calculation_dag_factory.py @@ -13,14 +13,14 @@ SQL code is deployed for the calculation. """ from airflow import DAG + +from common.constants import DAG_DEFAULT_ARGS, POSTGRES_CONN_ID from popularity import sql from popularity.popularity_refresh_types import ( POPULARITY_REFRESH_CONFIGS, PopularityRefresh, ) -from common.constants import DAG_DEFAULT_ARGS, POSTGRES_CONN_ID - def create_recreate_popularity_calculation_dag(popularity_refresh: PopularityRefresh): media_type = popularity_refresh.media_type diff --git a/catalog/dags/popularity/sql.py b/catalog/dags/popularity/sql.py index bacf3bf6291..388bd02f92c 100644 --- a/catalog/dags/popularity/sql.py +++ b/catalog/dags/popularity/sql.py @@ -4,12 +4,12 @@ from airflow.decorators import task, task_group from airflow.models.abstractoperator import AbstractOperator -from popularity.popularity_refresh_types import PopularityRefresh from common.constants import DAG_DEFAULT_ARGS, SQLInfo from common.sql import PostgresHook, single_value from common.storage import columns as col from common.utils import setup_sql_info_for_media_type +from popularity.popularity_refresh_types import PopularityRefresh DEFAULT_PERCENTILE = 0.85 diff --git a/catalog/dags/providers/provider_dag_factory.py b/catalog/dags/providers/provider_dag_factory.py index 65f94b8078c..1e5a5d04068 100644 --- a/catalog/dags/providers/provider_dag_factory.py +++ b/catalog/dags/providers/provider_dag_factory.py @@ -76,9 +76,8 @@ from airflow.utils.task_group import TaskGroup from airflow.utils.trigger_rule import TriggerRule -from common.constants import AWS_CONN_ID, DAG_DEFAULT_ARGS +from common.constants import AWS_CONN_ID, DAG_DEFAULT_ARGS, XCOM_PULL_TEMPLATE from common.constants import POSTGRES_CONN_ID as DB_CONN_ID -from common.constants import XCOM_PULL_TEMPLATE from common.loader import loader, reporting, s3, sql from providers.factory_utils import date_partition_for_prefix, pull_media_wrapper from providers.provider_reingestion_workflows import ProviderReingestionWorkflow diff --git a/ingestion_server/ingestion_server/api.py b/ingestion_server/ingestion_server/api.py index dc004a8674a..7285fc695d6 100644 --- a/ingestion_server/ingestion_server/api.py +++ b/ingestion_server/ingestion_server/api.py @@ -1,10 +1,9 @@ """A small RPC API server for scheduling data refresh and indexing tasks.""" -from collections import defaultdict - import logging import os import time import uuid +from collections import defaultdict from multiprocessing import Process, Value from pathlib import Path from urllib.parse import urlparse @@ -17,8 +16,8 @@ from ingestion_server import slack from ingestion_server.constants.media_types import MEDIA_TYPES, MediaType from ingestion_server.db_helpers import ( - DB_UPSTREAM_CONFIG, DB_API_CONFIG, + DB_UPSTREAM_CONFIG, database_connect, ) from ingestion_server.es_helpers import elasticsearch_connect, get_stat diff --git a/ingestion_server/ingestion_server/cleanup.py b/ingestion_server/ingestion_server/cleanup.py index e7a5dfa3058..23f69f84390 100644 --- a/ingestion_server/ingestion_server/cleanup.py +++ b/ingestion_server/ingestion_server/cleanup.py @@ -14,8 +14,9 @@ import tldextract from psycopg2.extras import DictCursor, Json -from ingestion_server.indexer import DB_BUFFER_SIZE from ingestion_server.db_helpers import database_connect +from ingestion_server.indexer import DB_BUFFER_SIZE + # Number of records to buffer in memory at once CLEANUP_BUFFER_SIZE = DB_BUFFER_SIZE diff --git a/ingestion_server/ingestion_server/ingest.py b/ingestion_server/ingestion_server/ingest.py index f400f10ffd0..dacd7395d07 100644 --- a/ingestion_server/ingestion_server/ingest.py +++ b/ingestion_server/ingestion_server/ingest.py @@ -26,7 +26,7 @@ from ingestion_server import slack from ingestion_server.cleanup import clean_image_data from ingestion_server.constants.internal_types import ApproachType -from ingestion_server.db_helpers import database_connect, DB_UPSTREAM_CONFIG +from ingestion_server.db_helpers import DB_UPSTREAM_CONFIG, database_connect from ingestion_server.queries import ( get_copy_data_query, get_create_ext_query, diff --git a/ingestion_server/test/unit_tests/test_cleanup.py b/ingestion_server/test/unit_tests/test_cleanup.py index 8e6daa2c766..e8b8dcc10c2 100644 --- a/ingestion_server/test/unit_tests/test_cleanup.py +++ b/ingestion_server/test/unit_tests/test_cleanup.py @@ -1,9 +1,9 @@ -from test.unit_tests.conftest import create_mock_image from unittest.mock import MagicMock from psycopg2._json import Json from ingestion_server.cleanup import CleanupFunctions, TlsTest +from test.unit_tests.conftest import create_mock_image class TestCleanup: diff --git a/ingestion_server/test/unit_tests/test_es_image_model.py b/ingestion_server/test/unit_tests/test_es_image_model.py index ad79c950cc0..940261a85c6 100644 --- a/ingestion_server/test/unit_tests/test_es_image_model.py +++ b/ingestion_server/test/unit_tests/test_es_image_model.py @@ -1,6 +1,5 @@ -from test.unit_tests.conftest import create_mock_image - from ingestion_server.elasticsearch_models import Image +from test.unit_tests.conftest import create_mock_image class TestImage: diff --git a/pyproject.toml b/pyproject.toml index 3263fb57a41..cc0d058069a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,8 +10,9 @@ ignore-words = ".codespell/ignore_words.txt" # Default for `select` is ["E4", "E7", "E9", "F"] # All additional rules must be added using `extend-select`. extend-select = [ - "UP", + "UP", # pyupgrade, https://docs.astral.sh/ruff/rules/#pyupgrade-up "D", # pydocstyle, https://docs.astral.sh/ruff/rules/#pydocstyle-d + "I", # isort, https://docs.astral.sh/ruff/rules/#isort-i ] ignore = [ "D1", # D1xx: we do not want to force contributors to write redundant or useless docstrings @@ -45,10 +46,12 @@ known-first-party = [ "common", "data_refresh", "database", + "elasticsearch_cluster", "maintenance", "oauth2", "providers", "retired", + "test", ] [tool.ruff.lint.isort.sections]