diff --git a/.gitignore b/.gitignore index 24b6742..f2c979c 100644 --- a/.gitignore +++ b/.gitignore @@ -129,3 +129,4 @@ dmypy.json # editors .idea/ +source_raw_data/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..15c24eb --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "libs/pystac-monty"] + path = libs/pystac-monty + url = https://github.com/IFRCGo/pystac-monty.git diff --git a/Dockerfile b/Dockerfile index 6e4e9e7..c2eecb4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,8 @@ WORKDIR /code COPY pyproject.toml poetry.lock /code/ +COPY libs /code/libs + RUN apt-get update -y \ && apt-get install -y --no-install-recommends \ # Build required packages diff --git a/README.md b/README.md index e69de29..2ebdb58 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,25 @@ +## Getting started + +- Clone this repository: git@github.com:IFRCGo/montandon-etl.github +- Go the directory where manage.py exists. +- Create a .env file and copy all environment variable from sample.env. +- Set your own environment variables in .env file. +- Buiid docker using this command: + ```bash + docker compose up --build -d + ``` +- Run migration using this command: + ```bash + docker-compose exec web python manage.py migrate + ``` +- Command to import GDACS data. + ```bash + docker-compose exec web python manage.py import_gdacs_data + ``` +- To view the imported data in the admin panel you need to create yourself as a superuser: + ```bash + docker-compose exec web python manage.py createsuperuser + ``` + Fill up the form for creating super user. +- Once user is created, go the browser and request the link localhost:8000/admin/ to view the data in Extraction data table. +- To go to graphql server go to: localhost:8000/graphql diff --git a/apps/common/__init__.py b/apps/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/common/admin.py b/apps/common/admin.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/common/apps.py b/apps/common/apps.py new file mode 100644 index 0000000..df04969 --- /dev/null +++ b/apps/common/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class CommonConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "apps.common" diff --git a/apps/common/dataloaders.py b/apps/common/dataloaders.py new file mode 100644 index 0000000..62cb81b --- /dev/null +++ b/apps/common/dataloaders.py @@ -0,0 +1,14 @@ +import typing + +from django.db import models + +DjangoModel = typing.TypeVar("DjangoModel", bound=models.Model) + + +def load_model_objects( + Model: typing.Type[DjangoModel], + keys: list[int], +) -> list[DjangoModel]: + qs = Model.objects.filter(id__in=keys) + _map = {obj.pk: obj for obj in qs} + return [_map[key] for key in keys] diff --git a/apps/common/management/__init__.py b/apps/common/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/common/management/commands/generate_schema.py b/apps/common/management/commands/generate_schema.py new file mode 100644 index 0000000..d0e31bd --- /dev/null +++ b/apps/common/management/commands/generate_schema.py @@ -0,0 +1,23 @@ +import argparse + +from django.core.management.base import BaseCommand +from strawberry.printer import print_schema + +from main.graphql.schema import schema + + +class Command(BaseCommand): + help = "Create schema.graphql file" + + def add_arguments(self, parser): + parser.add_argument( + "--out", + type=argparse.FileType("w"), + default="schema.graphql", + ) + + def handle(self, *args, **options): + file = options["out"] + file.write(print_schema(schema)) + file.close() + self.stdout.write(self.style.SUCCESS(f"{file.name} file generated")) diff --git a/apps/common/migrations/0001_initial.py b/apps/common/migrations/0001_initial.py new file mode 100644 index 0000000..348d076 --- /dev/null +++ b/apps/common/migrations/0001_initial.py @@ -0,0 +1,37 @@ +# Generated by Django 5.1.3 on 2024-11-21 11:08 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Region', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.IntegerField(choices=[(0, 'Africa'), (1, 'Americas'), (2, 'Asia Pacific'), (3, 'Europe'), (4, 'Middle East & North Africa')], verbose_name='name')), + ], + ), + migrations.CreateModel( + name='Country', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(blank=True, max_length=255, null=True, verbose_name='name')), + ('iso3', models.CharField(blank=True, max_length=3, null=True, verbose_name='iso3')), + ('iso', models.CharField(blank=True, max_length=2, null=True, verbose_name='iso2')), + ('record_type', models.IntegerField(blank=True, choices=[(1, 'Country'), (2, 'Cluster'), (3, 'Region'), (4, 'Country Office'), (5, 'Representative Office')], help_text='Type of entity', null=True, verbose_name='type')), + ('bbox', models.JSONField(blank=True, default=dict, null=True, verbose_name='bbox')), + ('centroid', models.JSONField(blank=True, default=dict, null=True, verbose_name='centroid')), + ('independent', models.BooleanField(default=None, help_text='Is this an independent country?', null=True)), + ('is_deprecated', models.BooleanField(default=False, help_text='Is this an active, valid country?')), + ('region', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='common.region', verbose_name='region')), + ], + ), + ] diff --git a/apps/common/migrations/__init__.py b/apps/common/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/common/models.py b/apps/common/models.py new file mode 100644 index 0000000..f236a66 --- /dev/null +++ b/apps/common/models.py @@ -0,0 +1,76 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + + +class UserResource(models.Model): + created_at = models.DateTimeField(auto_now_add=True) + modified_at = models.DateTimeField(auto_now=True) + # Typing + id: int + pk: int + + class Meta: + abstract = True + ordering = ["-id"] + + +class Region(models.Model): + class RegionName(models.IntegerChoices): + AFRICA = 0, _("Africa") + AMERICAS = 1, _("Americas") + ASIA_PACIFIC = 2, _("Asia Pacific") + EUROPE = 3, _("Europe") + MENA = 4, _("Middle East & North Africa") + + name = models.IntegerField( + verbose_name=_("name"), + choices=RegionName.choices, + ) + + def __str__(self): + return f"{self.name}" + + +class Country(models.Model): + class CountryType(models.IntegerChoices): + """ + We use the Country model for some things that are not "Countries". This helps classify the type. + """ + + COUNTRY = 1, _("Country") + CLUSTER = 2, _("Cluster") + REGION = 3, _("Region") + COUNTRY_OFFICE = 4, _("Country Office") + REPRESENTATIVE_OFFICE = 5, _("Representative Office") + + name = models.CharField(max_length=255, verbose_name=_("name"), null=True, blank=True) + iso3 = models.CharField(max_length=3, verbose_name=_("iso3"), null=True, blank=True) + iso = models.CharField(max_length=2, verbose_name=_("iso2"), null=True, blank=True) + record_type = models.IntegerField( + choices=CountryType.choices, verbose_name=_("type"), null=True, blank=True, help_text=_("Type of entity") + ) + region = models.ForeignKey(Region, verbose_name=_("region"), null=True, blank=True, on_delete=models.SET_NULL) + bbox = models.JSONField( + default=dict, + null=True, + blank=True, + verbose_name=_("bbox"), + ) + centroid = models.JSONField( + default=dict, + null=True, + blank=True, + verbose_name=_("centroid"), + ) + independent = models.BooleanField(default=None, null=True, help_text=_("Is this an independent country?")) + is_deprecated = models.BooleanField(default=False, help_text=_("Is this an active, valid country?")) + + def __str__(self): + return f"{self.name} - {self.iso3}" + + def save(self, *args, **kwargs): + if self.iso3: + self.iso3 = self.iso3.lower() + if self.iso: + self.iso = self.iso.lower() + return super().save(*args, **kwargs) diff --git a/apps/common/tests.py b/apps/common/tests.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/common/types.py b/apps/common/types.py new file mode 100644 index 0000000..a78b653 --- /dev/null +++ b/apps/common/types.py @@ -0,0 +1,14 @@ +import strawberry_django +from django.contrib.auth.models import User +from strawberry import auto + + +@strawberry_django.type(User) +class UserMeType: + id: auto + username: auto + first_name: auto + last_name: auto + email: auto + is_staff: auto + is_superuser: auto diff --git a/apps/common/views.py b/apps/common/views.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/etl/__init__.py b/apps/etl/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/etl/admin.py b/apps/etl/admin.py new file mode 100644 index 0000000..1678aee --- /dev/null +++ b/apps/etl/admin.py @@ -0,0 +1,47 @@ +from django.contrib import admin + +# Register your models here. +from .models import ExtractionData, GdacsTransformation + + +@admin.register(ExtractionData) +class ExtractionDataAdmin(admin.ModelAdmin): + def get_readonly_fields(self, request, obj=None): + # Use the model's fields to populate readonly_fields + if obj: # If the object exists (edit page) + return [field.name for field in self.model._meta.fields] + return [] + + list_display = ( + "id", + "source", + "resp_code", + "status", + "parent__id", + "resp_data_type", + "source_validation_status", + "hazard_type", + "created_at", + ) + list_filter = ("status",) + autocomplete_fields = ["parent"] + search_fields = ["parent"] + + +@admin.register(GdacsTransformation) +class GdacsTransformationAdmin(admin.ModelAdmin): + def get_readonly_fields(self, request, obj=None): + # Use the model's fields to populate readonly_fields + if obj: # If the object exists (edit page) + return [field.name for field in self.model._meta.fields] + return [] + + list_display = ( + "id", + "extraction", + "item_type", + "status", + ) + list_filter = ("status",) + autocomplete_fields = ["extraction"] + search_fields = ["extraction"] diff --git a/apps/etl/apps.py b/apps/etl/apps.py new file mode 100644 index 0000000..98ad1b8 --- /dev/null +++ b/apps/etl/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class EtlConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "apps.etl" diff --git a/apps/etl/dataloaders.py b/apps/etl/dataloaders.py new file mode 100644 index 0000000..e573dc3 --- /dev/null +++ b/apps/etl/dataloaders.py @@ -0,0 +1,21 @@ +import typing + +from asgiref.sync import sync_to_async +from common.dataloaders import load_model_objects +from django.utils.functional import cached_property +from strawberry.dataloader import DataLoader + +from .models import User + +if typing.TYPE_CHECKING: + from .types import ExtractionDataType + + +def load_extraction(keys: list[int]) -> list["ExtractionDataType"]: + return load_model_objects(User, keys) # type: ignore[reportReturnType] + + +class ExtractionDataLoader: + @cached_property + def load_extraction(self): + return DataLoader(load_fn=sync_to_async(load_extraction)) diff --git a/apps/etl/enums.py b/apps/etl/enums.py new file mode 100644 index 0000000..7a62336 --- /dev/null +++ b/apps/etl/enums.py @@ -0,0 +1,9 @@ +import strawberry + +from .models import ExtractionData + +ExtractionDataStatusTypeEnum = strawberry.enum(ExtractionData.Status, name="ExtractionDataStatusTypeEnum") +ExtractionValidationTypeEnum = strawberry.enum( + ExtractionData.ValidationStatus, name="ExtractionDataValidationStatusTypeEnum" +) +ExtractionSourceTypeEnum = strawberry.enum(ExtractionData.Source, name="ExtractionDataSourceTypeEnum") diff --git a/apps/etl/extract.py b/apps/etl/extract.py new file mode 100644 index 0000000..2442448 --- /dev/null +++ b/apps/etl/extract.py @@ -0,0 +1,85 @@ +import requests +from celery.utils.log import get_task_logger +from django.core.exceptions import ObjectDoesNotExist + +from .models import ExtractionData + +logger = get_task_logger(__name__) + + +class Extraction: + def __init__(self, url: str): + self.url = url + + def _get_file_extension(self, content_type): + mappings = { + "application/json": "json", + "text/html": "html", + "application/xml": "xml", + "text/csv": "csv", + } + return mappings.get(content_type, "txt") + + def pull_data(self, source: int, retry_count: int, timeout: int = 30, ext_object_id: int = None): + resp_status = ExtractionData.Status.IN_PROGRESS + source_validation_status = ExtractionData.ValidationStatus.NO_VALIDATION + + # Update extraction object status to in_progress + if ext_object_id: + try: + instance_obj = ExtractionData.objects.get(id=ext_object_id) + instance_obj.resp_code = resp_status + instance_obj.attempt_no = retry_count + instance_obj.save(update_fields=["resp_code", "attempt_no"]) + except ExtractionData.DoesNotExist: + raise ObjectDoesNotExist("ExtractionData object with ID {ext_object_id} not found") + + try: + response = requests.get(self.url, timeout=timeout) + resp_type = response.headers.get("Content-Type", "") + file_extension = self._get_file_extension(resp_type) + + # Try saving the data in case of failure + if response.status_code != 200: + data = { + "source": source, + "url": self.url, + "attempt_no": retry_count, + "resp_code": response.status_code, + "status": ExtractionData.Status.FAILED, + "resp_data": None, + "resp_data_type": "text", + "file_extension": None, + "source_validation_status": ExtractionData.ValidationStatus.NO_VALIDATION, + "content_validation": "", + "resp_text": response.text, + } + + for key, value in data.items(): + setattr(instance_obj, key, value) + instance_obj.save() + + logger.error(f"Request failed with status {response.status_code}") + raise Exception("Request failed") + + elif response.status_code == 204: + source_validation_status = ExtractionData.ValidationStatus.NO_DATA + + resp_status = ExtractionData.Status.SUCCESS + + return { + "source": source, + "url": self.url, + "attempt_no": retry_count, + "resp_code": response.status_code, + "status": resp_status, + "resp_data": response, + "resp_data_type": resp_type, + "file_extension": file_extension, + "source_validation_status": source_validation_status, + "content_validation": "", + "resp_text": "", + } + except requests.exceptions.RequestException as e: + logger.error(f"Extraction failed for source {source}: {str(e)}") + raise Exception(f"Request failed: {e}") diff --git a/apps/etl/extraction_validators/__init__.py b/apps/etl/extraction_validators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/etl/extraction_validators/gdacs_eventsdata.py b/apps/etl/extraction_validators/gdacs_eventsdata.py new file mode 100644 index 0000000..531e8b9 --- /dev/null +++ b/apps/etl/extraction_validators/gdacs_eventsdata.py @@ -0,0 +1,111 @@ +from datetime import datetime +from typing import List, Optional, Union + +from pydantic import BaseModel, HttpUrl + + +# Nested Models +class Coordinates(BaseModel): + type: str + coordinates: List[float] + + +class URLData(BaseModel): + geometry: HttpUrl + report: HttpUrl + media: HttpUrl + + +class AffectedCountry(BaseModel): + iso3: str + countryname: str + + +class SeverityData(BaseModel): + severity: float + severitytext: str + severityunit: str + + +class EpisodeDetails(BaseModel): + details: HttpUrl + + +class SendaiData(BaseModel): + latest: bool + sendaitype: str + sendainame: str + sendaivalue: Union[int, str] + country: str + region: str + dateinsert: datetime + description: str + onset_date: datetime + expires_date: datetime + effective_date: Optional[datetime] + + +class Images(BaseModel): + populationmap: Optional[HttpUrl] = None + floodmap_cached: Optional[HttpUrl] = None + thumbnailmap_cached: Optional[HttpUrl] = None + rainmap_cached: Optional[HttpUrl] = None + overviewmap_cached: Optional[HttpUrl] = None + overviewmap: Optional[HttpUrl] = None + floodmap: Optional[HttpUrl] = None + rainmap: Optional[HttpUrl] = None + rainmap_legend: Optional[HttpUrl] = None + floodmap_legend: Optional[HttpUrl] = None + overviewmap_legend: Optional[HttpUrl] = None + rainimage: Optional[HttpUrl] = None + meteoimages: Optional[HttpUrl] = None + mslpimages: Optional[HttpUrl] = None + event_icon_map: Optional[HttpUrl] = None + event_icon: Optional[HttpUrl] = None + thumbnailmap: Optional[HttpUrl] = None + npp_icon: Optional[HttpUrl] = None + + +# Main Schema +class FeatureProperties(BaseModel): + eventtype: str + eventid: int + episodeid: int + eventname: Optional[str] + glide: Optional[str] + name: str + description: str + htmldescription: str + icon: Optional[HttpUrl] + iconoverall: Optional[str] + url: URLData + alertlevel: str + alertscore: float + episodealertlevel: str + episodealertscore: float + istemporary: str + iscurrent: str + country: str + fromdate: datetime + todate: datetime + datemodified: datetime + iso3: str + source: str + sourceid: str + polygonlabel: str + Class: str + affectedcountries: List[AffectedCountry] + severitydata: SeverityData + episodes: List[EpisodeDetails] + sendai: Optional[List[SendaiData]] = None + impacts: List[dict] + images: Images + additionalinfos: dict + documents: dict + + +class GDacsEventDataValidator(BaseModel): + type: str + bbox: List[float] + geometry: Coordinates + properties: FeatureProperties diff --git a/apps/etl/extraction_validators/gdacs_geometry.py b/apps/etl/extraction_validators/gdacs_geometry.py new file mode 100644 index 0000000..6a70036 --- /dev/null +++ b/apps/etl/extraction_validators/gdacs_geometry.py @@ -0,0 +1,70 @@ +from datetime import datetime +from typing import List, Optional, Union + +from pydantic import BaseModel, HttpUrl + + +class URLDetails(BaseModel): + geometry: HttpUrl + report: HttpUrl + details: HttpUrl + + +class SeverityData(BaseModel): + severity: float + severitytext: str + severityunit: str + + +class AffectedCountry(BaseModel): + iso3: str + countryname: str + + +class Properties(BaseModel): + eventtype: str + eventid: int + episodeid: int + eventname: str + glide: Optional[str] + name: str + description: str + htmldescription: str + icon: HttpUrl + iconoverall: HttpUrl + url: URLDetails + alertlevel: str + alertscore: float + episodealertlevel: str + episodealertscore: float + istemporary: str + iscurrent: str + country: str + fromdate: datetime + todate: datetime + datemodified: datetime + iso3: str + source: str + sourceid: str + polygonlabel: str + Class: str + country: str + affectedcountries: List[AffectedCountry] + severitydata: SeverityData + + +class Geometry(BaseModel): + type: str + coordinates: Union[List[float], List[List[List[float]]], List[List[List[List[float]]]]] + + +class Feature(BaseModel): + type: str + bbox: List[float] + geometry: Geometry + properties: Properties + + +class GdacsEventsGeometryData(BaseModel): + type: str + features: List[Feature] diff --git a/apps/etl/extraction_validators/gdacs_main_source.py b/apps/etl/extraction_validators/gdacs_main_source.py new file mode 100644 index 0000000..a118928 --- /dev/null +++ b/apps/etl/extraction_validators/gdacs_main_source.py @@ -0,0 +1,69 @@ +from typing import List, Optional + +from pydantic import BaseModel, HttpUrl + + +class Urls(BaseModel): + geometry: HttpUrl + report: HttpUrl + details: HttpUrl + + +class SeverityData(BaseModel): + severity: float + severitytext: str + severityunit: str + + +class AffectedCountry(BaseModel): + iso3: str + countryname: str + + +class Properties(BaseModel): + eventtype: str + eventid: int + episodeid: int + eventname: str + glide: Optional[str] + name: str + description: str + htmldescription: str + icon: HttpUrl + iconoverall: HttpUrl + url: Urls + alertlevel: str + alertscore: float + episodealertlevel: str + episodealertscore: float + istemporary: str + iscurrent: str + country: str + fromdate: str + todate: str + datemodified: str + iso3: str + source: str + sourceid: Optional[str] + polygonlabel: str + Class: str + country: str + affectedcountries: List[AffectedCountry] + severitydata: SeverityData + + +class Geometry(BaseModel): + type: str + coordinates: List[float] + + +class Feature(BaseModel): + type: str + bbox: List[float] + geometry: Geometry + properties: Properties + + +class GdacsEventSourceValidator(BaseModel): + type: str + features: List[Feature] diff --git a/apps/etl/extraction_validators/gdacs_pop_exposure.py b/apps/etl/extraction_validators/gdacs_pop_exposure.py new file mode 100644 index 0000000..c17a21f --- /dev/null +++ b/apps/etl/extraction_validators/gdacs_pop_exposure.py @@ -0,0 +1,20 @@ +from typing import Optional, Union + +from pydantic import BaseModel + + +class GdacsPopulationExposureEQTC(BaseModel): + exposed_population: Optional[str] + + +class GdacsPopulationExposure_FL(BaseModel): + death: int + displaced: Optional[Union[int, str]] + + +class GdacsPopulationExposureDR(BaseModel): + impact: str + + +class GdacsPopulationExposureWF(BaseModel): + people_affected: str diff --git a/apps/etl/filters.py b/apps/etl/filters.py new file mode 100644 index 0000000..24394c6 --- /dev/null +++ b/apps/etl/filters.py @@ -0,0 +1,35 @@ +from typing import Optional + +import strawberry +import strawberry_django +from django.db import models + +from .enums import ExtractionDataStatusTypeEnum, ExtractionSourceTypeEnum +from .models import ExtractionData + + +@strawberry_django.filters.filter(ExtractionData, lookups=True) +class ExtractionDataFilter: + source: Optional[ExtractionSourceTypeEnum] + status: Optional[ExtractionDataStatusTypeEnum] + created_at: strawberry.auto + + @strawberry_django.filter_field + def created_at_lte( + self, + queryset: models.QuerySet, + value: strawberry.ID, + prefix: str, + ) -> tuple[models.QuerySet, models.Q]: + + return queryset, models.Q(**{f"{prefix}created_at__lte": value}) + + @strawberry_django.filter_field + def created_at_gte( + self, + queryset: models.QuerySet, + value: strawberry.ID, + prefix: str, + ) -> tuple[models.QuerySet, models.Q]: + + return queryset, models.Q(**{f"{prefix}created_at__gte": value}) diff --git a/apps/etl/loaders.py b/apps/etl/loaders.py new file mode 100644 index 0000000..1382cd4 --- /dev/null +++ b/apps/etl/loaders.py @@ -0,0 +1,60 @@ +import time +import uuid + +import requests +from celery import chain, shared_task +from celery.result import AsyncResult +from celery.utils.log import get_task_logger + +logger = get_task_logger(__name__) + + +@shared_task(bind=True, autoretry_for=(requests.exceptions.RequestException,), retry_kwargs={"max_retries": 3}) +def send_post_request_to_stac_api(self, result, collection_id): + try: + url = f"http://montandon-eoapi-stage.ifrc.org/stac/collections/{collection_id}/items" + response = requests.post( + url, json=result, headers={"Content-Type": "application/json"} # Send result as JSON payload + ) + response.raise_for_status() # Raise an exception for HTTP errors + logger.info(f"POST Response for {collection_id}:", response.status_code, response.text) + except requests.exceptions.RequestException as e: + logger.info(f"Error posting data for {collection_id}: {e}") + + +@shared_task(bind=True) +def process_load_data(self, task_id, task_name): + while True: + result = AsyncResult(task_id) + if result.state == "SUCCESS": + result = result.result + break + elif result.state == "FAILURE": + raise Exception(f"Fetching {task_name} data failed with error: {result.result}") + time.sleep(2) + + if not result == [] or {}: + if "properties" not in result: + result["properties"] = {} + result["properties"]["monty:etl_id"] = str(uuid.uuid4()) + result["id"] = f"{task_name}-{uuid.uuid4()}" # Done for testing purpose to make id unique. + + send_post_request_to_stac_api.delay(result, f"{task_name}") + + return "Data loaded successfully" + + +@shared_task(bind=True) +def load_data(self, event_result_id, geo_result_id, impact_result_id): + """Load data by chaining tasks instead of blocking calls.""" + try: + # Create a chain of tasks for processing and posting data + chain( + process_load_data.si(event_result_id, "gdacs-events"), + process_load_data.si(geo_result_id, "gdacs-hazards"), + process_load_data.si(impact_result_id, "gdacs-impacts"), + )() + return "Data loading process started successfully." + except Exception as e: + logger.error(f"Error loading data: {e}") + raise diff --git a/apps/etl/management/commands/__init__.py b/apps/etl/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/etl/management/commands/import_gdacs_data.py b/apps/etl/management/commands/import_gdacs_data.py new file mode 100644 index 0000000..30cf071 --- /dev/null +++ b/apps/etl/management/commands/import_gdacs_data.py @@ -0,0 +1,19 @@ +import logging + +from django.core.management.base import BaseCommand + +from apps.etl.models import HazardType +from apps.etl.tasks import import_hazard_data + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Import data from gdacs api" + + def handle(self, *args, **options): + import_hazard_data.delay("EQ", HazardType.EARTHQUAKE) + import_hazard_data.delay("TC", HazardType.CYCLONE) + import_hazard_data.delay("FL", HazardType.FLOOD) + import_hazard_data.delay("DR", HazardType.DROUGHT) + import_hazard_data.delay("WF", HazardType.WILDFIRE) diff --git a/apps/etl/migrations/0001_initial.py b/apps/etl/migrations/0001_initial.py new file mode 100644 index 0000000..cd325a9 --- /dev/null +++ b/apps/etl/migrations/0001_initial.py @@ -0,0 +1,35 @@ +# Generated by Django 5.1.3 on 2024-11-19 09:44 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='ExtractionData', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('source', models.IntegerField(choices=[(1, 'Gdacs'), (2, 'pdc')], verbose_name='source')), + ('url', models.URLField(blank=True, verbose_name='url')), + ('attempt_no', models.IntegerField(blank=True, verbose_name='attempt number')), + ('resp_code', models.IntegerField(blank=True, verbose_name='response code')), + ('status', models.IntegerField(choices=[(1, 'Pending'), (2, 'In progress'), (3, 'Success'), (4, 'Failed')], verbose_name='status')), + ('resp_data', models.FileField(blank=True, null=True, upload_to='source_raw_data/', verbose_name='resp_data')), + ('resp_type', models.IntegerField(blank=True, choices=[(1, 'Json'), (2, 'CSV'), (3, 'Text'), (4, 'Html'), (5, 'XML'), (6, 'pdf')], null=True, verbose_name='response type')), + ('parent', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='child_extraction', to='etl.extractiondata')), + ], + options={ + 'ordering': ['-id'], + 'abstract': False, + }, + ), + ] diff --git a/apps/etl/migrations/0002_extractiondata_source_validation_fail_reason_and_more.py b/apps/etl/migrations/0002_extractiondata_source_validation_fail_reason_and_more.py new file mode 100644 index 0000000..ec1cead --- /dev/null +++ b/apps/etl/migrations/0002_extractiondata_source_validation_fail_reason_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 5.1.3 on 2024-11-21 11:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('etl', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='extractiondata', + name='source_validation_fail_reason', + field=models.TextField(blank=True, verbose_name='validation status fail reason'), + ), + migrations.AddField( + model_name='extractiondata', + name='source_validation_status', + field=models.IntegerField(choices=[(1, 'Success'), (2, 'Failed')], default=1, verbose_name='validation status'), + preserve_default=False, + ), + ] diff --git a/apps/etl/migrations/0003_extractiondata_resp_data_type.py b/apps/etl/migrations/0003_extractiondata_resp_data_type.py new file mode 100644 index 0000000..a2a50e3 --- /dev/null +++ b/apps/etl/migrations/0003_extractiondata_resp_data_type.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.3 on 2024-11-22 11:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('etl', '0002_extractiondata_source_validation_fail_reason_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='extractiondata', + name='resp_data_type', + field=models.CharField(blank=True, verbose_name='response data type'), + ), + ] diff --git a/apps/etl/migrations/0004_alter_extractiondata_resp_type_and_more.py b/apps/etl/migrations/0004_alter_extractiondata_resp_type_and_more.py new file mode 100644 index 0000000..dc2092a --- /dev/null +++ b/apps/etl/migrations/0004_alter_extractiondata_resp_type_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 5.1.3 on 2024-11-26 12:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('etl', '0003_extractiondata_resp_data_type'), + ] + + operations = [ + migrations.AlterField( + model_name='extractiondata', + name='resp_type', + field=models.IntegerField(blank=True, choices=[(1, 'json'), (2, 'csv'), (3, 'text'), (4, 'html'), (5, 'xml'), (6, 'pdf')], null=True, verbose_name='response type'), + ), + migrations.AlterField( + model_name='extractiondata', + name='source', + field=models.IntegerField(choices=[(1, 'GDACS'), (2, 'PDC')], verbose_name='source'), + ), + migrations.AlterField( + model_name='extractiondata', + name='source_validation_status', + field=models.IntegerField(choices=[(1, 'Success'), (2, 'Failed'), (3, 'No data'), (4, 'No change')], verbose_name='validation status'), + ), + ] diff --git a/apps/etl/migrations/0005_rename_source_validation_fail_reason_extractiondata_content_validation.py b/apps/etl/migrations/0005_rename_source_validation_fail_reason_extractiondata_content_validation.py new file mode 100644 index 0000000..8d7e42a --- /dev/null +++ b/apps/etl/migrations/0005_rename_source_validation_fail_reason_extractiondata_content_validation.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.3 on 2024-11-27 05:26 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('etl', '0004_alter_extractiondata_resp_type_and_more'), + ] + + operations = [ + migrations.RenameField( + model_name='extractiondata', + old_name='source_validation_fail_reason', + new_name='content_validation', + ), + ] diff --git a/apps/etl/migrations/0006_extractiondata_file_hash_extractiondata_revision_id_and_more.py b/apps/etl/migrations/0006_extractiondata_file_hash_extractiondata_revision_id_and_more.py new file mode 100644 index 0000000..cf97901 --- /dev/null +++ b/apps/etl/migrations/0006_extractiondata_file_hash_extractiondata_revision_id_and_more.py @@ -0,0 +1,34 @@ +# Generated by Django 5.1.3 on 2024-11-28 04:47 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('etl', '0005_rename_source_validation_fail_reason_extractiondata_content_validation'), + ] + + operations = [ + migrations.AddField( + model_name='extractiondata', + name='file_hash', + field=models.CharField(blank=True, max_length=500, verbose_name='file hash value'), + ), + migrations.AddField( + model_name='extractiondata', + name='revision_id', + field=models.ForeignKey(blank=True, help_text='This id points to the extraction object having same file content', null=True, on_delete=django.db.models.deletion.CASCADE, to='etl.extractiondata', verbose_name='revision id'), + ), + migrations.AlterField( + model_name='extractiondata', + name='resp_data', + field=models.FileField(blank=True, null=True, upload_to='source_raw_data/', verbose_name='response data'), + ), + migrations.AlterField( + model_name='extractiondata', + name='source_validation_status', + field=models.IntegerField(choices=[(1, 'Success'), (2, 'Failed'), (3, 'No data'), (4, 'No change'), (5, 'No validation')], verbose_name='source data validation status'), + ), + ] diff --git a/apps/etl/migrations/0007_extractiondata_resp_text.py b/apps/etl/migrations/0007_extractiondata_resp_text.py new file mode 100644 index 0000000..9b70f41 --- /dev/null +++ b/apps/etl/migrations/0007_extractiondata_resp_text.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.3 on 2024-12-09 11:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('etl', '0006_extractiondata_file_hash_extractiondata_revision_id_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='extractiondata', + name='resp_text', + field=models.TextField(blank=True, verbose_name='response data in case failure occurs'), + ), + ] diff --git a/apps/etl/migrations/0008_extractiondata_hazard_type.py b/apps/etl/migrations/0008_extractiondata_hazard_type.py new file mode 100644 index 0000000..950f981 --- /dev/null +++ b/apps/etl/migrations/0008_extractiondata_hazard_type.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.3 on 2024-12-12 12:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('etl', '0007_extractiondata_resp_text'), + ] + + operations = [ + migrations.AddField( + model_name='extractiondata', + name='hazard_type', + field=models.CharField(blank=True, choices=[('EQ', 'Earthquake'), ('FL', 'Flood'), ('TC', 'Cyclone'), ('EP', 'Epidemic'), ('FI', 'Food Insecurity'), ('SS', 'Storm Surge'), ('DR', 'Drought'), ('TS', 'Tsunami'), ('CD', 'Cyclonic Wind'), ('WF', 'WildFire')], max_length=100, verbose_name='hazard type'), + ), + ] diff --git a/apps/etl/migrations/0009_gdacstransformation.py b/apps/etl/migrations/0009_gdacstransformation.py new file mode 100644 index 0000000..53d91db --- /dev/null +++ b/apps/etl/migrations/0009_gdacstransformation.py @@ -0,0 +1,31 @@ +# Generated by Django 5.1.3 on 2024-12-25 16:01 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('etl', '0008_extractiondata_hazard_type'), + ] + + operations = [ + migrations.CreateModel( + name='GdacsTransformation', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('item_type', models.CharField(choices=[(1, 'Event'), (2, 'Hazard'), (3, 'Impact')])), + ('data', models.JSONField(default=dict)), + ('status', models.CharField(choices=[(1, 'Failed'), (2, 'Success')])), + ('failed_reason', models.TextField(blank=True)), + ('extraction', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='etl.extractiondata')), + ], + options={ + 'ordering': ['-id'], + 'abstract': False, + }, + ), + ] diff --git a/apps/etl/migrations/0010_alter_gdacstransformation_item_type_and_more.py b/apps/etl/migrations/0010_alter_gdacstransformation_item_type_and_more.py new file mode 100644 index 0000000..707396d --- /dev/null +++ b/apps/etl/migrations/0010_alter_gdacstransformation_item_type_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 5.1.3 on 2024-12-25 16:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('etl', '0009_gdacstransformation'), + ] + + operations = [ + migrations.AlterField( + model_name='gdacstransformation', + name='item_type', + field=models.IntegerField(choices=[(1, 'Event'), (2, 'Hazard'), (3, 'Impact')]), + ), + migrations.AlterField( + model_name='gdacstransformation', + name='status', + field=models.IntegerField(choices=[(1, 'Failed'), (2, 'Success')]), + ), + ] diff --git a/apps/etl/migrations/__init__.py b/apps/etl/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/etl/models.py b/apps/etl/models.py new file mode 100644 index 0000000..41a149e --- /dev/null +++ b/apps/etl/models.py @@ -0,0 +1,94 @@ +# Create your models here. +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from apps.common.models import UserResource + + +class HazardType(models.TextChoices): + EARTHQUAKE = "EQ", "Earthquake" + FLOOD = "FL", "Flood" + CYCLONE = "TC", "Cyclone" + EPIDEMIC = "EP", "Epidemic" + FOOD_INSECURITY = "FI", "Food Insecurity" + STORM = "SS", "Storm Surge" + DROUGHT = "DR", "Drought" + TSUNAMI = "TS", "Tsunami" + WIND = "CD", "Cyclonic Wind" + WILDFIRE = "WF", "WildFire" + + +class ExtractionData(UserResource): + class ValidationStatus(models.IntegerChoices): + SUCCESS = 1, _("Success") + FAILED = 2, _("Failed") + NO_DATA = 3, _("No data") + NO_CHANGE = 4, _("No change") + NO_VALIDATION = 5, _("No validation") + + class ResponseDataType(models.IntegerChoices): + JSON = 1, _("json") + CSV = 2, _("csv") + TEXT = 3, _("text") + HTML = 4, _("html") + XML = 5, _("xml") + PDF = 6, _("pdf") + + class Source(models.IntegerChoices): + GDACS = 1, _("GDACS") + PDC = 2, _("PDC") + + class Status(models.IntegerChoices): + PENDING = 1, _("Pending") + IN_PROGRESS = 2, _("In progress") + SUCCESS = 3, _("Success") + FAILED = 4, _("Failed") + + source = models.IntegerField(verbose_name=_("source"), choices=Source.choices) + url = models.URLField(verbose_name=_("url"), blank=True) + attempt_no = models.IntegerField(verbose_name=_("attempt number"), blank=True) + resp_code = models.IntegerField(verbose_name=_("response code"), blank=True) + status = models.IntegerField(verbose_name=_("status"), choices=Status.choices) + resp_data = models.FileField(verbose_name=_("response data"), upload_to="source_raw_data/", blank=True, null=True) + file_hash = models.CharField( + verbose_name=_("file hash value"), + max_length=500, + blank=True, + ) + resp_type = models.IntegerField(verbose_name=_("response type"), choices=ResponseDataType.choices, blank=True, null=True) + resp_data_type = models.CharField(verbose_name=_("response data type"), blank=True) + resp_text = models.TextField(verbose_name=_("response data in case failure occurs"), blank=True) + parent = models.ForeignKey("self", on_delete=models.PROTECT, null=True, blank=True, related_name="child_extraction") + source_validation_status = models.IntegerField( + verbose_name=_("source data validation status"), choices=ValidationStatus.choices + ) + content_validation = models.TextField(verbose_name=_("validation status fail reason"), blank=True) + revision_id = models.ForeignKey( + "self", + verbose_name=_("revision id"), + help_text="This id points to the extraction object having same file content", + on_delete=models.CASCADE, + null=True, + blank=True, + ) + hazard_type = models.CharField(max_length=100, verbose_name=_("hazard type"), choices=HazardType.choices, blank=True) + + def __str__(self): + return str(self.id) + + +class GdacsTransformation(UserResource): + class ItemType(models.IntegerChoices): + EVENT = 1, "Event" + HAZARD = 2, "Hazard" + IMPACT = 3, "Impact" + + class TransformationStatus(models.IntegerChoices): + FAILED = 1, "Failed" + SUCCESS = 2, "Success" + + extraction = models.ForeignKey(ExtractionData, on_delete=models.PROTECT, null=True, blank=True) + item_type = models.IntegerField(choices=ItemType.choices) + data = models.JSONField(default=dict) + status = models.IntegerField(choices=TransformationStatus.choices) + failed_reason = models.TextField(blank=True) diff --git a/apps/etl/queries.py b/apps/etl/queries.py new file mode 100644 index 0000000..bc40d52 --- /dev/null +++ b/apps/etl/queries.py @@ -0,0 +1,24 @@ +import strawberry +from asgiref.sync import sync_to_async +from strawberry.types import Info + +from apps.common.types import UserMeType +from apps.etl.types import ExtractionDataType +from main.graphql.paginations import CountList, pagination_field + +from .filters import ExtractionDataFilter + + +@strawberry.type +class PrivateQuery: + extraction_list: CountList[ExtractionDataType] = pagination_field( + pagination=True, + filters=ExtractionDataFilter, + ) + + @strawberry.field + @sync_to_async + def me(self, info: Info) -> UserMeType | None: + user = info.context.request.user + if user.is_authenticated: + return user # type: ignore[reportGeneralTypeIssues] diff --git a/apps/etl/tasks.py b/apps/etl/tasks.py new file mode 100644 index 0000000..59ea6c0 --- /dev/null +++ b/apps/etl/tasks.py @@ -0,0 +1,435 @@ +import hashlib +import json +import logging +import typing +from datetime import datetime, timedelta + +import numpy as np +import pandas as pd +import requests +from celery import chain, shared_task +from django.core.files.base import ContentFile +from django.core.management import call_command +from pydantic import ValidationError + +from apps.etl.extract import Extraction +from apps.etl.extraction_validators.gdacs_eventsdata import GDacsEventDataValidator +from apps.etl.extraction_validators.gdacs_geometry import GdacsEventsGeometryData +from apps.etl.extraction_validators.gdacs_main_source import GdacsEventSourceValidator +from apps.etl.extraction_validators.gdacs_pop_exposure import ( + GdacsPopulationExposure_FL, + GdacsPopulationExposureDR, + GdacsPopulationExposureEQTC, + GdacsPopulationExposureWF, +) +from apps.etl.loaders import load_data +from apps.etl.models import ExtractionData, HazardType +from apps.etl.transformer import ( + transform_event_data, + transform_geo_data, + transform_impact_data, +) + +logger = logging.getLogger(__name__) + + +@shared_task +def fetch_gdacs_data(): + call_command("import_gdacs_data") + + +def get_as_int(value: typing.Optional[str]) -> typing.Optional[int]: + if value is None: + return + if value == "-": + return + return int(value) + + +def validate_source_data(resp_data): + try: + resp_data_for_validation = json.loads(resp_data.decode("utf-8")) + GdacsEventSourceValidator(**resp_data_for_validation) + validation_error = "" + except ValidationError as e: + validation_error = e.json() + validation_data = { + "status": ExtractionData.ValidationStatus.FAILED if validation_error else ExtractionData.ValidationStatus.SUCCESS, + "validation_error": validation_error if validation_error else "", + } + return validation_data + + +def validate_event_data(resp_data): + try: + resp_data_for_validation = json.loads(resp_data.decode("utf-8")) + GDacsEventDataValidator(**resp_data_for_validation) + validation_error = "" + except ValidationError as e: + validation_error = e.json() + validation_data = { + "status": ExtractionData.ValidationStatus.FAILED if validation_error else ExtractionData.ValidationStatus.SUCCESS, + "validation_error": validation_error if validation_error else "", + } + return validation_data + + +def validate_population_exposure(html_content, hazard_type=None): + tables = pd.read_html(html_content) + population_exposure = {} + + displacement_data_raw = tables[0].replace({np.nan: None}).to_dict() + displacement_data = dict( + zip( + displacement_data_raw[0].values(), # First column are keys + displacement_data_raw[1].values(), # Second column are values + ) + ) + + validation_error = "" + try: + if hazard_type == "EQ": + population_exposure["exposed_population"] = displacement_data.get("Exposed Population:") + GdacsPopulationExposureEQTC(**population_exposure) + + elif hazard_type == "TC": + population_exposure["exposed_population"] = displacement_data.get("Exposed population") + GdacsPopulationExposureEQTC(**population_exposure) + + elif hazard_type == "FL": + population_exposure["death"] = get_as_int(displacement_data.get("Death:")) + population_exposure["displaced"] = get_as_int(displacement_data.get("Displaced:")) + GdacsPopulationExposure_FL(**population_exposure) + + elif hazard_type == "DR": + population_exposure["impact"] = displacement_data.get("Impact:") + GdacsPopulationExposureDR(**population_exposure) + + elif hazard_type == "WF": + population_exposure["people_affected"] = displacement_data.get("People affected:") + GdacsPopulationExposureWF(**population_exposure) + + except ValidationError as e: + validation_error = e.json() + + validation_data = { + "status": ExtractionData.ValidationStatus.FAILED if validation_error else ExtractionData.ValidationStatus.SUCCESS, + "validation_error": validation_error, + } + return validation_data + + +def validate_gdacs_geometry_data(resp_data): + try: + resp_data_for_validation = json.loads(resp_data.decode("utf-8")) + GdacsEventsGeometryData(**resp_data_for_validation) + validation_error = "" + except ValidationError as e: + validation_error = e.json() + validation_data = { + "status": ExtractionData.ValidationStatus.FAILED if validation_error else ExtractionData.ValidationStatus.SUCCESS, + "validation_error": validation_error if validation_error else "", + } + return validation_data + + +def manage_duplicate_file_content(source, hash_content, instance, response_data, file_name): + """ + if duplicate file content exists then do not create a new file, but point the url to + the previous file. + """ + duplicate_file_content = ExtractionData.objects.filter(source=source, file_hash=hash_content) + if duplicate_file_content: + instance.resp_data = duplicate_file_content.first().resp_data + instance.revision_id = duplicate_file_content.first() + else: + instance.resp_data.save(file_name, ContentFile(response_data)) + instance.save() + + +def hash_file_content(content): + """ + Compute the hash of a file using the specified algorithm. + :return: Hexadecimal hash of the file + """ + file_hash = hashlib.sha256(content).hexdigest() + return file_hash + + +def store_extraction_data( + response, validate_source_func, parent_id=None, instance_id=None, hazard_type=None, requires_hazard_type=False +): + file_extension = response.pop("file_extension") + file_name = f"gdacs.{file_extension}" + resp_data = response.pop("resp_data") + + # save the additional response data after the data is fetched from api. + gdacs_instance = ExtractionData.objects.get(id=instance_id) + for key, value in response.items(): + setattr(gdacs_instance, key, value) + gdacs_instance.save() + + # save parent id if it is child extraction object + if parent_id: + gdacs_instance.parent_id = parent_id + gdacs_instance.save(update_fields=["parent_id"]) + + # Validate the non empty response data. + if resp_data and not response["resp_code"] == 204: + resp_data_content = resp_data.content + # Source validation + # if the validate function requires hazard type as argument pass it as argument else don't. + if requires_hazard_type: + gdacs_instance.source_validation_status = validate_source_func(resp_data_content, hazard_type)["status"] + gdacs_instance.content_validation = validate_source_func(resp_data_content, hazard_type)["validation_error"] + else: + gdacs_instance.source_validation_status = validate_source_func(resp_data_content)["status"] + gdacs_instance.content_validation = validate_source_func(resp_data_content)["validation_error"] + + # manage duplicate file content. + hash_content = hash_file_content(resp_data_content) + manage_duplicate_file_content( + source=ExtractionData.Source.GDACS, + hash_content=hash_content, + instance=gdacs_instance, + response_data=resp_data_content, + file_name=file_name, + ) + return gdacs_instance + + +@shared_task(bind=True, max_retries=3, default_retry_delay=5) +def fetch_event_data(self, parent_id, event_id: int, hazard_type: str, **kwargs): + # url = f"https://www.gdacs.org/report.aspx?eventid={event_id}&eventtype={hazard_type}" + url = f"https://www.gdacs.org/gdacsapi/api/events/geteventdata?eventtype={hazard_type}&eventid={event_id}" + + # instance_id is passed in this func in kwargs during retry from self.retry() method. + # It forbids creating new extraction object during retry. + instance_id = kwargs.get("instance_id", None) + hazard_type = ExtractionData.objects.get(id=parent_id).hazard_type + if not instance_id: + gdacs_instance = ExtractionData.objects.create( + source=ExtractionData.Source.GDACS, + status=ExtractionData.Status.PENDING, + source_validation_status=ExtractionData.ValidationStatus.NO_VALIDATION, + attempt_no=0, + resp_code=0, + hazard_type=hazard_type, + ) + else: + gdacs_instance = ExtractionData.objects.get(id=instance_id) + + # Extract the data from api. + gdacs_extraction = Extraction(url=url) + response = None + try: + response = gdacs_extraction.pull_data( + source=ExtractionData.Source.GDACS, + ext_object_id=gdacs_instance.id, + retry_count=0, + ) + except Exception as exc: + self.retry(exc=exc, kwargs={"instance_id": gdacs_instance.id, "retry_count": self.request.retries}) + + # Save the extracted data into the existing gdacs object + if response: + gdacs_instance = store_extraction_data( + response=response, + validate_source_func=validate_event_data, + instance_id=gdacs_instance.id, + parent_id=parent_id, + requires_hazard_type=False, + hazard_type=hazard_type, + ) + with open(gdacs_instance.resp_data.path, "r") as file: + data = file.read() + + return {"extraction_id": gdacs_instance.id, "extracted_data": data} + + +@shared_task(bind=True, max_retries=3, default_retry_delay=5) +def scrape_population_exposure_data(self, parent_id, event_id: int, hazard_type: str, parent_transform_id: str, **kwargs): + url = f"https://www.gdacs.org/report.aspx?eventid={event_id}&eventtype={hazard_type}" + + # instance_id is passed in this func in kwargs during retry from self.retry() method. + # It forbids creating new extraction object during retry. + instance_id = kwargs.get("instance_id", None) + hazard_type = ExtractionData.objects.get(id=parent_id).hazard_type + if not instance_id: + gdacs_instance = ExtractionData.objects.create( + source=ExtractionData.Source.GDACS, + status=ExtractionData.Status.PENDING, + source_validation_status=ExtractionData.ValidationStatus.NO_VALIDATION, + attempt_no=0, + resp_code=0, + hazard_type=hazard_type, + ) + else: + gdacs_instance = ExtractionData.objects.get(id=instance_id) + + # Extract the data from api. + gdacs_extraction = Extraction(url=url) + response = None + try: + response = gdacs_extraction.pull_data( + source=ExtractionData.Source.GDACS, + ext_object_id=gdacs_instance.id, + retry_count=0, + ) + except Exception as exc: + self.retry(exc=exc, kwargs={"instance_id": gdacs_instance.id, "retry_count": self.request.retries}) + + # Save the extracted data into the existing gdacs object + if response: + gdacs_instance = store_extraction_data( + response=response, + validate_source_func=validate_population_exposure, + instance_id=gdacs_instance.id, + parent_id=parent_id, + requires_hazard_type=True, + hazard_type=hazard_type, + ) + + +@shared_task(bind=True, max_retries=3, default_retry_delay=5) +def fetch_gdacs_geometry_data(self, parent_id, footprint_url, **kwargs): + + # instance_id is passed in this func in kwargs during retry from self.retry() method. + # It forbids creating new extraction object during retry. + instance_id = kwargs.get("instance_id", None) + hazard_type = ExtractionData.objects.get(id=parent_id).hazard_type + if not instance_id: + gdacs_instance = ExtractionData.objects.create( + source=ExtractionData.Source.GDACS, + status=ExtractionData.Status.PENDING, + source_validation_status=ExtractionData.ValidationStatus.NO_VALIDATION, + attempt_no=0, + resp_code=0, + hazard_type=hazard_type, + ) + else: + gdacs_instance = ExtractionData.objects.get(id=instance_id) + + gdacs_extraction = Extraction(url=footprint_url) + response = None + try: + response = gdacs_extraction.pull_data( + source=ExtractionData.Source.GDACS, + ext_object_id=gdacs_instance.id, + retry_count=0, + ) + except Exception as exc: + self.retry(exc=exc, kwargs={"instance_id": gdacs_instance.id, "retry_count": self.request.retries}) + + if response: + gdacs_instance = store_extraction_data( + response=response, + validate_source_func=validate_gdacs_geometry_data, + instance_id=gdacs_instance.id, + parent_id=parent_id, + ) + + with open(gdacs_instance.resp_data.path, "r") as file: + data = file.read() + + return {"extraction_id": gdacs_instance.id, "extracted_data": data} + + +@shared_task(bind=True, max_retries=3, default_retry_delay=5) +def import_hazard_data(self, hazard_type: str, hazard_type_str: str, **kwargs): + """ + Import hazard data from gdacs api + """ + logger.info(f"Importing {hazard_type} data") + + today = datetime.now().date() + yesterday = today - timedelta(days=1) + gdacs_url = f"https://www.gdacs.org/gdacsapi/api/events/geteventlist/SEARCH?eventlist={hazard_type}&fromDate={yesterday}&toDate={today}&alertlevel=Green;Orange;Red" # noqa: E501 + + # Create a Extraction object in the begining + instance_id = kwargs.get("instance_id", None) + retry_count = kwargs.get("retry_count", None) + + gdacs_instance = ( + ExtractionData.objects.get(id=instance_id) + if instance_id + else ExtractionData.objects.create( + source=ExtractionData.Source.GDACS, + status=ExtractionData.Status.PENDING, + source_validation_status=ExtractionData.ValidationStatus.NO_VALIDATION, + hazard_type=hazard_type_str, + attempt_no=0, + resp_code=0, + ) + ) + + # Extract the data from api. + gdacs_extraction = Extraction(url=gdacs_url) + response = None + try: + response = gdacs_extraction.pull_data( + source=ExtractionData.Source.GDACS, + ext_object_id=gdacs_instance.id, + retry_count=retry_count if retry_count else 1, + ) + except requests.exceptions.RequestException as exc: + self.retry(exc=exc, kwargs={"instance_id": gdacs_instance.id, "retry_count": self.request.retries}) + + if response: + resp_data_content = response["resp_data"].content + + # decode the byte object(response data) into json + try: + resp_data_json = json.loads(resp_data_content.decode("utf-8")) + except json.JSONDecodeError as e: + logger.info(f"JSON decode error: {e}") + resp_data_json = {} + + # Save the extracted data into the existing gdacs object + gdacs_instance = store_extraction_data( + response=response, + validate_source_func=validate_source_data, + instance_id=gdacs_instance.id, + ) + + # Fetch geometry and population exposure data + if gdacs_instance.resp_code == 200 and gdacs_instance.status == ExtractionData.Status.SUCCESS and resp_data_json: + for feature in resp_data_json["features"]: + event_id = feature["properties"]["eventid"] + episode_id = feature["properties"]["episodeid"] + footprint_url = feature["properties"]["url"]["geometry"] + if hazard_type == HazardType.CYCLONE and event_id and episode_id: + footprint_url = f"https://www.gdacs.org/contentdata/resources/{hazard_type_str}/{event_id}/geojson_{event_id}_{episode_id}.geojson" # noqa: E501 + + event_workflow = chain( + fetch_event_data.s( + parent_id=gdacs_instance.id, + event_id=event_id, + hazard_type=hazard_type, + ), + transform_event_data.s(), + ) + event_result = event_workflow.apply_async() + + geo_workflow = chain( + fetch_gdacs_geometry_data.s( + parent_id=gdacs_instance.id, + footprint_url=footprint_url, + ), + transform_geo_data.s(event_result.parent.id), + ) + geo_result = geo_workflow.apply_async() + + impact_workflow = chain( + fetch_event_data.s( + parent_id=gdacs_instance.id, + event_id=event_id, + hazard_type=hazard_type, + ), + transform_impact_data.s(), + ) + impact_result = impact_workflow.apply_async() + + load_data.s(event_result.id, geo_result.id, impact_result.id).apply_async() + + logger.info(f"{hazard_type} data imported sucessfully") diff --git a/apps/etl/tests.py b/apps/etl/tests.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/etl/transformer.py b/apps/etl/transformer.py new file mode 100644 index 0000000..cb3c64e --- /dev/null +++ b/apps/etl/transformer.py @@ -0,0 +1,175 @@ +import logging +import time + +from celery import shared_task +from celery.result import AsyncResult +from pystac_monty.sources.gdacs import ( + GDACSDataSource, + GDACSDataSourceType, + GDACSTransformer, +) + +from apps.etl.models import ExtractionData, GdacsTransformation + +logger = logging.getLogger(__name__) + + +@shared_task +def transform_event_data(data): + logger.info("Trandformation started for event data") + + gdacs_instance = ExtractionData.objects.get(id=data["extraction_id"]) + + data_file_path = gdacs_instance.resp_data.path # Absolute file path + + try: + with open(data_file_path, "r") as file: + data = file.read() + except FileNotFoundError: + logger.error(f"File not found: {data_file_path}") + raise + except IOError as e: + logger.error(f"I/O error while reading file: {str(e)}") + raise + + transformer = GDACSTransformer( + [GDACSDataSource(type=GDACSDataSourceType.EVENT, source_url=gdacs_instance.url, data=data)] + ) + + transformed_item_dict = {} + try: + transformed_event_item = transformer.make_source_event_item() + transformed_item_dict = transformed_event_item.to_dict() + + GdacsTransformation.objects.create( + extraction=gdacs_instance, + item_type=GdacsTransformation.ItemType.EVENT, + data=transformed_item_dict, + status=GdacsTransformation.TransformationStatus.SUCCESS, + failed_reason="", + ) + except Exception as e: + GdacsTransformation.objects.create( + extraction=gdacs_instance, + item_type=GdacsTransformation.ItemType.EVENT, + status=GdacsTransformation.TransformationStatus.FAILED, + failed_reason=str(e), + ) + + if transformed_item_dict: + logger.info("Trandformation ended for event data") + return transformed_item_dict + else: + raise Exception("Transformation failed. Check logs for details.") + + +@shared_task +def transform_geo_data(geo_data, event_task_id): + logger.info("Transformation started for hazard data") + + timeout = 300 # 5 minutes + start_time = time.time() + while True: + result = AsyncResult(event_task_id) + if result.state == "SUCCESS": + # Fetch the output of event task + event_data = result.result + break + elif result.state == "FAILURE": + raise Exception(f"Fetching event data failed with error: {result.result}") + elif time.time() - start_time > timeout: + raise TimeoutError("Fetching event data timed out.") + time.sleep(1) + + gdacs_instance = ExtractionData.objects.get(id=geo_data["extraction_id"]) + data_file_path = gdacs_instance.resp_data.path # Absolute file path + + try: + with open(data_file_path, "r") as file: + data = file.read() + except FileNotFoundError: + logger.error(f"File not found: {data_file_path}") + raise + except IOError as e: + logger.error(f"I/O error while reading file: {str(e)}") + raise + + transformer = GDACSTransformer( + [ + GDACSDataSource( + type=GDACSDataSourceType.EVENT, source_url=gdacs_instance.url, data=event_data["extracted_data"] + ), + GDACSDataSource(type=GDACSDataSourceType.GEOMETRY, source_url=gdacs_instance.url, data=data), + ] + ) + transformed_item_dict = {} + try: + transformed_geo_item = transformer.make_hazard_event_item() + transformed_item_dict = transformed_geo_item.to_dict() + + GdacsTransformation.objects.create( + extraction=gdacs_instance, + item_type=GdacsTransformation.ItemType.HAZARD, + data=transformed_item_dict, + status=GdacsTransformation.TransformationStatus.SUCCESS, + failed_reason="", + ) + except Exception as e: + GdacsTransformation.objects.create( + extraction=gdacs_instance, + item_type=GdacsTransformation.ItemType.HAZARD, + status=GdacsTransformation.TransformationStatus.FAILED, + failed_reason=str(e), + ) + + if transformed_item_dict: + logger.info("Transformation ended for hazard data") + return transformed_item_dict + else: + raise Exception("Transformation failed. Check logs for details.") + + +@shared_task +def transform_impact_data(event_data): + logger.info("Transformation started for impact data") + + gdacs_instance = ExtractionData.objects.get(id=event_data["extraction_id"]) + data_file_path = gdacs_instance.resp_data.path # Absolute file path + try: + with open(data_file_path, "r") as file: + data = file.read() + except FileNotFoundError: + logger.error(f"File not found: {data_file_path}") + raise + except IOError as e: + logger.error(f"I/O error while reading file: {str(e)}") + raise + transformer = GDACSTransformer( + [GDACSDataSource(type=GDACSDataSourceType.EVENT, source_url=gdacs_instance.url, data=data)] + ) + + transformed_item_dict = {"data": []} + try: + transformed_impact_item = transformer.make_impact_items() + transformed_item_dict = {"data": transformed_impact_item} + + GdacsTransformation.objects.create( + extraction=gdacs_instance, + item_type=GdacsTransformation.ItemType.IMPACT, + data=transformed_item_dict, + status=GdacsTransformation.TransformationStatus.SUCCESS, + failed_reason="", + ) + except Exception as e: + GdacsTransformation.objects.create( + extraction=gdacs_instance, + item_type=GdacsTransformation.ItemType.IMPACT, + status=GdacsTransformation.TransformationStatus.FAILED, + failed_reason=str(e), + ) + + if transformed_item_dict: + logger.info("Transformation ended for impact data") + return transformed_item_dict + else: + raise Exception("Transformation failed. Check logs for details.") diff --git a/apps/etl/types.py b/apps/etl/types.py new file mode 100644 index 0000000..3ddd838 --- /dev/null +++ b/apps/etl/types.py @@ -0,0 +1,27 @@ +from typing import Optional + +import strawberry_django +from strawberry import auto + +from .enums import ExtractionDataStatusTypeEnum, ExtractionSourceTypeEnum +from .models import ExtractionData + + +@strawberry_django.type(ExtractionData) +class ExtractionDataType: + id: auto + source: ExtractionSourceTypeEnum + url: auto + resp_code: auto + status: ExtractionDataStatusTypeEnum + resp_data_type: auto + parent_id: Optional[int] + source_validation_status: auto + revision_id: Optional[int] + hazard_type: auto + + def resolve_parent(self, root): + return root.parent.id + + def resolve_revision_id(self, root): + return root.revision_id.id diff --git a/apps/etl/views.py b/apps/etl/views.py new file mode 100644 index 0000000..e69de29 diff --git a/docker-compose.yml b/docker-compose.yml index 0a48872..b38fc80 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,32 @@ +x-server: &base_server_setup + restart: unless-stopped + image: etl-montandon/server + build: + context: ./ + # Used for python debugging. + stdin_open: true + tty: true + env_file: + - .env + environment: + DJANGO_DEBUG: ${DJANGO_DEBUG:-true} + DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY:?err} + DB_NAME: ${DB_NAME:-postgres} + DB_USER: ${DB_USER:-postgres} + DB_PASSWORD: ${DB_PASSWORD:-postgres} + DB_HOST: ${DB_HOST:-db} + DB_PORT: ${DB_PORT:-5432} + CELERY_REDIS_URL: ${CELERY_REDIS_URL:-redis://redis:6379/0} + DJANGO_CACHE_REDIS_URL: ${DJANGO_CACHE_REDIS_URL:-redis://redis:6379/1} + APP_DOMAIN: localhost:8000 + APP_HTTP_PROTOCOL: ${APP_HTTP_PROTOCOL:-http} + + volumes: + - .:/code + depends_on: + - db + - redis + services: db: image: postgis/postgis:15-3.5-alpine @@ -9,28 +38,31 @@ services: volumes: - postgres-data:/var/lib/postgresql/data + redis: + image: redis:6-alpine + volumes: + - redis-data:/data + web: - restart: unless-stopped - image: etl-montandon/server - build: - context: ./ - tty: true - environment: - DJANGO_DEBUG: ${DJANGO_DEBUG:-true} - DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY:?err} - DB_NAME: ${DB_NAME:-postgres} - DB_USER: ${DB_USER:-postgres} - DB_PASSWORD: ${DB_PASSWORD:-postgres} - DB_HOST: ${DB_HOST:-db} - DB_PORT: ${DB_PORT:-5432} + <<: *base_server_setup command: bash -c "/code/scripts/run_develop.sh" - volumes: - - ./:/code ports: - - '8000:8000' + - 8000:8000 + depends_on: - db + - redis + + celery-beat: + <<: *base_server_setup + command: bash -c "/code/scripts/run_worker_beat.sh" + + worker: + <<: *base_server_setup + # command: celery -A main worker --loglevel=info + command: bash -c "/code/scripts/run_worker.sh" volumes: postgres-data: + redis-data: diff --git a/libs/pystac-monty b/libs/pystac-monty new file mode 160000 index 0000000..753ada4 --- /dev/null +++ b/libs/pystac-monty @@ -0,0 +1 @@ +Subproject commit 753ada466aeed886e16f79f0e5f639bb4009bf1b diff --git a/main/__init__.py b/main/__init__.py index e69de29..53f4ccb 100644 --- a/main/__init__.py +++ b/main/__init__.py @@ -0,0 +1,3 @@ +from .celery import app as celery_app + +__all__ = ("celery_app",) diff --git a/main/celery.py b/main/celery.py new file mode 100644 index 0000000..dc94ff2 --- /dev/null +++ b/main/celery.py @@ -0,0 +1,13 @@ +import os + +from celery import Celery + +# Set the default Django settings module for the 'celery' program. +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "main.settings") + +app = Celery("main") + +app.config_from_object("django.conf:settings", namespace="CELERY") + +# Load task modules from all registered Django app configs. +app.autodiscover_tasks() diff --git a/main/graphql/context.py b/main/graphql/context.py new file mode 100644 index 0000000..18d37c3 --- /dev/null +++ b/main/graphql/context.py @@ -0,0 +1,16 @@ +from dataclasses import dataclass + +from strawberry.django.context import StrawberryDjangoContext +from strawberry.types import Info as _Info + +from .dataloaders import GlobalDataLoader + + +@dataclass +class GraphQLContext(StrawberryDjangoContext): + dl: GlobalDataLoader + + +# NOTE: This is for type support only, There is a better way? +class Info(_Info): + context: GraphQLContext diff --git a/main/graphql/dataloaders.py b/main/graphql/dataloaders.py new file mode 100644 index 0000000..9ada4c7 --- /dev/null +++ b/main/graphql/dataloaders.py @@ -0,0 +1,2 @@ +class GlobalDataLoader: + pass diff --git a/main/graphql/paginations.py b/main/graphql/paginations.py new file mode 100644 index 0000000..9eaa743 --- /dev/null +++ b/main/graphql/paginations.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +from functools import cached_property +from typing import Any, Callable, Generic, Type, TypeVar + +import strawberry +from asgiref.sync import sync_to_async +from django.conf import settings +from django.db import models +from strawberry.types import Info +from strawberry_django.fields.field import StrawberryDjangoField +from strawberry_django.filters import apply as apply_filters +from strawberry_django.ordering import apply as apply_orders +from strawberry_django.pagination import ( + OffsetPaginationInput, + StrawberryDjangoPagination, +) +from strawberry_django.resolvers import django_resolver +from strawberry_django.utils.typing import unwrap_type + + +def process_pagination(pagination: OffsetPaginationInput): + """ + Mutate pagination object to make sure limit are under given threshold + """ + if pagination is strawberry.UNSET or pagination is None: + pagination = OffsetPaginationInput( + offset=0, + limit=settings.STRAWBERRY_DEFAULT_PAGINATION_LIMIT, + ) + if pagination.limit == -1: + pagination.limit = settings.STRAWBERRY_DEFAULT_PAGINATION_LIMIT + pagination.limit = min(pagination.limit, settings.STRAWBERRY_MAX_PAGINATION_LIMIT) + return pagination + + +def apply_pagination(pagination, queryset): + pagination = process_pagination(pagination) + start = pagination.offset + stop = start + pagination.limit + return queryset[start:stop] + + +class CountBeforePaginationMonkeyPatch(StrawberryDjangoPagination): + def get_queryset( + self, + queryset: models.QuerySet[Any], + info, + pagination=strawberry.UNSET, + **kwargs, + ) -> models.QuerySet: + queryset = apply_pagination(pagination, queryset) + return super(StrawberryDjangoPagination, self).get_queryset( + queryset, + info, + **kwargs, + ) + + +StrawberryDjangoPagination.get_queryset = ( # type: ignore[reportGeneralTypeIssues] + CountBeforePaginationMonkeyPatch.get_queryset +) +OffsetPaginationInput.limit = 1 # TODO: This is not working + +DjangoModelTypeVar = TypeVar("DjangoModelTypeVar") + + +@strawberry.type +class CountList(Generic[DjangoModelTypeVar]): + limit: int + offset: int + queryset: strawberry.Private[ + models.QuerySet[DjangoModelTypeVar] | list[DjangoModelTypeVar] # type: ignore[reportGeneralTypeIssues] + ] + get_count: strawberry.Private[Callable] + + @strawberry.field + async def count(self) -> int: + return await self.get_count() + + @strawberry.field + async def items(self) -> list[DjangoModelTypeVar]: + queryset = self.queryset + if type(queryset) in [list, tuple]: + return list(queryset) + return [d async for d in queryset] # type: ignore[reportGeneralTypeIssues] + + +class StrawberryDjangoCountList(StrawberryDjangoField): + @cached_property + def is_list(self) -> bool: + return True + + @cached_property + def django_model(self) -> type[models.Model] | None: + super().django_model + # Hack to get the nested type of `CountList` to register + # as the type of this field + items_type = [ + f.type + for f in self.type.__strawberry_definition__.fields # type: ignore[reportGeneralTypeIssues] + if f.name == "items" + ] + if len(items_type) > 0: + type_ = unwrap_type(items_type[0]) + self._base_type = type_ + return type_.__strawberry_django_definition__.model # type: ignore[reportGeneralTypeIssues] + return None + + def get_result( + self, + source: models.Model | None, + info: Any, + args: list[Any], + kwargs: dict[str, Any], + ): + return self.resolver(source, info, args, kwargs) + + def resolver( + self, + source: Any, + info: Info | None, + args: list[Any], + kwargs: dict[str, Any], + ) -> Any: + pk: int = kwargs.get("pk", strawberry.UNSET) + filters: Type = kwargs.get("filters", strawberry.UNSET) + order: Type = kwargs.get("order", strawberry.UNSET) + pagination: OffsetPaginationInput = kwargs.get("pagination", strawberry.UNSET) + + if self.django_model is None or self._base_type is None: + # This needs to be fixed by developers + raise Exception("django_model should be defined!!") + + queryset = self.django_model.objects.all() + + type_ = self._base_type + type_ = unwrap_type(type_) + get_queryset = getattr(type_, "get_queryset", None) + if get_queryset: + queryset = get_queryset(type_, queryset, info) + + queryset = apply_filters(filters, queryset, info, pk) + + queryset = apply_orders(order, queryset, info=info) + # Add a default order_by id if there is none defined/used + if not queryset.query.order_by: + queryset = queryset.order_by("-pk") + + _current_queryset = queryset._chain() # type: ignore[reportGeneralTypeIssues] + + @sync_to_async + def get_count(): + return _current_queryset.values("pk").count() + + pagination = process_pagination(pagination) + + queryset = self.apply_pagination(queryset, pagination) + return CountList[self._base_type]( # type: ignore[reportGeneralTypeIssues] + get_count=get_count, + queryset=queryset, + limit=pagination.limit, + offset=pagination.offset, + ) + + +def pagination_field( + resolver=None, + *, + name=None, + field_name=None, + filters=strawberry.UNSET, + default=strawberry.UNSET, + **kwargs, +) -> Any: + field_ = StrawberryDjangoCountList( + python_name=None, + graphql_name=name, + type_annotation=None, + filters=filters, + django_name=field_name, + default=default, + **kwargs, + ) + if resolver: + resolver = django_resolver(resolver) + return field_(resolver) + return field_ diff --git a/main/graphql/permission.py b/main/graphql/permission.py new file mode 100644 index 0000000..eaae273 --- /dev/null +++ b/main/graphql/permission.py @@ -0,0 +1,14 @@ +import typing + +from asgiref.sync import sync_to_async +from strawberry.permission import BasePermission +from strawberry.types import Info + + +class IsAuthenticated(BasePermission): + message = "User is not authenticated" + + @sync_to_async + def has_permission(self, source: typing.Any, info: Info, **_) -> bool: + user = info.context.request.user + return bool(user and user.is_authenticated) diff --git a/main/graphql/schema.py b/main/graphql/schema.py new file mode 100644 index 0000000..4a094e3 --- /dev/null +++ b/main/graphql/schema.py @@ -0,0 +1,57 @@ +import strawberry +from strawberry.django.views import AsyncGraphQLView + +from apps.etl import queries as etl_queries + +from .context import GraphQLContext +from .dataloaders import GlobalDataLoader +from .permission import IsAuthenticated + + +class CustomAsyncGraphQLView(AsyncGraphQLView): + async def get_context(self, *args, **kwargs) -> GraphQLContext: + return GraphQLContext( + *args, + **kwargs, + dl=GlobalDataLoader(), + ) + + +@strawberry.type +class PublicQuery: + id: strawberry.ID = strawberry.ID("public") + + +@strawberry.type +class PrivateQuery(etl_queries.PrivateQuery): + id: strawberry.ID = strawberry.ID("private") + + +@strawberry.type +class PublicMutation: + id: strawberry.ID = strawberry.ID("public") + + +@strawberry.type +class PrivateMutation: + id: strawberry.ID = strawberry.ID("private") + + +@strawberry.type +class Query: + public: PublicQuery = strawberry.field(resolver=lambda: PublicQuery()) + private: PrivateQuery = strawberry.field(permission_classes=[IsAuthenticated], resolver=lambda: PrivateQuery()) + + +@strawberry.type +class Mutation: + public: PublicMutation = strawberry.field(resolver=lambda: PublicMutation()) + private: PrivateMutation = strawberry.field( + resolver=lambda: PrivateMutation(), + permission_classes=[IsAuthenticated], + ) + + +schema = strawberry.Schema( + query=Query, +) diff --git a/main/settings.py b/main/settings.py index 3939a91..9dd4a1c 100644 --- a/main/settings.py +++ b/main/settings.py @@ -10,9 +10,11 @@ https://docs.djangoproject.com/en/5.1/ref/settings/ """ +import os from pathlib import Path import environ +from celery.schedules import crontab # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -27,8 +29,18 @@ DB_PASSWORD=str, DB_HOST=str, DB_PORT=int, + DJANGO_TIME_ZONE=(str, "UTC"), + # Redis + CELERY_REDIS_URL=str, + DJANGO_STATIC_ROOT=(str, os.path.join(BASE_DIR, "assets/static")), # Where to store + DJANGO_STATIC_URL=(str, "/static/"), + APP_DOMAIN=str, + APP_HTTP_PROTOCOL=str, + DJANGO_CORS_ORIGIN_REGEX_WHITELIST=(list, []), ) +TIME_ZONE = env("DJANGO_TIME_ZONE") + SECRET_KEY = env("DJANGO_SECRET_KEY") # SECURITY WARNING: don't run with debug turned on in production! @@ -39,6 +51,20 @@ ALLOWED_HOSTS = env("DJANGO_ALLOWED_HOST") +# Redis +CELERY_REDIS_URL = env("CELERY_REDIS_URL") + +# Celery +CELERY_BROKER_URL = CELERY_REDIS_URL +CELERY_RESULT_BACKEND = CELERY_REDIS_URL +CELERY_TIMEZONE = TIME_ZONE +CELERY_EVENT_QUEUE_PREFIX = "etl-celery-" +CELERY_ACKS_LATE = True +CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True + +# Strawberry +STRAWBERRY_DEFAULT_PAGINATION_LIMIT = 50 +STRAWBERRY_MAX_PAGINATION_LIMIT = 100 # Application definition INSTALLED_APPS = [ @@ -48,6 +74,11 @@ "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", + # third party apps + "corsheaders", + # internal apps + "apps.common", + "apps.etl", ] MIDDLEWARE = [ @@ -130,9 +161,59 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/5.1/howto/static-files/ -STATIC_URL = "static/" +STATIC_URL = env("DJANGO_STATIC_URL") + +MEDIA_URL = "/media/" +MEDIA_ROOT = BASE_DIR / "media" # Default primary key field type # https://docs.djangoproject.com/en/5.1/ref/settings/#default-auto-field DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + +if not env("DJANGO_CORS_ORIGIN_REGEX_WHITELIST"): + CORS_ORIGIN_ALLOW_ALL = True +else: + # Example ^https://[\w-]+\.mapswipe\.org$ + CORS_ORIGIN_REGEX_WHITELIST = env("DJANGO_CORS_ORIGIN_REGEX_WHITELIST") + +CORS_ALLOW_CREDENTIALS = True +CORS_URLS_REGEX = r"(^/media/.*$)|(^/graphql/$)" +CSRF_COOKIE_NAME = "ifrc-monty-csrftoken" +CORS_ALLOW_METHODS = ( + "DELETE", + "GET", + "OPTIONS", + "PATCH", + "POST", + "PUT", +) + +CORS_ALLOW_HEADERS = ( + "accept", + "accept-encoding", + "authorization", + "content-type", + "dnt", + "origin", + "user-agent", + "x-csrftoken", + "x-requested-with", + "sentry-trace", +) + +APP_HTTP_PROTOCOL = env("APP_HTTP_PROTOCOL") +APP_DOMAIN = env("APP_DOMAIN") + +if APP_HTTP_PROTOCOL == "https": + CSRF_TRUSTED_ORIGINS = [ + f"{APP_HTTP_PROTOCOL}://{APP_DOMAIN}", + ] + + +CELERY_BEAT_SCHEDULE = { + "import_gdacs_data": { + "task": "apps.etl.tasks.fetch_gdacs_data", + "schedule": crontab(minute=0, hour=0), # This task execute daily at 12 AM (UTC) + } +} diff --git a/main/urls.py b/main/urls.py index 1eaff61..863481a 100644 --- a/main/urls.py +++ b/main/urls.py @@ -15,9 +15,20 @@ 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ +from django.conf import settings +from django.conf.urls.static import static from django.contrib import admin from django.urls import path +from django.views.decorators.csrf import csrf_exempt + +from main.graphql.schema import CustomAsyncGraphQLView, schema urlpatterns = [ path("admin/", admin.site.urls), ] + +if settings.DEBUG: + urlpatterns.append(path("graphiql/", csrf_exempt(CustomAsyncGraphQLView.as_view(schema=schema)))) + # Static and media file URLs + urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) + urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) diff --git a/poetry.lock b/poetry.lock index 0089a59..55c0873 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,29 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. + +[[package]] +name = "amqp" +version = "5.3.1" +description = "Low-level AMQP client for Python (fork of amqplib)." +optional = false +python-versions = ">=3.6" +files = [ + {file = "amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2"}, + {file = "amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432"}, +] + +[package.dependencies] +vine = ">=5.0.0,<6.0.0" + +[[package]] +name = "annotated-types" +version = "0.7.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, + {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, +] [[package]] name = "asgiref" @@ -14,6 +39,183 @@ files = [ [package.extras] tests = ["mypy (>=0.800)", "pytest", "pytest-asyncio"] +[[package]] +name = "beautifulsoup4" +version = "4.12.3" +description = "Screen-scraping library" +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"}, + {file = "beautifulsoup4-4.12.3.tar.gz", hash = "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051"}, +] + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +cchardet = ["cchardet"] +chardet = ["chardet"] +charset-normalizer = ["charset-normalizer"] +html5lib = ["html5lib"] +lxml = ["lxml"] + +[[package]] +name = "billiard" +version = "4.2.1" +description = "Python multiprocessing fork with improvements and bugfixes" +optional = false +python-versions = ">=3.7" +files = [ + {file = "billiard-4.2.1-py3-none-any.whl", hash = "sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb"}, + {file = "billiard-4.2.1.tar.gz", hash = "sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f"}, +] + +[[package]] +name = "celery" +version = "5.4.0" +description = "Distributed Task Queue." +optional = false +python-versions = ">=3.8" +files = [ + {file = "celery-5.4.0-py3-none-any.whl", hash = "sha256:369631eb580cf8c51a82721ec538684994f8277637edde2dfc0dacd73ed97f64"}, + {file = "celery-5.4.0.tar.gz", hash = "sha256:504a19140e8d3029d5acad88330c541d4c3f64c789d85f94756762d8bca7e706"}, +] + +[package.dependencies] +billiard = ">=4.2.0,<5.0" +click = ">=8.1.2,<9.0" +click-didyoumean = ">=0.3.0" +click-plugins = ">=1.1.1" +click-repl = ">=0.2.0" +kombu = ">=5.3.4,<6.0" +python-dateutil = ">=2.8.2" +redis = {version = ">=4.5.2,<4.5.5 || >4.5.5,<6.0.0", optional = true, markers = "extra == \"redis\""} +tzdata = ">=2022.7" +vine = ">=5.1.0,<6.0" + +[package.extras] +arangodb = ["pyArango (>=2.0.2)"] +auth = ["cryptography (==42.0.5)"] +azureblockblob = ["azure-storage-blob (>=12.15.0)"] +brotli = ["brotli (>=1.0.0)", "brotlipy (>=0.7.0)"] +cassandra = ["cassandra-driver (>=3.25.0,<4)"] +consul = ["python-consul2 (==0.1.5)"] +cosmosdbsql = ["pydocumentdb (==2.3.5)"] +couchbase = ["couchbase (>=3.0.0)"] +couchdb = ["pycouchdb (==1.14.2)"] +django = ["Django (>=2.2.28)"] +dynamodb = ["boto3 (>=1.26.143)"] +elasticsearch = ["elastic-transport (<=8.13.0)", "elasticsearch (<=8.13.0)"] +eventlet = ["eventlet (>=0.32.0)"] +gcs = ["google-cloud-storage (>=2.10.0)"] +gevent = ["gevent (>=1.5.0)"] +librabbitmq = ["librabbitmq (>=2.0.0)"] +memcache = ["pylibmc (==1.6.3)"] +mongodb = ["pymongo[srv] (>=4.0.2)"] +msgpack = ["msgpack (==1.0.8)"] +pymemcache = ["python-memcached (>=1.61)"] +pyro = ["pyro4 (==4.82)"] +pytest = ["pytest-celery[all] (>=1.0.0)"] +redis = ["redis (>=4.5.2,!=4.5.5,<6.0.0)"] +s3 = ["boto3 (>=1.26.143)"] +slmq = ["softlayer-messaging (>=1.0.3)"] +solar = ["ephem (==4.1.5)"] +sqlalchemy = ["sqlalchemy (>=1.4.48,<2.1)"] +sqs = ["boto3 (>=1.26.143)", "kombu[sqs] (>=5.3.4)", "pycurl (>=7.43.0.5)", "urllib3 (>=1.26.16)"] +tblib = ["tblib (>=1.3.0)", "tblib (>=1.5.0)"] +yaml = ["PyYAML (>=3.10)"] +zookeeper = ["kazoo (>=1.3.1)"] +zstd = ["zstandard (==0.22.0)"] + +[[package]] +name = "click" +version = "8.1.7" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "click-didyoumean" +version = "0.3.1" +description = "Enables git-like *did-you-mean* feature in click" +optional = false +python-versions = ">=3.6.2" +files = [ + {file = "click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c"}, + {file = "click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463"}, +] + +[package.dependencies] +click = ">=7" + +[[package]] +name = "click-plugins" +version = "1.1.1" +description = "An extension module for click to enable registering CLI commands via setuptools entry-points." +optional = false +python-versions = "*" +files = [ + {file = "click-plugins-1.1.1.tar.gz", hash = "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b"}, + {file = "click_plugins-1.1.1-py2.py3-none-any.whl", hash = "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8"}, +] + +[package.dependencies] +click = ">=4.0" + +[package.extras] +dev = ["coveralls", "pytest (>=3.6)", "pytest-cov", "wheel"] + +[[package]] +name = "click-repl" +version = "0.3.0" +description = "REPL plugin for Click" +optional = false +python-versions = ">=3.6" +files = [ + {file = "click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9"}, + {file = "click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812"}, +] + +[package.dependencies] +click = ">=7.0" +prompt-toolkit = ">=3.0.36" + +[package.extras] +testing = ["pytest (>=7.2.1)", "pytest-cov (>=4.0.0)", "tox (>=4.4.3)"] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "cron-descriptor" +version = "1.4.5" +description = "A Python library that converts cron expressions into human readable strings." +optional = false +python-versions = "*" +files = [ + {file = "cron_descriptor-1.4.5-py3-none-any.whl", hash = "sha256:736b3ae9d1a99bc3dbfc5b55b5e6e7c12031e7ba5de716625772f8b02dcd6013"}, + {file = "cron_descriptor-1.4.5.tar.gz", hash = "sha256:f51ce4ffc1d1f2816939add8524f206c376a42c87a5fca3091ce26725b3b1bca"}, +] + +[package.extras] +dev = ["polib"] + [[package]] name = "django" version = "5.1.3" @@ -34,6 +236,40 @@ tzdata = {version = "*", markers = "sys_platform == \"win32\""} argon2 = ["argon2-cffi (>=19.1.0)"] bcrypt = ["bcrypt"] +[[package]] +name = "django-celery-beat" +version = "2.7.0" +description = "Database-backed Periodic Tasks." +optional = false +python-versions = ">=3.8" +files = [ + {file = "django_celery_beat-2.7.0-py3-none-any.whl", hash = "sha256:851c680d8fbf608ca5fecd5836622beea89fa017bc2b3f94a5b8c648c32d84b1"}, + {file = "django_celery_beat-2.7.0.tar.gz", hash = "sha256:8482034925e09b698c05ad61c36ed2a8dbc436724a3fe119215193a4ca6dc967"}, +] + +[package.dependencies] +celery = ">=5.2.3,<6.0" +cron-descriptor = ">=1.2.32" +Django = ">=2.2,<5.2" +django-timezone-field = ">=5.0" +python-crontab = ">=2.3.4" +tzdata = "*" + +[[package]] +name = "django-cors-headers" +version = "4.6.0" +description = "django-cors-headers is a Django application for handling the server headers required for Cross-Origin Resource Sharing (CORS)." +optional = false +python-versions = ">=3.9" +files = [ + {file = "django_cors_headers-4.6.0-py3-none-any.whl", hash = "sha256:8edbc0497e611c24d5150e0055d3b178c6534b8ed826fb6f53b21c63f5d48ba3"}, + {file = "django_cors_headers-4.6.0.tar.gz", hash = "sha256:14d76b4b4c8d39375baeddd89e4f08899051eeaf177cb02a29bd6eae8cf63aa8"}, +] + +[package.dependencies] +asgiref = ">=3.6" +django = ">=4.2" + [[package]] name = "django-environ" version = "0.11.2" @@ -50,6 +286,38 @@ develop = ["coverage[toml] (>=5.0a4)", "furo (>=2021.8.17b43,<2021.9.dev0)", "py docs = ["furo (>=2021.8.17b43,<2021.9.dev0)", "sphinx (>=3.5.0)", "sphinx-notfound-page"] testing = ["coverage[toml] (>=5.0a4)", "pytest (>=4.6.11)"] +[[package]] +name = "django-redis" +version = "5.4.0" +description = "Full featured redis cache backend for Django." +optional = false +python-versions = ">=3.6" +files = [ + {file = "django-redis-5.4.0.tar.gz", hash = "sha256:6a02abaa34b0fea8bf9b707d2c363ab6adc7409950b2db93602e6cb292818c42"}, + {file = "django_redis-5.4.0-py3-none-any.whl", hash = "sha256:ebc88df7da810732e2af9987f7f426c96204bf89319df4c6da6ca9a2942edd5b"}, +] + +[package.dependencies] +Django = ">=3.2" +redis = ">=3,<4.0.0 || >4.0.0,<4.0.1 || >4.0.1" + +[package.extras] +hiredis = ["redis[hiredis] (>=3,!=4.0.0,!=4.0.1)"] + +[[package]] +name = "django-timezone-field" +version = "7.0" +description = "A Django app providing DB, form, and REST framework fields for zoneinfo and pytz timezone objects." +optional = false +python-versions = "<4.0,>=3.8" +files = [ + {file = "django_timezone_field-7.0-py3-none-any.whl", hash = "sha256:3232e7ecde66ba4464abb6f9e6b8cc739b914efb9b29dc2cf2eee451f7cc2acb"}, + {file = "django_timezone_field-7.0.tar.gz", hash = "sha256:aa6f4965838484317b7f08d22c0d91a53d64e7bbbd34264468ae83d4023898a7"}, +] + +[package.dependencies] +Django = ">=3.2,<6.0" + [[package]] name = "flake8" version = "7.1.1" @@ -66,6 +334,230 @@ mccabe = ">=0.7.0,<0.8.0" pycodestyle = ">=2.12.0,<2.13.0" pyflakes = ">=3.2.0,<3.3.0" +[[package]] +name = "geojson" +version = "3.1.0" +description = "Python bindings and utilities for GeoJSON" +optional = false +python-versions = ">=3.7" +files = [ + {file = "geojson-3.1.0-py3-none-any.whl", hash = "sha256:68a9771827237adb8c0c71f8527509c8f5bef61733aa434cefc9c9d4f0ebe8f3"}, + {file = "geojson-3.1.0.tar.gz", hash = "sha256:58a7fa40727ea058efc28b0e9ff0099eadf6d0965e04690830208d3ef571adac"}, +] + +[[package]] +name = "graphql-core" +version = "3.2.5" +description = "GraphQL implementation for Python, a port of GraphQL.js, the JavaScript reference implementation for GraphQL." +optional = false +python-versions = "<4,>=3.6" +files = [ + {file = "graphql_core-3.2.5-py3-none-any.whl", hash = "sha256:2f150d5096448aa4f8ab26268567bbfeef823769893b39c1a2e1409590939c8a"}, + {file = "graphql_core-3.2.5.tar.gz", hash = "sha256:e671b90ed653c808715645e3998b7ab67d382d55467b7e2978549111bbabf8d5"}, +] + +[[package]] +name = "kombu" +version = "5.4.2" +description = "Messaging library for Python." +optional = false +python-versions = ">=3.8" +files = [ + {file = "kombu-5.4.2-py3-none-any.whl", hash = "sha256:14212f5ccf022fc0a70453bb025a1dcc32782a588c49ea866884047d66e14763"}, + {file = "kombu-5.4.2.tar.gz", hash = "sha256:eef572dd2fd9fc614b37580e3caeafdd5af46c1eff31e7fba89138cdb406f2cf"}, +] + +[package.dependencies] +amqp = ">=5.1.1,<6.0.0" +tzdata = {version = "*", markers = "python_version >= \"3.9\""} +vine = "5.1.0" + +[package.extras] +azureservicebus = ["azure-servicebus (>=7.10.0)"] +azurestoragequeues = ["azure-identity (>=1.12.0)", "azure-storage-queue (>=12.6.0)"] +confluentkafka = ["confluent-kafka (>=2.2.0)"] +consul = ["python-consul2 (==0.1.5)"] +librabbitmq = ["librabbitmq (>=2.0.0)"] +mongodb = ["pymongo (>=4.1.1)"] +msgpack = ["msgpack (==1.1.0)"] +pyro = ["pyro4 (==4.82)"] +qpid = ["qpid-python (>=0.26)", "qpid-tools (>=0.26)"] +redis = ["redis (>=4.5.2,!=4.5.5,!=5.0.2)"] +slmq = ["softlayer-messaging (>=1.0.3)"] +sqlalchemy = ["sqlalchemy (>=1.4.48,<2.1)"] +sqs = ["boto3 (>=1.26.143)", "pycurl (>=7.43.0.5)", "urllib3 (>=1.26.16)"] +yaml = ["PyYAML (>=3.10)"] +zookeeper = ["kazoo (>=2.8.0)"] + +[[package]] +name = "lxml" +version = "5.3.0" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +optional = false +python-versions = ">=3.6" +files = [ + {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656"}, + {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:501d0d7e26b4d261fca8132854d845e4988097611ba2531408ec91cf3fd9d20a"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb66442c2546446944437df74379e9cf9e9db353e61301d1a0e26482f43f0dd8"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9e41506fec7a7f9405b14aa2d5c8abbb4dbbd09d88f9496958b6d00cb4d45330"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f7d4a670107d75dfe5ad080bed6c341d18c4442f9378c9f58e5851e86eb79965"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41ce1f1e2c7755abfc7e759dc34d7d05fd221723ff822947132dc934d122fe22"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:44264ecae91b30e5633013fb66f6ddd05c006d3e0e884f75ce0b4755b3e3847b"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_ppc64le.whl", hash = "sha256:3c174dc350d3ec52deb77f2faf05c439331d6ed5e702fc247ccb4e6b62d884b7"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_s390x.whl", hash = "sha256:2dfab5fa6a28a0b60a20638dc48e6343c02ea9933e3279ccb132f555a62323d8"}, + {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b1c8c20847b9f34e98080da785bb2336ea982e7f913eed5809e5a3c872900f32"}, + {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2c86bf781b12ba417f64f3422cfc302523ac9cd1d8ae8c0f92a1c66e56ef2e86"}, + {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:c162b216070f280fa7da844531169be0baf9ccb17263cf5a8bf876fcd3117fa5"}, + {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:36aef61a1678cb778097b4a6eeae96a69875d51d1e8f4d4b491ab3cfb54b5a03"}, + {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f65e5120863c2b266dbcc927b306c5b78e502c71edf3295dfcb9501ec96e5fc7"}, + {file = "lxml-5.3.0-cp310-cp310-win32.whl", hash = "sha256:ef0c1fe22171dd7c7c27147f2e9c3e86f8bdf473fed75f16b0c2e84a5030ce80"}, + {file = "lxml-5.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:052d99051e77a4f3e8482c65014cf6372e61b0a6f4fe9edb98503bb5364cfee3"}, + {file = "lxml-5.3.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74bcb423462233bc5d6066e4e98b0264e7c1bed7541fff2f4e34fe6b21563c8b"}, + {file = "lxml-5.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a3d819eb6f9b8677f57f9664265d0a10dd6551d227afb4af2b9cd7bdc2ccbf18"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b8f5db71b28b8c404956ddf79575ea77aa8b1538e8b2ef9ec877945b3f46442"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c3406b63232fc7e9b8783ab0b765d7c59e7c59ff96759d8ef9632fca27c7ee4"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ecdd78ab768f844c7a1d4a03595038c166b609f6395e25af9b0f3f26ae1230f"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:168f2dfcfdedf611eb285efac1516c8454c8c99caf271dccda8943576b67552e"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa617107a410245b8660028a7483b68e7914304a6d4882b5ff3d2d3eb5948d8c"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:69959bd3167b993e6e710b99051265654133a98f20cec1d9b493b931942e9c16"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:bd96517ef76c8654446fc3db9242d019a1bb5fe8b751ba414765d59f99210b79"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:ab6dd83b970dc97c2d10bc71aa925b84788c7c05de30241b9e96f9b6d9ea3080"}, + {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:eec1bb8cdbba2925bedc887bc0609a80e599c75b12d87ae42ac23fd199445654"}, + {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6a7095eeec6f89111d03dabfe5883a1fd54da319c94e0fb104ee8f23616b572d"}, + {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f651ebd0b21ec65dfca93aa629610a0dbc13dbc13554f19b0113da2e61a4763"}, + {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:f422a209d2455c56849442ae42f25dbaaba1c6c3f501d58761c619c7836642ec"}, + {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:62f7fdb0d1ed2065451f086519865b4c90aa19aed51081979ecd05a21eb4d1be"}, + {file = "lxml-5.3.0-cp311-cp311-win32.whl", hash = "sha256:c6379f35350b655fd817cd0d6cbeef7f265f3ae5fedb1caae2eb442bbeae9ab9"}, + {file = "lxml-5.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:9c52100e2c2dbb0649b90467935c4b0de5528833c76a35ea1a2691ec9f1ee7a1"}, + {file = "lxml-5.3.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:e99f5507401436fdcc85036a2e7dc2e28d962550afe1cbfc07c40e454256a859"}, + {file = "lxml-5.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:384aacddf2e5813a36495233b64cb96b1949da72bef933918ba5c84e06af8f0e"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:874a216bf6afaf97c263b56371434e47e2c652d215788396f60477540298218f"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65ab5685d56914b9a2a34d67dd5488b83213d680b0c5d10b47f81da5a16b0b0e"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aac0bbd3e8dd2d9c45ceb82249e8bdd3ac99131a32b4d35c8af3cc9db1657179"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b369d3db3c22ed14c75ccd5af429086f166a19627e84a8fdade3f8f31426e52a"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c24037349665434f375645fa9d1f5304800cec574d0310f618490c871fd902b3"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:62d172f358f33a26d6b41b28c170c63886742f5b6772a42b59b4f0fa10526cb1"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:c1f794c02903c2824fccce5b20c339a1a14b114e83b306ff11b597c5f71a1c8d"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:5d6a6972b93c426ace71e0be9a6f4b2cfae9b1baed2eed2006076a746692288c"}, + {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:3879cc6ce938ff4eb4900d901ed63555c778731a96365e53fadb36437a131a99"}, + {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:74068c601baff6ff021c70f0935b0c7bc528baa8ea210c202e03757c68c5a4ff"}, + {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ecd4ad8453ac17bc7ba3868371bffb46f628161ad0eefbd0a855d2c8c32dd81a"}, + {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:7e2f58095acc211eb9d8b5771bf04df9ff37d6b87618d1cbf85f92399c98dae8"}, + {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e63601ad5cd8f860aa99d109889b5ac34de571c7ee902d6812d5d9ddcc77fa7d"}, + {file = "lxml-5.3.0-cp312-cp312-win32.whl", hash = "sha256:17e8d968d04a37c50ad9c456a286b525d78c4a1c15dd53aa46c1d8e06bf6fa30"}, + {file = "lxml-5.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:c1a69e58a6bb2de65902051d57fde951febad631a20a64572677a1052690482f"}, + {file = "lxml-5.3.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8c72e9563347c7395910de6a3100a4840a75a6f60e05af5e58566868d5eb2d6a"}, + {file = "lxml-5.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e92ce66cd919d18d14b3856906a61d3f6b6a8500e0794142338da644260595cd"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d04f064bebdfef9240478f7a779e8c5dc32b8b7b0b2fc6a62e39b928d428e51"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c2fb570d7823c2bbaf8b419ba6e5662137f8166e364a8b2b91051a1fb40ab8b"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c120f43553ec759f8de1fee2f4794452b0946773299d44c36bfe18e83caf002"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:562e7494778a69086f0312ec9689f6b6ac1c6b65670ed7d0267e49f57ffa08c4"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:423b121f7e6fa514ba0c7918e56955a1d4470ed35faa03e3d9f0e3baa4c7e492"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:c00f323cc00576df6165cc9d21a4c21285fa6b9989c5c39830c3903dc4303ef3"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:1fdc9fae8dd4c763e8a31e7630afef517eab9f5d5d31a278df087f307bf601f4"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:658f2aa69d31e09699705949b5fc4719cbecbd4a97f9656a232e7d6c7be1a367"}, + {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:1473427aff3d66a3fa2199004c3e601e6c4500ab86696edffdbc84954c72d832"}, + {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a87de7dd873bf9a792bf1e58b1c3887b9264036629a5bf2d2e6579fe8e73edff"}, + {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0d7b36afa46c97875303a94e8f3ad932bf78bace9e18e603f2085b652422edcd"}, + {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:cf120cce539453ae086eacc0130a324e7026113510efa83ab42ef3fcfccac7fb"}, + {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:df5c7333167b9674aa8ae1d4008fa4bc17a313cc490b2cca27838bbdcc6bb15b"}, + {file = "lxml-5.3.0-cp313-cp313-win32.whl", hash = "sha256:c802e1c2ed9f0c06a65bc4ed0189d000ada8049312cfeab6ca635e39c9608957"}, + {file = "lxml-5.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:406246b96d552e0503e17a1006fd27edac678b3fcc9f1be71a2f94b4ff61528d"}, + {file = "lxml-5.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8f0de2d390af441fe8b2c12626d103540b5d850d585b18fcada58d972b74a74e"}, + {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1afe0a8c353746e610bd9031a630a95bcfb1a720684c3f2b36c4710a0a96528f"}, + {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56b9861a71575f5795bde89256e7467ece3d339c9b43141dbdd54544566b3b94"}, + {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:9fb81d2824dff4f2e297a276297e9031f46d2682cafc484f49de182aa5e5df99"}, + {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:2c226a06ecb8cdef28845ae976da407917542c5e6e75dcac7cc33eb04aaeb237"}, + {file = "lxml-5.3.0-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:7d3d1ca42870cdb6d0d29939630dbe48fa511c203724820fc0fd507b2fb46577"}, + {file = "lxml-5.3.0-cp36-cp36m-win32.whl", hash = "sha256:094cb601ba9f55296774c2d57ad68730daa0b13dc260e1f941b4d13678239e70"}, + {file = "lxml-5.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:eafa2c8658f4e560b098fe9fc54539f86528651f61849b22111a9b107d18910c"}, + {file = "lxml-5.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cb83f8a875b3d9b458cada4f880fa498646874ba4011dc974e071a0a84a1b033"}, + {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:25f1b69d41656b05885aa185f5fdf822cb01a586d1b32739633679699f220391"}, + {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23e0553b8055600b3bf4a00b255ec5c92e1e4aebf8c2c09334f8368e8bd174d6"}, + {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ada35dd21dc6c039259596b358caab6b13f4db4d4a7f8665764d616daf9cc1d"}, + {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:81b4e48da4c69313192d8c8d4311e5d818b8be1afe68ee20f6385d0e96fc9512"}, + {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:2bc9fd5ca4729af796f9f59cd8ff160fe06a474da40aca03fcc79655ddee1a8b"}, + {file = "lxml-5.3.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:07da23d7ee08577760f0a71d67a861019103e4812c87e2fab26b039054594cc5"}, + {file = "lxml-5.3.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:ea2e2f6f801696ad7de8aec061044d6c8c0dd4037608c7cab38a9a4d316bfb11"}, + {file = "lxml-5.3.0-cp37-cp37m-win32.whl", hash = "sha256:5c54afdcbb0182d06836cc3d1be921e540be3ebdf8b8a51ee3ef987537455f84"}, + {file = "lxml-5.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:f2901429da1e645ce548bf9171784c0f74f0718c3f6150ce166be39e4dd66c3e"}, + {file = "lxml-5.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c56a1d43b2f9ee4786e4658c7903f05da35b923fb53c11025712562d5cc02753"}, + {file = "lxml-5.3.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ee8c39582d2652dcd516d1b879451500f8db3fe3607ce45d7c5957ab2596040"}, + {file = "lxml-5.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fdf3a3059611f7585a78ee10399a15566356116a4288380921a4b598d807a22"}, + {file = "lxml-5.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:146173654d79eb1fc97498b4280c1d3e1e5d58c398fa530905c9ea50ea849b22"}, + {file = "lxml-5.3.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:0a7056921edbdd7560746f4221dca89bb7a3fe457d3d74267995253f46343f15"}, + {file = "lxml-5.3.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:9e4b47ac0f5e749cfc618efdf4726269441014ae1d5583e047b452a32e221920"}, + {file = "lxml-5.3.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:f914c03e6a31deb632e2daa881fe198461f4d06e57ac3d0e05bbcab8eae01945"}, + {file = "lxml-5.3.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:213261f168c5e1d9b7535a67e68b1f59f92398dd17a56d934550837143f79c42"}, + {file = "lxml-5.3.0-cp38-cp38-win32.whl", hash = "sha256:218c1b2e17a710e363855594230f44060e2025b05c80d1f0661258142b2add2e"}, + {file = "lxml-5.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:315f9542011b2c4e1d280e4a20ddcca1761993dda3afc7a73b01235f8641e903"}, + {file = "lxml-5.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:1ffc23010330c2ab67fac02781df60998ca8fe759e8efde6f8b756a20599c5de"}, + {file = "lxml-5.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2b3778cb38212f52fac9fe913017deea2fdf4eb1a4f8e4cfc6b009a13a6d3fcc"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b0c7a688944891086ba192e21c5229dea54382f4836a209ff8d0a660fac06be"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:747a3d3e98e24597981ca0be0fd922aebd471fa99d0043a3842d00cdcad7ad6a"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86a6b24b19eaebc448dc56b87c4865527855145d851f9fc3891673ff97950540"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b11a5d918a6216e521c715b02749240fb07ae5a1fefd4b7bf12f833bc8b4fe70"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68b87753c784d6acb8a25b05cb526c3406913c9d988d51f80adecc2b0775d6aa"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:109fa6fede314cc50eed29e6e56c540075e63d922455346f11e4d7a036d2b8cf"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_ppc64le.whl", hash = "sha256:02ced472497b8362c8e902ade23e3300479f4f43e45f4105c85ef43b8db85229"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_s390x.whl", hash = "sha256:6b038cc86b285e4f9fea2ba5ee76e89f21ed1ea898e287dc277a25884f3a7dfe"}, + {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:7437237c6a66b7ca341e868cda48be24b8701862757426852c9b3186de1da8a2"}, + {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7f41026c1d64043a36fda21d64c5026762d53a77043e73e94b71f0521939cc71"}, + {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:482c2f67761868f0108b1743098640fbb2a28a8e15bf3f47ada9fa59d9fe08c3"}, + {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:1483fd3358963cc5c1c9b122c80606a3a79ee0875bcac0204149fa09d6ff2727"}, + {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2dec2d1130a9cda5b904696cec33b2cfb451304ba9081eeda7f90f724097300a"}, + {file = "lxml-5.3.0-cp39-cp39-win32.whl", hash = "sha256:a0eabd0a81625049c5df745209dc7fcef6e2aea7793e5f003ba363610aa0a3ff"}, + {file = "lxml-5.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:89e043f1d9d341c52bf2af6d02e6adde62e0a46e6755d5eb60dc6e4f0b8aeca2"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7b1cd427cb0d5f7393c31b7496419da594fe600e6fdc4b105a54f82405e6626c"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51806cfe0279e06ed8500ce19479d757db42a30fd509940b1701be9c86a5ff9a"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee70d08fd60c9565ba8190f41a46a54096afa0eeb8f76bd66f2c25d3b1b83005"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:8dc2c0395bea8254d8daebc76dcf8eb3a95ec2a46fa6fae5eaccee366bfe02ce"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:6ba0d3dcac281aad8a0e5b14c7ed6f9fa89c8612b47939fc94f80b16e2e9bc83"}, + {file = "lxml-5.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:6e91cf736959057f7aac7adfc83481e03615a8e8dd5758aa1d95ea69e8931dba"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:94d6c3782907b5e40e21cadf94b13b0842ac421192f26b84c45f13f3c9d5dc27"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c300306673aa0f3ed5ed9372b21867690a17dba38c68c44b287437c362ce486b"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78d9b952e07aed35fe2e1a7ad26e929595412db48535921c5013edc8aa4a35ce"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:01220dca0d066d1349bd6a1726856a78f7929f3878f7e2ee83c296c69495309e"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:2d9b8d9177afaef80c53c0a9e30fa252ff3036fb1c6494d427c066a4ce6a282f"}, + {file = "lxml-5.3.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:20094fc3f21ea0a8669dc4c61ed7fa8263bd37d97d93b90f28fc613371e7a875"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ace2c2326a319a0bb8a8b0e5b570c764962e95818de9f259ce814ee666603f19"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92e67a0be1639c251d21e35fe74df6bcc40cba445c2cda7c4a967656733249e2"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd5350b55f9fecddc51385463a4f67a5da829bc741e38cf689f38ec9023f54ab"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4c1fefd7e3d00921c44dc9ca80a775af49698bbfd92ea84498e56acffd4c5469"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:71a8dd38fbd2f2319136d4ae855a7078c69c9a38ae06e0c17c73fd70fc6caad8"}, + {file = "lxml-5.3.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:97acf1e1fd66ab53dacd2c35b319d7e548380c2e9e8c54525c6e76d21b1ae3b1"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:68934b242c51eb02907c5b81d138cb977b2129a0a75a8f8b60b01cb8586c7b21"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b710bc2b8292966b23a6a0121f7a6c51d45d2347edcc75f016ac123b8054d3f2"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18feb4b93302091b1541221196a2155aa296c363fd233814fa11e181adebc52f"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:3eb44520c4724c2e1a57c0af33a379eee41792595023f367ba3952a2d96c2aab"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:609251a0ca4770e5a8768ff902aa02bf636339c5a93f9349b48eb1f606f7f3e9"}, + {file = "lxml-5.3.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:516f491c834eb320d6c843156440fe7fc0d50b33e44387fcec5b02f0bc118a4c"}, + {file = "lxml-5.3.0.tar.gz", hash = "sha256:4e109ca30d1edec1ac60cdbe341905dc3b8f55b16855e03a54aaf59e51ec8c6f"}, +] + +[package.extras] +cssselect = ["cssselect (>=0.7)"] +html-clean = ["lxml-html-clean"] +html5 = ["html5lib"] +htmlsoup = ["BeautifulSoup4"] +source = ["Cython (>=3.0.11)"] + +[[package]] +name = "markdownify" +version = "0.14.1" +description = "Convert HTML to markdown." +optional = false +python-versions = "*" +files = [ + {file = "markdownify-0.14.1-py3-none-any.whl", hash = "sha256:4c46a6c0c12c6005ddcd49b45a5a890398b002ef51380cd319db62df5e09bc2a"}, + {file = "markdownify-0.14.1.tar.gz", hash = "sha256:a62a7a216947ed0b8dafb95b99b2ef4a0edd1e18d5653c656f68f03db2bfb2f1"}, +] + +[package.dependencies] +beautifulsoup4 = ">=4.9,<5" +six = ">=1.15,<2" + [[package]] name = "mccabe" version = "0.7.0" @@ -77,6 +569,166 @@ files = [ {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, ] +[[package]] +name = "numpy" +version = "2.1.3" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.10" +files = [ + {file = "numpy-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c894b4305373b9c5576d7a12b473702afdf48ce5369c074ba304cc5ad8730dff"}, + {file = "numpy-2.1.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b47fbb433d3260adcd51eb54f92a2ffbc90a4595f8970ee00e064c644ac788f5"}, + {file = "numpy-2.1.3-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:825656d0743699c529c5943554d223c021ff0494ff1442152ce887ef4f7561a1"}, + {file = "numpy-2.1.3-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:6a4825252fcc430a182ac4dee5a505053d262c807f8a924603d411f6718b88fd"}, + {file = "numpy-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e711e02f49e176a01d0349d82cb5f05ba4db7d5e7e0defd026328e5cfb3226d3"}, + {file = "numpy-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78574ac2d1a4a02421f25da9559850d59457bac82f2b8d7a44fe83a64f770098"}, + {file = "numpy-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c7662f0e3673fe4e832fe07b65c50342ea27d989f92c80355658c7f888fcc83c"}, + {file = "numpy-2.1.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:fa2d1337dc61c8dc417fbccf20f6d1e139896a30721b7f1e832b2bb6ef4eb6c4"}, + {file = "numpy-2.1.3-cp310-cp310-win32.whl", hash = "sha256:72dcc4a35a8515d83e76b58fdf8113a5c969ccd505c8a946759b24e3182d1f23"}, + {file = "numpy-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:ecc76a9ba2911d8d37ac01de72834d8849e55473457558e12995f4cd53e778e0"}, + {file = "numpy-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4d1167c53b93f1f5d8a139a742b3c6f4d429b54e74e6b57d0eff40045187b15d"}, + {file = "numpy-2.1.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c80e4a09b3d95b4e1cac08643f1152fa71a0a821a2d4277334c88d54b2219a41"}, + {file = "numpy-2.1.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:576a1c1d25e9e02ed7fa5477f30a127fe56debd53b8d2c89d5578f9857d03ca9"}, + {file = "numpy-2.1.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:973faafebaae4c0aaa1a1ca1ce02434554d67e628b8d805e61f874b84e136b09"}, + {file = "numpy-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:762479be47a4863e261a840e8e01608d124ee1361e48b96916f38b119cfda04a"}, + {file = "numpy-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc6f24b3d1ecc1eebfbf5d6051faa49af40b03be1aaa781ebdadcbc090b4539b"}, + {file = "numpy-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:17ee83a1f4fef3c94d16dc1802b998668b5419362c8a4f4e8a491de1b41cc3ee"}, + {file = "numpy-2.1.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:15cb89f39fa6d0bdfb600ea24b250e5f1a3df23f901f51c8debaa6a5d122b2f0"}, + {file = "numpy-2.1.3-cp311-cp311-win32.whl", hash = "sha256:d9beb777a78c331580705326d2367488d5bc473b49a9bc3036c154832520aca9"}, + {file = "numpy-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:d89dd2b6da69c4fff5e39c28a382199ddedc3a5be5390115608345dec660b9e2"}, + {file = "numpy-2.1.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f55ba01150f52b1027829b50d70ef1dafd9821ea82905b63936668403c3b471e"}, + {file = "numpy-2.1.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:13138eadd4f4da03074851a698ffa7e405f41a0845a6b1ad135b81596e4e9958"}, + {file = "numpy-2.1.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:a6b46587b14b888e95e4a24d7b13ae91fa22386c199ee7b418f449032b2fa3b8"}, + {file = "numpy-2.1.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:0fa14563cc46422e99daef53d725d0c326e99e468a9320a240affffe87852564"}, + {file = "numpy-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8637dcd2caa676e475503d1f8fdb327bc495554e10838019651b76d17b98e512"}, + {file = "numpy-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2312b2aa89e1f43ecea6da6ea9a810d06aae08321609d8dc0d0eda6d946a541b"}, + {file = "numpy-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a38c19106902bb19351b83802531fea19dee18e5b37b36454f27f11ff956f7fc"}, + {file = "numpy-2.1.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:02135ade8b8a84011cbb67dc44e07c58f28575cf9ecf8ab304e51c05528c19f0"}, + {file = "numpy-2.1.3-cp312-cp312-win32.whl", hash = "sha256:e6988e90fcf617da2b5c78902fe8e668361b43b4fe26dbf2d7b0f8034d4cafb9"}, + {file = "numpy-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:0d30c543f02e84e92c4b1f415b7c6b5326cbe45ee7882b6b77db7195fb971e3a"}, + {file = "numpy-2.1.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96fe52fcdb9345b7cd82ecd34547fca4321f7656d500eca497eb7ea5a926692f"}, + {file = "numpy-2.1.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f653490b33e9c3a4c1c01d41bc2aef08f9475af51146e4a7710c450cf9761598"}, + {file = "numpy-2.1.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:dc258a761a16daa791081d026f0ed4399b582712e6fc887a95af09df10c5ca57"}, + {file = "numpy-2.1.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:016d0f6f5e77b0f0d45d77387ffa4bb89816b57c835580c3ce8e099ef830befe"}, + {file = "numpy-2.1.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c181ba05ce8299c7aa3125c27b9c2167bca4a4445b7ce73d5febc411ca692e43"}, + {file = "numpy-2.1.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5641516794ca9e5f8a4d17bb45446998c6554704d888f86df9b200e66bdcce56"}, + {file = "numpy-2.1.3-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ea4dedd6e394a9c180b33c2c872b92f7ce0f8e7ad93e9585312b0c5a04777a4a"}, + {file = "numpy-2.1.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0df3635b9c8ef48bd3be5f862cf71b0a4716fa0e702155c45067c6b711ddcef"}, + {file = "numpy-2.1.3-cp313-cp313-win32.whl", hash = "sha256:50ca6aba6e163363f132b5c101ba078b8cbd3fa92c7865fd7d4d62d9779ac29f"}, + {file = "numpy-2.1.3-cp313-cp313-win_amd64.whl", hash = "sha256:747641635d3d44bcb380d950679462fae44f54b131be347d5ec2bce47d3df9ed"}, + {file = "numpy-2.1.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:996bb9399059c5b82f76b53ff8bb686069c05acc94656bb259b1d63d04a9506f"}, + {file = "numpy-2.1.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:45966d859916ad02b779706bb43b954281db43e185015df6eb3323120188f9e4"}, + {file = "numpy-2.1.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:baed7e8d7481bfe0874b566850cb0b85243e982388b7b23348c6db2ee2b2ae8e"}, + {file = "numpy-2.1.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:a9f7f672a3388133335589cfca93ed468509cb7b93ba3105fce780d04a6576a0"}, + {file = "numpy-2.1.3-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7aac50327da5d208db2eec22eb11e491e3fe13d22653dce51b0f4109101b408"}, + {file = "numpy-2.1.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4394bc0dbd074b7f9b52024832d16e019decebf86caf909d94f6b3f77a8ee3b6"}, + {file = "numpy-2.1.3-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:50d18c4358a0a8a53f12a8ba9d772ab2d460321e6a93d6064fc22443d189853f"}, + {file = "numpy-2.1.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:14e253bd43fc6b37af4921b10f6add6925878a42a0c5fe83daee390bca80bc17"}, + {file = "numpy-2.1.3-cp313-cp313t-win32.whl", hash = "sha256:08788d27a5fd867a663f6fc753fd7c3ad7e92747efc73c53bca2f19f8bc06f48"}, + {file = "numpy-2.1.3-cp313-cp313t-win_amd64.whl", hash = "sha256:2564fbdf2b99b3f815f2107c1bbc93e2de8ee655a69c261363a1172a79a257d4"}, + {file = "numpy-2.1.3-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4f2015dfe437dfebbfce7c85c7b53d81ba49e71ba7eadbf1df40c915af75979f"}, + {file = "numpy-2.1.3-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:3522b0dfe983a575e6a9ab3a4a4dfe156c3e428468ff08ce582b9bb6bd1d71d4"}, + {file = "numpy-2.1.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c006b607a865b07cd981ccb218a04fc86b600411d83d6fc261357f1c0966755d"}, + {file = "numpy-2.1.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:e14e26956e6f1696070788252dcdff11b4aca4c3e8bd166e0df1bb8f315a67cb"}, + {file = "numpy-2.1.3.tar.gz", hash = "sha256:aa08e04e08aaf974d4458def539dece0d28146d866a39da5639596f4921fd761"}, +] + +[[package]] +name = "pandas" +version = "2.2.3" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pandas-2.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1948ddde24197a0f7add2bdc4ca83bf2b1ef84a1bc8ccffd95eda17fd836ecb5"}, + {file = "pandas-2.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:381175499d3802cde0eabbaf6324cce0c4f5d52ca6f8c377c29ad442f50f6348"}, + {file = "pandas-2.2.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d9c45366def9a3dd85a6454c0e7908f2b3b8e9c138f5dc38fed7ce720d8453ed"}, + {file = "pandas-2.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86976a1c5b25ae3f8ccae3a5306e443569ee3c3faf444dfd0f41cda24667ad57"}, + {file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b8661b0238a69d7aafe156b7fa86c44b881387509653fdf857bebc5e4008ad42"}, + {file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:37e0aced3e8f539eccf2e099f65cdb9c8aa85109b0be6e93e2baff94264bdc6f"}, + {file = "pandas-2.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:56534ce0746a58afaf7942ba4863e0ef81c9c50d3f0ae93e9497d6a41a057645"}, + {file = "pandas-2.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66108071e1b935240e74525006034333f98bcdb87ea116de573a6a0dccb6c039"}, + {file = "pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c2875855b0ff77b2a64a0365e24455d9990730d6431b9e0ee18ad8acee13dbd"}, + {file = "pandas-2.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd8d0c3be0515c12fed0bdbae072551c8b54b7192c7b1fda0ba56059a0179698"}, + {file = "pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c124333816c3a9b03fbeef3a9f230ba9a737e9e5bb4060aa2107a86cc0a497fc"}, + {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:63cc132e40a2e084cf01adf0775b15ac515ba905d7dcca47e9a251819c575ef3"}, + {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29401dbfa9ad77319367d36940cd8a0b3a11aba16063e39632d98b0e931ddf32"}, + {file = "pandas-2.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:3fc6873a41186404dad67245896a6e440baacc92f5b716ccd1bc9ed2995ab2c5"}, + {file = "pandas-2.2.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b1d432e8d08679a40e2a6d8b2f9770a5c21793a6f9f47fdd52c5ce1948a5a8a9"}, + {file = "pandas-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a5a1595fe639f5988ba6a8e5bc9649af3baf26df3998a0abe56c02609392e0a4"}, + {file = "pandas-2.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5de54125a92bb4d1c051c0659e6fcb75256bf799a732a87184e5ea503965bce3"}, + {file = "pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319"}, + {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfcb5ee8d4d50c06a51c2fffa6cff6272098ad6540aed1a76d15fb9318194d8"}, + {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a"}, + {file = "pandas-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:59ef3764d0fe818125a5097d2ae867ca3fa64df032331b7e0917cf5d7bf66b13"}, + {file = "pandas-2.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f00d1345d84d8c86a63e476bb4955e46458b304b9575dcf71102b5c705320015"}, + {file = "pandas-2.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3508d914817e153ad359d7e069d752cdd736a247c322d932eb89e6bc84217f28"}, + {file = "pandas-2.2.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22a9d949bfc9a502d320aa04e5d02feab689d61da4e7764b62c30b991c42c5f0"}, + {file = "pandas-2.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3a255b2c19987fbbe62a9dfd6cff7ff2aa9ccab3fc75218fd4b7530f01efa24"}, + {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:800250ecdadb6d9c78eae4990da62743b857b470883fa27f652db8bdde7f6659"}, + {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6374c452ff3ec675a8f46fd9ab25c4ad0ba590b71cf0656f8b6daa5202bca3fb"}, + {file = "pandas-2.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:61c5ad4043f791b61dd4752191d9f07f0ae412515d59ba8f005832a532f8736d"}, + {file = "pandas-2.2.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3b71f27954685ee685317063bf13c7709a7ba74fc996b84fc6821c59b0f06468"}, + {file = "pandas-2.2.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:38cf8125c40dae9d5acc10fa66af8ea6fdf760b2714ee482ca691fc66e6fcb18"}, + {file = "pandas-2.2.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba96630bc17c875161df3818780af30e43be9b166ce51c9a18c1feae342906c2"}, + {file = "pandas-2.2.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1db71525a1538b30142094edb9adc10be3f3e176748cd7acc2240c2f2e5aa3a4"}, + {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:15c0e1e02e93116177d29ff83e8b1619c93ddc9c49083f237d4312337a61165d"}, + {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ad5b65698ab28ed8d7f18790a0dc58005c7629f227be9ecc1072aa74c0c1d43a"}, + {file = "pandas-2.2.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc6b93f9b966093cb0fd62ff1a7e4c09e6d546ad7c1de191767baffc57628f39"}, + {file = "pandas-2.2.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5dbca4c1acd72e8eeef4753eeca07de9b1db4f398669d5994086f788a5d7cc30"}, + {file = "pandas-2.2.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8cd6d7cc958a3910f934ea8dbdf17b2364827bb4dafc38ce6eef6bb3d65ff09c"}, + {file = "pandas-2.2.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99df71520d25fade9db7c1076ac94eb994f4d2673ef2aa2e86ee039b6746d20c"}, + {file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:31d0ced62d4ea3e231a9f228366919a5ea0b07440d9d4dac345376fd8e1477ea"}, + {file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7eee9e7cea6adf3e3d24e304ac6b8300646e2a5d1cd3a3c2abed9101b0846761"}, + {file = "pandas-2.2.3-cp39-cp39-win_amd64.whl", hash = "sha256:4850ba03528b6dd51d6c5d273c46f183f39a9baf3f0143e566b89450965b105e"}, + {file = "pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667"}, +] + +[package.dependencies] +numpy = {version = ">=1.26.0", markers = "python_version >= \"3.12\""} +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.7" + +[package.extras] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] +aws = ["s3fs (>=2022.11.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] +compression = ["zstandard (>=0.19.0)"] +computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] +feather = ["pyarrow (>=10.0.1)"] +fss = ["fsspec (>=2022.11.0)"] +gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] +hdf5 = ["tables (>=3.8.0)"] +html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] +mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=10.0.1)"] +performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] +plot = ["matplotlib (>=3.6.3)"] +postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] +pyarrow = ["pyarrow (>=10.0.1)"] +spss = ["pyreadstat (>=1.2.0)"] +sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.9.2)"] + +[[package]] +name = "prompt-toolkit" +version = "3.0.48" +description = "Library for building powerful interactive command lines in Python" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "prompt_toolkit-3.0.48-py3-none-any.whl", hash = "sha256:f49a827f90062e411f1ce1f854f2aedb3c23353244f8108b89283587397ac10e"}, + {file = "prompt_toolkit-3.0.48.tar.gz", hash = "sha256:d6623ab0477a80df74e646bdbc93621143f5caf104206aa29294d53de1a03d90"}, +] + +[package.dependencies] +wcwidth = "*" + [[package]] name = "psycopg2-binary" version = "2.9.10" @@ -164,6 +816,138 @@ files = [ {file = "pycodestyle-2.12.1.tar.gz", hash = "sha256:6838eae08bbce4f6accd5d5572075c63626a15ee3e6f842df996bf62f6d73521"}, ] +[[package]] +name = "pydantic" +version = "2.10.2" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic-2.10.2-py3-none-any.whl", hash = "sha256:cfb96e45951117c3024e6b67b25cdc33a3cb7b2fa62e239f7af1378358a1d99e"}, + {file = "pydantic-2.10.2.tar.gz", hash = "sha256:2bc2d7f17232e0841cbba4641e65ba1eb6fafb3a08de3a091ff3ce14a197c4fa"}, +] + +[package.dependencies] +annotated-types = ">=0.6.0" +pydantic-core = "2.27.1" +typing-extensions = ">=4.12.2" + +[package.extras] +email = ["email-validator (>=2.0.0)"] +timezone = ["tzdata"] + +[[package]] +name = "pydantic-core" +version = "2.27.1" +description = "Core functionality for Pydantic validation and serialization" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_core-2.27.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:71a5e35c75c021aaf400ac048dacc855f000bdfed91614b4a726f7432f1f3d6a"}, + {file = "pydantic_core-2.27.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f82d068a2d6ecfc6e054726080af69a6764a10015467d7d7b9f66d6ed5afa23b"}, + {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:121ceb0e822f79163dd4699e4c54f5ad38b157084d97b34de8b232bcaad70278"}, + {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4603137322c18eaf2e06a4495f426aa8d8388940f3c457e7548145011bb68e05"}, + {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a33cd6ad9017bbeaa9ed78a2e0752c5e250eafb9534f308e7a5f7849b0b1bfb4"}, + {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15cc53a3179ba0fcefe1e3ae50beb2784dede4003ad2dfd24f81bba4b23a454f"}, + {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45d9c5eb9273aa50999ad6adc6be5e0ecea7e09dbd0d31bd0c65a55a2592ca08"}, + {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8bf7b66ce12a2ac52d16f776b31d16d91033150266eb796967a7e4621707e4f6"}, + {file = "pydantic_core-2.27.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:655d7dd86f26cb15ce8a431036f66ce0318648f8853d709b4167786ec2fa4807"}, + {file = "pydantic_core-2.27.1-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:5556470f1a2157031e676f776c2bc20acd34c1990ca5f7e56f1ebf938b9ab57c"}, + {file = "pydantic_core-2.27.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f69ed81ab24d5a3bd93861c8c4436f54afdf8e8cc421562b0c7504cf3be58206"}, + {file = "pydantic_core-2.27.1-cp310-none-win32.whl", hash = "sha256:f5a823165e6d04ccea61a9f0576f345f8ce40ed533013580e087bd4d7442b52c"}, + {file = "pydantic_core-2.27.1-cp310-none-win_amd64.whl", hash = "sha256:57866a76e0b3823e0b56692d1a0bf722bffb324839bb5b7226a7dbd6c9a40b17"}, + {file = "pydantic_core-2.27.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:ac3b20653bdbe160febbea8aa6c079d3df19310d50ac314911ed8cc4eb7f8cb8"}, + {file = "pydantic_core-2.27.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a5a8e19d7c707c4cadb8c18f5f60c843052ae83c20fa7d44f41594c644a1d330"}, + {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f7059ca8d64fea7f238994c97d91f75965216bcbe5f695bb44f354893f11d52"}, + {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bed0f8a0eeea9fb72937ba118f9db0cb7e90773462af7962d382445f3005e5a4"}, + {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a3cb37038123447cf0f3ea4c74751f6a9d7afef0eb71aa07bf5f652b5e6a132c"}, + {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:84286494f6c5d05243456e04223d5a9417d7f443c3b76065e75001beb26f88de"}, + {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acc07b2cfc5b835444b44a9956846b578d27beeacd4b52e45489e93276241025"}, + {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4fefee876e07a6e9aad7a8c8c9f85b0cdbe7df52b8a9552307b09050f7512c7e"}, + {file = "pydantic_core-2.27.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:258c57abf1188926c774a4c94dd29237e77eda19462e5bb901d88adcab6af919"}, + {file = "pydantic_core-2.27.1-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:35c14ac45fcfdf7167ca76cc80b2001205a8d5d16d80524e13508371fb8cdd9c"}, + {file = "pydantic_core-2.27.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d1b26e1dff225c31897696cab7d4f0a315d4c0d9e8666dbffdb28216f3b17fdc"}, + {file = "pydantic_core-2.27.1-cp311-none-win32.whl", hash = "sha256:2cdf7d86886bc6982354862204ae3b2f7f96f21a3eb0ba5ca0ac42c7b38598b9"}, + {file = "pydantic_core-2.27.1-cp311-none-win_amd64.whl", hash = "sha256:3af385b0cee8df3746c3f406f38bcbfdc9041b5c2d5ce3e5fc6637256e60bbc5"}, + {file = "pydantic_core-2.27.1-cp311-none-win_arm64.whl", hash = "sha256:81f2ec23ddc1b476ff96563f2e8d723830b06dceae348ce02914a37cb4e74b89"}, + {file = "pydantic_core-2.27.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:9cbd94fc661d2bab2bc702cddd2d3370bbdcc4cd0f8f57488a81bcce90c7a54f"}, + {file = "pydantic_core-2.27.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5f8c4718cd44ec1580e180cb739713ecda2bdee1341084c1467802a417fe0f02"}, + {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15aae984e46de8d376df515f00450d1522077254ef6b7ce189b38ecee7c9677c"}, + {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1ba5e3963344ff25fc8c40da90f44b0afca8cfd89d12964feb79ac1411a260ac"}, + {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:992cea5f4f3b29d6b4f7f1726ed8ee46c8331c6b4eed6db5b40134c6fe1768bb"}, + {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0325336f348dbee6550d129b1627cb8f5351a9dc91aad141ffb96d4937bd9529"}, + {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7597c07fbd11515f654d6ece3d0e4e5093edc30a436c63142d9a4b8e22f19c35"}, + {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3bbd5d8cc692616d5ef6fbbbd50dbec142c7e6ad9beb66b78a96e9c16729b089"}, + {file = "pydantic_core-2.27.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:dc61505e73298a84a2f317255fcc72b710b72980f3a1f670447a21efc88f8381"}, + {file = "pydantic_core-2.27.1-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:e1f735dc43da318cad19b4173dd1ffce1d84aafd6c9b782b3abc04a0d5a6f5bb"}, + {file = "pydantic_core-2.27.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f4e5658dbffe8843a0f12366a4c2d1c316dbe09bb4dfbdc9d2d9cd6031de8aae"}, + {file = "pydantic_core-2.27.1-cp312-none-win32.whl", hash = "sha256:672ebbe820bb37988c4d136eca2652ee114992d5d41c7e4858cdd90ea94ffe5c"}, + {file = "pydantic_core-2.27.1-cp312-none-win_amd64.whl", hash = "sha256:66ff044fd0bb1768688aecbe28b6190f6e799349221fb0de0e6f4048eca14c16"}, + {file = "pydantic_core-2.27.1-cp312-none-win_arm64.whl", hash = "sha256:9a3b0793b1bbfd4146304e23d90045f2a9b5fd5823aa682665fbdaf2a6c28f3e"}, + {file = "pydantic_core-2.27.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f216dbce0e60e4d03e0c4353c7023b202d95cbaeff12e5fd2e82ea0a66905073"}, + {file = "pydantic_core-2.27.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a2e02889071850bbfd36b56fd6bc98945e23670773bc7a76657e90e6b6603c08"}, + {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42b0e23f119b2b456d07ca91b307ae167cc3f6c846a7b169fca5326e32fdc6cf"}, + {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:764be71193f87d460a03f1f7385a82e226639732214b402f9aa61f0d025f0737"}, + {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c00666a3bd2f84920a4e94434f5974d7bbc57e461318d6bb34ce9cdbbc1f6b2"}, + {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ccaa88b24eebc0f849ce0a4d09e8a408ec5a94afff395eb69baf868f5183107"}, + {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c65af9088ac534313e1963443d0ec360bb2b9cba6c2909478d22c2e363d98a51"}, + {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:206b5cf6f0c513baffaeae7bd817717140770c74528f3e4c3e1cec7871ddd61a"}, + {file = "pydantic_core-2.27.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:062f60e512fc7fff8b8a9d680ff0ddaaef0193dba9fa83e679c0c5f5fbd018bc"}, + {file = "pydantic_core-2.27.1-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:a0697803ed7d4af5e4c1adf1670af078f8fcab7a86350e969f454daf598c4960"}, + {file = "pydantic_core-2.27.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:58ca98a950171f3151c603aeea9303ef6c235f692fe555e883591103da709b23"}, + {file = "pydantic_core-2.27.1-cp313-none-win32.whl", hash = "sha256:8065914ff79f7eab1599bd80406681f0ad08f8e47c880f17b416c9f8f7a26d05"}, + {file = "pydantic_core-2.27.1-cp313-none-win_amd64.whl", hash = "sha256:ba630d5e3db74c79300d9a5bdaaf6200172b107f263c98a0539eeecb857b2337"}, + {file = "pydantic_core-2.27.1-cp313-none-win_arm64.whl", hash = "sha256:45cf8588c066860b623cd11c4ba687f8d7175d5f7ef65f7129df8a394c502de5"}, + {file = "pydantic_core-2.27.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:5897bec80a09b4084aee23f9b73a9477a46c3304ad1d2d07acca19723fb1de62"}, + {file = "pydantic_core-2.27.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d0165ab2914379bd56908c02294ed8405c252250668ebcb438a55494c69f44ab"}, + {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b9af86e1d8e4cfc82c2022bfaa6f459381a50b94a29e95dcdda8442d6d83864"}, + {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f6c8a66741c5f5447e047ab0ba7a1c61d1e95580d64bce852e3df1f895c4067"}, + {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a42d6a8156ff78981f8aa56eb6394114e0dedb217cf8b729f438f643608cbcd"}, + {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64c65f40b4cd8b0e049a8edde07e38b476da7e3aaebe63287c899d2cff253fa5"}, + {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdcf339322a3fae5cbd504edcefddd5a50d9ee00d968696846f089b4432cf78"}, + {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bf99c8404f008750c846cb4ac4667b798a9f7de673ff719d705d9b2d6de49c5f"}, + {file = "pydantic_core-2.27.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8f1edcea27918d748c7e5e4d917297b2a0ab80cad10f86631e488b7cddf76a36"}, + {file = "pydantic_core-2.27.1-cp38-cp38-musllinux_1_1_armv7l.whl", hash = "sha256:159cac0a3d096f79ab6a44d77a961917219707e2a130739c64d4dd46281f5c2a"}, + {file = "pydantic_core-2.27.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:029d9757eb621cc6e1848fa0b0310310de7301057f623985698ed7ebb014391b"}, + {file = "pydantic_core-2.27.1-cp38-none-win32.whl", hash = "sha256:a28af0695a45f7060e6f9b7092558a928a28553366519f64083c63a44f70e618"}, + {file = "pydantic_core-2.27.1-cp38-none-win_amd64.whl", hash = "sha256:2d4567c850905d5eaaed2f7a404e61012a51caf288292e016360aa2b96ff38d4"}, + {file = "pydantic_core-2.27.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:e9386266798d64eeb19dd3677051f5705bf873e98e15897ddb7d76f477131967"}, + {file = "pydantic_core-2.27.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4228b5b646caa73f119b1ae756216b59cc6e2267201c27d3912b592c5e323b60"}, + {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b3dfe500de26c52abe0477dde16192ac39c98f05bf2d80e76102d394bd13854"}, + {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:aee66be87825cdf72ac64cb03ad4c15ffef4143dbf5c113f64a5ff4f81477bf9"}, + {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b748c44bb9f53031c8cbc99a8a061bc181c1000c60a30f55393b6e9c45cc5bd"}, + {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ca038c7f6a0afd0b2448941b6ef9d5e1949e999f9e5517692eb6da58e9d44be"}, + {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e0bd57539da59a3e4671b90a502da9a28c72322a4f17866ba3ac63a82c4498e"}, + {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ac6c2c45c847bbf8f91930d88716a0fb924b51e0c6dad329b793d670ec5db792"}, + {file = "pydantic_core-2.27.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b94d4ba43739bbe8b0ce4262bcc3b7b9f31459ad120fb595627eaeb7f9b9ca01"}, + {file = "pydantic_core-2.27.1-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:00e6424f4b26fe82d44577b4c842d7df97c20be6439e8e685d0d715feceb9fb9"}, + {file = "pydantic_core-2.27.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:38de0a70160dd97540335b7ad3a74571b24f1dc3ed33f815f0880682e6880131"}, + {file = "pydantic_core-2.27.1-cp39-none-win32.whl", hash = "sha256:7ccebf51efc61634f6c2344da73e366c75e735960b5654b63d7e6f69a5885fa3"}, + {file = "pydantic_core-2.27.1-cp39-none-win_amd64.whl", hash = "sha256:a57847b090d7892f123726202b7daa20df6694cbd583b67a592e856bff603d6c"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3fa80ac2bd5856580e242dbc202db873c60a01b20309c8319b5c5986fbe53ce6"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d950caa237bb1954f1b8c9227b5065ba6875ac9771bb8ec790d956a699b78676"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e4216e64d203e39c62df627aa882f02a2438d18a5f21d7f721621f7a5d3611d"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02a3d637bd387c41d46b002f0e49c52642281edacd2740e5a42f7017feea3f2c"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:161c27ccce13b6b0c8689418da3885d3220ed2eae2ea5e9b2f7f3d48f1d52c27"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:19910754e4cc9c63bc1c7f6d73aa1cfee82f42007e407c0f413695c2f7ed777f"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:e173486019cc283dc9778315fa29a363579372fe67045e971e89b6365cc035ed"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:af52d26579b308921b73b956153066481f064875140ccd1dfd4e77db89dbb12f"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:981fb88516bd1ae8b0cbbd2034678a39dedc98752f264ac9bc5839d3923fa04c"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5fde892e6c697ce3e30c61b239330fc5d569a71fefd4eb6512fc6caec9dd9e2f"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:816f5aa087094099fff7edabb5e01cc370eb21aa1a1d44fe2d2aefdfb5599b31"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c10c309e18e443ddb108f0ef64e8729363adbfd92d6d57beec680f6261556f3"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98476c98b02c8e9b2eec76ac4156fd006628b1b2d0ef27e548ffa978393fd154"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c3027001c28434e7ca5a6e1e527487051136aa81803ac812be51802150d880dd"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:7699b1df36a48169cdebda7ab5a2bac265204003f153b4bd17276153d997670a"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:1c39b07d90be6b48968ddc8c19e7585052088fd7ec8d568bb31ff64c70ae3c97"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:46ccfe3032b3915586e469d4972973f893c0a2bb65669194a5bdea9bacc088c2"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:62ba45e21cf6571d7f716d903b5b7b6d2617e2d5d67c0923dc47b9d41369f840"}, + {file = "pydantic_core-2.27.1.tar.gz", hash = "sha256:62a763352879b84aa31058fc931884055fd75089cccbd9d58bb6afd01141b235"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + [[package]] name = "pyflakes" version = "3.2.0" @@ -175,6 +959,186 @@ files = [ {file = "pyflakes-3.2.0.tar.gz", hash = "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f"}, ] +[[package]] +name = "pystac" +version = "1.11.0" +description = "Python library for working with the SpatioTemporal Asset Catalog (STAC) specification" +optional = false +python-versions = ">=3.10" +files = [ + {file = "pystac-1.11.0-py3-none-any.whl", hash = "sha256:10ac7c7b4ea6c5ec8333829a09ec1a33b596f02d1a97ffbbd72cd1b6c10598c1"}, + {file = "pystac-1.11.0.tar.gz", hash = "sha256:acb1e04be398a0cda2d8870ab5e90457783a8014a206590233171d8b2ae0d9e7"}, +] + +[package.dependencies] +python-dateutil = ">=2.7.0" + +[package.extras] +jinja2 = ["jinja2 (<4.0)"] +orjson = ["orjson (>=3.5)"] +urllib3 = ["urllib3 (>=1.26)"] +validation = ["jsonschema (>=4.18,<5.0)"] + +[[package]] +name = "pystac_monty" +version = "0.1.0" +description = "Python library for working with the SpatioTemporal Asset Catalog (STAC) extension for Montandon" +optional = false +python-versions = ">=3.10" +files = [] +develop = true + +[package.dependencies] +geojson = ">=2.5.0" +markdownify = ">=0.14.1" +pandas = ">=2.2.0" +pystac = ">=1.11.0" +python-dateutil = ">=2.7.0" +pytz = ">=2021.1" +shapely = ">=2.0.0" + +[package.source] +type = "directory" +url = "libs/pystac-monty" + +[[package]] +name = "python-crontab" +version = "3.2.0" +description = "Python Crontab API" +optional = false +python-versions = "*" +files = [ + {file = "python_crontab-3.2.0-py3-none-any.whl", hash = "sha256:82cb9b6a312d41ff66fd3caf3eed7115c28c195bfb50711bc2b4b9592feb9fe5"}, + {file = "python_crontab-3.2.0.tar.gz", hash = "sha256:40067d1dd39ade3460b2ad8557c7651514cd3851deffff61c5c60e1227c5c36b"}, +] + +[package.dependencies] +python-dateutil = "*" + +[package.extras] +cron-description = ["cron-descriptor"] +cron-schedule = ["croniter"] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2024.2" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, + {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, +] + +[[package]] +name = "redis" +version = "5.2.0" +description = "Python client for Redis database and key-value store" +optional = false +python-versions = ">=3.8" +files = [ + {file = "redis-5.2.0-py3-none-any.whl", hash = "sha256:ae174f2bb3b1bf2b09d54bf3e51fbc1469cf6c10aa03e21141f51969801a7897"}, + {file = "redis-5.2.0.tar.gz", hash = "sha256:0b1087665a771b1ff2e003aa5bdd354f15a70c9e25d5a7dbf9c722c16528a7b0"}, +] + +[package.extras] +hiredis = ["hiredis (>=3.0.0)"] +ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==23.2.1)", "requests (>=2.31.0)"] + +[[package]] +name = "shapely" +version = "2.0.6" +description = "Manipulation and analysis of geometric objects" +optional = false +python-versions = ">=3.7" +files = [ + {file = "shapely-2.0.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29a34e068da2d321e926b5073539fd2a1d4429a2c656bd63f0bd4c8f5b236d0b"}, + {file = "shapely-2.0.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c84c3f53144febf6af909d6b581bc05e8785d57e27f35ebaa5c1ab9baba13b"}, + {file = "shapely-2.0.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ad2fae12dca8d2b727fa12b007e46fbc522148a584f5d6546c539f3464dccde"}, + {file = "shapely-2.0.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3304883bd82d44be1b27a9d17f1167fda8c7f5a02a897958d86c59ec69b705e"}, + {file = "shapely-2.0.6-cp310-cp310-win32.whl", hash = "sha256:3ec3a0eab496b5e04633a39fa3d5eb5454628228201fb24903d38174ee34565e"}, + {file = "shapely-2.0.6-cp310-cp310-win_amd64.whl", hash = "sha256:28f87cdf5308a514763a5c38de295544cb27429cfa655d50ed8431a4796090c4"}, + {file = "shapely-2.0.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5aeb0f51a9db176da9a30cb2f4329b6fbd1e26d359012bb0ac3d3c7781667a9e"}, + {file = "shapely-2.0.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9a7a78b0d51257a367ee115f4d41ca4d46edbd0dd280f697a8092dd3989867b2"}, + {file = "shapely-2.0.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f32c23d2f43d54029f986479f7c1f6e09c6b3a19353a3833c2ffb226fb63a855"}, + {file = "shapely-2.0.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3dc9fb0eb56498912025f5eb352b5126f04801ed0e8bdbd867d21bdbfd7cbd0"}, + {file = "shapely-2.0.6-cp311-cp311-win32.whl", hash = "sha256:d93b7e0e71c9f095e09454bf18dad5ea716fb6ced5df3cb044564a00723f339d"}, + {file = "shapely-2.0.6-cp311-cp311-win_amd64.whl", hash = "sha256:c02eb6bf4cfb9fe6568502e85bb2647921ee49171bcd2d4116c7b3109724ef9b"}, + {file = "shapely-2.0.6-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cec9193519940e9d1b86a3b4f5af9eb6910197d24af02f247afbfb47bcb3fab0"}, + {file = "shapely-2.0.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:83b94a44ab04a90e88be69e7ddcc6f332da7c0a0ebb1156e1c4f568bbec983c3"}, + {file = "shapely-2.0.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:537c4b2716d22c92036d00b34aac9d3775e3691f80c7aa517c2c290351f42cd8"}, + {file = "shapely-2.0.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98fea108334be345c283ce74bf064fa00cfdd718048a8af7343c59eb40f59726"}, + {file = "shapely-2.0.6-cp312-cp312-win32.whl", hash = "sha256:42fd4cd4834747e4990227e4cbafb02242c0cffe9ce7ef9971f53ac52d80d55f"}, + {file = "shapely-2.0.6-cp312-cp312-win_amd64.whl", hash = "sha256:665990c84aece05efb68a21b3523a6b2057e84a1afbef426ad287f0796ef8a48"}, + {file = "shapely-2.0.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:42805ef90783ce689a4dde2b6b2f261e2c52609226a0438d882e3ced40bb3013"}, + {file = "shapely-2.0.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6d2cb146191a47bd0cee8ff5f90b47547b82b6345c0d02dd8b25b88b68af62d7"}, + {file = "shapely-2.0.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3fdef0a1794a8fe70dc1f514440aa34426cc0ae98d9a1027fb299d45741c381"}, + {file = "shapely-2.0.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c665a0301c645615a107ff7f52adafa2153beab51daf34587170d85e8ba6805"}, + {file = "shapely-2.0.6-cp313-cp313-win32.whl", hash = "sha256:0334bd51828f68cd54b87d80b3e7cee93f249d82ae55a0faf3ea21c9be7b323a"}, + {file = "shapely-2.0.6-cp313-cp313-win_amd64.whl", hash = "sha256:d37d070da9e0e0f0a530a621e17c0b8c3c9d04105655132a87cfff8bd77cc4c2"}, + {file = "shapely-2.0.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fa7468e4f5b92049c0f36d63c3e309f85f2775752e076378e36c6387245c5462"}, + {file = "shapely-2.0.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed5867e598a9e8ac3291da6cc9baa62ca25706eea186117034e8ec0ea4355653"}, + {file = "shapely-2.0.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81d9dfe155f371f78c8d895a7b7f323bb241fb148d848a2bf2244f79213123fe"}, + {file = "shapely-2.0.6-cp37-cp37m-win32.whl", hash = "sha256:fbb7bf02a7542dba55129062570211cfb0defa05386409b3e306c39612e7fbcc"}, + {file = "shapely-2.0.6-cp37-cp37m-win_amd64.whl", hash = "sha256:837d395fac58aa01aa544495b97940995211e3e25f9aaf87bc3ba5b3a8cd1ac7"}, + {file = "shapely-2.0.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c6d88ade96bf02f6bfd667ddd3626913098e243e419a0325ebef2bbd481d1eb6"}, + {file = "shapely-2.0.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8b3b818c4407eaa0b4cb376fd2305e20ff6df757bf1356651589eadc14aab41b"}, + {file = "shapely-2.0.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bbc783529a21f2bd50c79cef90761f72d41c45622b3e57acf78d984c50a5d13"}, + {file = "shapely-2.0.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2423f6c0903ebe5df6d32e0066b3d94029aab18425ad4b07bf98c3972a6e25a1"}, + {file = "shapely-2.0.6-cp38-cp38-win32.whl", hash = "sha256:2de00c3bfa80d6750832bde1d9487e302a6dd21d90cb2f210515cefdb616e5f5"}, + {file = "shapely-2.0.6-cp38-cp38-win_amd64.whl", hash = "sha256:3a82d58a1134d5e975f19268710e53bddd9c473743356c90d97ce04b73e101ee"}, + {file = "shapely-2.0.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:392f66f458a0a2c706254f473290418236e52aa4c9b476a072539d63a2460595"}, + {file = "shapely-2.0.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eba5bae271d523c938274c61658ebc34de6c4b33fdf43ef7e938b5776388c1be"}, + {file = "shapely-2.0.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7060566bc4888b0c8ed14b5d57df8a0ead5c28f9b69fb6bed4476df31c51b0af"}, + {file = "shapely-2.0.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b02154b3e9d076a29a8513dffcb80f047a5ea63c897c0cd3d3679f29363cf7e5"}, + {file = "shapely-2.0.6-cp39-cp39-win32.whl", hash = "sha256:44246d30124a4f1a638a7d5419149959532b99dfa25b54393512e6acc9c211ac"}, + {file = "shapely-2.0.6-cp39-cp39-win_amd64.whl", hash = "sha256:2b542d7f1dbb89192d3512c52b679c822ba916f93479fa5d4fc2fe4fa0b3c9e8"}, + {file = "shapely-2.0.6.tar.gz", hash = "sha256:997f6159b1484059ec239cacaa53467fd8b5564dabe186cd84ac2944663b0bf6"}, +] + +[package.dependencies] +numpy = ">=1.14,<3" + +[package.extras] +docs = ["matplotlib", "numpydoc (==1.1.*)", "sphinx", "sphinx-book-theme", "sphinx-remove-toctrees"] +test = ["pytest", "pytest-cov"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "soupsieve" +version = "2.6" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = false +python-versions = ">=3.8" +files = [ + {file = "soupsieve-2.6-py3-none-any.whl", hash = "sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9"}, + {file = "soupsieve-2.6.tar.gz", hash = "sha256:e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb"}, +] + [[package]] name = "sqlparse" version = "0.5.1" @@ -190,6 +1154,71 @@ files = [ dev = ["build", "hatch"] doc = ["sphinx"] +[[package]] +name = "strawberry-graphql" +version = "0.254.0" +description = "A library for creating GraphQL APIs" +optional = false +python-versions = "<4.0,>=3.8" +files = [ + {file = "strawberry_graphql-0.254.0-py3-none-any.whl", hash = "sha256:1bb98982d55cf09ebea4db0f77b65db1625a7dc2e33c7ce2139f8879e91d95a7"}, + {file = "strawberry_graphql-0.254.0.tar.gz", hash = "sha256:f0a5229d71ea09b5648e38c474c1a1185751ba7e3fdd9c59d5615f34d668a21f"}, +] + +[package.dependencies] +graphql-core = ">=3.2.0,<3.4.0" +python-dateutil = ">=2.7.0,<3.0.0" +typing-extensions = ">=4.5.0" + +[package.extras] +aiohttp = ["aiohttp (>=3.7.4.post0,<4.0.0)"] +asgi = ["python-multipart (>=0.0.7)", "starlette (>=0.18.0)"] +chalice = ["chalice (>=1.22,<2.0)"] +channels = ["asgiref (>=3.2,<4.0)", "channels (>=3.0.5)"] +cli = ["graphlib_backport", "libcst (>=0.4.7)", "pygments (>=2.3,<3.0)", "rich (>=12.0.0)", "typer (>=0.7.0)"] +debug = ["libcst (>=0.4.7)", "rich (>=12.0.0)"] +debug-server = ["libcst (>=0.4.7)", "pygments (>=2.3,<3.0)", "python-multipart (>=0.0.7)", "rich (>=12.0.0)", "starlette (>=0.18.0)", "typer (>=0.7.0)", "uvicorn (>=0.11.6)"] +django = ["Django (>=3.2)", "asgiref (>=3.2,<4.0)"] +fastapi = ["fastapi (>=0.65.2)", "python-multipart (>=0.0.7)"] +flask = ["flask (>=1.1)"] +litestar = ["litestar (>=2)"] +opentelemetry = ["opentelemetry-api (<2)", "opentelemetry-sdk (<2)"] +pydantic = ["pydantic (>1.6.1)"] +pyinstrument = ["pyinstrument (>=4.0.0)"] +quart = ["quart (>=0.19.3)"] +sanic = ["sanic (>=20.12.2)"] + +[[package]] +name = "strawberry-graphql-django" +version = "0.47.1" +description = "Strawberry GraphQL Django extension" +optional = false +python-versions = "<4.0,>=3.8" +files = [ + {file = "strawberry_graphql_django-0.47.1-py3-none-any.whl", hash = "sha256:9afef91933c6b7a87b80a61b6634a278086f8880b3fbc0d2aa56f78747043cbc"}, + {file = "strawberry_graphql_django-0.47.1.tar.gz", hash = "sha256:864c3f41de741639ce1b33107ee16ccd400167d5d0bc4e1fe01b1f3f556e127e"}, +] + +[package.dependencies] +asgiref = ">=3.8" +django = ">=3.2" +strawberry-graphql = ">=0.236.0" + +[package.extras] +debug-toolbar = ["django-debug-toolbar (>=3.4)"] +enum = ["django-choices-field (>=2.2.2)"] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, +] + [[package]] name = "tzdata" version = "2024.2" @@ -201,7 +1230,47 @@ files = [ {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"}, ] +[[package]] +name = "vine" +version = "5.1.0" +description = "Python promises." +optional = false +python-versions = ">=3.6" +files = [ + {file = "vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc"}, + {file = "vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0"}, +] + +[[package]] +name = "wcwidth" +version = "0.2.13" +description = "Measures the displayed width of unicode strings in a terminal" +optional = false +python-versions = "*" +files = [ + {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, + {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, +] + [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "6871e0539b17308ed7a721c31f1340f48774463bde31d76836c7920438ab7c25" +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +content-hash = "ef012eb09177f59b37c9526b4d98351695a7c3f19519bc978bf5d4fa5ab96654" +||||||| parent of 6d74328 (Add api for extraction data.) +content-hash = "e683fe738af6c2d979df924179474381d1c076b8207c17951d7a78934d8784d9" +======= +content-hash = "59f6313ef290e9662fd784673c300572f6260517bcdc0d8cf4719d6affc48033" +>>>>>>> 6d74328 (Add api for extraction data.) +||||||| parent of c3df16d (Resolve cors issue.) +content-hash = "59f6313ef290e9662fd784673c300572f6260517bcdc0d8cf4719d6affc48033" +======= +content-hash = "e2ba4ed92035036dde8a76e2aca1b5a2981f8e4bd33d2bc94598f8545fc4c8ed" +>>>>>>> c3df16d (Resolve cors issue.) +||||||| parent of e566049 (- Add extraction queries) +content-hash = "e2ba4ed92035036dde8a76e2aca1b5a2981f8e4bd33d2bc94598f8545fc4c8ed" +======= +content-hash = "1b33bbc22b14fdc73c0f2713e294396e88a5aa833cd6ccb65dd196b69887fd2e" +>>>>>>> e566049 (- Add extraction queries) diff --git a/pyproject.toml b/pyproject.toml index 8bddb96..bf0ec35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,15 @@ django = "^5.1.3" django-environ = "*" psycopg2-binary = "^2.9.9" flake8 = "^7.1.1" - +celery = {extras = ["redis"], version = "^5.4.0"} +django-redis = "^5.4.0" +django-celery-beat = "^2.7.0" +pandas = "^2.2.3" +lxml = "^5.3.0" +pydantic = "^2.10.2" +pystac-monty = {path = "libs/pystac-monty", develop = true} +strawberry-graphql-django = {extras = ["strawberry-graphql"], version = "0.47.1"} +django-cors-headers = "^4.6.0" [build-system] requires = ["poetry-core"] diff --git a/sample.env b/sample.env new file mode 100644 index 0000000..9dea96a --- /dev/null +++ b/sample.env @@ -0,0 +1,2 @@ +DJANGO_SECRET_KEY=test +DJANGO_DEBUG=true diff --git a/schema.graphql b/schema.graphql new file mode 100644 index 0000000..39a16bf --- /dev/null +++ b/schema.graphql @@ -0,0 +1,155 @@ +"""Date with time (isoformat)""" +scalar DateTime + +input DatetimeDatetimeFilterLookup { + """Exact match. Filter will be skipped on `null` value""" + exact: DateTime + + """Assignment test. Filter will be skipped on `null` value""" + isNull: Boolean + + """ + Exact match of items in a given list. Filter will be skipped on `null` value + """ + inList: [DateTime!] + + """Greater than. Filter will be skipped on `null` value""" + gt: DateTime + + """Greater than or equal to. Filter will be skipped on `null` value""" + gte: DateTime + + """Less than. Filter will be skipped on `null` value""" + lt: DateTime + + """Less than or equal to. Filter will be skipped on `null` value""" + lte: DateTime + + """Inclusive range test (between)""" + range: DatetimeRangeLookup + year: IntComparisonFilterLookup + month: IntComparisonFilterLookup + day: IntComparisonFilterLookup + weekDay: IntComparisonFilterLookup + isoWeekDay: IntComparisonFilterLookup + week: IntComparisonFilterLookup + isoYear: IntComparisonFilterLookup + quarter: IntComparisonFilterLookup + hour: IntComparisonFilterLookup + minute: IntComparisonFilterLookup + second: IntComparisonFilterLookup + date: IntComparisonFilterLookup + time: IntComparisonFilterLookup +} + +input DatetimeRangeLookup { + start: DateTime = null + end: DateTime = null +} + +input ExtractionDataFilter { + source: ExtractionDataSourceTypeEnum + status: ExtractionDataStatusTypeEnum + createdAt: DatetimeDatetimeFilterLookup + AND: ExtractionDataFilter + OR: ExtractionDataFilter + NOT: ExtractionDataFilter + DISTINCT: Boolean + createdAtLte: ID + createdAtGte: ID +} + +enum ExtractionDataSourceTypeEnum { + GDACS + PDC +} + +enum ExtractionDataStatusTypeEnum { + PENDING + IN_PROGRESS + SUCCESS + FAILED +} + +type ExtractionDataType { + id: ID! + source: ExtractionDataSourceTypeEnum! + url: String! + respCode: Int! + status: ExtractionDataStatusTypeEnum! + respDataType: String! + parentId: Int + sourceValidationStatus: Int! + revisionId: Int + hazardType: String! +} + +type ExtractionDataTypeCountList { + limit: Int! + offset: Int! + count: Int! + items: [ExtractionDataType!]! +} + +input IntComparisonFilterLookup { + """Exact match. Filter will be skipped on `null` value""" + exact: Int + + """Assignment test. Filter will be skipped on `null` value""" + isNull: Boolean + + """ + Exact match of items in a given list. Filter will be skipped on `null` value + """ + inList: [Int!] + + """Greater than. Filter will be skipped on `null` value""" + gt: Int + + """Greater than or equal to. Filter will be skipped on `null` value""" + gte: Int + + """Less than. Filter will be skipped on `null` value""" + lt: Int + + """Less than or equal to. Filter will be skipped on `null` value""" + lte: Int + + """Inclusive range test (between)""" + range: IntRangeLookup +} + +input IntRangeLookup { + start: Int = null + end: Int = null +} + +input OffsetPaginationInput { + offset: Int! = 0 + limit: Int! = -1 +} + +type PrivateQuery { + extractionList(filters: ExtractionDataFilter, pagination: OffsetPaginationInput): ExtractionDataTypeCountList! + me: UserMeType + id: ID! +} + +type PublicQuery { + id: ID! +} + +type Query { + public: PublicQuery! + private: PrivateQuery! +} + +type UserMeType { + id: ID! + username: String! + firstName: String! + lastName: String! + email: String! + isStaff: Boolean! + isSuperuser: Boolean! +} \ No newline at end of file diff --git a/scripts/run_worker.sh b/scripts/run_worker.sh new file mode 100755 index 0000000..0a11f81 --- /dev/null +++ b/scripts/run_worker.sh @@ -0,0 +1,3 @@ +#!/bin/bash -x + +celery -A main worker --loglevel=info diff --git a/scripts/run_worker_beat.sh b/scripts/run_worker_beat.sh new file mode 100755 index 0000000..fc57deb --- /dev/null +++ b/scripts/run_worker_beat.sh @@ -0,0 +1,3 @@ +#!/bin/bash -x + +celery -A main beat --loglevel=info