From ed2d7d10f7696e162368e4e90776c98b0e5acc80 Mon Sep 17 00:00:00 2001 From: David Paul Graham <43794491+dpgraham4401@users.noreply.github.com> Date: Thu, 26 Oct 2023 15:47:56 -0400 Subject: [PATCH] Dev fixtures (#621) * add Transporter and TSDFs mock RcraSite model instances * add error handling when HandlerSearchView is unable to connect to RCRAInfo This error handling now searches for known (by haztrak) RcraSites when haztrak cannot pull sites from RCRAInfo * refactor manifest services to return predictable types and raise uniform exceptions * add check for if task results are string and parse JSON --- client/package-lock.json | 4 +- .../Manifest/Handler/HandlerSearchForm.tsx | 29 +++++++- .../Manifest/UpdateRcra/UpdateRcra.tsx | 5 +- server/apps/trak/models/manifest_models.py | 9 +-- server/apps/trak/serializers/manifest_ser.py | 7 ++ server/apps/trak/services/__init__.py | 2 +- server/apps/trak/services/manifest_service.py | 71 ++++++++++++------- server/apps/trak/tasks/manifest_task.py | 25 +++---- server/apps/trak/views/manifest_view.py | 6 +- server/fixtures/dev_data.yaml | 50 ++++++++++++- 10 files changed, 155 insertions(+), 53 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index 37454802c..dfa86229a 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -1,12 +1,12 @@ { "name": "haztrak", - "version": "0.7.0", + "version": "0.6.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "haztrak", - "version": "0.7.0", + "version": "0.6.2", "dependencies": { "@formkit/auto-animate": "^0.8.0", "@fortawesome/fontawesome-svg-core": "^6.4.2", diff --git a/client/src/components/Manifest/Handler/HandlerSearchForm.tsx b/client/src/components/Manifest/Handler/HandlerSearchForm.tsx index 4706fca58..ef8ba001f 100644 --- a/client/src/components/Manifest/Handler/HandlerSearchForm.tsx +++ b/client/src/components/Manifest/Handler/HandlerSearchForm.tsx @@ -3,7 +3,7 @@ import { ManifestContext, ManifestContextType } from 'components/Manifest/Manife import { Manifest, SiteType, Transporter } from 'components/Manifest/manifestSchema'; import { RcraSite } from 'components/RcraSite'; import React, { useContext, useState } from 'react'; -import { Button } from 'react-bootstrap'; +import { Alert, Button } from 'react-bootstrap'; import { Controller, SubmitHandler, @@ -40,6 +40,13 @@ export function HandlerSearchForm({ const dispatch = useAppDispatch(); const { setGeneratorStateCode, setTsdfStateCode } = useContext(ManifestContext); + const [searchMessage, setSearchMessage] = useState< + | { + message: string; + variant: 'success' | 'danger'; + } + | undefined + >(undefined); const [options, setOptions] = useState([]); const [rcrainfoSitesLoading, setRcrainfoSitesLoading] = useState(false); @@ -76,6 +83,21 @@ export function HandlerSearchForm({ siteId: value, }) ); + if (rcrainfoSites.isError) { + setSearchMessage({ + message: 'Sorry, could not connect to RCRAInfo. Showing known handlers.', + variant: 'danger', + }); + const knownRcraSites = await dispatch( + siteApi.endpoints?.searchRcraSites.initiate({ + siteType: handlerType, + siteId: value, + }) + ); + setOptions(knownRcraSites.data as Array); + setRcrainfoSitesLoading(false); + return; + } setOptions(rcrainfoSites.data as Array); setRcrainfoSitesLoading(false); } @@ -89,6 +111,11 @@ export function HandlerSearchForm({ <> + {searchMessage && ( +
+ {searchMessage.message} +
+ )} EPA ID Number ; } diff --git a/server/apps/trak/models/manifest_models.py b/server/apps/trak/models/manifest_models.py index 9a9fdc556..f72c4a5f6 100644 --- a/server/apps/trak/models/manifest_models.py +++ b/server/apps/trak/models/manifest_models.py @@ -22,12 +22,9 @@ def draft_mtn(): A callable that returns a timestamped draft MTN in lieu of an official MTN from e-Manifest """ - manifests = Manifest.objects.all() - if not manifests: - max_id = 1 - else: - max_id = manifests.aggregate(Max("id")) - return f"{str(max_id).zfill(9)}DFT" + mtn_count: int = Manifest.objects.all().count() + print(mtn_count) + return f"{str(mtn_count).zfill(9)}DFT" def validate_mtn(value): diff --git a/server/apps/trak/serializers/manifest_ser.py b/server/apps/trak/serializers/manifest_ser.py index 64aee705c..ee7d4a77d 100644 --- a/server/apps/trak/serializers/manifest_ser.py +++ b/server/apps/trak/serializers/manifest_ser.py @@ -10,6 +10,7 @@ CorrectionInfo, ImportInfo, PortOfEntry, + draft_mtn, ) from apps.trak.serializers.handler_ser import HandlerSerializer from apps.trak.serializers.signature_ser import ESignatureSerializer @@ -221,6 +222,12 @@ def update(self, instance, validated_data: Dict) -> Manifest: def create(self, validated_data: Dict) -> Manifest: return self.Meta.model.objects.save(**validated_data) + def validate(self, data): + if data["mtn"] == "" and data["status"] == "NotAssigned": + data["mtn"] = draft_mtn() + print("data", data) + return super().validate(data) + # https://www.django-rest-framework.org/api-guide/serializers/#overriding-serialization-and-deserialization-behavior def to_representation(self, instance) -> str: """ diff --git a/server/apps/trak/services/__init__.py b/server/apps/trak/services/__init__.py index 10720d180..8512a1d96 100644 --- a/server/apps/trak/services/__init__.py +++ b/server/apps/trak/services/__init__.py @@ -1 +1 @@ -from .manifest_service import ManifestService +from .manifest_service import ManifestService, ManifestServiceError diff --git a/server/apps/trak/services/manifest_service.py b/server/apps/trak/services/manifest_service.py index 384023ea2..4fb18b70a 100644 --- a/server/apps/trak/services/manifest_service.py +++ b/server/apps/trak/services/manifest_service.py @@ -5,6 +5,7 @@ from django.db import transaction from emanifest import RcrainfoResponse # type: ignore from requests import RequestException # type: ignore +from rest_framework.exceptions import ValidationError from apps.core.services import RcrainfoService # type: ignore from apps.trak.models import Manifest, QuickerSign # type: ignore @@ -14,6 +15,14 @@ logger = logging.getLogger(__name__) +class ManifestServiceError(Exception): + """Base class for ManifestService exceptions""" + + def __init__(self, message: str = None, *args): + self.message = message + super().__init__(*args) + + class PullManifestsResult(TypedDict): """Type definition for the results returned from pulling manifests from RCRAInfo""" @@ -36,27 +45,6 @@ def __repr__(self): f"<{self.__class__.__name__}(username='{self.username}', rcrainfo='{self.rcrainfo}')>" ) - def _retrieve_manifest(self, mtn: str): - response = self.rcrainfo.get_manifest(mtn) - if response.ok: - logger.debug(f"manifest pulled {mtn}") - return response.json() - else: - logger.warning(f"error retrieving manifest {mtn}") - raise RequestException(response.json()) - - @transaction.atomic - def _save_manifest(self, manifest_json: dict) -> Manifest: - serializer = ManifestSerializer(data=manifest_json) - if serializer.is_valid(): - logger.debug("manifest serializer is valid") - manifest = serializer.save() - logger.info(f"saved manifest {manifest.mtn}") - return manifest - else: - logger.warning(f"malformed serializer data: {serializer.errors}") - raise Exception(serializer.errors) - def search_rcra_mtn( self, *, @@ -124,7 +112,7 @@ def pull_manifests(self, tracking_numbers: List[str]) -> PullManifestsResult: for mtn in tracking_numbers: try: manifest_json: dict = self._retrieve_manifest(mtn) - manifest = self._save_manifest(manifest_json) + manifest = self._save_manifest_to_db(manifest_json) results["success"].append(manifest.mtn) except Exception as exc: logger.warning(f"error pulling manifest {mtn}: {exc}") @@ -161,13 +149,23 @@ def sign_manifest(self, signature: QuickerSign) -> PullManifestsResult: results["error"].extend(results["success"]) # Temporary return results - def create_rcra_manifest(self, *, manifest: dict) -> RcrainfoResponse: + def create_rcra_manifest(self, *, manifest: dict) -> dict | None: """ Create a manifest in RCRAInfo through the RESTful API. :param manifest: Dict :return: """ - logger.debug(f"start create rcra manifest with arguments {manifest}") + if self.rcrainfo.has_api_user: + logger.warning("POSTing manifest to RCRAInfo.") + return self._save_manifest_to_rcrainfo(manifest) + else: + logger.warning("RCRAInfo API credentials not found, RCRAInfo manifest creation") + saved_manifest = self._save_manifest_to_db(manifest) + logger.info(f"saved manifest {saved_manifest.mtn}") + return ManifestSerializer(saved_manifest).data + + def _save_manifest_to_rcrainfo(self, manifest: dict) -> dict: + logger.info(f"start save manifest to rcrainfo with arguments {manifest}") create_resp: RcrainfoResponse = self.rcrainfo.save_manifest(manifest) try: if create_resp.ok: @@ -176,7 +174,8 @@ def create_rcra_manifest(self, *, manifest: dict) -> RcrainfoResponse: f"{create_resp.json()['manifestTrackingNumber']} in RCRAInfo" ) self.pull_manifests([create_resp.json()["manifestTrackingNumber"]]) - return create_resp + return create_resp.json() + raise ManifestServiceError(message=f"error creating manifest: {create_resp.json()}") except KeyError: logger.error( f"error retrieving manifestTrackingNumber from response: {create_resp.json()}" @@ -192,3 +191,25 @@ def _filter_mtn(signature: QuickerSign) -> PullManifestsResult: results["error"].extend(list(set(signature.mtn).difference(set(results["success"])))) logger.warning(f"MTN not found or site not listed as site type {results['error']}") return results + + def _retrieve_manifest(self, mtn: str): + response = self.rcrainfo.get_manifest(mtn) + if response.ok: + logger.debug(f"manifest pulled {mtn}") + return response.json() + else: + logger.warning(f"error retrieving manifest {mtn}") + raise RequestException(response.json()) + + @transaction.atomic + def _save_manifest_to_db(self, manifest_json: dict) -> Manifest: + """Save manifest to Haztrak database""" + serializer = ManifestSerializer(data=manifest_json) + if serializer.is_valid(): + logger.debug("manifest serializer is valid") + manifest = serializer.save() + logger.info(f"saved manifest {manifest.mtn}") + return manifest + else: + logger.warning(f"malformed serializer data: {serializer.errors}") + raise ValidationError(serializer.errors) diff --git a/server/apps/trak/tasks/manifest_task.py b/server/apps/trak/tasks/manifest_task.py index a47ca53ad..4eb31aaf3 100644 --- a/server/apps/trak/tasks/manifest_task.py +++ b/server/apps/trak/tasks/manifest_task.py @@ -4,7 +4,6 @@ from celery import Task, shared_task, states from celery.exceptions import Ignore, Reject -from emanifest import RcrainfoResponse from apps.sites.models import RcraSiteType from apps.trak.models import QuickerSign @@ -33,7 +32,7 @@ def pull_manifest(self: Task, *, mtn: List[str], username: str) -> dict: raise Reject() except Exception as exc: task_status.update_task_status(status="FAILURE") - self.update_state(state=states.FAILURE, meta=f"unknown error: {exc}") + self.update_state(state=states.FAILURE, meta={"unknown error": str(exc)}) raise Ignore() @@ -70,7 +69,7 @@ def sign_manifest( except (ConnectionError, TimeoutError) as exc: raise Reject(exc) except ValueError as exc: - self.update_state(state=states.FAILURE, meta={"Error": f"{repr(exc)}"}) + self.update_state(state=states.FAILURE, meta={"error": f"{repr(exc)}"}) raise Ignore() except Exception as exc: self.update_state(state=states.FAILURE, meta={"unknown error": f"{exc}"}) @@ -88,7 +87,7 @@ def sync_site_manifests(self, *, site_id: str, username: str): return results except Exception as exc: logger.error(f"failed to sync {site_id} manifest") - self.update_state(state=states.FAILURE, meta={f"Error: {exc}"}) + self.update_state(state=states.FAILURE, meta={f"error: {exc}"}) raise Ignore() @@ -100,19 +99,21 @@ def create_rcra_manifest(self, *, manifest: dict, username: str): user who is creating the manifest """ from apps.core.services import TaskService - from apps.trak.services import ManifestService + from apps.trak.services import ManifestService, ManifestServiceError logger.info(f"start task: {self.name}") task_status = TaskService(task_id=self.request.id, task_name=self.name, status="STARTED") try: manifest_service = ManifestService(username=username) - resp: RcrainfoResponse = manifest_service.create_rcra_manifest(manifest=manifest) - if resp.ok: - task_status.update_task_status(status="SUCCESS", results=resp.json()) - return resp.json() - logger.error(f"failed to create manifest ({manifest}): {resp.json()}") - task_status.update_task_status(status="FAILURE", results=resp.json()) - return resp.json() + new_manifest = manifest_service.create_rcra_manifest(manifest=manifest) + if new_manifest: + task_status.update_task_status(status="SUCCESS", results=new_manifest) + return new_manifest + raise ManifestServiceError("error creating manifest") + except ManifestServiceError as exc: + logger.error(f"failed to create manifest ({manifest}): {exc.message}") + task_status.update_task_status(status="FAILURE", results=exc.message) + return {"error": exc.message} except Exception as exc: logger.error("error: ", exc) task_status.update_task_status(status="FAILURE", results={"result": str(exc)}) diff --git a/server/apps/trak/views/manifest_view.py b/server/apps/trak/views/manifest_view.py index dc5ff887c..354165c5b 100644 --- a/server/apps/trak/views/manifest_view.py +++ b/server/apps/trak/views/manifest_view.py @@ -127,12 +127,12 @@ class CreateRcraManifestView(GenericAPIView): def post(self, request: Request) -> Response: manifest_serializer = self.serializer_class(data=request.data) if manifest_serializer.is_valid(): - logger.debug( - f"manifest data submitted for creation in RCRAInfo: {manifest_serializer.data}" - ) task: AsyncResult = create_rcra_manifest.delay( manifest=manifest_serializer.data, username=str(request.user) ) + logger.debug( + f"manifest data submitted for creation in RCRAInfo: {manifest_serializer.data}" + ) TaskService(task_id=task.id, task_name=task.name).update_task_status("PENDING") return Response(data={"taskId": task.id}, status=status.HTTP_201_CREATED) else: diff --git a/server/fixtures/dev_data.yaml b/server/fixtures/dev_data.yaml index 62d78a25a..e24e0abf6 100644 --- a/server/fixtures/dev_data.yaml +++ b/server/fixtures/dev_data.yaml @@ -78,12 +78,58 @@ can_esign: true limited_esign: true registered_emanifest_user: true +- model: sites.rcrasite + pk: 2 + fields: + site_type: Transporter + epa_id: MOCKTRANS001 + name: Mock Transporter 01 + site_address: 1 + mail_address: 1 + modified: false + registered: false + contact: 1 + emergency_phone: null + gis_primary: false + can_esign: true + limited_esign: true + registered_emanifest_user: true +- model: sites.rcrasite + pk: 3 + fields: + site_type: Transporter + epa_id: MOCKTRANS002 + name: Mock Transporter 02 + site_address: 1 + mail_address: 1 + modified: false + registered: false + contact: 1 + emergency_phone: null + gis_primary: false + can_esign: true + limited_esign: true + registered_emanifest_user: true +- model: sites.rcrasite + pk: 4 + fields: + site_type: Tsdf + epa_id: MOCKTSDFS001 + name: Mock haztrak TSDF 01 + site_address: 1 + mail_address: 1 + modified: false + registered: false + contact: 1 + emergency_phone: null + gis_primary: false + can_esign: true + limited_esign: true + registered_emanifest_user: true - model: core.rcraprofile pk: 1 fields: user: 2 - rcra_api_key: mockRcraKey - rcra_api_id: mockRcraInfoId rcra_username: dpgraham4401 phone_number: null email: test.user@haztrak.net