Failure webhooks (#2)

* failure webhooks

* deps

* fix

* fix

---------

Co-authored-by: Josh Smith <[email protected]>

app/api/webhooks/paypal.py

@@ -1,3 +1,4 @@
+import asyncio
 import logging
 import time
 import urllib.parse
@@ -11,9 +12,13 @@
 from fastapi import Header
 from fastapi import Request
 from fastapi import Response
+from tenacity import retry
+from tenacity import stop_after_attempt
+from tenacity import wait_exponential_jitter
 
 from app import clients
 from app import settings
+from app.reliability import retry_if_exception_network_related
 from app.repositories import notifications
 from app.repositories import user_badges
 from app.repositories import users
@@ -70,6 +75,29 @@ def supporter_to_premium(donor_time_remaining: float) -> float:
     return donor_time_remaining * exchange_rate
 
 
+@retry(
+    stop=stop_after_attempt(7),
+    wait=wait_exponential_jitter(initial=1, max=60, exp_base=2, jitter=1),
+    retry=retry_if_exception_network_related(),
+)
+async def send_discord_webhook(webhook: AsyncDiscordWebhook) -> None:
+    await webhook.execute()
+
+
+def schedule_failure_webhook(**data: Any) -> None:
+    webhook = AsyncDiscordWebhook(
+        url=settings.DISCORD_WEBHOOK_URL,
+        embeds=[
+            DiscordEmbed(
+                title="Failed to grant donation perks to user",
+                fields=[{"name": k, "value": str(v)} for k, v in data.items()],
+                color=0xFF0000,
+            ),
+        ],
+    )
+    asyncio.create_task(send_discord_webhook(webhook))
+
+
 @router.post("/webhooks/paypal_ipn")
 async def process_notification(
     request: Request,
@@ -97,6 +125,7 @@ async def process_notification(
         logging.warning(
             "PayPal IPN invalid",
             extra={
+                "reason": "ipn_verification_failed",
                 "response_text": response.text,
                 "will_grant_donor": will_grant_donor,
                 "request_id": x_request_id,
@@ -106,6 +135,11 @@ async def process_notification(
         if settings.SHOULD_REQUIRE_IPN_VERIFICATION:
             # Do not process the request any further.
             # Return a 2xx code to prevent PayPal from retrying.
+            schedule_failure_webhook(
+                reason="ipn_verification_failed",
+                response_text=response.text,
+                request_id=x_request_id,
+            )
             return Response(status_code=200)
         else:
             pass
@@ -121,6 +155,11 @@ async def process_notification(
                 "request_id": x_request_id,
             },
         )
+        schedule_failure_webhook(
+            reason="incomplete_payment",
+            payment_status=notification["payment_status"],
+            request_id=x_request_id,
+        )
         return Response(status_code=200)
 
     transaction_id = notification["txn_id"]
@@ -136,6 +175,11 @@ async def process_notification(
                 "request_id": x_request_id,
             },
         )
+        schedule_failure_webhook(
+            reason="transaction_already_processed",
+            transaction_id=transaction_id,
+            request_id=x_request_id,
+        )
         return Response(status_code=200)
 
     if notification["business"] != settings.PAYPAL_BUSINESS_EMAIL:
@@ -148,6 +192,12 @@ async def process_notification(
                 "request_id": x_request_id,
             },
         )
+        schedule_failure_webhook(
+            reason="wrong_paypal_business_email",
+            business=notification["business"],
+            expected_business=settings.PAYPAL_BUSINESS_EMAIL,
+            request_id=x_request_id,
+        )
         return Response(status_code=200)
 
     donation_currency = notification["mc_currency"]
@@ -161,6 +211,12 @@ async def process_notification(
                 "request_id": x_request_id,
             },
         )
+        schedule_failure_webhook(
+            reason="non_accpeted_currency",
+            currency=donation_currency,
+            accepted_currencies=ACCEPTED_CURRENCIES,
+            request_id=x_request_id,
+        )
         return Response(status_code=200)
 
     custom_fields = dict(urllib.parse.parse_qsl(notification["custom"]))
@@ -177,6 +233,10 @@ async def process_notification(
                 "request_id": x_request_id,
             },
         )
+        schedule_failure_webhook(
+            reason="no_user_identification",
+            request_id=x_request_id,
+        )
         return Response(status_code=200)
 
     if user is None:
@@ -188,6 +248,11 @@ async def process_notification(
                 "request_id": x_request_id,
             },
         )
+        schedule_failure_webhook(
+            reason="user_not_found",
+            custom_fields=custom_fields,
+            request_id=x_request_id,
+        )
         return Response(status_code=200)
 
     user_id = user["id"]
@@ -221,6 +286,11 @@ async def process_notification(
                 "request_id": x_request_id,
             },
         )
+        schedule_failure_webhook(
+            reason="invalid_donation_tier",
+            donation_tier=donation_tier,
+            request_id=x_request_id,
+        )
         return Response(status_code=200)
 
     donation_amount = float(notification["mc_gross"])
@@ -234,6 +304,12 @@ async def process_notification(
                 "request_id": x_request_id,
             },
         )
+        schedule_failure_webhook(
+            reason="invalid_donation_amount",
+            donation_amount=donation_amount,
+            calculated_price=calculated_price,
+            request_id=x_request_id,
+        )
         return Response(status_code=200)
 
     privileges = user["privileges"]

app/reliability.py

@@ -0,0 +1,20 @@
+import httpx
+from fastapi import status
+from tenacity import retry_if_exception
+
+
+class retry_if_exception_network_related(retry_if_exception):
+    """Retries if an exception is from a network related failure."""
+
+    def __init__(self) -> None:
+        def predicate(exc: BaseException) -> bool:
+            if isinstance(exc, httpx.HTTPStatusError):
+                if exc.response.status_code == status.HTTP_429_TOO_MANY_REQUESTS:
+                    # TODO: 3rd parties may have specific retry-after headers
+                    #       that we should/can respect for better performance
+                    return True
+            elif isinstance(exc, httpx.NetworkError):
+                return True
+            return False
+
+        super().__init__(predicate)

requirements.txt

@@ -6,4 +6,5 @@ pydantic
 python-dotenv
 python-json-logger
 pyyaml
+tenacity
 uvicorn