From 2a2598ddd980ced10a0c1801dc251c77c3e24430 Mon Sep 17 00:00:00 2001 From: Josh Smith Date: Sun, 17 Dec 2023 05:42:41 -0500 Subject: [PATCH] Failure webhooks (#2) * failure webhooks * deps * fix * fix --------- Co-authored-by: Josh Smith --- app/api/webhooks/paypal.py | 76 ++++++++++++++++++++++++++++++++++++++ app/reliability.py | 20 ++++++++++ requirements.txt | 1 + 3 files changed, 97 insertions(+) create mode 100644 app/reliability.py diff --git a/app/api/webhooks/paypal.py b/app/api/webhooks/paypal.py index 2685eaf..42fbbf1 100644 --- a/app/api/webhooks/paypal.py +++ b/app/api/webhooks/paypal.py @@ -1,3 +1,4 @@ +import asyncio import logging import time import urllib.parse @@ -11,9 +12,13 @@ from fastapi import Header from fastapi import Request from fastapi import Response +from tenacity import retry +from tenacity import stop_after_attempt +from tenacity import wait_exponential_jitter from app import clients from app import settings +from app.reliability import retry_if_exception_network_related from app.repositories import notifications from app.repositories import user_badges from app.repositories import users @@ -70,6 +75,29 @@ def supporter_to_premium(donor_time_remaining: float) -> float: return donor_time_remaining * exchange_rate +@retry( + stop=stop_after_attempt(7), + wait=wait_exponential_jitter(initial=1, max=60, exp_base=2, jitter=1), + retry=retry_if_exception_network_related(), +) +async def send_discord_webhook(webhook: AsyncDiscordWebhook) -> None: + await webhook.execute() + + +def schedule_failure_webhook(**data: Any) -> None: + webhook = AsyncDiscordWebhook( + url=settings.DISCORD_WEBHOOK_URL, + embeds=[ + DiscordEmbed( + title="Failed to grant donation perks to user", + fields=[{"name": k, "value": str(v)} for k, v in data.items()], + color=0xFF0000, + ), + ], + ) + asyncio.create_task(send_discord_webhook(webhook)) + + @router.post("/webhooks/paypal_ipn") async def process_notification( request: Request, @@ -97,6 +125,7 @@ async def process_notification( logging.warning( "PayPal IPN invalid", extra={ + "reason": "ipn_verification_failed", "response_text": response.text, "will_grant_donor": will_grant_donor, "request_id": x_request_id, @@ -106,6 +135,11 @@ async def process_notification( if settings.SHOULD_REQUIRE_IPN_VERIFICATION: # Do not process the request any further. # Return a 2xx code to prevent PayPal from retrying. + schedule_failure_webhook( + reason="ipn_verification_failed", + response_text=response.text, + request_id=x_request_id, + ) return Response(status_code=200) else: pass @@ -121,6 +155,11 @@ async def process_notification( "request_id": x_request_id, }, ) + schedule_failure_webhook( + reason="incomplete_payment", + payment_status=notification["payment_status"], + request_id=x_request_id, + ) return Response(status_code=200) transaction_id = notification["txn_id"] @@ -136,6 +175,11 @@ async def process_notification( "request_id": x_request_id, }, ) + schedule_failure_webhook( + reason="transaction_already_processed", + transaction_id=transaction_id, + request_id=x_request_id, + ) return Response(status_code=200) if notification["business"] != settings.PAYPAL_BUSINESS_EMAIL: @@ -148,6 +192,12 @@ async def process_notification( "request_id": x_request_id, }, ) + schedule_failure_webhook( + reason="wrong_paypal_business_email", + business=notification["business"], + expected_business=settings.PAYPAL_BUSINESS_EMAIL, + request_id=x_request_id, + ) return Response(status_code=200) donation_currency = notification["mc_currency"] @@ -161,6 +211,12 @@ async def process_notification( "request_id": x_request_id, }, ) + schedule_failure_webhook( + reason="non_accpeted_currency", + currency=donation_currency, + accepted_currencies=ACCEPTED_CURRENCIES, + request_id=x_request_id, + ) return Response(status_code=200) custom_fields = dict(urllib.parse.parse_qsl(notification["custom"])) @@ -177,6 +233,10 @@ async def process_notification( "request_id": x_request_id, }, ) + schedule_failure_webhook( + reason="no_user_identification", + request_id=x_request_id, + ) return Response(status_code=200) if user is None: @@ -188,6 +248,11 @@ async def process_notification( "request_id": x_request_id, }, ) + schedule_failure_webhook( + reason="user_not_found", + custom_fields=custom_fields, + request_id=x_request_id, + ) return Response(status_code=200) user_id = user["id"] @@ -221,6 +286,11 @@ async def process_notification( "request_id": x_request_id, }, ) + schedule_failure_webhook( + reason="invalid_donation_tier", + donation_tier=donation_tier, + request_id=x_request_id, + ) return Response(status_code=200) donation_amount = float(notification["mc_gross"]) @@ -234,6 +304,12 @@ async def process_notification( "request_id": x_request_id, }, ) + schedule_failure_webhook( + reason="invalid_donation_amount", + donation_amount=donation_amount, + calculated_price=calculated_price, + request_id=x_request_id, + ) return Response(status_code=200) privileges = user["privileges"] diff --git a/app/reliability.py b/app/reliability.py new file mode 100644 index 0000000..61cf0e5 --- /dev/null +++ b/app/reliability.py @@ -0,0 +1,20 @@ +import httpx +from fastapi import status +from tenacity import retry_if_exception + + +class retry_if_exception_network_related(retry_if_exception): + """Retries if an exception is from a network related failure.""" + + def __init__(self) -> None: + def predicate(exc: BaseException) -> bool: + if isinstance(exc, httpx.HTTPStatusError): + if exc.response.status_code == status.HTTP_429_TOO_MANY_REQUESTS: + # TODO: 3rd parties may have specific retry-after headers + # that we should/can respect for better performance + return True + elif isinstance(exc, httpx.NetworkError): + return True + return False + + super().__init__(predicate) diff --git a/requirements.txt b/requirements.txt index 804f908..01daef8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ pydantic python-dotenv python-json-logger pyyaml +tenacity uvicorn