[mock_uss/flight_planning] Do not fail op intent injection or deletio…

…n when a notification fail (interuss#355)

* [mock_uss/flight_planning] Do not fail injection and deletion when notification fail

* narrow down scope of exceptions caught

monitoring/mock_uss/f3548v21/flight_planning.py

@@ -1,8 +1,9 @@
 import uuid
 from datetime import datetime
-from typing import Optional, List, Callable
+from typing import Optional, List, Callable, Dict, Tuple
 
 import arrow
+import requests
 
 from monitoring.mock_uss import webapp
 from monitoring.mock_uss.config import KEY_BASE_URL
@@ -459,7 +460,18 @@ def share_op_intent(
     existing_flight: Optional[FlightRecord],
     key: List[f3548_v21.EntityOVN],
     log: Callable[[str], None],
-):
+) -> Tuple[FlightRecord, Dict[f3548_v21.SubscriptionUssBaseURL, Exception]]:
+    """Share the operational intent reference with the DSS in compliance with ASTM F3548-21.
+
+    Returns:
+        The flight record shared;
+        Notification errors if any, by subscriber.
+
+    Raises:
+        * QueryError
+        * ConnectionError
+        * requests.exceptions.ConnectionError
+    """
     # Create operational intent in DSS
     log("Sharing operational intent with DSS")
     base_url = new_flight.op_intent.reference.uss_base_url
@@ -498,31 +510,27 @@ def share_op_intent(
         mod_op_sharing_behavior=new_flight.mod_op_sharing_behavior,
     )
     operational_intent = op_intent_from_flightrecord(record, "POST")
-    for subscriber in result.subscribers:
-        if subscriber.uss_base_url == base_url:
-            # Do not notify ourselves
-            continue
-        update = f3548_v21.PutOperationalIntentDetailsParameters(
-            operational_intent_id=result.operational_intent_reference.id,
-            operational_intent=operational_intent,
-            subscriptions=subscriber.subscriptions,
-        )
-        log(f"Notifying subscriber at {subscriber.uss_base_url}")
-        scd_client.notify_operational_intent_details_changed(
-            utm_client, subscriber.uss_base_url, update
-        )
-    return record
+    notif_errors = notify_subscribers(
+        result.operational_intent_reference.id,
+        operational_intent,
+        result.subscribers,
+        log,
+    )
+    return record, notif_errors
 
 
 def delete_op_intent(
     op_intent_ref: f3548_v21.OperationalIntentReference, log: Callable[[str], None]
-):
+) -> Dict[f3548_v21.SubscriptionUssBaseURL, Exception]:
     """Remove the operational intent reference from the DSS in compliance with ASTM F3548-21.
 
     Args:
         op_intent_ref: Operational intent reference to remove.
         log: Means of indicating debugging information.
 
+    Returns:
+        Notification errors if any, by subscriber.
+
     Raises:
         * QueryError
         * ConnectionError
@@ -533,17 +541,47 @@ def delete_op_intent(
         op_intent_ref.id,
         op_intent_ref.ovn,
     )
+    return notify_subscribers(
+        result.operational_intent_reference.id, None, result.subscribers, log
+    )
+
+
+def notify_subscribers(
+    op_intent_id: f3548_v21.EntityID,
+    op_intent: Optional[f3548_v21.OperationalIntent],
+    subscribers: List[f3548_v21.SubscriberToNotify],
+    log: Callable[[str], None],
+) -> Dict[f3548_v21.SubscriptionUssBaseURL, Exception]:
+    """
+    Notify subscribers of a changed or deleted operational intent.
+    This function will attempt all notifications, even if some of them fail.
 
+    :return: Notification errors if any, by subscriber.
+    """
+    notif_errors: Dict[f3548_v21.SubscriptionUssBaseURL, Exception] = {}
     base_url = "{}/mock/scd".format(webapp.config[KEY_BASE_URL])
-    for subscriber in result.subscribers:
+    for subscriber in subscribers:
         if subscriber.uss_base_url == base_url:
             # Do not notify ourselves
             continue
         update = f3548_v21.PutOperationalIntentDetailsParameters(
-            operational_intent_id=result.operational_intent_reference.id,
+            operational_intent_id=op_intent_id,
+            operational_intent=op_intent,
             subscriptions=subscriber.subscriptions,
         )
         log(f"Notifying {subscriber.uss_base_url}")
-        scd_client.notify_operational_intent_details_changed(
-            utm_client, subscriber.uss_base_url, update
-        )
+        try:
+            scd_client.notify_operational_intent_details_changed(
+                utm_client, subscriber.uss_base_url, update
+            )
+        except (
+            ValueError,
+            ConnectionError,
+            requests.exceptions.ConnectionError,
+            QueryError,
+        ) as e:
+            log(f"Failed to notify {subscriber.uss_base_url}: {str(e)}")
+            notif_errors[subscriber.uss_base_url] = e
+
+    log(f"{len(notif_errors) if notif_errors else 'No'} notifications failed")
+    return notif_errors

monitoring/mock_uss/scd_injection/routes_injection.py

@@ -180,6 +180,7 @@ def unsuccessful(
         return unsuccessful(PlanningActivityResult.Rejected, str(e))
 
     step_name = "performing unknown operation"
+    notes: Optional[str] = None
     try:
         step_name = "checking F3548-21 operational intent"
         try:
@@ -188,7 +189,13 @@ def unsuccessful(
             return unsuccessful(PlanningActivityResult.Rejected, str(e))
 
         step_name = "sharing operational intent in DSS"
-        record = share_op_intent(new_flight, existing_flight, key, log)
+        record, notif_errors = share_op_intent(new_flight, existing_flight, key, log)
+        if notif_errors:
+            notif_errors_messages = [
+                f"{url}: {str(err)}" for url, err in notif_errors.items()
+            ]
+            notes = f"Injection succeeded, but notification to some subscribers failed: {'; '.join(notif_errors_messages)}"
+            log(notes)
 
         # Store flight in database
         step_name = "storing flight in database"
@@ -204,6 +211,7 @@ def unsuccessful(
             queries=[],  # TODO: Add queries used
             activity_result=PlanningActivityResult.Completed,
             flight_plan_status=FlightPlanStatus.from_flightinfo(record.flight_info),
+            notes=notes,
         )
     except (ValueError, ConnectionError) as e:
         notes = (
@@ -277,10 +285,18 @@ def unsuccessful(msg: str) -> PlanningActivityResponse:
 
     # Delete operational intent from DSS
     step_name = "performing unknown operation"
+    notes: Optional[str] = None
     try:
         step_name = f"deleting operational intent {flight.op_intent.reference.id} with OVN {flight.op_intent.reference.ovn} from DSS"
         log(step_name)
-        delete_op_intent(flight.op_intent.reference, log)
+        notif_errors = delete_op_intent(flight.op_intent.reference, log)
+        if notif_errors:
+            notif_errors_messages = [
+                f"{url}: {str(err)}" for url, err in notif_errors.items()
+            ]
+            notes = f"Deletion succeeded, but notification to some subscribers failed: {'; '.join(notif_errors_messages)}"
+            log(notes)
+
     except (ValueError, ConnectionError) as e:
         notes = (
             f"{e.__class__.__name__} while {step_name} for flight {flight_id}: {str(e)}"
@@ -307,6 +323,7 @@ def unsuccessful(msg: str) -> PlanningActivityResponse:
         queries=[],
         activity_result=PlanningActivityResult.Completed,
         flight_plan_status=FlightPlanStatus.Closed,
+        notes=notes,
     )