From 9f4c155d2b44567a296f3dfcc6a1d125ad08d86d Mon Sep 17 00:00:00 2001
From: Theophile du Laz <theophile.dulaz@gmail.com>
Date: Thu, 28 Sep 2023 15:54:47 -0700
Subject: [PATCH] Temporary: use all candidates (not just prv_candidates) to
 generate lightcurve  (#249)

While we are retrieving data in prod for the prv_candidates, make sure to post all the candidates for an object as detections.
---
 kowalski/alert_brokers/alert_broker.py | 77 +++++++++++++++++++++++++-
 1 file changed, 75 insertions(+), 2 deletions(-)

diff --git a/kowalski/alert_brokers/alert_broker.py b/kowalski/alert_brokers/alert_broker.py
index 6d6cc4be..a9b94cfc 100644
--- a/kowalski/alert_brokers/alert_broker.py
+++ b/kowalski/alert_brokers/alert_broker.py
@@ -45,6 +45,9 @@
     time_stamp,
     timer,
 )
+from warnings import simplefilter
+
+simplefilter(action="ignore", category=pd.errors.PerformanceWarning)
 
 # Tensorflow is problematic for Mac's currently, so we can add an option to disable it
 USE_TENSORFLOW = os.environ.get("USE_TENSORFLOW", True) in [
@@ -1680,6 +1683,37 @@ def alert_sentinel_skyportal(self, alert, prv_candidates, passed_filters):
                     log(e)
                     alert["prv_candidates"] = prv_candidates
 
+                # also get all the alerts for this object, to make sure to have all the detections
+                try:
+                    all_alerts = list(
+                        retry(self.mongo.db[self.collection_alerts].find)(
+                            {
+                                "objectId": alert["objectId"],
+                                "candid": {"$ne": alert["candid"]},
+                            },
+                            {
+                                "candidate": 1,
+                            },
+                        )
+                    )
+                    all_alerts = [
+                        {**a["candidate"]} for a in all_alerts if "candidate" in a
+                    ]
+                    # add to prv_candidates the detections that are not already in there
+                    # use the jd and the fid to match
+                    for a in all_alerts:
+                        if not any(
+                            [
+                                (a["jd"] == p["jd"]) and (a["fid"] == p["fid"])
+                                for p in alert["prv_candidates"]
+                            ]
+                        ):
+                            alert["prv_candidates"].append(a)
+                    del all_alerts
+                except Exception as e:
+                    # this should never happen, but just in case
+                    log(f"Failed to get all alerts for {alert['objectId']}: {e}")
+
                 self.alert_put_photometry(alert)
 
                 # post thumbnails
@@ -1768,6 +1802,37 @@ def alert_sentinel_skyportal(self, alert, prv_candidates, passed_filters):
             # post alert photometry in single call to /api/photometry
             alert["prv_candidates"] = prv_candidates
 
+            # also get all the alerts for this object, to make sure to have all the detections
+            try:
+                all_alerts = list(
+                    retry(self.mongo.db[self.collection_alerts].find)(
+                        {
+                            "objectId": alert["objectId"],
+                            "candid": {"$ne": alert["candid"]},
+                        },
+                        {
+                            "candidate": 1,
+                        },
+                    )
+                )
+                all_alerts = [
+                    {**a["candidate"]} for a in all_alerts if "candidate" in a
+                ]
+                # add to prv_candidates the detections that are not already in there
+                # use the jd and the fid to match
+                for a in all_alerts:
+                    if not any(
+                        [
+                            (a["jd"] == p["jd"]) and (a["fid"] == p["fid"])
+                            for p in alert["prv_candidates"]
+                        ]
+                    ):
+                        alert["prv_candidates"].append(a)
+                del all_alerts
+            except Exception as e:
+                # this should never happen, but just in case
+                log(f"Failed to get all alerts for {alert['objectId']}: {e}")
+
             self.alert_put_photometry(alert)
 
         if len(autosave_group_ids):
@@ -1888,14 +1953,22 @@ def alert_sentinel_skyportal(self, alert, prv_candidates, passed_filters):
                                             )
                                             if response.json()["status"] != "success":
                                                 raise ValueError(
-                                                    response.json()["message"]
+                                                    response.json().get(
+                                                        "message",
+                                                        "unknow error posting comment",
+                                                    )
                                                 )
                                         except Exception as e:
                                             log(
                                                 f"Failed to post followup comment {comment['text']} for {alert['objectId']} to SkyPortal: {e}"
                                             )
                             else:
-                                raise ValueError(response.json()["message"])
+                                raise ValueError(
+                                    response.json().get(
+                                        "message",
+                                        "unknow error posting followup request",
+                                    )
+                                )
                         except Exception as e:
                             log(
                                 f"Failed to post followup request for {alert['objectId']} to SkyPortal: {e}"