Merge pull request #998 from linea-it/995-fix-eventos-duplicados

Fixed #995 - Now events are removed regardless of the execution result.

backend/tno/views/occultation.py

@@ -159,7 +159,7 @@ class OccultationViewSet(viewsets.ReadOnlyModelViewSet):
         "ra_star_candidate",
         "dec_star_candidate",
         "ra_target",
-        " dec_target",
+        "dec_target",
         "closest_approach",
         "position_angle",
         "velocity",

predict_occultation/src/asteroid/asteroid.py

@@ -825,13 +825,18 @@ def remove_previus_results(self, remove_inputs=False):
 
     def register_occultations(self, start_period: str, end_period: str, jobid: int):
         log = self.get_log()
-
-        if "filename" not in self.predict_occultation:
-            # Nao executou a etapa de predicao.
-            return 0
+        t0 = dt.now(tz=timezone.utc)
 
         try:
-            t0 = dt.now(tz=timezone.utc)
+            dao = OccultationDao(log=log)
+            # Apaga as occultations já registradas para este asteroid antes de inserir.
+            # IMPORTANTE! apaga mesmo que não tenham sido gerados resultados.
+            dao.delete_by_asteroid_name_period(self.name, start_period, end_period)
+
+            if "filename" not in self.predict_occultation:
+                log.warning("There is no file with the predictions.")
+                # Nao executou a etapa de predicao.
+                return 0
 
             predict_table_path = pathlib.Path(
                 self.path, self.predict_occultation["filename"]
@@ -841,11 +846,6 @@ def register_occultations(self, start_period: str, end_period: str, jobid: int):
                 # Arquivo com resultados da predicao nao foi criado
                 return 0
 
-            dao = OccultationDao()
-
-            # Apaga as occultations já registradas para este asteroid antes de inserir.
-            dao.delete_by_asteroid_name(self.name, start_period, end_period)
-
             # Le o arquivo occultation table e cria um dataframe
             # occultation_date;ra_star_candidate;dec_star_candidate;ra_object;dec_object;ca;pa;vel;delta;g;j;h;k;long;loc_t;off_ra;off_de;pm;ct;f;e_ra;e_de;pmra;pmde
             df = pd.read_csv(

predict_occultation/src/dao/occultation.py

@@ -3,11 +3,13 @@
 
 
 class OccultationDao(DBBase):
-    def __init__(self):
+    def __init__(self, log):
         super(OccultationDao, self).__init__()
 
         self.tbl = self.get_table("tno_occultation")
 
+        self.log = log
+
     def delete_by_asteroid_name(self, name):
 
         stm = delete(self.tbl).where(and_(self.tbl.c.name == name))
@@ -18,25 +20,30 @@ def delete_by_asteroid_name(self, name):
 
             return rows
 
-    def delete_by_asteroid_name(self, id, start_period: str, end_period: str):
+    def delete_by_asteroid_name_period(
+        self, name: str, start_period: str, end_period: str
+    ):
 
         stm = delete(self.tbl).where(
             and_(
-                self.tbl.c.name == id,
+                self.tbl.c.name == name,
                 self.tbl.c.date_time.between(start_period, end_period),
             )
         )
 
         engine = self.get_db_engine()
         with engine.connect() as con:
-            rows = con.execute(stm)
-
+            result = con.execute(stm)
+            rows = result.rowcount
+            self.log.info(
+                f"Removed {rows} events for {name} in period {start_period} - {end_period}"
+            )
             return rows
 
     def import_occultations(self, data):
 
         # Sql Copy com todas as colunas que vão ser importadas e o formato do csv.
-        # IMPORTANTA! A ORDEM DAS COLUNAS PRECISA SER IDENTICA A COMO ESTA NO DB!
+        # IMPORTANTE! A ORDEM DAS COLUNAS PRECISA SER IDENTICA A COMO ESTA NO DB!
         sql = (
             f"COPY {self.tbl} (name, number, date_time, gaia_source_id, ra_star_candidate, dec_star_candidate, ra_target, dec_target, closest_approach, position_angle, velocity, delta, g, j_star, h, k_star, long, loc_t, off_ra, off_dec, proper_motion, ct, multiplicity_flag, e_ra, e_dec, pmra, pmdec, ra_star_deg, dec_star_deg, ra_target_deg, dec_target_deg, created_at, apparent_diameter, aphelion, apparent_magnitude, dec_star_to_date, dec_star_with_pm, dec_target_apparent, diameter, e_dec_target, e_ra_target, eccentricity, ephemeris_version, g_mag_vel_corrected, h_mag_vel_corrected, inclination, instant_uncertainty, magnitude_drop, perihelion, ra_star_to_date, ra_star_with_pm, ra_target_apparent, rp_mag_vel_corrected, semimajor_axis, have_path_coeff, occ_path_max_longitude, occ_path_min_longitude, occ_path_coeff, occ_path_is_nightside, occ_path_max_latitude, occ_path_min_latitude, base_dynclass, bsp_planetary, bsp_source, catalog, dynclass, job_id, leap_seconds, nima, obs_source, orb_ele_source, predict_step, albedo, albedo_err_max, albedo_err_min, alias, arg_perihelion, astorb_dynbaseclass, astorb_dynsubclass, density, density_err_max, density_err_min, diameter_err_max, diameter_err_min, epoch, last_obs_included, long_asc_node, mass, mass_err_max, mass_err_min, mean_anomaly, mean_daily_motion, mpc_critical_list, pha_flag, principal_designation, rms, g_star, h_star, event_duration, moon_separation, sun_elongation) "
             "FROM STDIN with (FORMAT CSV, DELIMITER '|', HEADER);"

predict_occultation/src/run_pred_occ.py

@@ -953,8 +953,13 @@ def submit_tasks(jobid: int):
                         )
                     else:
                         step2_success += 1
-                        start_date = str(PREDICT_START.date())
-                        end_date = str(PREDICT_END.date())
+
+                        start_date = str(
+                            PREDICT_START.replace(hour=0, minute=0, second=0)
+                        )
+                        end_date = str(
+                            PREDICT_END.replace(hour=23, minute=59, second=59)
+                        )
 
                         ingested_occ_count = ast_obj.register_occultations(
                             start_date, end_date, jobid