Merge pull request #184 from lsst-uk/163-database-tweaks

163 database tweaks
lsst-uk · Dec 3, 2024 · 0d3804d · 0d3804d
2 parents 2942783 + 55c6e9f
commit 0d3804d
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 185 deletions.
diff --git a/src/adler/objectdata/AdlerData.py b/src/adler/objectdata/AdlerData.py
@@ -4,7 +4,7 @@
 import re
 import numpy as np
 from dataclasses import dataclass, field
-from datetime import datetime, timezone
+from astropy.time import Time
 
 
 FILTER_DEPENDENT_KEYS = ["phaseAngle_min", "phaseAngle_range", "nobs", "arc"]
@@ -319,7 +319,7 @@ def _get_database_connection(self, filepath, create_new=False):
         if not database_exists and create_new:  # we need to make the table and a couple of starter columns
             con = sqlite3.connect(filepath)
             cur = con.cursor()
-            cur.execute("CREATE TABLE AdlerData(ssObjectId, timestamp)")
+            cur.execute("CREATE TABLE AdlerData(ssObjectId INTEGER PRIMARY KEY, timestamp REAL)")
         elif not database_exists and not create_new:
             logger.error("ValueError: Database cannot be found at given filepath.")
             raise ValueError("Database cannot be found at given filepath.")
@@ -365,7 +365,7 @@ def _get_row_data_and_columns(self):
 
         """
         required_columns = ["ssObjectId", "timestamp"]
-        row_data = [self.ssObjectId, str(datetime.now(timezone.utc))]
+        row_data = [int(self.ssObjectId), Time.now().mjd]
 
         for f, filter_name in enumerate(self.filter_list):
             columns_by_filter = ["_".join([filter_name, filter_key]) for filter_key in FILTER_DEPENDENT_KEYS]
@@ -437,8 +437,12 @@ def write_row_to_database(self, filepath, table_name="AdlerData"):
 
         column_names = ",".join(required_columns)
         column_spaces = ",".join(["?"] * len(required_columns))
-        sql_command = "INSERT INTO %s (%s) values(%s)" % (table_name, column_names, column_spaces)
-
+        update_clause = ", ".join([f"{col} = excluded.{col}" for col in required_columns[1:]])
+        sql_command = f"""
+                        INSERT INTO {table_name} ({column_names})
+                        VALUES ({column_spaces})
+                        ON CONFLICT(ssObjectId) DO UPDATE SET {update_clause};
+                        """
         cur = con.cursor()
         cur.execute(sql_command, row_data)
         con.commit()

diff --git a/tests/adler/objectdata/test_AdlerData.py b/tests/adler/objectdata/test_AdlerData.py
@@ -11,7 +11,6 @@
 
 # setting up the AdlerData object to be used for testing
 
-
 test_object = AdlerData("8268570668335894776", ["g", "r", "i"])
 model_1 = "HG12_Pen16"
 model_2 = "HG"
@@ -104,53 +103,6 @@
     "phase_parameter_2_err": np.nan,
 }
 
-# u_model_1 = {
-#     "model_name": "model_1",
-#     "phaseAngle_min": 11.0,
-#     "phaseAngle_range": 12.0,
-#     "nobs": 13,
-#     "arc": 14.0,
-#     "H": 15.0,
-#     "H_err": 16.0,
-#     "phase_parameter_1": 17.0,
-#     "phase_parameter_1_err": 18.0,
-# }
-
-# u_model_2 = {
-#     "model_name": "model_2",
-#     "H": 25.0,
-#     "H_err": 26.0,
-#     "phase_parameter_1": 27.0,
-#     "phase_parameter_1_err": 28.0,
-#     "phase_parameter_2": 29.0,
-#     "phase_parameter_2_err": 30.0,
-# }
-
-# g_model_1 = {
-#     "model_name": "model_1",
-#     "phaseAngle_min": 31.0,
-#     "phaseAngle_range": 32.0,
-#     "nobs": 33,
-#     "arc": 34.0,
-#     "H": 35.0,
-#     "H_err": 36.0,
-#     "phase_parameter_1": 37.0,
-#     "phase_parameter_1_err": 38.0,
-# }
-
-# r_model_2 = {
-#     "model_name": "model_2",
-#     "phaseAngle_min": 41.0,
-#     "phaseAngle_range": 42.0,
-#     "nobs": 43,
-#     "arc": 44.0,
-#     "H": 45.0,
-#     "H_err": 46.0,
-#     "phase_parameter_1": 47.0,
-#     "phase_parameter_1_err": 48.0,
-#     "phase_parameter_2": 49.0,
-#     "phase_parameter_2_err": 50.0,
-# }
 
 _g_model_1 = g_model_1.copy()
 del _g_model_1["filter_name"]
@@ -192,54 +144,6 @@ def test_populate_phase_parameters():
         [g_model_1["arc"], r_model_1["arc"], i_model_1["arc"]],
     )
 
-    # assert test_object.filter_dependent_values[0].model_dependent_values[0].__dict__ == g_model_1
-    # {
-    #     "filter_name": "u",
-    #     "model_name": "model_1",
-    #     "H": 15.0,
-    #     "H_err": 16.0,
-    #     "phase_parameter_1": 17.0,
-    #     "phase_parameter_1_err": 18.0,
-    #     "phase_parameter_2": np.nan,
-    #     "phase_parameter_2_err": np.nan,
-    # }
-
-    # assert test_object.filter_dependent_values[0].model_dependent_values[1].__dict__ == g_model_2
-    # {
-    #     "filter_name": "u",
-    #     "model_name": "model_2",
-    #     "H": 25.0,
-    #     "H_err": 26.0,
-    #     "phase_parameter_1": 27.0,
-    #     "phase_parameter_1_err": 28.0,
-    #     "phase_parameter_2": 29.0,
-    #     "phase_parameter_2_err": 30.0,
-    # }
-
-    # assert test_object.filter_dependent_values[1].model_dependent_values[0].__dict__ == r_model_1
-    # {
-    #     "filter_name": "g",
-    #     "model_name": "model_1",
-    #     "H": 35.0,
-    #     "H_err": 36.0,
-    #     "phase_parameter_1": 37.0,
-    #     "phase_parameter_1_err": 38.0,
-    #     "phase_parameter_2": np.nan,
-    #     "phase_parameter_2_err": np.nan,
-    # }
-
-    # assert test_object.filter_dependent_values[2].model_dependent_values[0].__dict__ == i_model_2
-    # {
-    #     "filter_name": "r",
-    #     "model_name": "model_2",
-    #     "H": 45.0,
-    #     "H_err": 46.0,
-    #     "phase_parameter_1": 47.0,
-    #     "phase_parameter_1_err": 48.0,
-    #     "phase_parameter_2": 49.0,
-    #     "phase_parameter_2_err": 50.0,
-    # }
-
     test_dict_list = [
         test_object.get_phase_parameters_in_filter("g", model_1).__dict__,
         test_object.get_phase_parameters_in_filter("g", model_2).__dict__,
@@ -264,17 +168,10 @@ def test_populate_phase_parameters():
     test_object.populate_phase_parameters("g", model_name=model_1, H=g_model_1["H"])
 
     # check to make sure filter-dependent parameter is correctly updated (then return it to previous)
-    # print(test_object.__dict__)
+
     test_object.populate_phase_parameters("r", model_name=model_1, nobs=99)
-    # print(test_object.__dict__)
-    # print(r_model_1["nobs"])
-    # print(test_object.get_phase_parameters_in_filter("r",model_1).__dict__)
-    # print(test_object.get_phase_parameters_in_filter("r",model_1).nobs)
-    # assert test_object.filter_dependent_values[0].nobs == 99 # TODO: the indexing in this line does not access the correct filter
     assert test_object.get_phase_parameters_in_filter("r", model_1).nobs == 99
-    # print(r_model_1["nobs"])
     test_object.populate_phase_parameters("r", model_name=model_1, nobs=r_model_1["nobs"])
-    # print(test_object.get_phase_parameters_in_filter("r",model_1).nobs)
 
     # testing to make sure the correct error messages trigger
     test_filt = "y"
@@ -292,69 +189,6 @@ def test_populate_phase_parameters():
 
 
 def test_get_phase_parameters_in_filter():
-    # assert test_object.get_phase_parameters_in_filter("g", model_1).__dict__ == g_model_1
-    # {
-    #     "filter_name": "u",
-    #     "phaseAngle_min": 11.0,
-    #     "phaseAngle_range": 12.0,
-    #     "nobs": 13,
-    #     "arc": 14.0,
-    #     "model_name": "model_1",
-    #     "H": 15.0,
-    #     "H_err": 16.0,
-    #     "phase_parameter_1": 17.0,
-    #     "phase_parameter_1_err": 18.0,
-    #     "phase_parameter_2": np.nan,
-    #     "phase_parameter_2_err": np.nan,
-    # }
-
-    # assert test_object.get_phase_parameters_in_filter("g", model_2).__dict__ == g_model_2
-    # {
-    #     "filter_name": "u",
-    #     "phaseAngle_min": 11.0,
-    #     "phaseAngle_range": 12.0,
-    #     "nobs": 13,
-    #     "arc": 14.0,
-    #     "model_name": "model_2",
-    #     "H": 25.0,
-    #     "H_err": 26.0,
-    #     "phase_parameter_1": 27.0,
-    #     "phase_parameter_1_err": 28.0,
-    #     "phase_parameter_2": 29.0,
-    #     "phase_parameter_2_err": 30.0,
-    # }
-
-    # assert test_object.get_phase_parameters_in_filter("r", model_1).__dict__ == r_model_1
-    # {
-    #     "filter_name": "g",
-    #     "phaseAngle_min": 31.0,
-    #     "phaseAngle_range": 32.0,
-    #     "nobs": 33,
-    #     "arc": 34.0,
-    #     "model_name": "model_1",
-    #     "H": 35.0,
-    #     "H_err": 36.0,
-    #     "phase_parameter_1": 37.0,
-    #     "phase_parameter_1_err": 38.0,
-    #     "phase_parameter_2": np.nan,
-    #     "phase_parameter_2_err": np.nan,
-    # }
-
-    # assert test_object.get_phase_parameters_in_filter("i", model_2).__dict__ == i_model_2
-    # {
-    #     "filter_name": "r",
-    #     "phaseAngle_min": 41.0,
-    #     "phaseAngle_range": 42.0,
-    #     "nobs": 43,
-    #     "arc": 44.0,
-    #     "model_name": "model_2",
-    #     "H": 45.0,
-    #     "H_err": 46.0,
-    #     "phase_parameter_1": 47.0,
-    #     "phase_parameter_1_err": 48.0,
-    #     "phase_parameter_2": 49.0,
-    #     "phase_parameter_2_err": 50.0,
-    # }
 
     test_dict_list = [
         test_object.get_phase_parameters_in_filter("g", model_1).__dict__,
@@ -410,8 +244,6 @@ def test_print_data(capsys):
 
 def test_write_row_to_database(tmp_path):
     db_location = os.path.join(tmp_path, "test_AdlerData_database.db")
-    # print(db_location)
-    # print(test_object.__dict__)
     test_object.write_row_to_database(db_location)
 
     con = sqlite3.connect(db_location)
@@ -431,21 +263,39 @@ def test_write_row_to_database(tmp_path):
     # expected_data = expected_data.replace({np.nan: None}) # TODO: fix weirdness between nan and None?
     written_data = written_data.replace({None: np.nan})  # TODO: fix weirdness between nan and None?
     expected_data = expected_data.astype({"ssObjectId": str})
-    # print(expected_data.dtypes)
-    # print(written_data.dtypes)
-    # print(expected_data.iloc[0].to_dict())
-    # print(written_data.iloc[0].to_dict())
+    written_data = written_data.astype({"ssObjectId": str})
+
     for x in written_data.columns:
-        # print(x)
-        # print(expected_data.iloc[0][x],written_data.iloc[0][x])
-        # print(type(expected_data.iloc[0][x]),type(written_data.iloc[0][x]))
-        # print(expected_data.iloc[0][x]==written_data.iloc[0][x])
         if type(written_data.iloc[0][x]) == str:
             assert expected_data.iloc[0][x] == written_data.iloc[0][x]
         else:
             assert_almost_equal(expected_data.iloc[0][x], written_data.iloc[0][x])
-    # pd.testing.assert_frame_equal(expected_data, written_data, check_dtype=False)
-    # assert expected_data.iloc[0].to_dict() == pytest.approx(written_data.iloc[0].to_dict())
+
+
+def test_overwriting_rows_in_database(tmp_path):
+    # a test to ensure we're overwriting correctly
+
+    # write the initial object
+    db_location = os.path.join(tmp_path, "test_AdlerData_database.db")
+    test_object.write_row_to_database(db_location)
+
+    # make a change to a value, then create a new AdlerData object
+    # and write that to the database
+    test_object_2 = AdlerData("8268570668335894776", ["g"])
+    _g_model_1["nobs"] = 666
+    test_object_2.populate_phase_parameters("g", **_g_model_1)
+    test_object_2.write_row_to_database(db_location)
+
+    con = sqlite3.connect(db_location)
+    written_data = pd.read_sql_query("SELECT * from AdlerData", con)
+    con.close()
+
+    # should only have one row in the database
+    assert len(written_data) == 1
+
+    # we should have a new value for g_nobs but r_nobs should be unchanged
+    assert written_data["g_nobs"][0] == 666
+    assert written_data["r_nobs"][0] == 38
 
 
 def test_read_row_from_database():

diff --git a/tests/adler/science/test_PhaseCurve.py b/tests/adler/science/test_PhaseCurve.py
@@ -117,7 +117,7 @@ def test_PhaseCurve_FitModel_HG_fixed():
     pc2 = pc1.InitModelSbpy(pc_fit)
 
     # the new fitted model should have the same parameters as the input model, but G is fixed
-    assert pc_fit._fixed["G"] is True
+    assert pc_fit.fixed["G"] is True
     assert pc2.H == pc1.H
     assert pc2.phase_parameter_1 == pc1.phase_parameter_1
     assert pc2.phase_parameter_1_err is None  # the fitted model has no uncertainties when param is fixed