Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

163 database tweaks #184

Merged
merged 8 commits into from
Dec 3, 2024
Merged

163 database tweaks #184

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
fcf0aa5
Overwriting database.
astronomerritt Nov 25, 2024
c908ba4
Adding test to make sure row overwrites
astronomerritt Nov 25, 2024
dea9c71
Changing timestamp to MJD
astronomerritt Nov 25, 2024
c9a35d6
Using UPDATE correctly
astronomerritt Nov 25, 2024
28724ea
Unit test.
astronomerritt Nov 25, 2024
3e2b834
timestamp should be stored as REAL
astronomerritt Nov 25, 2024
fade9c7
Pinning sbpy
astronomerritt Nov 25, 2024
55c6e9f
Tweak to pass unit tests
astronomerritt Nov 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions src/adler/objectdata/AdlerData.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import re
import numpy as np
from dataclasses import dataclass, field
from datetime import datetime, timezone
from astropy.time import Time


FILTER_DEPENDENT_KEYS = ["phaseAngle_min", "phaseAngle_range", "nobs", "arc"]
Expand Down Expand Up @@ -319,7 +319,7 @@ def _get_database_connection(self, filepath, create_new=False):
if not database_exists and create_new: # we need to make the table and a couple of starter columns
con = sqlite3.connect(filepath)
cur = con.cursor()
cur.execute("CREATE TABLE AdlerData(ssObjectId, timestamp)")
cur.execute("CREATE TABLE AdlerData(ssObjectId INTEGER PRIMARY KEY, timestamp REAL)")
elif not database_exists and not create_new:
logger.error("ValueError: Database cannot be found at given filepath.")
raise ValueError("Database cannot be found at given filepath.")
Expand Down Expand Up @@ -365,7 +365,7 @@ def _get_row_data_and_columns(self):

"""
required_columns = ["ssObjectId", "timestamp"]
row_data = [self.ssObjectId, str(datetime.now(timezone.utc))]
row_data = [int(self.ssObjectId), Time.now().mjd]

for f, filter_name in enumerate(self.filter_list):
columns_by_filter = ["_".join([filter_name, filter_key]) for filter_key in FILTER_DEPENDENT_KEYS]
Expand Down Expand Up @@ -437,8 +437,12 @@ def write_row_to_database(self, filepath, table_name="AdlerData"):

column_names = ",".join(required_columns)
column_spaces = ",".join(["?"] * len(required_columns))
sql_command = "INSERT INTO %s (%s) values(%s)" % (table_name, column_names, column_spaces)

update_clause = ", ".join([f"{col} = excluded.{col}" for col in required_columns[1:]])
sql_command = f"""
INSERT INTO {table_name} ({column_names})
VALUES ({column_spaces})
ON CONFLICT(ssObjectId) DO UPDATE SET {update_clause};
"""
cur = con.cursor()
cur.execute(sql_command, row_data)
con.commit()
Expand Down
208 changes: 29 additions & 179 deletions tests/adler/objectdata/test_AdlerData.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@

# setting up the AdlerData object to be used for testing


test_object = AdlerData("8268570668335894776", ["g", "r", "i"])
model_1 = "HG12_Pen16"
model_2 = "HG"
Expand Down Expand Up @@ -104,53 +103,6 @@
"phase_parameter_2_err": np.nan,
}

# u_model_1 = {
# "model_name": "model_1",
# "phaseAngle_min": 11.0,
# "phaseAngle_range": 12.0,
# "nobs": 13,
# "arc": 14.0,
# "H": 15.0,
# "H_err": 16.0,
# "phase_parameter_1": 17.0,
# "phase_parameter_1_err": 18.0,
# }

# u_model_2 = {
# "model_name": "model_2",
# "H": 25.0,
# "H_err": 26.0,
# "phase_parameter_1": 27.0,
# "phase_parameter_1_err": 28.0,
# "phase_parameter_2": 29.0,
# "phase_parameter_2_err": 30.0,
# }

# g_model_1 = {
# "model_name": "model_1",
# "phaseAngle_min": 31.0,
# "phaseAngle_range": 32.0,
# "nobs": 33,
# "arc": 34.0,
# "H": 35.0,
# "H_err": 36.0,
# "phase_parameter_1": 37.0,
# "phase_parameter_1_err": 38.0,
# }

# r_model_2 = {
# "model_name": "model_2",
# "phaseAngle_min": 41.0,
# "phaseAngle_range": 42.0,
# "nobs": 43,
# "arc": 44.0,
# "H": 45.0,
# "H_err": 46.0,
# "phase_parameter_1": 47.0,
# "phase_parameter_1_err": 48.0,
# "phase_parameter_2": 49.0,
# "phase_parameter_2_err": 50.0,
# }

_g_model_1 = g_model_1.copy()
del _g_model_1["filter_name"]
Expand Down Expand Up @@ -192,54 +144,6 @@ def test_populate_phase_parameters():
[g_model_1["arc"], r_model_1["arc"], i_model_1["arc"]],
)

# assert test_object.filter_dependent_values[0].model_dependent_values[0].__dict__ == g_model_1
# {
# "filter_name": "u",
# "model_name": "model_1",
# "H": 15.0,
# "H_err": 16.0,
# "phase_parameter_1": 17.0,
# "phase_parameter_1_err": 18.0,
# "phase_parameter_2": np.nan,
# "phase_parameter_2_err": np.nan,
# }

# assert test_object.filter_dependent_values[0].model_dependent_values[1].__dict__ == g_model_2
# {
# "filter_name": "u",
# "model_name": "model_2",
# "H": 25.0,
# "H_err": 26.0,
# "phase_parameter_1": 27.0,
# "phase_parameter_1_err": 28.0,
# "phase_parameter_2": 29.0,
# "phase_parameter_2_err": 30.0,
# }

# assert test_object.filter_dependent_values[1].model_dependent_values[0].__dict__ == r_model_1
# {
# "filter_name": "g",
# "model_name": "model_1",
# "H": 35.0,
# "H_err": 36.0,
# "phase_parameter_1": 37.0,
# "phase_parameter_1_err": 38.0,
# "phase_parameter_2": np.nan,
# "phase_parameter_2_err": np.nan,
# }

# assert test_object.filter_dependent_values[2].model_dependent_values[0].__dict__ == i_model_2
# {
# "filter_name": "r",
# "model_name": "model_2",
# "H": 45.0,
# "H_err": 46.0,
# "phase_parameter_1": 47.0,
# "phase_parameter_1_err": 48.0,
# "phase_parameter_2": 49.0,
# "phase_parameter_2_err": 50.0,
# }

test_dict_list = [
test_object.get_phase_parameters_in_filter("g", model_1).__dict__,
test_object.get_phase_parameters_in_filter("g", model_2).__dict__,
Expand All @@ -264,17 +168,10 @@ def test_populate_phase_parameters():
test_object.populate_phase_parameters("g", model_name=model_1, H=g_model_1["H"])

# check to make sure filter-dependent parameter is correctly updated (then return it to previous)
# print(test_object.__dict__)

test_object.populate_phase_parameters("r", model_name=model_1, nobs=99)
# print(test_object.__dict__)
# print(r_model_1["nobs"])
# print(test_object.get_phase_parameters_in_filter("r",model_1).__dict__)
# print(test_object.get_phase_parameters_in_filter("r",model_1).nobs)
# assert test_object.filter_dependent_values[0].nobs == 99 # TODO: the indexing in this line does not access the correct filter
assert test_object.get_phase_parameters_in_filter("r", model_1).nobs == 99
# print(r_model_1["nobs"])
test_object.populate_phase_parameters("r", model_name=model_1, nobs=r_model_1["nobs"])
# print(test_object.get_phase_parameters_in_filter("r",model_1).nobs)

# testing to make sure the correct error messages trigger
test_filt = "y"
Expand All @@ -292,69 +189,6 @@ def test_populate_phase_parameters():


def test_get_phase_parameters_in_filter():
# assert test_object.get_phase_parameters_in_filter("g", model_1).__dict__ == g_model_1
# {
# "filter_name": "u",
# "phaseAngle_min": 11.0,
# "phaseAngle_range": 12.0,
# "nobs": 13,
# "arc": 14.0,
# "model_name": "model_1",
# "H": 15.0,
# "H_err": 16.0,
# "phase_parameter_1": 17.0,
# "phase_parameter_1_err": 18.0,
# "phase_parameter_2": np.nan,
# "phase_parameter_2_err": np.nan,
# }

# assert test_object.get_phase_parameters_in_filter("g", model_2).__dict__ == g_model_2
# {
# "filter_name": "u",
# "phaseAngle_min": 11.0,
# "phaseAngle_range": 12.0,
# "nobs": 13,
# "arc": 14.0,
# "model_name": "model_2",
# "H": 25.0,
# "H_err": 26.0,
# "phase_parameter_1": 27.0,
# "phase_parameter_1_err": 28.0,
# "phase_parameter_2": 29.0,
# "phase_parameter_2_err": 30.0,
# }

# assert test_object.get_phase_parameters_in_filter("r", model_1).__dict__ == r_model_1
# {
# "filter_name": "g",
# "phaseAngle_min": 31.0,
# "phaseAngle_range": 32.0,
# "nobs": 33,
# "arc": 34.0,
# "model_name": "model_1",
# "H": 35.0,
# "H_err": 36.0,
# "phase_parameter_1": 37.0,
# "phase_parameter_1_err": 38.0,
# "phase_parameter_2": np.nan,
# "phase_parameter_2_err": np.nan,
# }

# assert test_object.get_phase_parameters_in_filter("i", model_2).__dict__ == i_model_2
# {
# "filter_name": "r",
# "phaseAngle_min": 41.0,
# "phaseAngle_range": 42.0,
# "nobs": 43,
# "arc": 44.0,
# "model_name": "model_2",
# "H": 45.0,
# "H_err": 46.0,
# "phase_parameter_1": 47.0,
# "phase_parameter_1_err": 48.0,
# "phase_parameter_2": 49.0,
# "phase_parameter_2_err": 50.0,
# }

test_dict_list = [
test_object.get_phase_parameters_in_filter("g", model_1).__dict__,
Expand Down Expand Up @@ -410,8 +244,6 @@ def test_print_data(capsys):

def test_write_row_to_database(tmp_path):
db_location = os.path.join(tmp_path, "test_AdlerData_database.db")
# print(db_location)
# print(test_object.__dict__)
test_object.write_row_to_database(db_location)

con = sqlite3.connect(db_location)
Expand All @@ -431,21 +263,39 @@ def test_write_row_to_database(tmp_path):
# expected_data = expected_data.replace({np.nan: None}) # TODO: fix weirdness between nan and None?
written_data = written_data.replace({None: np.nan}) # TODO: fix weirdness between nan and None?
expected_data = expected_data.astype({"ssObjectId": str})
# print(expected_data.dtypes)
# print(written_data.dtypes)
# print(expected_data.iloc[0].to_dict())
# print(written_data.iloc[0].to_dict())
written_data = written_data.astype({"ssObjectId": str})

for x in written_data.columns:
# print(x)
# print(expected_data.iloc[0][x],written_data.iloc[0][x])
# print(type(expected_data.iloc[0][x]),type(written_data.iloc[0][x]))
# print(expected_data.iloc[0][x]==written_data.iloc[0][x])
if type(written_data.iloc[0][x]) == str:
assert expected_data.iloc[0][x] == written_data.iloc[0][x]
else:
assert_almost_equal(expected_data.iloc[0][x], written_data.iloc[0][x])
# pd.testing.assert_frame_equal(expected_data, written_data, check_dtype=False)
# assert expected_data.iloc[0].to_dict() == pytest.approx(written_data.iloc[0].to_dict())


def test_overwriting_rows_in_database(tmp_path):
# a test to ensure we're overwriting correctly

# write the initial object
db_location = os.path.join(tmp_path, "test_AdlerData_database.db")
test_object.write_row_to_database(db_location)

# make a change to a value, then create a new AdlerData object
# and write that to the database
test_object_2 = AdlerData("8268570668335894776", ["g"])
_g_model_1["nobs"] = 666
test_object_2.populate_phase_parameters("g", **_g_model_1)
test_object_2.write_row_to_database(db_location)

con = sqlite3.connect(db_location)
written_data = pd.read_sql_query("SELECT * from AdlerData", con)
con.close()

# should only have one row in the database
assert len(written_data) == 1

# we should have a new value for g_nobs but r_nobs should be unchanged
assert written_data["g_nobs"][0] == 666
assert written_data["r_nobs"][0] == 38


def test_read_row_from_database():
Expand Down
2 changes: 1 addition & 1 deletion tests/adler/science/test_PhaseCurve.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ def test_PhaseCurve_FitModel_HG_fixed():
pc2 = pc1.InitModelSbpy(pc_fit)

# the new fitted model should have the same parameters as the input model, but G is fixed
assert pc_fit._fixed["G"] is True
assert pc_fit.fixed["G"] is True
assert pc2.H == pc1.H
assert pc2.phase_parameter_1 == pc1.phase_parameter_1
assert pc2.phase_parameter_1_err is None # the fitted model has no uncertainties when param is fixed
Expand Down
Loading