diff --git a/notebooks/adler_demo_rsp/adler_demo_rsp.ipynb b/notebooks/adler_demo_rsp/adler_demo_rsp.ipynb index 168aeee..fc16b8b 100644 --- a/notebooks/adler_demo_rsp/adler_demo_rsp.ipynb +++ b/notebooks/adler_demo_rsp/adler_demo_rsp.ipynb @@ -1,16 +1,25 @@ { "cells": [ + { + "cell_type": "markdown", + "id": "cf09023a-8a2b-4225-a556-24f1ab3261bf", + "metadata": {}, + "source": [ + "# Running Adler on the RSP\n", + "This notebook demonstrates running adler on the Rubin Science Platform to search for outlying photometry, using DP0.3. As a test we select a particular night of the survey and identify all `ssObjectId`s that got a detection, these are the objects that require processing." + ] + }, { "cell_type": "code", "execution_count": null, "id": "f50b8545-2147-46f2-9586-e66ce97dbbfe", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:55:53.950740Z", - "iopub.status.busy": "2025-01-06T11:55:53.950331Z", - "iopub.status.idle": "2025-01-06T11:55:55.765168Z", - "shell.execute_reply": "2025-01-06T11:55:55.764142Z", - "shell.execute_reply.started": "2025-01-06T11:55:53.950697Z" + "iopub.execute_input": "2025-01-07T14:24:17.570472Z", + "iopub.status.busy": "2025-01-07T14:24:17.570190Z", + "iopub.status.idle": "2025-01-07T14:24:19.395549Z", + "shell.execute_reply": "2025-01-07T14:24:19.394070Z", + "shell.execute_reply.started": "2025-01-07T14:24:17.570449Z" } }, "outputs": [], @@ -24,8 +33,16 @@ "from sbpy.photometry import HG, HG1G2, HG12, HG12_Pen16, LinearPhaseFunc\n", "from astropy.modeling.fitting import LevMarLSQFitter\n", "import sqlite3\n", + "import json\n", + "import time\n", + "\n", + "from lsst.rsp import get_tap_service\n", "\n", - "from lsst.rsp import get_tap_service" + "from adler.objectdata.AdlerPlanetoid import AdlerPlanetoid\n", + "from adler.science.PhaseCurve import PhaseCurve\n", + "from adler.objectdata.AdlerData import AdlerData\n", + "from adler.utilities.plotting_utilities import plot_errorbar, plot_phasecurve\n", + "import adler.utilities.science_utilities as sci_utils" ] }, { @@ -34,15 +51,16 @@ "id": "f5fc5248-9b1c-4af1-b3e2-64d8b9f4b7c1", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:55:55.767246Z", - "iopub.status.busy": "2025-01-06T11:55:55.766606Z", - "iopub.status.idle": "2025-01-06T11:55:55.818160Z", - "shell.execute_reply": "2025-01-06T11:55:55.817109Z", - "shell.execute_reply.started": "2025-01-06T11:55:55.767207Z" + "iopub.execute_input": "2025-01-07T14:24:19.398455Z", + "iopub.status.busy": "2025-01-07T14:24:19.397487Z", + "iopub.status.idle": "2025-01-07T14:24:19.463933Z", + "shell.execute_reply": "2025-01-07T14:24:19.463000Z", + "shell.execute_reply.started": "2025-01-07T14:24:19.398406Z" } }, "outputs": [], "source": [ + "# set up service for querying RSP\n", "service = get_tap_service(\"ssotap\")\n", "assert service is not None" ] @@ -50,54 +68,44 @@ { "cell_type": "code", "execution_count": null, - "id": "ae0560bc-0878-4756-94c5-b87ddc4f8313", + "id": "2b7bdbac-8264-4880-8f58-824919646139", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:55:55.819629Z", - "iopub.status.busy": "2025-01-06T11:55:55.819339Z", - "iopub.status.idle": "2025-01-06T11:55:55.823113Z", - "shell.execute_reply": "2025-01-06T11:55:55.822272Z", - "shell.execute_reply.started": "2025-01-06T11:55:55.819605Z" + "iopub.execute_input": "2025-01-07T14:24:19.466311Z", + "iopub.status.busy": "2025-01-07T14:24:19.466013Z", + "iopub.status.idle": "2025-01-07T14:24:19.472127Z", + "shell.execute_reply": "2025-01-07T14:24:19.470920Z", + "shell.execute_reply.started": "2025-01-07T14:24:19.466288Z" } }, "outputs": [], "source": [ - "# # count total rows in each table\n", - "# for tab in [\"SSObject\",\n", - "# \"DiaSource\",\n", - "# \"SSSource\",\n", - "# \"MPCORB\"]:\n", + "# set up variables\n", + "night = 61562 # night to test - this night has the most number of detections in DP0.3, see lsst-adler/in_progress/SSO_alerts_per_visit/alerts_per_night.ipynb\n", + "time_bounds = 0.5 # time shift to set bounds to get only observations from the night\n", "\n", - "# qry = \"\"\"SELECT COUNT(*) FROM dp03_catalogs_10yr.{}\"\"\".format(tab)\n", - "# _df = service.search(qry).to_table().to_pandas()\n", + "fname_id = \"df_id_{}.csv\".format(night) # filename to save ssObjectIds observed on night\n", + "fname_obj = \"df_obj_{}.csv\".format(night) # filename to save full 10yr DP0.3 object details\n", + "fname_sso = \"df_sso_{}.csv\".format(\n", + " night\n", + ") # filename to save the object details calculated on data prior to night\n", + "adler_data_db = \"adler_data_{}.db\".format(night) # database file to save adler data\n", + "adler_out_dir = \"adler_out_{}\".format(night) # directory to save an adler outlier detections\n", + "if not os.path.isdir(adler_out_dir):\n", + " os.mkdir(adler_out_dir)\n", "\n", - "# print(tab,_df)" + "qry_chunk = 5000 # number of objects to query at a time, dividing query to get it to run\n", + "G12_start = 0.62 # assumed value of G12 (P16) which is closest to G=0.15\n", + "fitter = LevMarLSQFitter() # phase curve fitting function\n", + "diff_cut = 1.0 # magnitude difference used to identify outliers" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "2b7bdbac-8264-4880-8f58-824919646139", - "metadata": { - "execution": { - "iopub.execute_input": "2025-01-06T11:55:55.825822Z", - "iopub.status.busy": "2025-01-06T11:55:55.825492Z", - "iopub.status.idle": "2025-01-06T11:55:55.841356Z", - "shell.execute_reply": "2025-01-06T11:55:55.840475Z", - "shell.execute_reply.started": "2025-01-06T11:55:55.825794Z" - } - }, - "outputs": [], + "cell_type": "markdown", + "id": "fe14c988-b69d-4266-8e2b-24a74e1c4e8d", + "metadata": {}, "source": [ - "night = 61562 # night to test\n", - "time_bounds = 0.5 # shift to get only observations from the night\n", - "fname_id = \"df_id_{}.csv\".format(night) # filename to save ids on night\n", - "fname_obj = \"df_obj_{}.csv\".format(night) # filename to save object details on the night\n", - "fname_sso = \"df_sso_{}.csv\".format(night)\n", - "qry_chunk = 5000 # number of objects to query at a time, dividing query to get it to run\n", - "N_min = 3 # minimum number of observations to fit phase curve\n", - "G12_start = 0.62 # assumed value of G12 (P16) which is closest to G=0.15\n", - "fitter = LevMarLSQFitter()" + "# Find all objects on the night" ] }, { @@ -106,15 +114,16 @@ "id": "9b14a440-5008-47e2-8c76-4c229a89786d", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:55:55.842728Z", - "iopub.status.busy": "2025-01-06T11:55:55.842407Z", - "iopub.status.idle": "2025-01-06T11:55:55.857417Z", - "shell.execute_reply": "2025-01-06T11:55:55.856466Z", - "shell.execute_reply.started": "2025-01-06T11:55:55.842700Z" + "iopub.execute_input": "2025-01-07T14:24:19.473492Z", + "iopub.status.busy": "2025-01-07T14:24:19.473208Z", + "iopub.status.idle": "2025-01-07T14:24:19.486318Z", + "shell.execute_reply": "2025-01-07T14:24:19.485179Z", + "shell.execute_reply.started": "2025-01-07T14:24:19.473471Z" } }, "outputs": [], "source": [ + "# query RSP DP0.3 for all ssObjectIds that were detected on night\n", "query = \"\"\"SELECT DISTINCT dia.ssObjectId\n", " FROM dp03_catalogs_10yr.DiaSource as dia\n", " WHERE dia.midPointMjdTai > {} AND dia.midPointMjdTai < {}\n", @@ -124,39 +133,22 @@ "print(query)" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "1966bab8-b863-4190-b58a-9f5a4e2e0ba2", - "metadata": { - "execution": { - "iopub.execute_input": "2025-01-06T11:55:55.858685Z", - "iopub.status.busy": "2025-01-06T11:55:55.858409Z", - "iopub.status.idle": "2025-01-06T11:55:55.870629Z", - "shell.execute_reply": "2025-01-06T11:55:55.869861Z", - "shell.execute_reply.started": "2025-01-06T11:55:55.858663Z" - } - }, - "outputs": [], - "source": [ - "# it takes ~3 mins to query this night" - ] - }, { "cell_type": "code", "execution_count": null, "id": "addc88c5-c4bc-4c37-aff1-40e2a60298f3", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:55:55.872309Z", - "iopub.status.busy": "2025-01-06T11:55:55.871648Z", - "iopub.status.idle": "2025-01-06T11:55:56.084915Z", - "shell.execute_reply": "2025-01-06T11:55:56.084129Z", - "shell.execute_reply.started": "2025-01-06T11:55:55.872274Z" + "iopub.execute_input": "2025-01-07T14:24:19.487623Z", + "iopub.status.busy": "2025-01-07T14:24:19.487324Z", + "iopub.status.idle": "2025-01-07T14:24:19.636245Z", + "shell.execute_reply": "2025-01-07T14:24:19.635201Z", + "shell.execute_reply.started": "2025-01-07T14:24:19.487600Z" } }, "outputs": [], "source": [ + "# it takes ~3 mins to query this night, load from file if available\n", "if os.path.isfile(fname_id):\n", " print(\"load {}\".format(fname_id))\n", " df_id = pd.read_csv(fname_id, index_col=0)\n", @@ -168,39 +160,24 @@ "df_id" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "b70acf82-fdc5-4a0b-930e-807cb93fb245", - "metadata": { - "execution": { - "iopub.execute_input": "2025-01-06T11:55:56.086173Z", - "iopub.status.busy": "2025-01-06T11:55:56.085911Z", - "iopub.status.idle": "2025-01-06T11:55:56.089606Z", - "shell.execute_reply": "2025-01-06T11:55:56.088842Z", - "shell.execute_reply.started": "2025-01-06T11:55:56.086150Z" - } - }, - "outputs": [], - "source": [ - "# it takes around 16 minutes to retreive all object data (in chunks)" - ] - }, { "cell_type": "code", "execution_count": null, "id": "fc6449d2-18f8-4369-80ad-7ca4c53ed5e8", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:55:56.091082Z", - "iopub.status.busy": "2025-01-06T11:55:56.090479Z", - "iopub.status.idle": "2025-01-06T11:55:59.701310Z", - "shell.execute_reply": "2025-01-06T11:55:59.700366Z", - "shell.execute_reply.started": "2025-01-06T11:55:56.091056Z" + "iopub.execute_input": "2025-01-07T14:24:19.637628Z", + "iopub.status.busy": "2025-01-07T14:24:19.637333Z", + "iopub.status.idle": "2025-01-07T14:24:22.803122Z", + "shell.execute_reply": "2025-01-07T14:24:22.802260Z", + "shell.execute_reply.started": "2025-01-07T14:24:19.637604Z" } }, "outputs": [], "source": [ + "# for each of these objects get the full DP0.3 SSObject details\n", + "# note that these parameters have been calculated on ALL DP0.3 data (using the full 10 years of survey data)\n", + "# it takes around 16 minutes to retreive all object data (in chunks), load from file if available\n", "if os.path.isfile(fname_obj):\n", " print(\"load {}\".format(fname_obj))\n", " df_obj = pd.read_csv(fname_obj, index_col=0)\n", @@ -258,11 +235,11 @@ "id": "0399adcb-b873-4dba-bece-c342e8a64318", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:55:59.702562Z", - "iopub.status.busy": "2025-01-06T11:55:59.702281Z", - "iopub.status.idle": "2025-01-06T11:55:59.794156Z", - "shell.execute_reply": "2025-01-06T11:55:59.793422Z", - "shell.execute_reply.started": "2025-01-06T11:55:59.702540Z" + "iopub.execute_input": "2025-01-07T14:24:22.804492Z", + "iopub.status.busy": "2025-01-07T14:24:22.804215Z", + "iopub.status.idle": "2025-01-07T14:24:22.903032Z", + "shell.execute_reply": "2025-01-07T14:24:22.902283Z", + "shell.execute_reply.started": "2025-01-07T14:24:22.804471Z" } }, "outputs": [], @@ -276,15 +253,16 @@ "id": "9f068ccc-d684-4629-938d-a602979ea23a", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:55:59.795386Z", - "iopub.status.busy": "2025-01-06T11:55:59.795114Z", - "iopub.status.idle": "2025-01-06T11:56:00.285944Z", - "shell.execute_reply": "2025-01-06T11:56:00.285170Z", - "shell.execute_reply.started": "2025-01-06T11:55:59.795363Z" + "iopub.execute_input": "2025-01-07T14:24:22.904295Z", + "iopub.status.busy": "2025-01-07T14:24:22.904027Z", + "iopub.status.idle": "2025-01-07T14:24:23.385127Z", + "shell.execute_reply": "2025-01-07T14:24:23.384291Z", + "shell.execute_reply.started": "2025-01-07T14:24:22.904274Z" } }, "outputs": [], "source": [ + "# plot of orbital elements (limited to within Jupiter Trojans)\n", "x_plot = \"a\"\n", "y_plot = \"e\"\n", "\n", @@ -312,36 +290,50 @@ "id": "2f573948-2b3c-43b1-b90a-839da18820a6", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:56:00.287318Z", - "iopub.status.busy": "2025-01-06T11:56:00.287013Z", - "iopub.status.idle": "2025-01-06T11:56:00.346034Z", - "shell.execute_reply": "2025-01-06T11:56:00.345188Z", - "shell.execute_reply.started": "2025-01-06T11:56:00.287293Z" + "iopub.execute_input": "2025-01-07T14:24:23.386446Z", + "iopub.status.busy": "2025-01-07T14:24:23.386165Z", + "iopub.status.idle": "2025-01-07T14:24:23.447289Z", + "shell.execute_reply": "2025-01-07T14:24:23.446427Z", + "shell.execute_reply.started": "2025-01-07T14:24:23.386426Z" } }, "outputs": [], "source": [ - "# some objects are missing from MPCORB, they appear only in SSObject\n", - "# these are spacecraft, with DiaSource.nameTrue beginning with ET\n", + "# There is a number difference between df_id and df_obj\n", + "# The join in the df_obj query loses some objects which are missing from MPCORB, they appear only in SSObject\n", + "# these are alien spacecraft, with DiaSource.nameTrue beginning with ET\n", "df_missing = df_id[~np.isin(df_id[\"ssObjectId\"], df_obj[\"ssObjectId\"])]\n", "df_missing" ] }, + { + "cell_type": "markdown", + "id": "be62f93c-cbf7-4989-a084-3ae3f48ad6bc", + "metadata": {}, + "source": [ + "# Calculate SSObject on the night" + ] + }, { "cell_type": "code", "execution_count": null, "id": "3640cc09-7730-409a-8b43-0a5a21c940a2", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:56:00.347673Z", - "iopub.status.busy": "2025-01-06T11:56:00.347279Z", - "iopub.status.idle": "2025-01-06T11:56:03.388170Z", - "shell.execute_reply": "2025-01-06T11:56:03.387180Z", - "shell.execute_reply.started": "2025-01-06T11:56:00.347648Z" + "iopub.execute_input": "2025-01-07T14:24:23.448520Z", + "iopub.status.busy": "2025-01-07T14:24:23.448271Z", + "iopub.status.idle": "2025-01-07T14:24:26.020617Z", + "shell.execute_reply": "2025-01-07T14:24:26.019624Z", + "shell.execute_reply.started": "2025-01-07T14:24:23.448501Z" } }, "outputs": [], "source": [ + "# Here we calculate what the SSObject parameters should be like on the night\n", + "# We fit phase curves to observations with mjd < night\n", + "# NB we have assumed that DP0.3 was calculated with HG12_Pen16, not HG12. See https://community.lsst.org/t/phase-curve-model-hg12-or-hg12-pen16-used-in-dp0-3/9674\n", + "# this takes around ~30 hours, load from file if available\n", + "\n", "error_list = []\n", "N_tot = len(df_id)\n", "\n", @@ -368,11 +360,11 @@ " AND dia.midPointMjdTai < {}\n", " ORDER by dia.midPointMjdTai\n", " \"\"\".format(\n", - " ssoid, night + time_bounds\n", + " ssoid,\n", + " night - time_bounds, # TODO: this should be night - time_bounds to exclude data on the night?\n", " )\n", "\n", " df_obs = service.search(query).to_table().to_pandas()\n", - " # print(len(df_obs),df_obj[df_obj[\"ssObjectId\"]==ssoid].iloc[0][\"numObs\"])\n", "\n", " # calculate reduced mag\n", " thdist = df_obs[\"topocentricDist\"] * df_obs[\"heliocentricDist\"]\n", @@ -383,68 +375,72 @@ " sso[\"ssObjectId\"] = ssoid\n", " sso[\"arc\"] = np.ptp(df_obs[\"midPointMjdTai\"])\n", " sso[\"numObs\"] = len(df_obs)\n", - " # also phaseAngle_max/min etc?\n", + " # TODO: also calculate phaseAngle_max/min etc?\n", "\n", " # fit phase curve to each filter\n", " for filt in \"ugrizy\":\n", " mask = df_obs[\"band\"] == filt\n", " _df_obs = df_obs[mask]\n", " _N = len(_df_obs)\n", - " # print(filt,_N)\n", "\n", " sso[\"{}_Ndata\".format(filt)] = _N\n", "\n", - " if _N < N_min:\n", + " # if _N < N_min:\n", + " # sso[\"{}_G12\".format(filt)] = np.nan\n", + " # sso[\"{}_G12Err\".format(filt)] = np.nan\n", + " # sso[\"{}_H\".format(filt)] = np.nan\n", + " # sso[\"{}_HErr\".format(filt)] = np.nan\n", + " # else:\n", + "\n", + " # Define the model with starting values for H and G12\n", + " model = HG12_Pen16(H=np.amin(_df_obs[\"reduced_mag\"]), G12=G12_start)\n", + " try:\n", + " # fit the model to the data\n", + " model_fit = fitter(\n", + " model,\n", + " np.radians(_df_obs[\"phaseAngle\"]), # no units, hence radians\n", + " np.array(_df_obs[\"reduced_mag\"]),\n", + " weights=1.0 / np.array(_df_obs[\"magErr\"]),\n", + " )\n", + " except:\n", + " # if the fit fails, store nan values\n", + " print(\"{} fit error\".format(ssoid))\n", " sso[\"{}_G12\".format(filt)] = np.nan\n", " sso[\"{}_G12Err\".format(filt)] = np.nan\n", " sso[\"{}_H\".format(filt)] = np.nan\n", " sso[\"{}_HErr\".format(filt)] = np.nan\n", - " else:\n", - " model = HG12_Pen16(H=np.amin(_df_obs[\"reduced_mag\"]), G12=G12_start)\n", - " # print(model)\n", - " try:\n", - " model_fit = fitter(\n", - " model,\n", - " np.radians(_df_obs[\"phaseAngle\"]),\n", - " np.array(_df_obs[\"reduced_mag\"]),\n", - " weights=1.0 / np.array(_df_obs[\"magErr\"]),\n", - " )\n", - " except:\n", - " print(\"{} fit error\".format(ssoid))\n", - " sso[\"{}_G12\".format(filt)] = np.nan\n", - " sso[\"{}_G12Err\".format(filt)] = np.nan\n", - " sso[\"{}_H\".format(filt)] = np.nan\n", - " sso[\"{}_HErr\".format(filt)] = np.nan\n", - " error_list.append(ssoid)\n", - " continue\n", - "\n", - " # print(model_fit)\n", - " covariance = fitter.fit_info[\"param_cov\"]\n", - " fit_errs = np.sqrt(np.diag(covariance))\n", - " # print(model_fit.param_names)\n", + " # error_list.append(ssoid)\n", + " continue\n", "\n", - " sso[\"{}_G12\".format(filt)] = model_fit.G12.value\n", - " sso[\"{}_G12Err\".format(filt)] = fit_errs[1]\n", - " sso[\"{}_H\".format(filt)] = model_fit.H.value\n", + " # store the model parameters\n", + " sso[\"{}_G12\".format(filt)] = model_fit.G12.value\n", + " sso[\"{}_H\".format(filt)] = model_fit.H.value\n", + "\n", + " # determine the parameter uncertainties\n", + " covariance = fitter.fit_info[\"param_cov\"]\n", + " if covariance is not None:\n", + " fit_errs = np.sqrt(np.diag(covariance))\n", " sso[\"{}_HErr\".format(filt)] = fit_errs[0]\n", + " sso[\"{}_G12Err\".format(filt)] = fit_errs[1]\n", + " else:\n", + " sso[\"{}_HErr\".format(filt)] = np.nan\n", + " sso[\"{}_G12Err\".format(filt)] = np.nan\n", "\n", + " # check that all required values have an entry in the sso dict\n", " if len(sso) != 33:\n", " print(\"{} error\".format(ssoid))\n", " error_list.append(ssoid)\n", " continue\n", "\n", " # save the data to file\n", - " # print(sso)\n", " df_sso = pd.DataFrame([sso])\n", " if i == 0:\n", " df_sso.to_csv(fname_sso)\n", " else:\n", " df_sso.to_csv(fname_sso, mode=\"a\", header=False)\n", "\n", - " # print(df_obs.iloc[0][\"nameTrue\"])\n", - "\n", - " # if i>5:\n", - " # break\n", + " if i > 5:\n", + " break\n", "\n", "df_sso = pd.read_csv(fname_sso, index_col=0).reset_index(drop=True)" ] @@ -455,11 +451,11 @@ "id": "e2d38b8f-53b4-47d4-bfc0-47e4237e110b", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:56:03.389770Z", - "iopub.status.busy": "2025-01-06T11:56:03.389449Z", - "iopub.status.idle": "2025-01-06T11:56:03.395116Z", - "shell.execute_reply": "2025-01-06T11:56:03.394145Z", - "shell.execute_reply.started": "2025-01-06T11:56:03.389743Z" + "iopub.execute_input": "2025-01-07T14:24:26.022046Z", + "iopub.status.busy": "2025-01-07T14:24:26.021742Z", + "iopub.status.idle": "2025-01-07T14:24:26.027405Z", + "shell.execute_reply": "2025-01-07T14:24:26.026531Z", + "shell.execute_reply.started": "2025-01-07T14:24:26.022010Z" } }, "outputs": [], @@ -473,11 +469,11 @@ "id": "83f77a51-369c-47bb-9de5-ffe160223c13", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:56:03.396650Z", - "iopub.status.busy": "2025-01-06T11:56:03.396328Z", - "iopub.status.idle": "2025-01-06T11:56:03.533015Z", - "shell.execute_reply": "2025-01-06T11:56:03.531955Z", - "shell.execute_reply.started": "2025-01-06T11:56:03.396619Z" + "iopub.execute_input": "2025-01-07T14:24:26.031473Z", + "iopub.status.busy": "2025-01-07T14:24:26.031046Z", + "iopub.status.idle": "2025-01-07T14:24:26.160993Z", + "shell.execute_reply": "2025-01-07T14:24:26.160133Z", + "shell.execute_reply.started": "2025-01-07T14:24:26.031448Z" } }, "outputs": [], @@ -491,16 +487,17 @@ "id": "2802ad4a-c56d-4b2f-9adc-3995cb1806da", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:56:03.534388Z", - "iopub.status.busy": "2025-01-06T11:56:03.534098Z", - "iopub.status.idle": "2025-01-06T11:56:04.855659Z", - "shell.execute_reply": "2025-01-06T11:56:04.854618Z", - "shell.execute_reply.started": "2025-01-06T11:56:03.534354Z" + "iopub.execute_input": "2025-01-07T14:24:26.162341Z", + "iopub.status.busy": "2025-01-07T14:24:26.162050Z", + "iopub.status.idle": "2025-01-07T14:24:27.529059Z", + "shell.execute_reply": "2025-01-07T14:24:27.528145Z", + "shell.execute_reply.started": "2025-01-07T14:24:26.162317Z" } }, "outputs": [], "source": [ - "# x_plot = \"numObs\"\n", + "# compare the distributions between phase curve parameters on the night, vs the full 10 yr parameters\n", + "\n", "y_plot = \"number\"\n", "df_plot = df_sso\n", "df_plot2 = df_obj\n", @@ -514,6 +511,7 @@ " if \"G12\" in x_plot:\n", " bins = np.linspace(-1.0, 1.5, n_bins)\n", " ax1.axvline(0.2, c=\"r\")\n", + " ax1.axvline(0.55, c=\"r\", ls=\":\")\n", " else:\n", " bins = n_bins\n", "\n", @@ -535,11 +533,11 @@ "id": "458922d0-dc24-418f-bf9d-bbb4c6f160d1", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:56:04.857048Z", - "iopub.status.busy": "2025-01-06T11:56:04.856763Z", - "iopub.status.idle": "2025-01-06T11:56:04.861252Z", - "shell.execute_reply": "2025-01-06T11:56:04.860327Z", - "shell.execute_reply.started": "2025-01-06T11:56:04.857025Z" + "iopub.execute_input": "2025-01-07T14:24:27.530380Z", + "iopub.status.busy": "2025-01-07T14:24:27.530107Z", + "iopub.status.idle": "2025-01-07T14:24:27.534141Z", + "shell.execute_reply": "2025-01-07T14:24:27.533298Z", + "shell.execute_reply.started": "2025-01-07T14:24:27.530358Z" } }, "outputs": [], @@ -550,37 +548,39 @@ { "cell_type": "code", "execution_count": null, - "id": "3a6330c3-5af3-4bab-a5a1-82ea8addb370", + "id": "cc7ff426-f2c3-439e-b3b3-4c00a97ffa3f", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:56:04.862666Z", - "iopub.status.busy": "2025-01-06T11:56:04.862372Z", - "iopub.status.idle": "2025-01-06T11:56:04.876080Z", - "shell.execute_reply": "2025-01-06T11:56:04.875054Z", - "shell.execute_reply.started": "2025-01-06T11:56:04.862642Z" + "iopub.execute_input": "2025-01-07T14:24:27.535617Z", + "iopub.status.busy": "2025-01-07T14:24:27.535284Z", + "iopub.status.idle": "2025-01-07T14:24:27.547769Z", + "shell.execute_reply": "2025-01-07T14:24:27.546406Z", + "shell.execute_reply.started": "2025-01-07T14:24:27.535592Z" } }, "outputs": [], "source": [ - "# there is a peak in df_obj at r_G12=0.2 (or very close to 0.2)" + "# The peak near G12~0.55 probably corresponds to the HG=0.15 model used to generate the observations\n", + "# This is similar (but not the same) as the predicted value of G12=0.62 from Robinson et al. 2024 eqn B1a\n", + "# See notebooks/tutorial-notebooks/DP03_04a_Introduction_to_Phase_Curves.ipynb" ] }, { "cell_type": "code", "execution_count": null, - "id": "e015e86f-f236-411f-a78c-b70f15acd117", + "id": "3a6330c3-5af3-4bab-a5a1-82ea8addb370", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:56:04.880511Z", - "iopub.status.busy": "2025-01-06T11:56:04.880173Z", - "iopub.status.idle": "2025-01-06T11:56:04.889194Z", - "shell.execute_reply": "2025-01-06T11:56:04.887866Z", - "shell.execute_reply.started": "2025-01-06T11:56:04.880485Z" + "iopub.execute_input": "2025-01-07T14:24:27.549496Z", + "iopub.status.busy": "2025-01-07T14:24:27.549183Z", + "iopub.status.idle": "2025-01-07T14:24:27.561297Z", + "shell.execute_reply": "2025-01-07T14:24:27.560326Z", + "shell.execute_reply.started": "2025-01-07T14:24:27.549472Z" } }, "outputs": [], "source": [ - "# df_obj[df_obj[\"r_G12\"]==0.2]" + "# there is a peak in df_obj at r_G12=0.2 (or very close to 0.2) probably due to how objects were simulated and fit in DP0.3" ] }, { @@ -589,11 +589,11 @@ "id": "ab15eec9-6633-4f7c-b023-5215ca88189c", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:56:04.890662Z", - "iopub.status.busy": "2025-01-06T11:56:04.890338Z", - "iopub.status.idle": "2025-01-06T11:56:04.904923Z", - "shell.execute_reply": "2025-01-06T11:56:04.903927Z", - "shell.execute_reply.started": "2025-01-06T11:56:04.890632Z" + "iopub.execute_input": "2025-01-07T14:24:27.562684Z", + "iopub.status.busy": "2025-01-07T14:24:27.562384Z", + "iopub.status.idle": "2025-01-07T14:24:27.573527Z", + "shell.execute_reply": "2025-01-07T14:24:27.572535Z", + "shell.execute_reply.started": "2025-01-07T14:24:27.562660Z" } }, "outputs": [], @@ -614,11 +614,11 @@ "id": "ae430930-fc5c-43cc-b105-1a08965aa1c7", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:56:04.906505Z", - "iopub.status.busy": "2025-01-06T11:56:04.906194Z", - "iopub.status.idle": "2025-01-06T11:56:04.992015Z", - "shell.execute_reply": "2025-01-06T11:56:04.990866Z", - "shell.execute_reply.started": "2025-01-06T11:56:04.906480Z" + "iopub.execute_input": "2025-01-07T14:24:27.575661Z", + "iopub.status.busy": "2025-01-07T14:24:27.574836Z", + "iopub.status.idle": "2025-01-07T14:24:27.645772Z", + "shell.execute_reply": "2025-01-07T14:24:27.644615Z", + "shell.execute_reply.started": "2025-01-07T14:24:27.575631Z" } }, "outputs": [], @@ -632,11 +632,11 @@ "id": "4f249404-5eff-4804-a7d2-38905dd3985d", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:56:04.993828Z", - "iopub.status.busy": "2025-01-06T11:56:04.993378Z", - "iopub.status.idle": "2025-01-06T11:56:05.282330Z", - "shell.execute_reply": "2025-01-06T11:56:05.281428Z", - "shell.execute_reply.started": "2025-01-06T11:56:04.993789Z" + "iopub.execute_input": "2025-01-07T14:24:27.647292Z", + "iopub.status.busy": "2025-01-07T14:24:27.646993Z", + "iopub.status.idle": "2025-01-07T14:24:27.933608Z", + "shell.execute_reply": "2025-01-07T14:24:27.932598Z", + "shell.execute_reply.started": "2025-01-07T14:24:27.647267Z" } }, "outputs": [], @@ -650,11 +650,11 @@ "id": "e72302c7-85e5-4214-8a39-ae98764832a2", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:56:05.283828Z", - "iopub.status.busy": "2025-01-06T11:56:05.283398Z", - "iopub.status.idle": "2025-01-06T11:56:05.587097Z", - "shell.execute_reply": "2025-01-06T11:56:05.585881Z", - "shell.execute_reply.started": "2025-01-06T11:56:05.283791Z" + "iopub.execute_input": "2025-01-07T14:24:27.935134Z", + "iopub.status.busy": "2025-01-07T14:24:27.934841Z", + "iopub.status.idle": "2025-01-07T14:24:28.231813Z", + "shell.execute_reply": "2025-01-07T14:24:28.230941Z", + "shell.execute_reply.started": "2025-01-07T14:24:27.935110Z" } }, "outputs": [], @@ -669,16 +669,17 @@ "id": "5af8f58e-e139-40a3-b2bb-dbf7950a4a40", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:56:05.588740Z", - "iopub.status.busy": "2025-01-06T11:56:05.588432Z", - "iopub.status.idle": "2025-01-06T11:56:06.870545Z", - "shell.execute_reply": "2025-01-06T11:56:06.869733Z", - "shell.execute_reply.started": "2025-01-06T11:56:05.588714Z" + "iopub.execute_input": "2025-01-07T14:24:28.233237Z", + "iopub.status.busy": "2025-01-07T14:24:28.232930Z", + "iopub.status.idle": "2025-01-07T14:24:29.541815Z", + "shell.execute_reply": "2025-01-07T14:24:29.540636Z", + "shell.execute_reply.started": "2025-01-07T14:24:28.233213Z" } }, "outputs": [], "source": [ - "# compare matched values\n", + "# compare matched values directly\n", + "\n", "df_plot = df_sso_obj\n", "print(len(df_plot))\n", "\n", @@ -699,11 +700,11 @@ " ax1.set_ylabel(\"df_obj {}\".format(x_plot))\n", "\n", " if \"G12\" in x_plot:\n", + " # use a mask to display a sensible range of G12 values\n", " _df_plot = df_plot.dropna(subset=[\"{}_sso\".format(x_plot), \"{}_obj\".format(x_plot)])\n", " _df_plot = _df_plot[\n", " (_df_plot[\"{}_obj\".format(x_plot)] > -1.0) & (_df_plot[\"{}_obj\".format(x_plot)] < 2.0)\n", " ]\n", - " print(len(_df_plot))\n", " ax1.hist2d(\n", " _df_plot[\"{}_sso\".format(x_plot)],\n", " _df_plot[\"{}_obj\".format(x_plot)],\n", @@ -712,23 +713,40 @@ " norm=colors.LogNorm(),\n", " )\n", "\n", - " # ax1.set_ylim(-2,2)\n", " ax1.axhline(0.2, c=\"r\", ls=\":\")\n", "\n", " plt.show()" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b4efc73-dea5-4ffd-b285-be20eea172fa", + "metadata": { + "execution": { + "iopub.execute_input": "2025-01-07T14:24:29.543780Z", + "iopub.status.busy": "2025-01-07T14:24:29.543419Z", + "iopub.status.idle": "2025-01-07T14:24:29.548442Z", + "shell.execute_reply": "2025-01-07T14:24:29.547333Z", + "shell.execute_reply.started": "2025-01-07T14:24:29.543751Z" + } + }, + "outputs": [], + "source": [ + "# df_sso has a number of objects with G12=0.62, which is the initial conditions we selected (what would happen if we used 0.2 ...?)" + ] + }, { "cell_type": "code", "execution_count": null, "id": "fd726932-83e7-4e36-9c19-a17f9c090be3", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:56:06.871897Z", - "iopub.status.busy": "2025-01-06T11:56:06.871599Z", - "iopub.status.idle": "2025-01-06T11:56:06.875752Z", - "shell.execute_reply": "2025-01-06T11:56:06.874847Z", - "shell.execute_reply.started": "2025-01-06T11:56:06.871873Z" + "iopub.execute_input": "2025-01-07T14:24:29.550204Z", + "iopub.status.busy": "2025-01-07T14:24:29.549843Z", + "iopub.status.idle": "2025-01-07T14:24:29.563319Z", + "shell.execute_reply": "2025-01-07T14:24:29.562158Z", + "shell.execute_reply.started": "2025-01-07T14:24:29.550174Z" } }, "outputs": [], @@ -742,11 +760,11 @@ "id": "3cce8fe4-a516-4a96-9de8-abf2a72f59df", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:56:06.877075Z", - "iopub.status.busy": "2025-01-06T11:56:06.876800Z", - "iopub.status.idle": "2025-01-06T11:56:06.927727Z", - "shell.execute_reply": "2025-01-06T11:56:06.926879Z", - "shell.execute_reply.started": "2025-01-06T11:56:06.877053Z" + "iopub.execute_input": "2025-01-07T14:24:29.565199Z", + "iopub.status.busy": "2025-01-07T14:24:29.564784Z", + "iopub.status.idle": "2025-01-07T14:24:29.612958Z", + "shell.execute_reply": "2025-01-07T14:24:29.612032Z", + "shell.execute_reply.started": "2025-01-07T14:24:29.565157Z" } }, "outputs": [], @@ -760,11 +778,11 @@ "id": "eee6f9a6-dff2-4080-8687-6d8671ac768f", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:56:06.929087Z", - "iopub.status.busy": "2025-01-06T11:56:06.928779Z", - "iopub.status.idle": "2025-01-06T11:56:06.952968Z", - "shell.execute_reply": "2025-01-06T11:56:06.951975Z", - "shell.execute_reply.started": "2025-01-06T11:56:06.929049Z" + "iopub.execute_input": "2025-01-07T14:24:29.614450Z", + "iopub.status.busy": "2025-01-07T14:24:29.614149Z", + "iopub.status.idle": "2025-01-07T14:24:29.637444Z", + "shell.execute_reply": "2025-01-07T14:24:29.636512Z", + "shell.execute_reply.started": "2025-01-07T14:24:29.614423Z" } }, "outputs": [], @@ -778,11 +796,11 @@ "id": "3a79ee9c-4dbd-4e18-849f-0c864c9ae13b", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:56:06.954481Z", - "iopub.status.busy": "2025-01-06T11:56:06.954177Z", - "iopub.status.idle": "2025-01-06T11:56:08.103095Z", - "shell.execute_reply": "2025-01-06T11:56:08.101988Z", - "shell.execute_reply.started": "2025-01-06T11:56:06.954453Z" + "iopub.execute_input": "2025-01-07T14:24:29.638974Z", + "iopub.status.busy": "2025-01-07T14:24:29.638678Z", + "iopub.status.idle": "2025-01-07T14:24:30.265634Z", + "shell.execute_reply": "2025-01-07T14:24:30.264733Z", + "shell.execute_reply.started": "2025-01-07T14:24:29.638950Z" } }, "outputs": [], @@ -791,22 +809,6 @@ "# ssoid = \"5903260517146040230\" # No low phase angle coverage?\n", "ssoid = \"496523111065891749\" # just a chill guy\n", "\n", - "# query = \"\"\"\n", - "# SELECT\n", - "# *\n", - "# FROM\n", - "# dp03_catalogs_10yr.DiaSource as dia\n", - "# INNER JOIN\n", - "# dp03_catalogs_10yr.SSSource as sss\n", - "# ON\n", - "# dia.diaSourceId = sss.diaSourceId\n", - "# WHERE\n", - "# dia.ssObjectId={}\n", - "# ORDER by dia.midPointMjdTai\n", - "# \"\"\".format(\n", - "# ssoid\n", - "# )\n", - "\n", "query = \"\"\"\n", "SELECT\n", " *\n", @@ -825,7 +827,6 @@ ")\n", "\n", "df_obs = service.search(query).to_table().to_pandas()\n", - "# print(len(df_obs),df_obj[df_obj[\"ssObjectId\"]==ssoid].iloc[0][\"numObs\"])\n", "\n", "# calculate reduced mag\n", "thdist = df_obs[\"topocentricDist\"] * df_obs[\"heliocentricDist\"]\n", @@ -838,11 +839,11 @@ "id": "db42e25e-a9bf-4188-8586-d1ba564b254e", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:56:08.104531Z", - "iopub.status.busy": "2025-01-06T11:56:08.104255Z", - "iopub.status.idle": "2025-01-06T11:56:08.110372Z", - "shell.execute_reply": "2025-01-06T11:56:08.109527Z", - "shell.execute_reply.started": "2025-01-06T11:56:08.104510Z" + "iopub.execute_input": "2025-01-07T14:24:30.267007Z", + "iopub.status.busy": "2025-01-07T14:24:30.266733Z", + "iopub.status.idle": "2025-01-07T14:24:30.272503Z", + "shell.execute_reply": "2025-01-07T14:24:30.271631Z", + "shell.execute_reply.started": "2025-01-07T14:24:30.266986Z" } }, "outputs": [], @@ -850,35 +851,17 @@ "df_obs.columns" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "fbc38bf4-9115-4701-82bb-1e7157ab30d3", - "metadata": { - "execution": { - "iopub.execute_input": "2025-01-06T11:56:08.111741Z", - "iopub.status.busy": "2025-01-06T11:56:08.111440Z", - "iopub.status.idle": "2025-01-06T11:56:08.128071Z", - "shell.execute_reply": "2025-01-06T11:56:08.127115Z", - "shell.execute_reply.started": "2025-01-06T11:56:08.111717Z" - } - }, - "outputs": [], - "source": [ - "df_obj.columns" - ] - }, { "cell_type": "code", "execution_count": null, "id": "69972c5f-181f-4a05-81ad-46cfeaf35210", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:56:08.129412Z", - "iopub.status.busy": "2025-01-06T11:56:08.129133Z", - "iopub.status.idle": "2025-01-06T11:56:08.785973Z", - "shell.execute_reply": "2025-01-06T11:56:08.785154Z", - "shell.execute_reply.started": "2025-01-06T11:56:08.129391Z" + "iopub.execute_input": "2025-01-07T14:24:30.273810Z", + "iopub.status.busy": "2025-01-07T14:24:30.273517Z", + "iopub.status.idle": "2025-01-07T14:24:30.707200Z", + "shell.execute_reply": "2025-01-07T14:24:30.706310Z", + "shell.execute_reply.started": "2025-01-07T14:24:30.273787Z" } }, "outputs": [], @@ -897,8 +880,6 @@ "gs = gridspec.GridSpec(1, 1)\n", "ax1 = plt.subplot(gs[0, 0])\n", "\n", - "# ax1.scatter(df_plot[x_plot],df_plot[y_plot])\n", - "\n", "alpha = np.linspace(0, np.amax(df_plot[x_plot]))\n", "\n", "for i, filt in enumerate(np.unique(df_obs[\"band\"])):\n", @@ -940,139 +921,651 @@ "plt.show()" ] }, + { + "cell_type": "markdown", + "id": "cf752451-f1ed-45da-9042-41d6032e9d67", + "metadata": {}, + "source": [ + "# Adler Day Operations - Fit phase parameters" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "205358c9-5d5a-4730-917c-886c0ab8818f", + "id": "54595dc8-cd69-4433-97af-b52f09e55191", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:56:08.787637Z", - "iopub.status.busy": "2025-01-06T11:56:08.787336Z", - "iopub.status.idle": "2025-01-06T11:56:08.791949Z", - "shell.execute_reply": "2025-01-06T11:56:08.790736Z", - "shell.execute_reply.started": "2025-01-06T11:56:08.787614Z" + "iopub.execute_input": "2025-01-07T14:24:30.708912Z", + "iopub.status.busy": "2025-01-07T14:24:30.708518Z", + "iopub.status.idle": "2025-01-07T14:24:30.715468Z", + "shell.execute_reply": "2025-01-07T14:24:30.714289Z", + "shell.execute_reply.started": "2025-01-07T14:24:30.708879Z" } }, "outputs": [], "source": [ "# use adler command to create the adler database\n", - "# use only the data before the test night" + "# use only the data before the test night\n", + "# for example:\n", + "ssoid = 6098332225018\n", + "cmd = \"adler -s {} -n {} -d 60000.0 {} -np\".format(ssoid, adler_data_db, night - time_bounds)\n", + "# cmd+=\" -i /Users/jrobinson/lsst-adler/notebooks/gen_test_data/adler_demo_testing_database.db\"\n", + "cmd" ] }, { "cell_type": "code", "execution_count": null, - "id": "a5996253-da66-4a73-a73f-5a1f70a347c7", + "id": "146efd71-5b7f-4a11-9d85-391dd1dcebae", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:56:08.793517Z", - "iopub.status.busy": "2025-01-06T11:56:08.793131Z", - "iopub.status.idle": "2025-01-06T11:56:08.809517Z", - "shell.execute_reply": "2025-01-06T11:56:08.808387Z", - "shell.execute_reply.started": "2025-01-06T11:56:08.793477Z" + "iopub.execute_input": "2025-01-07T14:24:30.716972Z", + "iopub.status.busy": "2025-01-07T14:24:30.716669Z", + "iopub.status.idle": "2025-01-07T14:24:31.478229Z", + "shell.execute_reply": "2025-01-07T14:24:31.477438Z", + "shell.execute_reply.started": "2025-01-07T14:24:30.716940Z" } }, "outputs": [], "source": [ - "night - time_bounds" + "# Use the Adler CLI to determine phase curve fits and save them all to an SQL database\n", + "with open(\"adler_cmds.sh\", \"w\") as f:\n", + " f.write(\"#!/bin/bash\\n\")\n", + " for i, ssoid in enumerate(np.array(df_id[\"ssObjectId\"])):\n", + " cmd = \"adler -s {} -n {} -d 60000.0 {} -np\".format(ssoid, adler_data_db, night - time_bounds)\n", + " # cmd+=\" -i /Users/jrobinson/lsst-adler/notebooks/gen_test_data/adler_demo_testing_database.db\"\n", + " f.write(cmd + \"\\n\")" ] }, { "cell_type": "code", "execution_count": null, - "id": "be376c51-b405-4523-a054-cb4e0c163766", + "id": "e75d5e79-3e88-40cd-ba08-ddc743bbd7d7", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:58:04.252515Z", - "iopub.status.busy": "2025-01-06T11:58:04.251783Z", - "iopub.status.idle": "2025-01-06T11:58:04.257376Z", - "shell.execute_reply": "2025-01-06T11:58:04.256012Z", - "shell.execute_reply.started": "2025-01-06T11:58:04.252474Z" + "iopub.execute_input": "2025-01-07T14:24:31.479503Z", + "iopub.status.busy": "2025-01-07T14:24:31.479216Z", + "iopub.status.idle": "2025-01-07T14:24:31.482922Z", + "shell.execute_reply": "2025-01-07T14:24:31.482166Z", + "shell.execute_reply.started": "2025-01-07T14:24:31.479481Z" } }, "outputs": [], "source": [ - "adler_data_db = \"adler_data_{}.db\".format(night)" + "# run the adler commands, e.g. in terminal\n", + "# chmod +x adler_cmds.sh\n", + "# ./adler_cmds.sh > adler_cmds.out 2>&1" ] }, { "cell_type": "code", "execution_count": null, - "id": "146efd71-5b7f-4a11-9d85-391dd1dcebae", + "id": "5348db29-8182-4683-83e6-82eba3cc47bc", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:58:04.716044Z", - "iopub.status.busy": "2025-01-06T11:58:04.715600Z", - "iopub.status.idle": "2025-01-06T11:58:04.722463Z", - "shell.execute_reply": "2025-01-06T11:58:04.721518Z", - "shell.execute_reply.started": "2025-01-06T11:58:04.716003Z" + "iopub.execute_input": "2025-01-07T14:24:31.484122Z", + "iopub.status.busy": "2025-01-07T14:24:31.483868Z", + "iopub.status.idle": "2025-01-07T14:24:31.875048Z", + "shell.execute_reply": "2025-01-07T14:24:31.874171Z", + "shell.execute_reply.started": "2025-01-07T14:24:31.484100Z" } }, "outputs": [], "source": [ - "cmd = \"adler -s {} -n {} -d 60000.0 {} -np\".format(ssoid, adler_data_db, night - time_bounds)\n", - "# cmd+=\" -i /Users/jrobinson/lsst-adler/notebooks/gen_test_data/adler_demo_testing_database.db\"\n", - "cmd" + "# read the AdlerData values that were saved to the database\n", + "conn = sqlite3.connect(adler_data_db)\n", + "df_ad = pd.read_sql(\"select * from AdlerData;\", conn)" ] }, { "cell_type": "code", "execution_count": null, - "id": "5348db29-8182-4683-83e6-82eba3cc47bc", + "id": "ea8170fa-ec27-4f2a-a7e3-6e2bb305bdf3", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:58:05.510410Z", - "iopub.status.busy": "2025-01-06T11:58:05.509934Z", - "iopub.status.idle": "2025-01-06T11:58:05.517161Z", - "shell.execute_reply": "2025-01-06T11:58:05.516114Z", - "shell.execute_reply.started": "2025-01-06T11:58:05.510369Z" + "iopub.execute_input": "2025-01-07T14:24:31.876528Z", + "iopub.status.busy": "2025-01-07T14:24:31.876239Z", + "iopub.status.idle": "2025-01-07T14:24:31.911699Z", + "shell.execute_reply": "2025-01-07T14:24:31.910757Z", + "shell.execute_reply.started": "2025-01-07T14:24:31.876504Z" } }, "outputs": [], "source": [ - "conn = sqlite3.connect(adler_data_db)" + "df_ad" ] }, { "cell_type": "code", "execution_count": null, - "id": "6fdd8dc0-529d-464c-8c1a-015c2c1cd09b", + "id": "6b765876-c43c-4cc6-a488-9677987206c9", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:58:06.945179Z", - "iopub.status.busy": "2025-01-06T11:58:06.944329Z", - "iopub.status.idle": "2025-01-06T11:58:06.956894Z", - "shell.execute_reply": "2025-01-06T11:58:06.955647Z", - "shell.execute_reply.started": "2025-01-06T11:58:06.945138Z" + "iopub.execute_input": "2025-01-07T14:24:31.913246Z", + "iopub.status.busy": "2025-01-07T14:24:31.912931Z", + "iopub.status.idle": "2025-01-07T14:24:31.917105Z", + "shell.execute_reply": "2025-01-07T14:24:31.916133Z", + "shell.execute_reply.started": "2025-01-07T14:24:31.913223Z" } }, "outputs": [], "source": [ - "df_ad = pd.read_sql(\"select * from AdlerData limit 1;\", conn)" + "# NB that there is an issue with some fields, phaseAngle_min and phaseAngle_range\n", + "# Probably a dtype issue, see https://github.com/lsst-uk/lsst-adler/issues/188" ] }, { "cell_type": "code", "execution_count": null, - "id": "ea8170fa-ec27-4f2a-a7e3-6e2bb305bdf3", + "id": "ffcfbcf6-dea5-48cf-98d3-05340205d6d3", "metadata": { "execution": { - "iopub.execute_input": "2025-01-06T11:58:09.169347Z", - "iopub.status.busy": "2025-01-06T11:58:09.168857Z", - "iopub.status.idle": "2025-01-06T11:58:09.193703Z", - "shell.execute_reply": "2025-01-06T11:58:09.192844Z", - "shell.execute_reply.started": "2025-01-06T11:58:09.169308Z" + "iopub.execute_input": "2025-01-07T14:24:31.918490Z", + "iopub.status.busy": "2025-01-07T14:24:31.918192Z", + "iopub.status.idle": "2025-01-07T14:24:31.939801Z", + "shell.execute_reply": "2025-01-07T14:24:31.938869Z", + "shell.execute_reply.started": "2025-01-07T14:24:31.918456Z" } }, "outputs": [], "source": [ - "df_ad" + "# calculate the total number of observations taken prior to night\n", + "df_ad[\"nobs\"] = df_ad[[x for x in list(df_ad) if \"_nobs\" in x]].sum(axis=1)" + ] + }, + { + "cell_type": "markdown", + "id": "e3bbf44f-4b75-40ca-9c9a-5903d16cff24", + "metadata": {}, + "source": [ + "# Compare phase curve parameters between a simple \"RSP\" fit and an Adler fit" ] }, { "cell_type": "code", "execution_count": null, "id": "635dc3a6-ff59-4832-8912-68c9beae9661", + "metadata": { + "execution": { + "iopub.execute_input": "2025-01-07T14:24:31.941162Z", + "iopub.status.busy": "2025-01-07T14:24:31.940891Z", + "iopub.status.idle": "2025-01-07T14:24:32.012546Z", + "shell.execute_reply": "2025-01-07T14:24:32.011647Z", + "shell.execute_reply.started": "2025-01-07T14:24:31.941141Z" + } + }, + "outputs": [], + "source": [ + "# merge to get all matches between dataframes\n", + "df_sso_ad = df_sso.merge(df_ad, on=\"ssObjectId\", suffixes=[\"_sso\", \"_obj\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a01a91a2-1136-4ab2-aee2-00d042ef270a", + "metadata": { + "execution": { + "iopub.execute_input": "2025-01-07T14:24:32.013971Z", + "iopub.status.busy": "2025-01-07T14:24:32.013698Z", + "iopub.status.idle": "2025-01-07T14:24:32.597644Z", + "shell.execute_reply": "2025-01-07T14:24:32.596849Z", + "shell.execute_reply.started": "2025-01-07T14:24:32.013949Z" + } + }, + "outputs": [], + "source": [ + "# compare matched values\n", + "df_plot = df_sso_ad\n", + "print(len(df_plot))\n", + "\n", + "filt = \"r\"\n", + "\n", + "for x_plot1, x_plot2 in zip(\n", + " [\"numObs\", \"{}_H\".format(filt), \"{}_G12\".format(filt)],\n", + " [\"nobs\", \"{}_HG12_Pen16_H\".format(filt), \"{}_HG12_Pen16_phase_parameter_1\".format(filt)],\n", + "):\n", + " fig = plt.figure()\n", + " gs = gridspec.GridSpec(1, 1)\n", + " ax1 = plt.subplot(gs[0, 0])\n", + "\n", + " ax1.scatter(df_plot[\"{}\".format(x_plot1)], df_plot[\"{}\".format(x_plot2)], s=1)\n", + "\n", + " ax1.plot(\n", + " [np.amin(df_plot[\"{}\".format(x_plot1)]), np.amax(df_plot[\"{}\".format(x_plot2)])],\n", + " [np.amin(df_plot[\"{}\".format(x_plot1)]), np.amax(df_plot[\"{}\".format(x_plot2)])],\n", + " c=\"r\",\n", + " )\n", + "\n", + " ax1.set_xlabel(\"df_sso {}\".format(x_plot1))\n", + " ax1.set_ylabel(\"df_ad {}\".format(x_plot2))\n", + "\n", + " if \"G12\" in x_plot1:\n", + " _df_plot = df_plot.dropna(subset=[\"{}\".format(x_plot1), \"{}\".format(x_plot2)])\n", + " _df_plot = _df_plot[(_df_plot[\"{}\".format(x_plot1)] > -1.0) & (_df_plot[\"{}\".format(x_plot2)] < 2.0)]\n", + " print(len(_df_plot))\n", + " ax1.hist2d(\n", + " _df_plot[\"{}\".format(x_plot1)],\n", + " _df_plot[\"{}\".format(x_plot2)],\n", + " bins=50,\n", + " # zorder = 0,\n", + " norm=colors.LogNorm(),\n", + " )\n", + "\n", + " # ax1.set_ylim(-2,2)\n", + " ax1.axhline(0.62, c=\"r\", ls=\":\")\n", + "\n", + " plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "f9cc3b3a-9f77-49cc-a8ba-6980930ec209", + "metadata": {}, + "source": [ + "# Inspect the observations and phase curves of a given object" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3e21c9a4-022c-4b02-9d41-e42e6d5ca03e", + "metadata": { + "execution": { + "iopub.execute_input": "2025-01-07T14:24:32.599278Z", + "iopub.status.busy": "2025-01-07T14:24:32.598973Z", + "iopub.status.idle": "2025-01-07T14:24:36.388797Z", + "shell.execute_reply": "2025-01-07T14:24:36.387869Z", + "shell.execute_reply.started": "2025-01-07T14:24:32.599249Z" + } + }, + "outputs": [], + "source": [ + "# ssoid=-6029030307982626991\n", + "# ssoid=6203998258168907131\n", + "ssoid = -8615240469118203769\n", + "\n", + "# load the planetoid from RSP\n", + "planetoid = AdlerPlanetoid.construct_from_RSP(ssoid, date_range=[60000.0, night + time_bounds])\n", + "# load AdlerData from database\n", + "planetoid.AdlerData.populate_from_database(adler_data_db)\n", + "\n", + "# get just the new observations\n", + "planetoid2 = AdlerPlanetoid.construct_from_RSP(ssoid, date_range=[night - time_bounds, night + time_bounds])\n", + "# load AdlerData from database\n", + "planetoid2.AdlerData.populate_from_database(adler_data_db)\n", + "obs_list = [pd.DataFrame(planetoid2.observations_in_filter(filt).__dict__) for filt in planetoid2.filter_list]\n", + "df_obs = pd.concat(obs_list)\n", + "\n", + "# TODO: fix the phaseAngle values - see issue https://github.com/lsst-uk/lsst-adler/issues/188\n", + "for pl in [planetoid, planetoid2]:\n", + " for filt in pl.AdlerData.filter_list:\n", + " p = pl.AdlerData.get_phase_parameters_in_filter(filt, \"HG12_Pen16\")\n", + " obs = pl.observations_in_filter(filt)\n", + " _df_obs = pd.DataFrame(obs.__dict__)\n", + " p.phaseAngle_min = np.amin(_df_obs[\"phaseAngle\"])\n", + " p.phaseAngle_range = np.ptp(_df_obs[\"phaseAngle\"])\n", + " p_dict = p.__dict__\n", + " del p_dict[\"filter_name\"]\n", + " pl.AdlerData.populate_phase_parameters(filt, **p_dict)\n", + "\n", + "print(planetoid.MPCORB.q / (1.0 - planetoid.MPCORB.e), planetoid.MPCORB.e, planetoid.MPCORB.q)\n", + "\n", + "fig = plot_errorbar(planetoid, filt_list=planetoid.filter_list, label_list=planetoid.filter_list)\n", + "fig = plot_errorbar(\n", + " planetoid2,\n", + " fig=fig,\n", + " filt_list=planetoid2.filter_list,\n", + " label_list=[\"{}:{}\".format(night, filt) for filt in planetoid2.filter_list],\n", + " col_list=[\"k\"] * len(planetoid2.filter_list),\n", + ")\n", + "fig = plot_phasecurve(\n", + " planetoid.AdlerData,\n", + " fig=fig,\n", + " filt_list=planetoid.filter_list,\n", + " col_list=[\"C{}\".format(i) for i in range(len(planetoid.filter_list))],\n", + ")\n", + "plt.gca().legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "028a12ac-8fe2-46a9-ac83-037319496186", + "metadata": {}, + "source": [ + "# Adler Night Operations - Outlier detection " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6cf48b16-4fdf-4eaa-80cd-56a74b22640a", + "metadata": { + "execution": { + "iopub.execute_input": "2025-01-07T14:24:36.393681Z", + "iopub.status.busy": "2025-01-07T14:24:36.393320Z", + "iopub.status.idle": "2025-01-07T14:24:36.397876Z", + "shell.execute_reply": "2025-01-07T14:24:36.396888Z", + "shell.execute_reply.started": "2025-01-07T14:24:36.393656Z" + } + }, + "outputs": [], + "source": [ + "# perform the adler outlier detection for only detections made that night\n", + "# Rather than reading from RSP (slow), this should simply involve ingesting the alert and reading from AdlerData" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "441de736-53ac-4894-92d7-b3d736093853", + "metadata": { + "execution": { + "iopub.execute_input": "2025-01-07T14:24:36.399299Z", + "iopub.status.busy": "2025-01-07T14:24:36.399008Z", + "iopub.status.idle": "2025-01-07T14:24:36.411185Z", + "shell.execute_reply": "2025-01-07T14:24:36.410406Z", + "shell.execute_reply.started": "2025-01-07T14:24:36.399277Z" + } + }, + "outputs": [], + "source": [ + "# profile the code to find bottlenecks\n", + "# see https://stackoverflow.com/questions/44734297/how-to-profile-python-3-5-code-line-by-line-in-jupyter-notebook-5" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e0a36373-e233-4727-b303-46de01136000", + "metadata": { + "execution": { + "iopub.execute_input": "2025-01-07T14:24:36.412558Z", + "iopub.status.busy": "2025-01-07T14:24:36.412268Z", + "iopub.status.idle": "2025-01-07T14:24:36.428658Z", + "shell.execute_reply": "2025-01-07T14:24:36.427770Z", + "shell.execute_reply.started": "2025-01-07T14:24:36.412536Z" + } + }, + "outputs": [], + "source": [ + "%load_ext line_profiler" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14f91852-fd1f-4c40-aaf2-512a5f346917", + "metadata": { + "execution": { + "iopub.execute_input": "2025-01-07T14:24:36.430037Z", + "iopub.status.busy": "2025-01-07T14:24:36.429774Z", + "iopub.status.idle": "2025-01-07T14:24:36.437910Z", + "shell.execute_reply": "2025-01-07T14:24:36.436923Z", + "shell.execute_reply.started": "2025-01-07T14:24:36.430016Z" + } + }, + "outputs": [], + "source": [ + "def adler_outlier_detection(ssoid):\n", + "\n", + " # get just the new observations\n", + " planetoid2 = AdlerPlanetoid.construct_from_RSP(\n", + " ssoid, date_range=[night - time_bounds, night + time_bounds]\n", + " )\n", + " # load AdlerData from database\n", + " planetoid2.AdlerData.populate_from_database(adler_data_db)\n", + " obs_list = [\n", + " pd.DataFrame(planetoid2.observations_in_filter(filt).__dict__) for filt in planetoid2.filter_list\n", + " ]\n", + " df_obs = pd.concat(obs_list)\n", + "\n", + " # for each filter determine residuals\n", + " for filt in planetoid2.filter_list:\n", + "\n", + " sso = planetoid2.SSObject_in_filter(filt)\n", + "\n", + " # get the observations\n", + " obs = planetoid2.observations_in_filter(filt)\n", + " df_obs = pd.DataFrame(obs.__dict__)\n", + " df_obs[\"outlier\"] = [False] * len(df_obs)\n", + "\n", + " # Load the phase curve model available in AdlerData\n", + " adler_data = planetoid2.AdlerData.get_phase_parameters_in_filter(filt, \"HG12_Pen16\")\n", + "\n", + " pc = PhaseCurve().InitModelDict(adler_data.__dict__)\n", + "\n", + " # find outliers in new data\n", + " # calculate data - model residuals\n", + " res = np.array(df_obs[\"reduced_mag\"]) - pc.ReducedMag(np.radians(np.array(df_obs[\"phaseAngle\"])))\n", + " outlier_flag = sci_utils.outlier_diff(res, diff_cut=diff_cut)\n", + " diaId = np.array(df_obs[\"diaSourceId\"])\n", + " # print(diaId)\n", + " # print(res)\n", + " # print(outlier_flag)\n", + " df_obs[\"outlier\"] = outlier_flag" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eba72c25-c57f-44b5-b0c0-21297c5cb144", + "metadata": { + "execution": { + "iopub.execute_input": "2025-01-07T14:24:36.439343Z", + "iopub.status.busy": "2025-01-07T14:24:36.439024Z", + "iopub.status.idle": "2025-01-07T14:24:38.473895Z", + "shell.execute_reply": "2025-01-07T14:24:38.472651Z", + "shell.execute_reply.started": "2025-01-07T14:24:36.439315Z" + } + }, + "outputs": [], + "source": [ + "%lprun -f adler_outlier_detection adler_outlier_detection(ssoid)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "57831395-4d88-4f01-8329-cb9cad811b09", + "metadata": { + "execution": { + "iopub.execute_input": "2025-01-07T14:24:38.475311Z", + "iopub.status.busy": "2025-01-07T14:24:38.474998Z", + "iopub.status.idle": "2025-01-07T14:24:38.479750Z", + "shell.execute_reply": "2025-01-07T14:24:38.478695Z", + "shell.execute_reply.started": "2025-01-07T14:24:38.475286Z" + } + }, + "outputs": [], + "source": [ + "# construct_from_RSP dominates the run time, this will be much faster if an alert is ingested from the kafka stream" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aab3ca7b-03e4-49a2-9658-f0b170c45155", + "metadata": { + "execution": { + "iopub.execute_input": "2025-01-07T14:24:38.481433Z", + "iopub.status.busy": "2025-01-07T14:24:38.481026Z", + "iopub.status.idle": "2025-01-07T14:24:38.493918Z", + "shell.execute_reply": "2025-01-07T14:24:38.492950Z", + "shell.execute_reply.started": "2025-01-07T14:24:38.481395Z" + } + }, + "outputs": [], + "source": [ + "# run the outlier detection for all objects\n", + "# without adding an activity to DP0.3 this should only pick up objects with bad phase curve fits" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bdfeea4b-1f46-4a0a-b39c-ae43e7790a95", + "metadata": { + "execution": { + "iopub.execute_input": "2025-01-07T14:24:38.495383Z", + "iopub.status.busy": "2025-01-07T14:24:38.495090Z", + "iopub.status.idle": "2025-01-07T14:24:40.557814Z", + "shell.execute_reply": "2025-01-07T14:24:40.556641Z", + "shell.execute_reply.started": "2025-01-07T14:24:38.495362Z" + } + }, + "outputs": [], + "source": [ + "# %%capture --no-stderr\n", + "\n", + "start = time.time()\n", + "\n", + "adler_out_list = []\n", + "sum_x = 0\n", + "sum_x2 = 0\n", + "sum_N = 0\n", + "\n", + "N_obj = len(df_ad)\n", + "\n", + "with open(\"{}/adler_out_ssoid.txt\".format(adler_out_dir), \"w\") as f:\n", + " f.write(\"ssObjectId\\n\")\n", + "\n", + "for i, ssoid in enumerate(np.array(df_ad[\"ssObjectId\"])):\n", + " # for i,ssoid in enumerate(np.array([-8615240469118203769])):\n", + "\n", + " print(\"{}/{}, {}\".format(i, N_obj, ssoid))\n", + "\n", + " # get just the new observations\n", + " planetoid2 = AdlerPlanetoid.construct_from_RSP(\n", + " ssoid, date_range=[night - time_bounds, night + time_bounds]\n", + " )\n", + " # load AdlerData from database\n", + " planetoid2.AdlerData.populate_from_database(adler_data_db)\n", + " obs_list = [\n", + " pd.DataFrame(planetoid2.observations_in_filter(filt).__dict__) for filt in planetoid2.filter_list\n", + " ]\n", + " df_obs = pd.concat(obs_list)\n", + "\n", + " # for each filter determine residuals\n", + " for filt in planetoid2.filter_list:\n", + "\n", + " sso = planetoid2.SSObject_in_filter(filt)\n", + "\n", + " # get the observations\n", + " obs = planetoid2.observations_in_filter(filt)\n", + " df_obs = pd.DataFrame(obs.__dict__)\n", + " df_obs[\"outlier\"] = [False] * len(df_obs)\n", + "\n", + " # Load the phase curve model available in AdlerData\n", + " adler_data = planetoid2.AdlerData.get_phase_parameters_in_filter(filt, \"HG12_Pen16\")\n", + "\n", + " pc = PhaseCurve().InitModelDict(adler_data.__dict__)\n", + "\n", + " # find outliers in new data\n", + " # calculate data - model residuals\n", + " res = np.array(df_obs[\"reduced_mag\"]) - pc.ReducedMag(np.radians(np.array(df_obs[\"phaseAngle\"])))\n", + " outlier_flag = sci_utils.outlier_diff(res, diff_cut=diff_cut)\n", + " diaId = np.array(df_obs[\"diaSourceId\"])\n", + " # print(diaId)\n", + " # print(res)\n", + " # print(outlier_flag)\n", + " df_obs[\"outlier\"] = outlier_flag\n", + "\n", + " # store the running mean and std of all residuals\n", + " sum_x += res.sum()\n", + " sum_x2 += (res**2.0).sum()\n", + " sum_N += len(res)\n", + "\n", + " # save the df_obs subset with outlier classification?\n", + " # record the diaSourceId of any outliers\n", + " # return the ssSourceId of any objects with possible outliers\n", + "\n", + " if outlier_flag.any():\n", + "\n", + " # adler_out = {}\n", + " # adler_out[\"ssObjectId\"] = planetoid2.ssObjectId\n", + " # adler_out[\"diaSourceId\"] = diaId[outlier_flag]\n", + " # adler_out[\"residual\"] = res[outlier_flag]\n", + "\n", + " # # print(adler_out)\n", + "\n", + " # adler_out_list.append(adler_out)\n", + "\n", + " # with open('{}/{}.json'.format(adler_out_dir,planetoid2.ssObjectId), 'w') as f:\n", + " # json.dump(adler_out, f)\n", + "\n", + " with open(\"{}/adler_out_ssoid.txt\".format(adler_out_dir), \"a\") as f:\n", + " f.write(\"{}\\n\".format(planetoid2.ssObjectId))\n", + "\n", + " break\n", + " # if i>(3-2):\n", + " # break\n", + "\n", + "end = time.time()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "da2bf70e-2aa9-4e3f-acc9-5abca7d56037", + "metadata": { + "execution": { + "iopub.execute_input": "2025-01-07T14:24:40.559287Z", + "iopub.status.busy": "2025-01-07T14:24:40.558998Z", + "iopub.status.idle": "2025-01-07T14:24:40.569209Z", + "shell.execute_reply": "2025-01-07T14:24:40.567786Z", + "shell.execute_reply.started": "2025-01-07T14:24:40.559264Z" + } + }, + "outputs": [], + "source": [ + "\"{} seconds per object\".format((end - start) / (i + 1))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02ae4028-6066-4124-8992-2929eb89854d", + "metadata": { + "execution": { + "iopub.execute_input": "2025-01-07T14:24:40.570599Z", + "iopub.status.busy": "2025-01-07T14:24:40.570307Z", + "iopub.status.idle": "2025-01-07T14:24:40.579259Z", + "shell.execute_reply": "2025-01-07T14:24:40.578106Z", + "shell.execute_reply.started": "2025-01-07T14:24:40.570556Z" + } + }, + "outputs": [], + "source": [ + "# objects with outliers on the night\n", + "adler_out_list" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2809b952-603b-4169-8dea-c9d13f51ad9a", + "metadata": { + "execution": { + "iopub.execute_input": "2025-01-07T14:24:40.580900Z", + "iopub.status.busy": "2025-01-07T14:24:40.580478Z", + "iopub.status.idle": "2025-01-07T14:24:40.591344Z", + "shell.execute_reply": "2025-01-07T14:24:40.590442Z", + "shell.execute_reply.started": "2025-01-07T14:24:40.580862Z" + } + }, + "outputs": [], + "source": [ + "# residual stats\n", + "res_mean, res_std = sci_utils.running_stats(sum_N, sum_x, sum_x2)\n", + "print(\"mean residual = {}, std residual = {}\".format(res_mean, res_std))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1e168f2c-9adf-47ba-b9f1-1a30a882c486", "metadata": {}, "outputs": [], "source": []