diff --git a/doc/changes.rst b/doc/changes.rst index 73d496e..897a0d6 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -8,6 +8,11 @@ Change Log *Planned*: Support loading ongoing daily reductions, in particular, updates to tiles already in the database. +1.2.1 (unreleased) +------------------ + +* Finalize patches of ``tiles-daily`` and ``exposures-daily`` files. + 1.2.0 (2024-09-26) ------------------ diff --git a/doc/conf.py b/doc/conf.py index f4bf071..d2d0aa7 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -20,7 +20,7 @@ project = 'specprod-db' python_project = 'specprodDB' -copyright = '2023, DESI Collaboration' +copyright = '2023-2024, DESI Collaboration' author = 'DESI Collaboration' @@ -35,7 +35,8 @@ 'sphinx.ext.todo', 'sphinx.ext.mathjax', 'sphinx.ext.viewcode', - 'sphinx.ext.napoleon' + 'sphinx.ext.napoleon', + 'sphinx_rtd_theme' ] # Configuration for intersphinx, copied from astropy. @@ -46,9 +47,10 @@ 'matplotlib': ('https://matplotlib.org/stable/', None), 'astropy': ('https://docs.astropy.org/en/stable/', None), 'h5py': ('https://docs.h5py.org/en/latest/', None), - 'sqlalchemy': ('https://docs.sqlalchemy.org/en/14/', None), + 'sqlalchemy': ('https://docs.sqlalchemy.org/en/20/', None), 'desispec': ('https://desispec.readthedocs.io/en/latest/', None), 'desiutil': ('https://desiutil.readthedocs.io/en/latest/', None), + 'desitarget': ('https://desitarget.readthedocs.io/en/latest/', None), } # Add any paths that contain templates here, relative to this directory. @@ -88,13 +90,9 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -# -try: - import sphinx_rtd_theme - html_theme = 'sphinx_rtd_theme' - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] -except ImportError: - html_theme = 'alabaster' +#html_theme = 'default' +#html_theme = 'haiku' +html_theme = 'sphinx_rtd_theme' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/doc/nb/TestPatchDaily.ipynb b/doc/nb/TestPatchDaily.ipynb new file mode 100644 index 0000000..3999801 --- /dev/null +++ b/doc/nb/TestPatchDaily.ipynb @@ -0,0 +1,407 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "31725415-8b51-49d7-a2d6-7100227808d5", + "metadata": {}, + "source": [ + "# Patch daily with jura or kibo" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f220285b-75d0-4109-a5f5-3709bfaeed79", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import datetime\n", + "import pytz\n", + "import numpy as np\n", + "from unittest.mock import patch\n", + "from astropy.table import Table, join\n", + "from astropy.io import fits\n", + "# from desispec.io import read_table\n", + "from desiutil.log import get_logger, DEBUG\n", + "from desispec.io.meta import faflavor2program\n", + "from specprodDB.util import cameraid\n", + "import specprodDB.patch as p" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3d9291ce-ae1b-4196-8d9b-f6069f353828", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "with patch('sys.argv', ['patch_specprod', '--source', 'kibo', '--destination', 'daily', '--overwrite', os.environ['SCRATCH']]):\n", + " options = p.get_options()\n", + "log = get_logger(DEBUG)\n", + "src, dst = p.get_data(options)" + ] + }, + { + "cell_type": "markdown", + "id": "1c9487e7-9b9d-4790-80ac-912104091180", + "metadata": { + "tags": [] + }, + "source": [ + "## QA on SURVEY, PROGRAM" + ] + }, + { + "cell_type": "markdown", + "id": "522008bd-0ee9-4d41-93bc-acc5e25d0e29", + "metadata": {}, + "source": [ + "## Do some QA on patch specprod" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dea48f18-e8f9-4019-a041-7e49938e67bd", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "for c in src['frames'].colnames:\n", + " if hasattr(src['frames'][c], 'mask'):\n", + " print(c)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0f0d2992-50fe-458b-954a-5402d5d7fc6c", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "src['frames']['MJD'].min()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "36e4cecb-64ac-4dc0-ac90-23b9a332a46e", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "src_exposures_bad_rows = list()\n", + "bad_columns = list()\n", + "for c in src['exposures'].colnames:\n", + " if hasattr(src['exposures'][c], 'mask'):\n", + " print(c)\n", + " bad_columns.append(c)\n", + " src_exposures_bad_rows.append(np.where(src['exposures'][c].mask)[0])\n", + " elif src['exposures'][c].dtype.kind == 'f' and not np.isfinite(src['exposures'][c]).all():\n", + " print(c)\n", + " bad_columns.append(c)\n", + " src_exposures_bad_rows.append(np.where(~np.isfinite(src['exposures'][c]))[0])\n", + " else:\n", + " pass\n", + "src_exposures_bad_rows\n", + "# src_exposures_bad_rows = np.unique(np.hstack(src_exposures_bad_rows))\n", + "# src['exposures'][src_exposures_bad_rows]" + ] + }, + { + "cell_type": "markdown", + "id": "27390294-d3ee-4227-9811-1d580cac1a45", + "metadata": {}, + "source": [ + "### All exposures for the tiles that contain bad exposures" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27fc0c27-f5d1-41b9-bf28-f7ae4c1aadae", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "src_exposures_bad_tiles = np.in1d(src['exposures']['TILEID'], src['exposures']['TILEID'][src_exposures_bad_rows])\n", + "src['exposures'][src_exposures_bad_tiles]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c0bc9051-03d4-4d2e-90bc-bfa7503975e4", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "src_tiles_bad_rows = np.in1d(src['tiles']['TILEID'], src['exposures']['TILEID'][src_exposures_bad_rows])\n", + "src['tiles'][src_tiles_bad_rows]" + ] + }, + { + "cell_type": "markdown", + "id": "1b969ba8-784e-40b2-98c5-440a24633c90", + "metadata": {}, + "source": [ + "### Consistency of SURVEY, PROGRAM, etc." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fec068e2-85d6-412c-8557-206f38a9a40d", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "for row in src['tiles']:\n", + " w = np.where(src['exposures']['TILEID'] == row['TILEID'])[0]\n", + " for c in ('SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'GOALTYPE'):\n", + " assert (src['exposures'][c][w] == row[c]).all()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b17dd4e1-5d9f-4f29-b71f-c968050f8ed4", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "for row in src['exposures']:\n", + " w = np.where(src['frames']['EXPID'] == row['EXPID'])[0]\n", + " for c in ('SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'GOALTYPE'):\n", + " if c in src['frames'].colnames:\n", + " assert (src['frames'][c][w] == row[c]).all()" + ] + }, + { + "cell_type": "markdown", + "id": "777a4cde-ab8e-4f24-aa31-323dbf819321", + "metadata": { + "tags": [] + }, + "source": [ + "## Find patch exposures not in daily, daily exposures not in patch" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "74d311b4-d697-4944-945a-d56a68d1a39e", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "assert (np.unique(src['exposures']['EXPID']) == sorted(src['exposures']['EXPID'])).all()\n", + "assert (np.unique(dst['exposures']['EXPID']) == sorted(dst['exposures']['EXPID'])).all()\n", + "assert (np.unique(src['frames']['EXPID']) == sorted(src['exposures']['EXPID'])).all()\n", + "assert (np.unique(dst['frames']['EXPID']) == sorted(dst['exposures']['EXPID'])).all()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ca1d655b-e315-4e14-9fdc-5d31f5dadcb1", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "first_src_exposure, last_src_exposure = src['exposures']['EXPID'].min(), src['exposures']['EXPID'].max()\n", + "first_src_exposure, last_src_exposure" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29c2d15b-3247-4320-bba7-a9b4e9fcc398", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "first_src_night = src['exposures']['NIGHT'][src['exposures']['EXPID'] == first_src_exposure].min()\n", + "last_src_night = src['exposures']['NIGHT'][src['exposures']['EXPID'] == last_src_exposure].max()\n", + "first_src_night, last_src_night" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "51366113-4bfe-46e4-be86-dd4699cd2218", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "src['tiles']['LASTNIGHT'].min(), src['tiles']['LASTNIGHT'].max()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ded2208b-f17e-4c7f-aeff-3e7cca0f77a4", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "src_expid_set = frozenset(src['exposures']['EXPID'].tolist())\n", + "dst_expid_set = frozenset(dst['exposures']['EXPID'].tolist())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "473307c7-30f6-40ca-b297-3b927577d5b1", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "src_not_in_dst = src_expid_set - dst_expid_set\n", + "src_not_in_dst" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "879e049d-3aca-484b-80e3-5b06344077c5", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "dst_not_in_src = dst_expid_set - src_expid_set\n", + "# dst_not_in_src" + ] + }, + { + "cell_type": "markdown", + "id": "96e7d429-3edc-4f7e-9e66-0ce8e3c8dcec", + "metadata": {}, + "source": [ + "## Perform initial patching and QA" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11f82181-9647-42e4-b05d-bd8427d20c9a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "timestamp = datetime.datetime.now(tz=pytz.timezone('US/Pacific'))\n", + "ymd = timestamp.strftime('%Y%m%d')\n", + "patched = dict()\n", + "patched['tiles_file'] = os.path.join(options.output, f'tiles-{options.dst}-patched-with-{options.src}-{ymd}.csv')\n", + "patched['exposures_file'] = os.path.join(options.output, f'exposures-{options.dst}-patched-with-{options.src}-{ymd}.fits')\n", + "patched['frames'] = p.patch_frames(src['frames'], dst['frames'])\n", + "patched['exposures'] = p.patch_exposures(src['exposures'], dst['exposures'])\n", + "patched['frames'] = p.patch_missing_frames_mjd(patched['exposures'], patched['frames'])\n", + "patched['tiles'] = p.patch_tiles(src['tiles'], dst['tiles'], timestamp)\n", + "back_exposures, back_tiles = p.back_patch_inconsistent_values(patched)\n", + "patched['exposures'] = p.patch_exposures_efftime_spec(src['exposures'], patched['exposures'], patched['tiles'])" + ] + }, + { + "cell_type": "markdown", + "id": "7f126e49-1ebc-40a6-a9c7-2d340783047c", + "metadata": {}, + "source": [ + "## Analyze tiles that are candidates for loading\n", + "\n", + "Note, 20201214: Earliest night in `jura`/`kibo`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12b33463-0eaf-4531-b96d-3e8e2d18682c", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "candidate_tiles = patched['tiles'][(patched['tiles']['LASTNIGHT'] >= 20201214) & (patched['tiles']['EFFTIME_SPEC'] > 0)]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "36d1c46a-5d10-45a1-96ee-698030dff465", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "for new_tile in candidate_tiles:\n", + " row_index = np.where((patched['exposures']['TILEID'] == new_tile['TILEID']) & (patched['exposures']['EFFTIME_SPEC'] > 0))[0]\n", + " if len(row_index) == 0:\n", + " print(\"ERROR: No valid exposures found for tile {0:d}, even though EFFTIME_SPEC == {1:f}!\".format(new_tile['TILEID'], new_tile['EFFTIME_SPEC']))\n", + " bad_index = np.where((patched['exposures']['TILEID'] == new_tile['TILEID']))[0]\n", + " print(patched['exposures'][['TILEID', 'EXPID', 'NIGHT', 'SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'EFFTIME_SPEC']][bad_index])\n", + " w = np.in1d(src['exposures']['EXPID'], patched['exposures']['EXPID'][bad_index])\n", + " n_src = w.sum()\n", + " print(src['exposures'][['TILEID', 'EXPID', 'NIGHT', 'SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'EFFTIME_SPEC']][w])\n", + " if n_src == 0:\n", + " print(\"ERROR: Tile {0:d} cannot be patched with upstream data.\".format(new_tile['TILEID']))\n", + " elif n_src == len(bad_index):\n", + " print(\"INFO: Tile {0:d} can be fully patched with upstream data.\".format(new_tile['TILEID']))\n", + " elif n_src < len(bad_index):\n", + " print(\"WARNING: Tile {0:d} can be partially patched with upstream data.\".format(new_tile['TILEID']))\n", + " else:\n", + " print(\"CRITICAL: This is weird.\")\n", + " print('+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e6c7934f-f87e-4567-adb2-4e8f7c1197f6", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "DESI custom", + "language": "python", + "name": "desi-custom" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/doc/nb/TestPatchDailyWithJura.ipynb b/doc/nb/TestPatchDailyWithJura.ipynb deleted file mode 100644 index 7e0bd20..0000000 --- a/doc/nb/TestPatchDailyWithJura.ipynb +++ /dev/null @@ -1,779 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "31725415-8b51-49d7-a2d6-7100227808d5", - "metadata": {}, - "source": [ - "# Patch daily with jura" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f220285b-75d0-4109-a5f5-3709bfaeed79", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "import os\n", - "import numpy as np\n", - "from astropy.table import Table, join\n", - "from astropy.io import fits\n", - "# from desispec.io import read_table\n", - "from desispec.io.meta import faflavor2program\n", - "from specprodDB.util import cameraid" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c2571c97-816c-4b12-a572-8e987c88b654", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "daily_tiles_file = os.path.join(os.environ['DESI_SPECTRO_REDUX'], 'daily', 'tiles-daily.csv')\n", - "daily_exposures_file = os.path.join(os.environ['DESI_SPECTRO_REDUX'], 'daily', 'exposures-daily.fits')\n", - "jura_tiles_file = os.path.join(os.environ['DESI_SPECTRO_REDUX'], 'jura', 'tiles-jura.csv')\n", - "jura_exposures_file = os.path.join(os.environ['DESI_SPECTRO_REDUX'], 'jura', 'exposures-jura.fits')\n", - "daily_tiles = Table.read(daily_tiles_file, format='ascii.csv')\n", - "daily_exposures = Table.read(daily_exposures_file, format='fits', hdu='EXPOSURES')\n", - "# daily_exposures = read_table(daily_exposures_file, ext='EXPOSURES')\n", - "daily_frames = Table.read(daily_exposures_file, format='fits', hdu='FRAMES')\n", - "# daily_frames = read_table(daily_exposures_file, ext='FRAMES')\n", - "jura_tiles = Table.read(jura_tiles_file, format='ascii.csv')\n", - "jura_exposures = Table.read(jura_exposures_file, format='fits', hdu='EXPOSURES')\n", - "# jura_exposures = read_table(jura_exposures_file, ext='EXPOSURES')\n", - "jura_frames = Table.read(jura_exposures_file, format='fits', hdu='FRAMES')\n", - "# jura_frames = read_table(jura_exposures_file, ext='FRAMES')" - ] - }, - { - "cell_type": "markdown", - "id": "522008bd-0ee9-4d41-93bc-acc5e25d0e29", - "metadata": {}, - "source": [ - "## Do some QA on jura" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dea48f18-e8f9-4019-a041-7e49938e67bd", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "for c in jura_frames.colnames:\n", - " if hasattr(jura_frames[c], 'mask'):\n", - " print(c)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0f0d2992-50fe-458b-954a-5402d5d7fc6c", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "jura_exposures['MJD'].min()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "36e4cecb-64ac-4dc0-ac90-23b9a332a46e", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "jura_exposures_bad_rows = list()\n", - "bad_columns = list()\n", - "for c in jura_exposures.colnames:\n", - " if hasattr(jura_exposures[c], 'mask'):\n", - " print(c)\n", - " bad_columns.append(c)\n", - " jura_exposures_bad_rows.append(np.where(jura_exposures[c].mask)[0])\n", - " elif jura_exposures[c].dtype.kind == 'f' and not np.isfinite(jura_exposures[c]).all():\n", - " print(c)\n", - " bad_columns.append(c)\n", - " jura_exposures_bad_rows.append(np.where(~np.isfinite(jura_exposures[c]))[0])\n", - " else:\n", - " pass\n", - "jura_exposures_bad_rows = np.unique(np.hstack(jura_exposures_bad_rows))\n", - "jura_exposures[jura_exposures_bad_rows]" - ] - }, - { - "cell_type": "markdown", - "id": "27390294-d3ee-4227-9811-1d580cac1a45", - "metadata": {}, - "source": [ - "### All exposures for the tiles that contain bad exposures" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "27fc0c27-f5d1-41b9-bf28-f7ae4c1aadae", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "jura_exposures_bad_tiles = np.in1d(jura_exposures['TILEID'], jura_exposures[jura_exposures_bad_rows]['TILEID'])\n", - "jura_exposures[jura_exposures_bad_tiles]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c0bc9051-03d4-4d2e-90bc-bfa7503975e4", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "jura_tiles_bad_rows = np.in1d(jura_tiles['TILEID'], jura_exposures[jura_exposures_bad_rows]['TILEID'])\n", - "jura_tiles[jura_tiles_bad_rows]" - ] - }, - { - "cell_type": "markdown", - "id": "777a4cde-ab8e-4f24-aa31-323dbf819321", - "metadata": { - "tags": [] - }, - "source": [ - "## Find jura exposures not in daily, daily exposures not in jura" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "74d311b4-d697-4944-945a-d56a68d1a39e", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "assert (np.unique(jura_exposures['EXPID']) == sorted(jura_exposures['EXPID'])).all()\n", - "assert (np.unique(daily_exposures['EXPID']) == sorted(daily_exposures['EXPID'])).all()\n", - "assert (np.unique(jura_frames['EXPID']) == sorted(jura_exposures['EXPID'])).all()\n", - "assert (np.unique(daily_frames['EXPID']) == sorted(daily_exposures['EXPID'])).all()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ca1d655b-e315-4e14-9fdc-5d31f5dadcb1", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "first_jura_exposure, last_jura_exposure = jura_exposures['EXPID'].min(), jura_exposures['EXPID'].max()\n", - "first_jura_exposure, last_jura_exposure" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "29c2d15b-3247-4320-bba7-a9b4e9fcc398", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "first_jura_night = jura_exposures['NIGHT'][jura_exposures['EXPID'] == first_jura_exposure].min()\n", - "last_jura_night = jura_exposures['NIGHT'][jura_exposures['EXPID'] == last_jura_exposure].max()\n", - "first_jura_night, last_jura_night" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "51366113-4bfe-46e4-be86-dd4699cd2218", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "jura_tiles['LASTNIGHT'].min(), jura_tiles['LASTNIGHT'].max()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ded2208b-f17e-4c7f-aeff-3e7cca0f77a4", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "jura_expid_set = frozenset(jura_exposures['EXPID'].tolist())\n", - "daily_expid_set = frozenset(daily_exposures['EXPID'].tolist())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "473307c7-30f6-40ca-b297-3b927577d5b1", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "jura_not_in_daily = jura_expid_set - daily_expid_set\n", - "jura_not_in_daily" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "879e049d-3aca-484b-80e3-5b06344077c5", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "daily_not_in_jura = daily_expid_set - jura_expid_set\n", - "# daily_not_in_jura" - ] - }, - { - "cell_type": "markdown", - "id": "96e7d429-3edc-4f7e-9e66-0ce8e3c8dcec", - "metadata": {}, - "source": [ - "## Patch frames\n", - "\n", - "We don't necessarily want to change the values in the `FRAMES` table, just make sure it is consistent with the `EXPOSURES` table." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "11f82181-9647-42e4-b05d-bd8427d20c9a", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "daily_frames_join = Table()\n", - "daily_frames_join['FRAMEID'] = np.array([100*row['EXPID'] + cameraid(row['CAMERA']) for row in daily_frames])\n", - "daily_frames_join['DAILY_INDEX'] = np.arange(len(daily_frames))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "333dbff1-65b2-4e66-a35f-b1d2b34a05c4", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "jura_frames_join = Table()\n", - "jura_frames_join['FRAMEID'] = np.array([100*row['EXPID'] + cameraid(row['CAMERA']) for row in jura_frames])\n", - "jura_frames_join['JURA_INDEX'] = np.arange(len(jura_frames))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8d6ca37d-2350-47b6-8e90-5ab8b0cc01ad", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "j = join(daily_frames_join, jura_frames_join, join_type='outer', keys='FRAMEID')\n", - "# j" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "32431f66-d899-490e-98ad-4d9c299f74d7", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "daily_frames_index = j[(~j['JURA_INDEX'].mask) & (~j['DAILY_INDEX'].mask)]['DAILY_INDEX']\n", - "jura_frames_index = j[(~j['JURA_INDEX'].mask) & (~j['DAILY_INDEX'].mask)]['JURA_INDEX']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "31e94315-0a93-454f-b228-7c10e14c22b1", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "daily_frames_patched = daily_frames.copy()\n", - "for column in daily_frames_patched.colnames:\n", - " if hasattr(daily_frames_patched[column], 'mask') and not column.startswith('TSNR2_'):\n", - " if np.any(daily_frames_patched[column].mask[daily_frames_index]):\n", - " print(\"Patching {0:d} rows in dst_frames column {1}.\".format(np.sum(daily_frames_patched[column].mask[daily_frames_index]), column))\n", - " jura_frames_matched = jura_frames[column][jura_frames_index]\n", - " daily_frames_matched = daily_frames_patched[column][daily_frames_index]\n", - " daily_frames_mask_matched = daily_frames_patched[column].mask[daily_frames_index]\n", - " assert np.sum(daily_frames_mask_matched) == np.sum(daily_frames_patched[column].mask[daily_frames_index])\n", - " daily_frames_matched[daily_frames_mask_matched] = jura_frames_matched[daily_frames_mask_matched]\n", - " daily_frames_matched.mask[daily_frames_mask_matched] = False\n", - " daily_frames_patched[column][daily_frames_index] = daily_frames_matched\n", - " daily_frames_patched[column].mask[daily_frames_index] = daily_frames_matched.mask\n", - " # print(type(daily_frames_patched[column].data.data))\n", - " assert not (daily_frames_patched[column].data.data == daily_frames[column].data.data).all()\n", - "daily_frames_patched[daily_frames['SEEING_ETC'].mask]" - ] - }, - { - "cell_type": "markdown", - "id": "ca7c8da8-2c0b-45df-ab15-6f8aac0c9c6a", - "metadata": {}, - "source": [ - "## Patch exposures\n", - "\n", - "We want to only *patch* and rows that:\n", - "\n", - "* Appear in `jura`.\n", - "* Have `NIGHT >= first_jura_night`.\n", - "* Have `EFFTIME_SPEC > 0`.\n", - "\n", - "We want to only *load* rows that:\n", - "\n", - "* Have `NIGHT >= first_jura_night`.\n", - "* Have `EFFTIME_SPEC > 0`.\n", - "\n", - "which is slightly different. However we don't want to *remove* rows that *don't* satisfy these criteria." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e800b0bf-3824-4b78-94e5-e1f3fdb1e9ef", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "daily_exposures_join = Table()\n", - "daily_exposures_join['EXPID'] = daily_exposures['EXPID']\n", - "# daily_exposures_join['NIGHT'] = daily_exposures['NIGHT']\n", - "# daily_exposures_join['EFFTIME_SPEC'] = daily_exposures['EFFTIME_SPEC']\n", - "daily_exposures_join['DAILY_INDEX'] = np.arange(len(daily_exposures))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "933274c3-bc01-4287-ba1f-bf138d22c901", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "jura_exposures_join = Table()\n", - "jura_exposures_join['EXPID'] = jura_exposures['EXPID']\n", - "jura_exposures_join['JURA_INDEX'] = np.arange(len(jura_exposures))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6fec01f6-f4b2-443b-9763-c663375bae25", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "j = join(daily_exposures_join, jura_exposures_join, join_type='outer', keys='EXPID')\n", - "j" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "83004567-d880-4ba6-968d-0555662ef9a2", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "daily_exposures_index = j[(~j['JURA_INDEX'].mask) & (~j['DAILY_INDEX'].mask)]['DAILY_INDEX']\n", - "jura_exposures_index = j[(~j['JURA_INDEX'].mask) & (~j['DAILY_INDEX'].mask)]['JURA_INDEX']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a63345ad-5660-4f46-9940-60f13370312a", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "daily_exposures_patched = daily_exposures.copy()\n", - "can_patch = ('NIGHT', 'EXPID', 'TILEID', 'TILERA', 'TILEDEC', 'MJD', 'SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'EXPTIME', 'GOALTIME', 'GOALTYPE', 'MINTFRAC', 'AIRMASS', 'EBV', 'SEEING_ETC', 'EFFTIME_ETC',\n", - " 'TRANSPARENCY_GFA', 'SEEING_GFA', 'FIBER_FRACFLUX_GFA', 'FIBER_FRACFLUX_ELG_GFA', 'FIBER_FRACFLUX_BGS_GFA', 'FIBERFAC_GFA', 'FIBERFAC_ELG_GFA', 'FIBERFAC_BGS_GFA', 'AIRMASS_GFA', 'SKY_MAG_AB_GFA',\n", - " 'EFFTIME_GFA', 'EFFTIME_DARK_GFA', 'EFFTIME_BRIGHT_GFA', 'EFFTIME_BACKUP_GFA')\n", - "for column in ['TILERA', 'TILEDEC', 'MJD', 'SURVEY'] + [c for c in daily_exposures_patched.colnames if hasattr(daily_exposures_patched[c], 'mask') and c in can_patch]:\n", - " print(f\"Patching {column}...\")\n", - " if hasattr(jura_exposures[column], 'mask'):\n", - " if np.any(jura_exposures[column].mask[jura_exposures_index]):\n", - " jura_exposures[column][jura_exposures[column].mask] = 0\n", - " jura_exposures[column].mask[jura_exposures[column].mask] = False\n", - " daily_exposures_patched[column][daily_exposures_index] = jura_exposures[column][jura_exposures_index]\n", - " if hasattr(daily_exposures_patched[column], 'mask'):\n", - " daily_exposures_patched[column].mask[daily_exposures_index] = False\n", - "\n", - "known_bad_exposures = np.in1d(daily_exposures_patched['EXPID'], np.array([74307, 79769, 83420, 110852, 190752]))\n", - "daily_exposures_patched[known_bad_exposures]" - ] - }, - { - "cell_type": "markdown", - "id": "2cad2268-9557-444e-897c-e04bc63b42c6", - "metadata": {}, - "source": [ - "### After patching are there still missing data?" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "00e2e401-93f6-4d87-8307-9c1f0db767e2", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "assert not (daily_exposures_patched['TILERA'] == daily_exposures['TILERA']).all()\n", - "assert not (daily_exposures_patched['TILEDEC'] == daily_exposures['TILEDEC']).all()\n", - "assert not (daily_exposures_patched['MJD'] == daily_exposures['MJD']).all()\n", - "assert not (daily_exposures_patched['SURVEY'] == daily_exposures['SURVEY']).all()\n", - "assert (daily_exposures_patched['PROGRAM'] == daily_exposures['PROGRAM']).all()\n", - "assert (daily_exposures_patched['FAPRGRM'] == daily_exposures['FAPRGRM']).all()\n", - "assert (daily_exposures_patched['FAFLAVOR'] == daily_exposures['FAFLAVOR']).all()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "60427b0b-913d-456b-b45e-6ccc3a46866b", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "missing_mjd = np.where((daily_exposures_patched['NIGHT'] >= first_jura_night) & (daily_exposures_patched['EFFTIME_SPEC'] > 0) & (daily_exposures_patched['MJD'] < 50000))[0]\n", - "for row in daily_exposures_patched[missing_mjd]:\n", - " raw_data_file = os.path.join(os.environ['DESI_SPECTRO_DATA'], \"{0:08d}\".format(row['NIGHT']), \"{0:08d}\".format(row['EXPID']), \"desi-{0:08d}.fits.fz\".format(row['EXPID']))\n", - " with fits.open(raw_data_file, mode='readonly') as hdulist:\n", - " mjd_obs = hdulist['SPEC'].header['MJD-OBS']\n", - " print(\"INFO: tile {0:d} exposure {1:d} has MJD-OBS = {2:f} in {3}!\".format(row['TILEID'], row['EXPID'], mjd_obs, raw_data_file))\n", - " w = np.where(daily_exposures_patched['EXPID'] == row['EXPID'])[0]\n", - " assert len(w) == 1\n", - " daily_exposures_patched['MJD'][w] = mjd_obs\n", - " print(daily_exposures_patched[w][['NIGHT', 'EXPID', 'TILEID', 'MJD']])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fdc2baa0-8cd9-4fbf-8c5f-ec83a931a72b", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "still_missing_mjd = np.where((daily_exposures_patched['NIGHT'] >= first_jura_night) & (daily_exposures_patched['EFFTIME_SPEC'] > 0) & (daily_exposures_patched['MJD'] < 50000))[0]\n", - "daily_exposures_patched[still_missing_mjd]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "be0fd08b-0f73-4bf5-a1ee-a955ace6c3ae", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "possible_expid_still_patchable = list()\n", - "for c in daily_exposures_patched.colnames:\n", - " if hasattr(daily_exposures_patched[c], 'mask'):\n", - " if daily_exposures_patched[c].mask.any():\n", - " n_masked = daily_exposures_patched[c].mask.sum()\n", - " not_in_jura = [e in daily_not_in_jura for e in daily_exposures_patched['EXPID'][daily_exposures_patched[c].mask]]\n", - " if all(not_in_jura):\n", - " print(f\"Column {c} still has {n_masked:d} masked values, but the exposures are not in jura and cannot be patched.\")\n", - " else:\n", - " if c in can_patch:\n", - " for e in daily_exposures_patched['EXPID'][daily_exposures_patched[c].mask]:\n", - " if e not in daily_not_in_jura:\n", - " possible_expid_still_patchable.append(int(e))\n", - " print(f\"Column {c} still has {n_masked:d} masked values, and some exposures can still be patched.\")\n", - " else:\n", - " print(f\"Column {c} still has {n_masked:d} masked values, and some exposures could still be patched, but they do not meet the patchable column guidelines.\")\n", - "\n", - "possible_expid_still_patchable = np.unique(np.array(possible_expid_still_patchable))\n", - "daily_exposures_patched[np.in1d(daily_exposures_patched['EXPID'], possible_expid_still_patchable)]" - ] - }, - { - "cell_type": "markdown", - "id": "ed22c786-df96-47f7-b167-fac29e15d8c4", - "metadata": {}, - "source": [ - "### Fill remaining masked values with zero." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9b8e3874-fbf7-4fce-b31f-0c31218dc6c5", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "for c in daily_exposures_patched.colnames:\n", - " if hasattr(daily_exposures_patched[c], 'mask'):\n", - " if daily_exposures_patched[c].mask.any():\n", - " daily_exposures_patched[c][daily_exposures_patched[c].mask] = 0\n", - " daily_exposures_patched[c].mask[daily_exposures_patched[c].mask] = False" - ] - }, - { - "cell_type": "markdown", - "id": "f708faa2-36c4-4b2d-9a93-1c5b76b415d1", - "metadata": {}, - "source": [ - "## Patch tiles\n", - "\n", - "Similar to the discussion above, we want `LASTNIGHT >= first_jura_night`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "76fd512d-ee77-4e0f-b6c1-3ba5eb5d77c0", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "assert (np.unique(jura_exposures['TILEID']) == sorted(jura_tiles['TILEID'])).all()\n", - "assert (np.unique(daily_exposures['TILEID']) == sorted(daily_tiles['TILEID'])).all()" - ] - }, - { - "cell_type": "markdown", - "id": "17112d1a-97dc-4dec-9b71-605c6c381896", - "metadata": {}, - "source": [ - "### First patch PROGRAM with faflavor2program()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "de4595e0-6549-44aa-a2b2-52462f6ec28d", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "daily_tiles_patched = daily_tiles.copy()\n", - "daily_tiles_patched['PROGRAM'] = faflavor2program(daily_tiles_patched['FAFLAVOR'])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "71a87a7e-10b9-4287-8cdd-9d65578471af", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "oddball_survey = np.where((daily_tiles_patched['SURVEY'] != 'cmx') & (daily_tiles_patched['SURVEY'] != 'sv1') & (daily_tiles_patched['SURVEY'] != 'sv2') & (daily_tiles_patched['SURVEY'] != 'sv3') & (daily_tiles_patched['SURVEY'] != 'main') & (daily_tiles_patched['SURVEY'] != 'special'))[0]\n", - "oddball_program = np.where((daily_tiles_patched['PROGRAM'] != 'backup') & (daily_tiles_patched['PROGRAM'] != 'bright') & (daily_tiles_patched['PROGRAM'] != 'dark') & (daily_tiles_patched['PROGRAM'] != 'other'))[0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fb685210-58eb-47d6-a17b-6fda9c9a9739", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "assert (daily_tiles['SURVEY'][oddball_survey] == 'unknown').all()\n", - "assert len(oddball_program) == 0" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3d9f64c2-d3a9-48f1-a4c9-70577f354c1c", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "unknown_cmx = np.where((daily_tiles['SURVEY'] == 'unknown') & (daily_tiles['FAFLAVOR'] != 'unknown'))[0]\n", - "print(daily_tiles[['TILEID', 'SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'GOALTYPE', 'LASTNIGHT']][unknown_cmx])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "eb5eaf4f-a295-47b0-bfe5-e388fecedda5", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "unknown_unknown = np.where((daily_tiles['SURVEY'] == 'unknown') & (daily_tiles['FAFLAVOR'] == 'unknown'))[0]\n", - "daily_tiles['LASTNIGHT'][unknown_unknown].max()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4aafa0e4-64c5-4344-99e5-7b87fb008612", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "print(daily_tiles[['TILEID', 'SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'GOALTYPE', 'LASTNIGHT']][unknown_unknown])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a3029d17-9611-4afc-8346-0404be3d2954", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# For now just patch with 'cmx'\n", - "daily_tiles['SURVEY'][oddball_survey] = 'cmx'" - ] - }, - { - "cell_type": "markdown", - "id": "27c9cc2d-c9fc-4393-964b-ce7296814590", - "metadata": {}, - "source": [ - "## Write out the patched files" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ac5fdfb7-4635-4e5e-ba2d-b77dd9603554", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "daily_tiles_patched.write(os.path.join(os.environ['DESI_ROOT'], 'users', os.environ['USER'], 'tiles-daily-patched-with-jura.csv'), format='ascii.csv', overwrite=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "384e6030-fcc1-453e-b990-a34688f12494", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "daily_exposures_patched.write(os.path.join(os.environ['DESI_ROOT'], 'users', os.environ['USER'], 'exposures-daily-patched-with-jura.csv'), format='ascii.csv', overwrite=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "26d7bad1-f89f-4cde-bfa1-a0cbcabcf866", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "daily_exposures_fits = fits.HDUList([fits.PrimaryHDU(), fits.table_to_hdu(daily_exposures_patched), fits.table_to_hdu(daily_frames_patched)])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d06fcfd7-c5d9-4d9c-9989-eb313c70bc07", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "daily_exposures_fits.info()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d4e5751a-895b-4ca7-8c44-994a4d7d7524", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "daily_exposures_fits.writeto(os.path.join(os.environ['DESI_ROOT'], 'users', os.environ['USER'], 'exposures-daily-patched-with-jura.fits'), overwrite=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1f2a17e0-074c-433d-9b45-a35d91cc6a93", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "DESI custom", - "language": "python", - "name": "desi-custom" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/py/specprodDB/patch.py b/py/specprodDB/patch.py index 9ccdd0a..c94e472 100644 --- a/py/specprodDB/patch.py +++ b/py/specprodDB/patch.py @@ -133,7 +133,7 @@ def patch_exposures(src_exposures, dst_exposures, first_night=None): Parameters ---------- src_exposures : :class:`~astropy.table.Table` - Source of tiles data. + Source of exposures data. dst_exposures : :class:`~astropy.table.Table` Data to be patched. first_night : :class:`int`, optional @@ -355,6 +355,18 @@ def patch_tiles(src_tiles, dst_tiles, timestamp): assert len(oddball_program) == 0 dst_tiles_patched['SURVEY'][oddball_survey] = 'cmx' # + # Patch GOALTYPE. + # + dst_tiles_patched['GOALTYPE'][dst_tiles_patched['TILEID'] == 83004] = 'dark' + dst_tiles_patched['GOALTYPE'][dst_tiles_patched['TILEID'] == 83024] = 'bright' + oddball_goaltype = np.where(dst_tiles_patched['GOALTYPE'] == 'unknown')[0] + dst_tiles_patched['GOALTYPE'][oddball_goaltype] = 'other' + # + # Patch EFFTIME_SPEC. + # + dst_tiles_patched['EFFTIME_SPEC'][dst_tiles_patched['TILEID'] == 1825] = 0 + dst_tiles_patched['EFFTIME_SPEC'][dst_tiles_patched['TILEID'] == 21273] = 0 + # # Add UPDATED. # dst_tiles_patched['UPDATED'] = np.array([timestamp.strftime("%Y-%m-%dT%H:%M:%S%z")]*len(dst_tiles_patched)) @@ -367,6 +379,97 @@ def patch_tiles(src_tiles, dst_tiles, timestamp): return dst_tiles_patched +def back_patch_inconsistent_values(patched): + """When the primary round of patching is done, copy some values back + into the exposures and frames files. + + Parameters + ---------- + patched : :class:`dict` + A dictionary containing tables for further patching. + + Returns + ------- + :class:`tuple` + A tuple containing the back-patched exposures and frames tables. + Not strictly necessary as this function will modify the tables in + `patched` in-place. + """ + log = get_logger() + back_patch = {'tiles': 'exposures', 'exposures': 'frames'} + for s, d in back_patch.items(): + for row in patched[s]: + key = 'TILEID' if s == 'tiles' else 'EXPID' + w = np.where(patched[d][key] == row[key])[0] + for column in ('SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'GOALTYPE'): + if column in patched[d].colnames: + if (patched[d][column][w] != row[column]).any(): + log.info("Patching %s associated with %s %d with %s = '%s'.", + d, ('tile' if s == 'tiles' else 'exposure'), row[key], + column, row[column]) + patched[d][column][w] = row[column] + # + # Run a QA step. + # + for s, d in back_patch.items(): + for row in patched[s]: + key = 'TILEID' if s == 'tiles' else 'EXPID' + w = np.where(patched[d][key] == row[key])[0] + for column in ('SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'GOALTYPE'): + if column in patched[d].colnames: + assert (patched[d][column][w] == row[column]).all() + return (patched['exposures'], patched['frames']) + + +def patch_exposures_efftime_spec(src_exposures, dst_exposures, dst_tiles): + """Patch exposures that have ``EFFTIME_SPEC == 0`` where the corresponding + tile has ``EFFTIME_SPEC > 0``. + + Parameters + ---------- + src_exposures : :class:`~astropy.table.Table` + Source of exposures data. + dst_exposures : :class:`~astropy.table.Table` + Data to be patched. + dst_tiles : :class:`~astropy.table.Table` + Tiles data for comparison. Should not be modified. + + Returns + ------- + :class:`~astropy.table.Table` + A *copy* of `dst_exposures` with data replaced from `src_exposures`. + """ + log = get_logger() + dst_exposures_patched = dst_exposures.copy() + candidate_tiles = dst_tiles[(dst_tiles['LASTNIGHT'] >= 20201214) & + (dst_tiles['EFFTIME_SPEC'] > 0)] + for t in candidate_tiles: + row_index = np.where((dst_exposures_patched['TILEID'] == t['TILEID']) & + (dst_exposures_patched['EFFTIME_SPEC'] > 0))[0] + if len(row_index) == 0: + log.error("No valid exposures found for tile %d, even though EFFTIME_SPEC == %.2f!", + t['TILEID'], t['EFFTIME_SPEC']) + bad_index = np.where((dst_exposures_patched['TILEID'] == t['TILEID']))[0] + w = np.in1d(src_exposures['EXPID'], dst_exposures_patched['EXPID'][bad_index]) + n_src = w.sum() + can_patch = False + if n_src == 0: + log.error("Tile %d cannot be patched with upstream data.", t['TILEID']) + elif n_src == len(bad_index): + log.info("Tile %d can be fully patched with upstream data.", t['TILEID']) + can_patch = True + elif n_src < len(bad_index): + log.warning("Tile %d can be partially patched with upstream data.", t['TILEID']) + can_patch = True + else: + log.critical("Should not actually reach this point. This is weird.") + if can_patch: + for row in src_exposures[w]: + ww = dst_exposures_patched['EXPID'] == row['EXPID'] + dst_exposures_patched['EFFTIME_SPEC'][ww] = row['EFFTIME_SPEC'] + return dst_exposures_patched + + def get_data(options): """Read in source and destination data. @@ -458,6 +561,8 @@ def main(): patched['exposures'] = patch_exposures(src['exposures'], dst['exposures']) patched['frames'] = patch_missing_frames_mjd(patched['exposures'], patched['frames']) patched['tiles'] = patch_tiles(src['tiles'], dst['tiles'], timestamp) + back_exposures, back_tiles = back_patch_inconsistent_values(patched) + patched['exposures'] = patch_exposures_efftime_spec(src['exposures'], patched['exposures'], patched['tiles']) # # Write out data. #